Commit
add debate cost estimator with tests
commit
b6c1920
2 changed files with +89 and −0
Jump to a changed file
- src/lib/cost-estimate.test.ts +37 −0
- src/lib/cost-estimate.ts +52 −0
added src/lib/cost-estimate.test.ts +37 −0
| @@ -0,0 +1,37 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import { estimateDebateCostUsd, type TokenPrice } from './cost-estimate'; | |
| 3 | + | |
| 4 | +const price = (): TokenPrice => ({ prompt: 1e-6, completion: 3e-6 }); | |
| 5 | + | |
| 6 | +describe('estimateDebateCostUsd', () => { | |
| 7 | + const base = { | |
| 8 | + councilModels: ['a/x', 'b/y', 'c/z'], | |
| 9 | + chairmanModel: 'd/chair', | |
| 10 | + convergenceModel: 'e/conv', | |
| 11 | + price, | |
| 12 | + }; | |
| 13 | + | |
| 14 | + it('is positive and grows with rounds', () => { | |
| 15 | + const r1 = estimateDebateCostUsd({ ...base, rounds: 1 }); | |
| 16 | + const r3 = estimateDebateCostUsd({ ...base, rounds: 3 }); | |
| 17 | + expect(r1).toBeGreaterThan(0); | |
| 18 | + expect(r3).toBeGreaterThan(r1); | |
| 19 | + }); | |
| 20 | + | |
| 21 | + it('grows with council size', () => { | |
| 22 | + const small = estimateDebateCostUsd({ ...base, councilModels: ['a/x', 'b/y', 'c/z'], rounds: 2 }); | |
| 23 | + const big = estimateDebateCostUsd({ ...base, councilModels: ['a/x', 'b/y', 'c/z', 'd/w', 'e/v'], rounds: 2 }); | |
| 24 | + expect(big).toBeGreaterThan(small); | |
| 25 | + }); | |
| 26 | + | |
| 27 | + it('falls back to a default price for unknown models', () => { | |
| 28 | + const withFallback = estimateDebateCostUsd({ ...base, rounds: 1, price: () => undefined }); | |
| 29 | + expect(withFallback).toBeGreaterThan(0); | |
| 30 | + }); | |
| 31 | + | |
| 32 | + it('uses more expensive models to produce higher estimates', () => { | |
| 33 | + const cheap = estimateDebateCostUsd({ ...base, rounds: 2, price: () => ({ prompt: 1e-7, completion: 4e-7 }) }); | |
| 34 | + const dear = estimateDebateCostUsd({ ...base, rounds: 2, price: () => ({ prompt: 3e-6, completion: 1.5e-5 }) }); | |
| 35 | + expect(dear).toBeGreaterThan(cheap); | |
| 36 | + }); | |
| 37 | +}); |
added src/lib/cost-estimate.ts +52 −0
| @@ -0,0 +1,52 @@ | ||
| 1 | +/** | |
| 2 | + * Rough upfront cost estimate for a debate, so the builder can warn before a | |
| 3 | + * user spends real credit. Uses typical per-stage token counts; the actual cost | |
| 4 | + * is usually lower because convergence can stop the debate early. | |
| 5 | + */ | |
| 6 | +export interface TokenPrice { | |
| 7 | + /** USD per prompt token. */ | |
| 8 | + prompt: number; | |
| 9 | + /** USD per completion token. */ | |
| 10 | + completion: number; | |
| 11 | +} | |
| 12 | + | |
| 13 | +type Stage = 'answer' | 'critique' | 'revision' | 'convergence' | 'synthesis'; | |
| 14 | + | |
| 15 | +const AVG: Record<Stage, { prompt: number; completion: number }> = { | |
| 16 | + answer: { prompt: 250, completion: 320 }, | |
| 17 | + critique: { prompt: 850, completion: 260 }, | |
| 18 | + revision: { prompt: 950, completion: 320 }, | |
| 19 | + convergence: { prompt: 1100, completion: 160 }, | |
| 20 | + synthesis: { prompt: 1300, completion: 420 }, | |
| 21 | +}; | |
| 22 | + | |
| 23 | +const FALLBACK: TokenPrice = { prompt: 1e-6, completion: 3e-6 }; | |
| 24 | + | |
| 25 | +export interface EstimateInput { | |
| 26 | + councilModels: string[]; | |
| 27 | + chairmanModel: string; | |
| 28 | + convergenceModel: string; | |
| 29 | + /** Maximum number of critique->revision rounds. */ | |
| 30 | + rounds: number; | |
| 31 | + price: (modelId: string) => TokenPrice | undefined; | |
| 32 | +} | |
| 33 | + | |
| 34 | +/** Estimated *maximum* USD cost (all rounds run, no early stop). */ | |
| 35 | +export function estimateDebateCostUsd(input: EstimateInput): number { | |
| 36 | + const cost = (modelId: string, stage: Stage) => { | |
| 37 | + const p = input.price(modelId) ?? FALLBACK; | |
| 38 | + return AVG[stage].prompt * p.prompt + AVG[stage].completion * p.completion; | |
| 39 | + }; | |
| 40 | + | |
| 41 | + let total = 0; | |
| 42 | + for (const m of input.councilModels) total += cost(m, 'answer'); | |
| 43 | + for (let r = 0; r < input.rounds; r++) { | |
| 44 | + for (const m of input.councilModels) { | |
| 45 | + total += cost(m, 'critique'); | |
| 46 | + total += cost(m, 'revision'); | |
| 47 | + } | |
| 48 | + total += cost(input.convergenceModel, 'convergence'); | |
| 49 | + } | |
| 50 | + total += cost(input.chairmanModel, 'synthesis'); | |
| 51 | + return total; | |
| 52 | +} |