demo-fixtures.ts
10,944 bytes
| 1 | /** |
|---|---|
| 2 | * Hand-authored, realistic demo debates. |
| 3 | * |
| 4 | * The deterministic mock client is great for tests but produces near-identical |
| 5 | * text for every model, which reads fake in the showcase. These fixtures are |
| 6 | * written by hand so each model has a genuinely different answer, the critiques |
| 7 | * are specific with varied scores, the revisions actually change, and the |
| 8 | * chairman delivers a real synthesized verdict plus an honest dissent. |
| 9 | * |
| 10 | * `buildDebateResult` expands a compact spec into the exact `DebateResult` shape |
| 11 | * a live debate produces, and `resultToStageEvents` turns it into the same |
| 12 | * events the persistence layer stores, so the seed writes byte-identical rows |
| 13 | * and the demo replays through the real UI. |
| 14 | */ |
| 15 | import type { DebateEvent } from '@/core/events'; |
| 16 | import { |
| 17 | displayNameForModel, |
| 18 | emptyUsage, |
| 19 | type AnswerRecord, |
| 20 | type ConvergenceRecord, |
| 21 | type CritiqueRecord, |
| 22 | type DebateConfig, |
| 23 | type DebateResult, |
| 24 | type Participant, |
| 25 | type PeerReview, |
| 26 | type ProvenanceRecord, |
| 27 | type RevisionRecord, |
| 28 | type RoundRecord, |
| 29 | type SynthesisRecord, |
| 30 | type Usage, |
| 31 | } from '@/core/types'; |
| 32 | import { sumUsage } from '@/core/usage'; |
| 33 | |
| 34 | // --- authoring format ------------------------------------------------------ |
| 35 | |
| 36 | interface FixtureReview { |
| 37 | p: number; // target participant index |
| 38 | score: number; |
| 39 | strengths: string[]; |
| 40 | weaknesses: string[]; |
| 41 | note: string; |
| 42 | } |
| 43 | interface FixtureCritique { |
| 44 | by: number; // reviewer participant index |
| 45 | on: FixtureReview[]; |
| 46 | } |
| 47 | interface FixtureRevision { |
| 48 | p: number; |
| 49 | content: string; |
| 50 | changed: boolean; |
| 51 | summary: string; |
| 52 | bullets?: string[]; |
| 53 | } |
| 54 | interface FixtureConvergence { |
| 55 | score: number; |
| 56 | disagreements: { topic: string; summary: string; positions: { p: number; stance: string }[] }[]; |
| 57 | } |
| 58 | interface FixtureRound { |
| 59 | critiques: FixtureCritique[]; |
| 60 | revisions: FixtureRevision[]; |
| 61 | convergence: FixtureConvergence; |
| 62 | } |
| 63 | export interface FixtureSpec { |
| 64 | question: string; |
| 65 | models: string[]; |
| 66 | chairmanModel: string; |
| 67 | convergenceModel: string; |
| 68 | convergenceThreshold?: number; |
| 69 | answers: string[]; |
| 70 | rounds: FixtureRound[]; |
| 71 | finalAnswer: string; |
| 72 | dissent: { topic: string; positions: { p: number; text: string }[] }[]; |
| 73 | /** Claim-provenance audit of the final answer; empty supportedBy = chairman addition. */ |
| 74 | provenance?: { text: string; supportedBy: number[]; contestedBy?: number[] }[]; |
| 75 | } |
| 76 | |
| 77 | // --- cost model ------------------------------------------------------------ |
| 78 | |
| 79 | const PRICE: Record<string, { p: number; c: number }> = { |
| 80 | 'openai/gpt-4o': { p: 2.5e-6, c: 1e-5 }, |
| 81 | 'openai/gpt-4o-mini': { p: 1.5e-7, c: 6e-7 }, |
| 82 | 'anthropic/claude-3.5-sonnet': { p: 3e-6, c: 1.5e-5 }, |
| 83 | 'google/gemini-pro-1.5': { p: 1.25e-6, c: 5e-6 }, |
| 84 | 'google/gemini-2.0-flash-001': { p: 1e-7, c: 4e-7 }, |
| 85 | 'x-ai/grok-2-1212': { p: 2e-6, c: 1e-5 }, |
| 86 | 'deepseek/deepseek-chat': { p: 1.4e-7, c: 2.8e-7 }, |
| 87 | }; |
| 88 | |
| 89 | function usageFor(model: string, promptChars: number, completionChars: number): Usage { |
| 90 | const promptTokens = Math.max(60, Math.round(promptChars / 4)); |
| 91 | const completionTokens = Math.max(1, Math.round(completionChars / 4)); |
| 92 | const price = PRICE[model] ?? { p: 1e-6, c: 3e-6 }; |
| 93 | return { |
| 94 | promptTokens, |
| 95 | completionTokens, |
| 96 | totalTokens: promptTokens + completionTokens, |
| 97 | costUsd: promptTokens * price.p + completionTokens * price.c, |
| 98 | }; |
| 99 | } |
| 100 | |
| 101 | const SYS_PAD = 520; |
| 102 | const reviewChars = (r: FixtureReview) => |
| 103 | r.strengths.join(' ').length + r.weaknesses.join(' ').length + r.note.length + 24; |
| 104 | |
| 105 | // --- expansion ------------------------------------------------------------- |
| 106 | |
| 107 | export function buildDebateResult(spec: FixtureSpec): DebateResult { |
| 108 | const models = spec.models; |
| 109 | const threshold = spec.convergenceThreshold ?? 85; |
| 110 | const participants: Participant[] = models.map((m, i) => ({ |
| 111 | id: `p${i}`, |
| 112 | model: m, |
| 113 | displayName: displayNameForModel(m), |
| 114 | })); |
| 115 | const config: DebateConfig = { |
| 116 | question: spec.question, |
| 117 | models, |
| 118 | chairmanModel: spec.chairmanModel, |
| 119 | convergenceModel: spec.convergenceModel, |
| 120 | maxRounds: Math.max(3, spec.rounds.length), |
| 121 | convergenceThreshold: threshold, |
| 122 | temperature: 0.7, |
| 123 | perModelTimeoutMs: 90_000, |
| 124 | }; |
| 125 | |
| 126 | const initialAnswers: AnswerRecord[] = spec.answers.map((content, i) => ({ |
| 127 | participantId: `p${i}`, |
| 128 | model: models[i]!, |
| 129 | round: 0, |
| 130 | content, |
| 131 | usage: usageFor(models[i]!, spec.question.length + SYS_PAD, content.length), |
| 132 | latencyMs: 1400 + i * 450, |
| 133 | })); |
| 134 | |
| 135 | const current = [...spec.answers]; // current answer per participant, for prompt sizing |
| 136 | |
| 137 | const rounds: RoundRecord[] = spec.rounds.map((r, ri) => { |
| 138 | const round = ri + 1; |
| 139 | |
| 140 | const critiques: CritiqueRecord[] = r.critiques.map((c) => { |
| 141 | const peersLen = c.on.reduce((s, rv) => s + (current[rv.p]?.length ?? 0), 0); |
| 142 | const compLen = c.on.reduce((s, rv) => s + reviewChars(rv), 24); |
| 143 | const reviews: PeerReview[] = c.on.map((rv, j) => ({ |
| 144 | label: String.fromCharCode(65 + j), |
| 145 | targetParticipantId: `p${rv.p}`, |
| 146 | weaknesses: rv.weaknesses, |
| 147 | strengths: rv.strengths, |
| 148 | score: rv.score, |
| 149 | justification: rv.note, |
| 150 | })); |
| 151 | return { |
| 152 | round, |
| 153 | reviewerParticipantId: `p${c.by}`, |
| 154 | reviewerModel: models[c.by]!, |
| 155 | reviews, |
| 156 | usage: usageFor(models[c.by]!, spec.question.length + peersLen + SYS_PAD, compLen), |
| 157 | latencyMs: 1800 + c.by * 320, |
| 158 | }; |
| 159 | }); |
| 160 | |
| 161 | const revisions: RevisionRecord[] = r.revisions.map((rev) => { |
| 162 | const incomingLen = critiques |
| 163 | .flatMap((cr) => cr.reviews.filter((x) => x.targetParticipantId === `p${rev.p}`)) |
| 164 | .reduce((t, x) => t + x.justification.length + x.strengths.join(' ').length + x.weaknesses.join(' ').length, 0); |
| 165 | const rec: RevisionRecord = { |
| 166 | round, |
| 167 | participantId: `p${rev.p}`, |
| 168 | model: models[rev.p]!, |
| 169 | content: rev.content, |
| 170 | changelog: { changed: rev.changed, summary: rev.summary, bullets: rev.bullets ?? [] }, |
| 171 | usage: usageFor( |
| 172 | models[rev.p]!, |
| 173 | spec.question.length + (current[rev.p]?.length ?? 0) + incomingLen + SYS_PAD, |
| 174 | rev.content.length + rev.summary.length, |
| 175 | ), |
| 176 | latencyMs: 1900 + rev.p * 420, |
| 177 | }; |
| 178 | current[rev.p] = rev.content; |
| 179 | return rec; |
| 180 | }); |
| 181 | |
| 182 | const answersLen = current.reduce((s, c) => s + c.length, 0); |
| 183 | const convLen = r.convergence.disagreements.reduce( |
| 184 | (s, d) => s + d.summary.length + d.topic.length + d.positions.reduce((t, p) => t + p.stance.length, 0), |
| 185 | 80, |
| 186 | ); |
| 187 | const convergence: ConvergenceRecord = { |
| 188 | round, |
| 189 | model: spec.convergenceModel, |
| 190 | score: r.convergence.score, |
| 191 | converged: r.convergence.score >= threshold, |
| 192 | disagreements: r.convergence.disagreements.map((d) => ({ |
| 193 | topic: d.topic, |
| 194 | summary: d.summary, |
| 195 | positions: d.positions.map((p) => ({ |
| 196 | label: String.fromCharCode(65 + p.p), |
| 197 | participantId: `p${p.p}`, |
| 198 | stance: p.stance, |
| 199 | })), |
| 200 | })), |
| 201 | usage: usageFor(spec.convergenceModel, answersLen + 300, convLen), |
| 202 | latencyMs: 820 + ri * 60, |
| 203 | }; |
| 204 | |
| 205 | return { round, critiques, revisions, convergence }; |
| 206 | }); |
| 207 | |
| 208 | const dissentLen = spec.dissent.reduce((s, d) => s + d.positions.reduce((t, p) => t + p.text.length, 0), 0); |
| 209 | const synthesis: SynthesisRecord = { |
| 210 | model: spec.chairmanModel, |
| 211 | finalAnswer: spec.finalAnswer, |
| 212 | dissent: spec.dissent.map((d) => ({ |
| 213 | topic: d.topic, |
| 214 | positions: d.positions.map((p) => ({ participantId: `p${p.p}`, model: models[p.p]!, position: p.text })), |
| 215 | })), |
| 216 | usage: usageFor( |
| 217 | spec.chairmanModel, |
| 218 | current.reduce((s, c) => s + c.length, 0) + 400, |
| 219 | spec.finalAnswer.length + dissentLen, |
| 220 | ), |
| 221 | latencyMs: 3200, |
| 222 | }; |
| 223 | |
| 224 | const finalAnswers: AnswerRecord[] = participants.map((p, i) => ({ |
| 225 | participantId: p.id, |
| 226 | model: p.model, |
| 227 | round: rounds.length, |
| 228 | content: current[i]!, |
| 229 | usage: emptyUsage(), |
| 230 | latencyMs: 0, |
| 231 | })); |
| 232 | |
| 233 | const member = (i: number) => ({ participantId: `p${i}`, model: models[i]! }); |
| 234 | const provenance: ProvenanceRecord | null = spec.provenance |
| 235 | ? { |
| 236 | model: spec.convergenceModel, |
| 237 | round: rounds.length, |
| 238 | claims: spec.provenance.map((c) => ({ |
| 239 | text: c.text, |
| 240 | supportedBy: c.supportedBy.map(member), |
| 241 | contestedBy: (c.contestedBy ?? []).map(member), |
| 242 | unsourced: c.supportedBy.length === 0, |
| 243 | })), |
| 244 | usage: usageFor( |
| 245 | spec.convergenceModel, |
| 246 | spec.finalAnswer.length + current.reduce((s, c) => s + c.length, 0) + 300, |
| 247 | spec.provenance.reduce((s, c) => s + c.text.length + 30, 0), |
| 248 | ), |
| 249 | latencyMs: 950, |
| 250 | } |
| 251 | : null; |
| 252 | |
| 253 | const allUsage: Usage[] = [ |
| 254 | ...initialAnswers.map((a) => a.usage), |
| 255 | ...rounds.flatMap((r) => [ |
| 256 | ...r.critiques.map((c) => c.usage), |
| 257 | ...r.revisions.map((x) => x.usage), |
| 258 | ...(r.convergence ? [r.convergence.usage] : []), |
| 259 | ]), |
| 260 | synthesis.usage, |
| 261 | ...(provenance ? [provenance.usage] : []), |
| 262 | ]; |
| 263 | const totals = sumUsage(allUsage); |
| 264 | const costByModel: Record<string, number> = {}; |
| 265 | const add = (m: string, u: Usage) => { |
| 266 | costByModel[m] = (costByModel[m] ?? 0) + u.costUsd; |
| 267 | }; |
| 268 | initialAnswers.forEach((a) => add(a.model, a.usage)); |
| 269 | rounds.forEach((r) => { |
| 270 | r.critiques.forEach((c) => add(c.reviewerModel, c.usage)); |
| 271 | r.revisions.forEach((x) => add(x.model, x.usage)); |
| 272 | if (r.convergence) add(r.convergence.model, r.convergence.usage); |
| 273 | }); |
| 274 | add(synthesis.model, synthesis.usage); |
| 275 | if (provenance) add(provenance.model, provenance.usage); |
| 276 | |
| 277 | return { |
| 278 | debateId: '', |
| 279 | config, |
| 280 | participants, |
| 281 | status: 'completed', |
| 282 | initialAnswers, |
| 283 | rounds, |
| 284 | synthesis, |
| 285 | provenance, |
| 286 | failures: [], |
| 287 | finalAnswers, |
| 288 | totals: { |
| 289 | costUsd: totals.costUsd, |
| 290 | promptTokens: totals.promptTokens, |
| 291 | completionTokens: totals.completionTokens, |
| 292 | rounds: rounds.length, |
| 293 | durationMs: 42_000 + rounds.length * 16_000 + models.length * 2_500, |
| 294 | costByModel, |
| 295 | }, |
| 296 | }; |
| 297 | } |
| 298 | |
| 299 | /** The durable events, in order, that reproduce this result via persistEvent. */ |
| 300 | export function resultToStageEvents(result: DebateResult): DebateEvent[] { |
| 301 | const events: DebateEvent[] = []; |
| 302 | for (const a of result.initialAnswers) events.push({ type: 'answer_completed', round: 0, record: a }); |
| 303 | for (const round of result.rounds) { |
| 304 | for (const c of round.critiques) events.push({ type: 'critique_completed', round: round.round, record: c }); |
| 305 | for (const rev of round.revisions) events.push({ type: 'revision_completed', round: round.round, record: rev }); |
| 306 | if (round.convergence) events.push({ type: 'convergence_result', round: round.round, record: round.convergence }); |
| 307 | } |
| 308 | if (result.synthesis) events.push({ type: 'synthesis_completed', record: result.synthesis }); |
| 309 | if (result.provenance) events.push({ type: 'provenance_completed', record: result.provenance }); |
| 310 | return events; |
| 311 | } |
| 312 | |
| 313 | export { FIXTURES } from './demo-fixtures.data'; |
| 314 | |