orchestrator.test.ts
11,224 bytes
| 1 | import { describe, expect, it } from 'vitest'; |
|---|---|
| 2 | import type { DebateEvent } from './events'; |
| 3 | import { MockLlmClient, type MockScenario } from './mock-client'; |
| 4 | import { runDebate } from './orchestrator'; |
| 5 | import type { DebateConfig, DebateResult } from './types'; |
| 6 | |
| 7 | function makeConfig(overrides: Partial<DebateConfig> = {}): DebateConfig { |
| 8 | return { |
| 9 | question: 'What is the best sorting algorithm for nearly-sorted data?', |
| 10 | models: ['openai/gpt-4o', 'anthropic/claude-3.5-sonnet', 'google/gemini-pro'], |
| 11 | chairmanModel: 'x-ai/grok-2', |
| 12 | convergenceModel: 'google/gemini-2.0-flash-001', |
| 13 | maxRounds: 3, |
| 14 | convergenceThreshold: 85, |
| 15 | temperature: 0.7, |
| 16 | perModelTimeoutMs: 5_000, |
| 17 | ...overrides, |
| 18 | }; |
| 19 | } |
| 20 | |
| 21 | interface RunHarness { |
| 22 | result: DebateResult; |
| 23 | events: DebateEvent[]; |
| 24 | of<T extends DebateEvent['type']>(type: T): Extract<DebateEvent, { type: T }>[]; |
| 25 | } |
| 26 | |
| 27 | async function run( |
| 28 | config: DebateConfig, |
| 29 | scenario: MockScenario = {}, |
| 30 | opts: { |
| 31 | signal?: AbortSignal; |
| 32 | gavelSignal?: AbortSignal; |
| 33 | debateId?: string; |
| 34 | onEvent?: (e: DebateEvent) => void; |
| 35 | } = {}, |
| 36 | ): Promise<RunHarness> { |
| 37 | const events: DebateEvent[] = []; |
| 38 | let clock = 1_000; |
| 39 | const result = await runDebate( |
| 40 | config, |
| 41 | { |
| 42 | llm: new MockLlmClient({ latencyMs: 50, ...scenario }), |
| 43 | emit: (e) => { |
| 44 | events.push(e); |
| 45 | opts.onEvent?.(e); |
| 46 | }, |
| 47 | now: () => (clock += 10), |
| 48 | }, |
| 49 | { debateId: opts.debateId ?? 'debate-test', signal: opts.signal, gavelSignal: opts.gavelSignal }, |
| 50 | ); |
| 51 | return { |
| 52 | result, |
| 53 | events, |
| 54 | of: (type) => events.filter((e) => e.type === type) as never, |
| 55 | }; |
| 56 | } |
| 57 | |
| 58 | describe('runDebate - happy path', () => { |
| 59 | it('runs answers → critique/revision rounds → synthesis and completes', async () => { |
| 60 | const { result, of } = await run(makeConfig()); |
| 61 | |
| 62 | expect(result.status).toBe('completed'); |
| 63 | expect(result.participants).toHaveLength(3); |
| 64 | expect(result.initialAnswers).toHaveLength(3); |
| 65 | expect(result.synthesis).not.toBeNull(); |
| 66 | expect(result.synthesis!.finalAnswer.length).toBeGreaterThan(0); |
| 67 | |
| 68 | // Default mock convergence: 60, 78, 96 → stops at round 2 by convergence. |
| 69 | expect(result.rounds).toHaveLength(2); |
| 70 | for (const round of result.rounds) { |
| 71 | expect(round.critiques).toHaveLength(3); |
| 72 | expect(round.revisions).toHaveLength(3); |
| 73 | expect(round.convergence).not.toBeNull(); |
| 74 | } |
| 75 | expect(result.rounds.at(-1)!.convergence!.converged).toBe(true); |
| 76 | |
| 77 | // Event stream sanity. |
| 78 | expect(of('debate_started')).toHaveLength(1); |
| 79 | expect(of('answer_completed')).toHaveLength(3); |
| 80 | expect(of('synthesis_completed')).toHaveLength(1); |
| 81 | expect(of('provenance_completed')).toHaveLength(1); |
| 82 | expect(of('debate_completed')[0]!.status).toBe('completed'); |
| 83 | |
| 84 | // Cost accrued and is attributed per model. |
| 85 | expect(result.totals.costUsd).toBeGreaterThan(0); |
| 86 | expect(Object.keys(result.totals.costByModel).length).toBeGreaterThanOrEqual(3); |
| 87 | }); |
| 88 | |
| 89 | it('anonymizes correctly: no reviewer scores itself, targets resolve to real peers', async () => { |
| 90 | const { result } = await run(makeConfig()); |
| 91 | const ids = new Set(result.participants.map((p) => p.id)); |
| 92 | for (const round of result.rounds) { |
| 93 | for (const critique of round.critiques) { |
| 94 | for (const review of critique.reviews) { |
| 95 | expect(review.targetParticipantId).not.toBe(critique.reviewerParticipantId); |
| 96 | expect(ids.has(review.targetParticipantId)).toBe(true); |
| 97 | } |
| 98 | } |
| 99 | } |
| 100 | }); |
| 101 | |
| 102 | it('streams answer token deltas to the UI', async () => { |
| 103 | const { of } = await run(makeConfig()); |
| 104 | const deltas = of('token_delta'); |
| 105 | expect(deltas.length).toBeGreaterThan(0); |
| 106 | expect(deltas.every((d) => d.stage === 'answer')).toBe(true); |
| 107 | }); |
| 108 | }); |
| 109 | |
| 110 | describe('runDebate - early stopping', () => { |
| 111 | it('stops after one round when convergence is immediate', async () => { |
| 112 | const { result } = await run(makeConfig(), { convergenceBase: 90, convergenceStep: 0 }); |
| 113 | expect(result.rounds).toHaveLength(1); |
| 114 | expect(result.status).toBe('completed'); |
| 115 | }); |
| 116 | |
| 117 | it('runs to max rounds when never converging', async () => { |
| 118 | const { result } = await run(makeConfig({ maxRounds: 2 }), { |
| 119 | convergenceBase: 10, |
| 120 | convergenceStep: 0, |
| 121 | }); |
| 122 | expect(result.rounds).toHaveLength(2); |
| 123 | expect(result.rounds.at(-1)!.convergence!.converged).toBe(false); |
| 124 | expect(result.synthesis).not.toBeNull(); |
| 125 | }); |
| 126 | }); |
| 127 | |
| 128 | describe('runDebate - failure handling', () => { |
| 129 | it('drops a model that fails its initial answer but continues with the rest', async () => { |
| 130 | const { result, of } = await run(makeConfig(), { |
| 131 | fail: [{ model: 'google/gemini-pro', stage: 'answer', round: 0 }], |
| 132 | }); |
| 133 | |
| 134 | expect(result.status).toBe('completed'); |
| 135 | expect(result.initialAnswers).toHaveLength(2); |
| 136 | expect(result.finalAnswers.map((a) => a.model)).not.toContain('google/gemini-pro'); |
| 137 | |
| 138 | const failed = of('model_failed'); |
| 139 | expect(failed).toHaveLength(1); |
| 140 | expect(failed[0]!.droppedFromDebate).toBe(true); |
| 141 | // Later rounds only involve the two survivors. |
| 142 | expect(result.rounds[0]!.critiques).toHaveLength(2); |
| 143 | }); |
| 144 | |
| 145 | it('fails the debate when fewer than two models produce an initial answer', async () => { |
| 146 | const { result } = await run(makeConfig(), { |
| 147 | fail: [ |
| 148 | { model: 'anthropic/claude-3.5-sonnet', stage: 'answer', round: 0 }, |
| 149 | { model: 'google/gemini-pro', stage: 'answer', round: 0 }, |
| 150 | ], |
| 151 | }); |
| 152 | expect(result.status).toBe('failed'); |
| 153 | expect(result.synthesis).toBeNull(); |
| 154 | expect(result.error).toMatch(/initial answer/i); |
| 155 | }); |
| 156 | |
| 157 | it('keeps a model in the debate when it only fails a single critique', async () => { |
| 158 | const { result, of } = await run(makeConfig(), { |
| 159 | fail: [{ model: 'google/gemini-pro', stage: 'critique', round: 1 }], |
| 160 | }); |
| 161 | expect(result.status).toBe('completed'); |
| 162 | // The model that failed a critique is still present in the final answers. |
| 163 | expect(result.finalAnswers.map((a) => a.model)).toContain('google/gemini-pro'); |
| 164 | const failure = of('model_failed').find((f) => f.stage === 'critique'); |
| 165 | expect(failure?.droppedFromDebate).toBe(false); |
| 166 | }); |
| 167 | }); |
| 168 | |
| 169 | describe('runDebate - JSON repair', () => { |
| 170 | it('recovers from malformed critique JSON via a repair attempt (no drop)', async () => { |
| 171 | const { result, of } = await run(makeConfig(), { |
| 172 | malformFirst: [{ model: 'openai/gpt-4o', stage: 'critique', round: 1 }], |
| 173 | }); |
| 174 | expect(result.status).toBe('completed'); |
| 175 | // Repaired, so no critique failure was recorded for that model. |
| 176 | expect(of('model_failed')).toHaveLength(0); |
| 177 | expect(result.failures).toHaveLength(0); |
| 178 | expect(result.rounds[0]!.critiques).toHaveLength(3); |
| 179 | }); |
| 180 | }); |
| 181 | |
| 182 | describe('runDebate - gavel', () => { |
| 183 | it('skips all rounds and synthesizes when the gavel is struck before round 1', async () => { |
| 184 | const { result, of } = await run(makeConfig(), {}, { gavelSignal: AbortSignal.abort() }); |
| 185 | expect(result.status).toBe('completed'); |
| 186 | expect(result.rounds).toHaveLength(0); |
| 187 | expect(result.synthesis).not.toBeNull(); |
| 188 | expect(of('gavel_struck')).toHaveLength(1); |
| 189 | expect(of('gavel_struck')[0]!.round).toBe(0); |
| 190 | }); |
| 191 | |
| 192 | it('finishes the phase in flight, records the partial round, and synthesizes', async () => { |
| 193 | const gavel = new AbortController(); |
| 194 | const { result, of } = await run( |
| 195 | makeConfig(), |
| 196 | {}, |
| 197 | { |
| 198 | gavelSignal: gavel.signal, |
| 199 | // Strike as soon as the first round-1 critique lands: the critique |
| 200 | // phase finishes, revision and convergence are skipped. |
| 201 | onEvent: (e) => { |
| 202 | if (e.type === 'critique_completed' && e.round === 1) gavel.abort(); |
| 203 | }, |
| 204 | }, |
| 205 | ); |
| 206 | expect(result.status).toBe('completed'); |
| 207 | expect(result.rounds).toHaveLength(1); |
| 208 | expect(result.rounds[0]!.critiques.length).toBeGreaterThan(0); |
| 209 | expect(result.rounds[0]!.revisions).toHaveLength(0); |
| 210 | expect(result.rounds[0]!.convergence).toBeNull(); |
| 211 | expect(result.synthesis).not.toBeNull(); |
| 212 | expect(of('gavel_struck')[0]!.round).toBe(1); |
| 213 | }); |
| 214 | }); |
| 215 | |
| 216 | describe('runDebate - budget cap', () => { |
| 217 | it('skips remaining rounds and synthesizes when spend crosses the cap', async () => { |
| 218 | // Round-0 answers alone cost more than this, so no cycle should run. |
| 219 | const { result, of } = await run(makeConfig({ maxCostUsd: 0.000001 })); |
| 220 | expect(result.status).toBe('completed'); |
| 221 | expect(result.rounds).toHaveLength(0); |
| 222 | expect(result.synthesis).not.toBeNull(); |
| 223 | |
| 224 | const budget = of('budget_reached'); |
| 225 | expect(budget).toHaveLength(1); |
| 226 | expect(budget[0]!.round).toBe(0); |
| 227 | expect(budget[0]!.totalCostUsd).toBeGreaterThanOrEqual(budget[0]!.maxCostUsd); |
| 228 | }); |
| 229 | |
| 230 | it('never fires without a cap configured', async () => { |
| 231 | const { of } = await run(makeConfig()); |
| 232 | expect(of('budget_reached')).toHaveLength(0); |
| 233 | }); |
| 234 | |
| 235 | it('runs rounds normally while under the cap', async () => { |
| 236 | const { result, of } = await run(makeConfig({ maxCostUsd: 100 })); |
| 237 | expect(result.rounds.length).toBeGreaterThan(0); |
| 238 | expect(of('budget_reached')).toHaveLength(0); |
| 239 | }); |
| 240 | }); |
| 241 | |
| 242 | describe('runDebate - provenance audit', () => { |
| 243 | it('traces final-answer claims to council members and flags chairman additions', async () => { |
| 244 | const { result } = await run(makeConfig()); |
| 245 | expect(result.provenance).not.toBeNull(); |
| 246 | const claims = result.provenance!.claims; |
| 247 | expect(claims.length).toBeGreaterThan(0); |
| 248 | const ids = new Set(result.participants.map((p) => p.id)); |
| 249 | for (const claim of claims) { |
| 250 | expect(claim.unsourced).toBe(claim.supportedBy.length === 0); |
| 251 | for (const m of [...claim.supportedBy, ...claim.contestedBy]) { |
| 252 | expect(ids.has(m.participantId)).toBe(true); |
| 253 | } |
| 254 | } |
| 255 | // The mock always emits one deliberately unsourced claim. |
| 256 | expect(claims.some((c) => c.unsourced)).toBe(true); |
| 257 | }); |
| 258 | |
| 259 | it('completes the debate even when the provenance audit fails', async () => { |
| 260 | const { result, of } = await run(makeConfig(), { |
| 261 | fail: [{ stage: 'provenance' }], |
| 262 | }); |
| 263 | expect(result.status).toBe('completed'); |
| 264 | expect(result.synthesis).not.toBeNull(); |
| 265 | expect(result.provenance).toBeNull(); |
| 266 | expect(of('provenance_completed')).toHaveLength(0); |
| 267 | expect(result.failures.some((f) => f.stage === 'provenance')).toBe(true); |
| 268 | }); |
| 269 | }); |
| 270 | |
| 271 | describe('runDebate - chairman self-preference', () => { |
| 272 | it('flags when the chairman shares a provider family with a council member', async () => { |
| 273 | const { of } = await run( |
| 274 | makeConfig({ chairmanModel: 'openai/gpt-4o-mini' }), // same "openai" family as a council member |
| 275 | ); |
| 276 | expect(of('debate_started')[0]!.chairmanProviderConflict).toBe(true); |
| 277 | }); |
| 278 | |
| 279 | it('does not flag a chairman from an independent provider', async () => { |
| 280 | const { of } = await run(makeConfig()); // chairman x-ai/grok-2 |
| 281 | expect(of('debate_started')[0]!.chairmanProviderConflict).toBe(false); |
| 282 | }); |
| 283 | }); |
| 284 | |
| 285 | describe('runDebate - cancellation', () => { |
| 286 | it('reports aborted status when the debate signal is already aborted', async () => { |
| 287 | const { result } = await run(makeConfig(), {}, { signal: AbortSignal.abort() }); |
| 288 | expect(result.status).toBe('aborted'); |
| 289 | expect(result.synthesis).toBeNull(); |
| 290 | }); |
| 291 | }); |
| 292 | |