orchestrator.ts
28,027 bytes
| 1 | /** |
|---|---|
| 2 | * The debate orchestrator - an explicit, framework-agnostic state machine. |
| 3 | * |
| 4 | * round 0 answers (independent, streamed) |
| 5 | * round 1..N critique -> revision -> convergence (repeat) |
| 6 | * final synthesis (chairman: answer + dissent) |
| 7 | * |
| 8 | * It depends only on an injected `LlmClient` and an `emit` callback, so it runs |
| 9 | * identically under the mock client in a unit test and under OpenRouter behind |
| 10 | * an SSE route. Every meaningful step is emitted as a `DebateEvent`; the return |
| 11 | * value is the fully-materialized, replayable `DebateResult`. |
| 12 | * |
| 13 | * Robustness contract: |
| 14 | * - Each model call is bounded by `perModelTimeoutMs` and cancelled on timeout. |
| 15 | * - A model that fails a stage is dropped from that stage, not the debate, |
| 16 | * whenever the debate can still proceed with ≥2 healthy members. |
| 17 | * - Structured outputs get one JSON-repair retry before the model is dropped. |
| 18 | * - Tokens spent on failed attempts are still billed. |
| 19 | */ |
| 20 | import type { z } from 'zod'; |
| 21 | import { anonymizePeers, hashSeed, type AnonymizationResult } from './anonymize'; |
| 22 | import { participantsFor } from './config'; |
| 23 | import { shouldStop, type StopReason } from './convergence'; |
| 24 | import type { DebateEvent, EmitFn } from './events'; |
| 25 | import type { LlmClient, LlmRequest } from './llm-client'; |
| 26 | import { chairmanSharesProvider } from './models'; |
| 27 | import { |
| 28 | buildAnswerPrompt, |
| 29 | buildConvergencePrompt, |
| 30 | buildCritiquePrompt, |
| 31 | buildProvenancePrompt, |
| 32 | buildRevisionPrompt, |
| 33 | buildSynthesisPrompt, |
| 34 | } from './prompts'; |
| 35 | import type { IncomingCritique } from './prompts/revision'; |
| 36 | import { |
| 37 | convergenceOutputSchema, |
| 38 | critiqueOutputSchema, |
| 39 | provenanceOutputSchema, |
| 40 | revisionOutputSchema, |
| 41 | synthesisOutputSchema, |
| 42 | } from './schemas'; |
| 43 | import { requestStructured, StructuredParseError } from './structured'; |
| 44 | import { withTimeout } from './timeout'; |
| 45 | import { |
| 46 | displayNameForModel, |
| 47 | emptyUsage, |
| 48 | type AnswerRecord, |
| 49 | type ConvergenceRecord, |
| 50 | type CritiqueRecord, |
| 51 | type DebateConfig, |
| 52 | type DebateResult, |
| 53 | type DebateStatus, |
| 54 | type Disagreement, |
| 55 | type FailureRecord, |
| 56 | type Participant, |
| 57 | type PeerReview, |
| 58 | type ProvenanceRecord, |
| 59 | type RevisionRecord, |
| 60 | type RoundRecord, |
| 61 | type StageType, |
| 62 | type SynthesisRecord, |
| 63 | type Usage, |
| 64 | } from './types'; |
| 65 | import { addUsage } from './usage'; |
| 66 | |
| 67 | export interface Logger { |
| 68 | info(msg: string, meta?: Record<string, unknown>): void; |
| 69 | warn(msg: string, meta?: Record<string, unknown>): void; |
| 70 | error(msg: string, meta?: Record<string, unknown>): void; |
| 71 | } |
| 72 | |
| 73 | const noopLogger: Logger = { info: () => {}, warn: () => {}, error: () => {} }; |
| 74 | |
| 75 | export interface OrchestratorDeps { |
| 76 | llm: LlmClient; |
| 77 | emit: EmitFn; |
| 78 | now?: () => number; |
| 79 | logger?: Logger; |
| 80 | } |
| 81 | |
| 82 | export interface RunOptions { |
| 83 | debateId: string; |
| 84 | /** Debate-level cancellation (server shutdown / explicit user cancel). */ |
| 85 | signal?: AbortSignal; |
| 86 | /** |
| 87 | * Graceful early conclusion ("the gavel"): when fired, the run finishes the |
| 88 | * phase in flight, skips everything else, and goes straight to synthesis - |
| 89 | * unlike `signal`, which aborts the debate outright. |
| 90 | */ |
| 91 | gavelSignal?: AbortSignal; |
| 92 | } |
| 93 | |
| 94 | /** Mutable run state, threaded through the phase helpers. */ |
| 95 | interface RunState { |
| 96 | current: Map<string, AnswerRecord>; |
| 97 | active: Set<string>; |
| 98 | initialAnswers: AnswerRecord[]; |
| 99 | rounds: RoundRecord[]; |
| 100 | failures: FailureRecord[]; |
| 101 | totals: Usage; |
| 102 | costByModel: Map<string, number>; |
| 103 | lastDisagreements: Disagreement[]; |
| 104 | /** Revisions produced by the in-flight round, folded into a RoundRecord. */ |
| 105 | pendingRevisions?: RevisionRecord[]; |
| 106 | synthesisResult: SynthesisRecord | null; |
| 107 | provenanceResult: ProvenanceRecord | null; |
| 108 | } |
| 109 | |
| 110 | export async function runDebate( |
| 111 | config: DebateConfig, |
| 112 | deps: OrchestratorDeps, |
| 113 | opts: RunOptions, |
| 114 | ): Promise<DebateResult> { |
| 115 | const now = deps.now ?? Date.now; |
| 116 | const logger = deps.logger ?? noopLogger; |
| 117 | const startedAt = now(); |
| 118 | |
| 119 | const participants: Participant[] = participantsFor(config.models); |
| 120 | const byId = new Map(participants.map((p) => [p.id, p])); |
| 121 | |
| 122 | const state: RunState = { |
| 123 | current: new Map(), |
| 124 | active: new Set(), |
| 125 | initialAnswers: [], |
| 126 | rounds: [], |
| 127 | failures: [], |
| 128 | totals: emptyUsage(), |
| 129 | costByModel: new Map(), |
| 130 | lastDisagreements: [], |
| 131 | synthesisResult: null, |
| 132 | provenanceResult: null, |
| 133 | }; |
| 134 | |
| 135 | // --- helpers ------------------------------------------------------------- |
| 136 | const emit = (e: DebateEvent) => deps.emit(e); |
| 137 | |
| 138 | const bill = (model: string, usage: Usage) => { |
| 139 | state.totals = addUsage(state.totals, usage); |
| 140 | state.costByModel.set(model, (state.costByModel.get(model) ?? 0) + usage.costUsd); |
| 141 | }; |
| 142 | |
| 143 | const emitCost = async () => { |
| 144 | await emit({ |
| 145 | type: 'cost_update', |
| 146 | totalCostUsd: state.totals.costUsd, |
| 147 | promptTokens: state.totals.promptTokens, |
| 148 | completionTokens: state.totals.completionTokens, |
| 149 | costByModel: Object.fromEntries(state.costByModel), |
| 150 | }); |
| 151 | }; |
| 152 | |
| 153 | const recordFailure = async ( |
| 154 | round: number, |
| 155 | stage: StageType, |
| 156 | p: Participant, |
| 157 | error: unknown, |
| 158 | droppedFromDebate: boolean, |
| 159 | ) => { |
| 160 | const message = error instanceof Error ? error.message : String(error); |
| 161 | // A structured-parse failure still cost tokens - bill them. |
| 162 | if (error instanceof StructuredParseError) bill(p.model, error.usage); |
| 163 | state.failures.push({ |
| 164 | round, |
| 165 | stage, |
| 166 | participantId: p.id, |
| 167 | model: p.model, |
| 168 | error: message, |
| 169 | droppedFromDebate, |
| 170 | }); |
| 171 | if (droppedFromDebate) state.active.delete(p.id); |
| 172 | logger.warn('model_failed', { round, stage, model: p.model, error: message, droppedFromDebate }); |
| 173 | await emit({ |
| 174 | type: 'model_failed', |
| 175 | round, |
| 176 | stage, |
| 177 | participantId: p.id, |
| 178 | model: p.model, |
| 179 | error: message, |
| 180 | droppedFromDebate, |
| 181 | }); |
| 182 | }; |
| 183 | |
| 184 | const structured = <S extends z.ZodTypeAny>(req: LlmRequest, schema: S, signal: AbortSignal) => |
| 185 | requestStructured(deps.llm, { ...req, signal }, schema); |
| 186 | |
| 187 | const finish = async (status: DebateStatus, error?: string): Promise<DebateResult> => { |
| 188 | const finalAnswers = [...state.active] |
| 189 | .map((id) => state.current.get(id)) |
| 190 | .filter((a): a is AnswerRecord => Boolean(a)); |
| 191 | const durationMs = now() - startedAt; |
| 192 | |
| 193 | await emit({ |
| 194 | type: 'debate_completed', |
| 195 | debateId: opts.debateId, |
| 196 | status, |
| 197 | totalCostUsd: state.totals.costUsd, |
| 198 | rounds: state.rounds.length, |
| 199 | durationMs, |
| 200 | }); |
| 201 | |
| 202 | return { |
| 203 | debateId: opts.debateId, |
| 204 | config, |
| 205 | participants, |
| 206 | status, |
| 207 | initialAnswers: state.initialAnswers, |
| 208 | rounds: state.rounds, |
| 209 | synthesis: state.synthesisResult ?? null, |
| 210 | provenance: state.provenanceResult ?? null, |
| 211 | failures: state.failures, |
| 212 | finalAnswers, |
| 213 | totals: { |
| 214 | costUsd: state.totals.costUsd, |
| 215 | promptTokens: state.totals.promptTokens, |
| 216 | completionTokens: state.totals.completionTokens, |
| 217 | rounds: state.rounds.length, |
| 218 | durationMs, |
| 219 | costByModel: Object.fromEntries(state.costByModel), |
| 220 | }, |
| 221 | ...(error ? { error } : {}), |
| 222 | }; |
| 223 | }; |
| 224 | |
| 225 | // --- start --------------------------------------------------------------- |
| 226 | await emit({ |
| 227 | type: 'debate_started', |
| 228 | debateId: opts.debateId, |
| 229 | config, |
| 230 | participants, |
| 231 | chairmanModel: config.chairmanModel, |
| 232 | chairmanProviderConflict: chairmanSharesProvider(config.chairmanModel, config.models), |
| 233 | }); |
| 234 | |
| 235 | try { |
| 236 | // === Round 0: independent, streamed answers =========================== |
| 237 | await emit({ type: 'round_started', round: 0, kind: 'answers' }); |
| 238 | await runForEach(participants, async (p) => { |
| 239 | await emit({ type: 'stage_started', round: 0, stage: 'answer', participantId: p.id, model: p.model }); |
| 240 | try { |
| 241 | const record = await streamAnswer(p, 0, config.question); |
| 242 | state.current.set(p.id, record); |
| 243 | state.active.add(p.id); |
| 244 | state.initialAnswers.push(record); |
| 245 | bill(p.model, record.usage); |
| 246 | await emit({ type: 'answer_completed', round: 0, record }); |
| 247 | } catch (err) { |
| 248 | // No prior answer exists → this member cannot participate at all. |
| 249 | await recordFailure(0, 'answer', p, err, true); |
| 250 | } |
| 251 | }); |
| 252 | await emitCost(); |
| 253 | throwIfAborted(opts.signal); |
| 254 | |
| 255 | if (state.active.size < 2) { |
| 256 | return finish('failed', 'Not enough models produced an initial answer (need ≥2).'); |
| 257 | } |
| 258 | |
| 259 | // === Rounds 1..maxRounds: critique -> revision -> convergence ========= |
| 260 | let stopReason: StopReason = 'max_rounds'; |
| 261 | const gavelled = () => opts.gavelSignal?.aborted === true; |
| 262 | const strikeGavel = async (round: number) => { |
| 263 | stopReason = 'gavel'; |
| 264 | logger.info('gavel_struck', { round }); |
| 265 | await emit({ type: 'gavel_struck', round }); |
| 266 | }; |
| 267 | |
| 268 | for (let round = 1; round <= config.maxRounds; round++) { |
| 269 | // Spend cap: skip remaining rounds and let the chairman synthesize what |
| 270 | // exists - a cheaper answer instead of a wasted debate. |
| 271 | if (config.maxCostUsd !== undefined && state.totals.costUsd >= config.maxCostUsd) { |
| 272 | stopReason = 'budget'; |
| 273 | logger.warn('budget_reached', { round: round - 1, totalCostUsd: state.totals.costUsd, maxCostUsd: config.maxCostUsd }); |
| 274 | await emit({ |
| 275 | type: 'budget_reached', |
| 276 | round: round - 1, |
| 277 | totalCostUsd: state.totals.costUsd, |
| 278 | maxCostUsd: config.maxCostUsd, |
| 279 | }); |
| 280 | break; |
| 281 | } |
| 282 | if (gavelled()) { |
| 283 | await strikeGavel(round - 1); |
| 284 | break; |
| 285 | } |
| 286 | await emit({ type: 'round_started', round, kind: 'cycle' }); |
| 287 | |
| 288 | const critiques = await critiquePhase(round); |
| 289 | throwIfAborted(opts.signal); |
| 290 | // The gavel is checked at phase boundaries so it takes effect within |
| 291 | // seconds, not at the next round. A partial round is still recorded. |
| 292 | if (gavelled()) { |
| 293 | state.rounds.push({ round, critiques, revisions: [], convergence: null }); |
| 294 | await strikeGavel(round); |
| 295 | break; |
| 296 | } |
| 297 | await revisionPhase(round, critiques); |
| 298 | throwIfAborted(opts.signal); |
| 299 | if (gavelled()) { |
| 300 | state.rounds.push({ round, critiques, revisions: state.pendingRevisions ?? [], convergence: null }); |
| 301 | state.pendingRevisions = undefined; |
| 302 | await strikeGavel(round); |
| 303 | break; |
| 304 | } |
| 305 | const convergence = await convergencePhase(round); |
| 306 | await emitCost(); |
| 307 | |
| 308 | const decision = shouldStop({ |
| 309 | score: convergence?.score ?? 0, |
| 310 | threshold: config.convergenceThreshold, |
| 311 | round, |
| 312 | maxRounds: config.maxRounds, |
| 313 | activeCount: state.active.size, |
| 314 | }); |
| 315 | state.rounds.push({ round, critiques, revisions: state.pendingRevisions ?? [], convergence }); |
| 316 | state.pendingRevisions = undefined; |
| 317 | |
| 318 | if (decision.stop) { |
| 319 | stopReason = decision.reason; |
| 320 | if (decision.reason === 'insufficient_models') { |
| 321 | return finish('failed', 'Too many models dropped out to continue the debate.'); |
| 322 | } |
| 323 | break; |
| 324 | } |
| 325 | } |
| 326 | logger.info('deliberation_complete', { stopReason, rounds: state.rounds.length }); |
| 327 | throwIfAborted(opts.signal); |
| 328 | |
| 329 | // === Synthesis ======================================================== |
| 330 | await synthesisPhase(); |
| 331 | |
| 332 | // === Provenance audit (advisory) ====================================== |
| 333 | await provenancePhase(); |
| 334 | await emitCost(); |
| 335 | |
| 336 | return finish('completed'); |
| 337 | } catch (err) { |
| 338 | if (isAbortError(err) || opts.signal?.aborted) { |
| 339 | logger.warn('debate_aborted', { debateId: opts.debateId }); |
| 340 | return finish('aborted', 'Debate was cancelled.'); |
| 341 | } |
| 342 | const message = err instanceof Error ? err.message : String(err); |
| 343 | logger.error('debate_failed', { debateId: opts.debateId, error: message }); |
| 344 | await emit({ type: 'debate_failed', debateId: opts.debateId, error: message }); |
| 345 | return finish('failed', message); |
| 346 | } |
| 347 | |
| 348 | // --- phase implementations (closures over state) ------------------------- |
| 349 | |
| 350 | async function streamAnswer(p: Participant, round: number, question: string): Promise<AnswerRecord> { |
| 351 | const t0 = now(); |
| 352 | return withTimeout(async (signal) => { |
| 353 | const handle = await deps.llm.streamComplete({ |
| 354 | model: p.model, |
| 355 | messages: buildAnswerPrompt(question), |
| 356 | temperature: config.temperature, |
| 357 | signal, |
| 358 | meta: { stage: 'answer', round, participantId: p.id, seed: p.id }, |
| 359 | }); |
| 360 | for await (const delta of handle.stream) { |
| 361 | await emit({ type: 'token_delta', round, stage: 'answer', participantId: p.id, delta }); |
| 362 | } |
| 363 | const result = await handle.result; |
| 364 | return { |
| 365 | participantId: p.id, |
| 366 | model: p.model, |
| 367 | round, |
| 368 | content: result.text, |
| 369 | usage: result.usage, |
| 370 | latencyMs: result.latencyMs || now() - t0, |
| 371 | }; |
| 372 | }, config.perModelTimeoutMs, opts.signal); |
| 373 | } |
| 374 | |
| 375 | async function critiquePhase(round: number): Promise<CritiqueRecord[]> { |
| 376 | const reviewers = participants.filter((p) => state.active.has(p.id)); |
| 377 | const results: CritiqueRecord[] = []; |
| 378 | |
| 379 | await runForEach(reviewers, async (reviewer) => { |
| 380 | const peers = reviewers |
| 381 | .filter((p) => p.id !== reviewer.id) |
| 382 | .map((p) => ({ participantId: p.id, content: state.current.get(p.id)!.content })); |
| 383 | if (peers.length === 0) return; |
| 384 | |
| 385 | const anon = anonymizePeers(peers, hashSeed(opts.debateId, round, 'critique', reviewer.id)); |
| 386 | await emit({ type: 'stage_started', round, stage: 'critique', participantId: reviewer.id, model: reviewer.model }); |
| 387 | try { |
| 388 | const out = await withTimeout( |
| 389 | (signal) => |
| 390 | structured( |
| 391 | { |
| 392 | model: reviewer.model, |
| 393 | messages: buildCritiquePrompt(config.question, anon.peers), |
| 394 | temperature: config.temperature, |
| 395 | meta: { stage: 'critique', round, participantId: reviewer.id }, |
| 396 | }, |
| 397 | critiqueOutputSchema, |
| 398 | signal, |
| 399 | ), |
| 400 | config.perModelTimeoutMs, |
| 401 | opts.signal, |
| 402 | ); |
| 403 | bill(reviewer.model, out.usage); |
| 404 | const record = buildCritiqueRecord(round, reviewer, out.value.reviews, anon, out.usage, out.latencyMs); |
| 405 | results.push(record); |
| 406 | await emit({ type: 'critique_completed', round, record }); |
| 407 | } catch (err) { |
| 408 | // Losing a reviewer's critique doesn't remove it from the debate. |
| 409 | await recordFailure(round, 'critique', reviewer, err, false); |
| 410 | } |
| 411 | }); |
| 412 | |
| 413 | return results; |
| 414 | } |
| 415 | |
| 416 | async function revisionPhase(round: number, critiques: CritiqueRecord[]): Promise<void> { |
| 417 | const revisers = participants.filter((p) => state.active.has(p.id)); |
| 418 | const revisions: RevisionRecord[] = []; |
| 419 | |
| 420 | await runForEach(revisers, async (p) => { |
| 421 | const incoming: IncomingCritique[] = collectIncoming(p.id, critiques); |
| 422 | const ownAnswer = state.current.get(p.id)!.content; |
| 423 | await emit({ type: 'stage_started', round, stage: 'revision', participantId: p.id, model: p.model }); |
| 424 | try { |
| 425 | const out = await withTimeout( |
| 426 | (signal) => |
| 427 | structured( |
| 428 | { |
| 429 | model: p.model, |
| 430 | messages: buildRevisionPrompt(config.question, ownAnswer, incoming), |
| 431 | temperature: config.temperature, |
| 432 | meta: { stage: 'revision', round, participantId: p.id }, |
| 433 | }, |
| 434 | revisionOutputSchema, |
| 435 | signal, |
| 436 | ), |
| 437 | config.perModelTimeoutMs, |
| 438 | opts.signal, |
| 439 | ); |
| 440 | bill(p.model, out.usage); |
| 441 | const record: RevisionRecord = { |
| 442 | round, |
| 443 | participantId: p.id, |
| 444 | model: p.model, |
| 445 | content: out.value.answer, |
| 446 | changelog: { |
| 447 | changed: out.value.changelog.changed, |
| 448 | summary: out.value.changelog.summary, |
| 449 | bullets: out.value.changelog.bullets, |
| 450 | }, |
| 451 | usage: out.usage, |
| 452 | latencyMs: out.latencyMs, |
| 453 | }; |
| 454 | // Adopt the revised answer as this participant's current answer. |
| 455 | state.current.set(p.id, { |
| 456 | participantId: p.id, |
| 457 | model: p.model, |
| 458 | round, |
| 459 | content: out.value.answer, |
| 460 | usage: out.usage, |
| 461 | latencyMs: out.latencyMs, |
| 462 | }); |
| 463 | revisions.push(record); |
| 464 | await emit({ type: 'revision_completed', round, record }); |
| 465 | } catch (err) { |
| 466 | // Keep the participant's previous answer; drop only this revision. |
| 467 | await recordFailure(round, 'revision', p, err, false); |
| 468 | } |
| 469 | }); |
| 470 | |
| 471 | state.pendingRevisions = revisions; |
| 472 | } |
| 473 | |
| 474 | async function convergencePhase(round: number): Promise<ConvergenceRecord | null> { |
| 475 | const answersList = participants |
| 476 | .filter((p) => state.active.has(p.id)) |
| 477 | .map((p) => ({ participantId: p.id, content: state.current.get(p.id)!.content })); |
| 478 | |
| 479 | const anon = anonymizePeers(answersList, hashSeed(opts.debateId, round, 'convergence')); |
| 480 | await emit({ type: 'stage_started', round, stage: 'convergence', model: config.convergenceModel }); |
| 481 | try { |
| 482 | const out = await withTimeout( |
| 483 | (signal) => |
| 484 | structured( |
| 485 | { |
| 486 | model: config.convergenceModel, |
| 487 | messages: buildConvergencePrompt(config.question, anon.peers), |
| 488 | temperature: 0, |
| 489 | meta: { stage: 'convergence', round }, |
| 490 | }, |
| 491 | convergenceOutputSchema, |
| 492 | signal, |
| 493 | ), |
| 494 | config.perModelTimeoutMs, |
| 495 | opts.signal, |
| 496 | ); |
| 497 | bill(config.convergenceModel, out.usage); |
| 498 | const disagreements = mapDisagreements(out.value.disagreements, anon); |
| 499 | state.lastDisagreements = disagreements; |
| 500 | const record: ConvergenceRecord = { |
| 501 | round, |
| 502 | model: config.convergenceModel, |
| 503 | score: out.value.score, |
| 504 | disagreements, |
| 505 | converged: out.value.score >= config.convergenceThreshold, |
| 506 | usage: out.usage, |
| 507 | latencyMs: out.latencyMs, |
| 508 | }; |
| 509 | await emit({ type: 'convergence_result', round, record }); |
| 510 | return record; |
| 511 | } catch (err) { |
| 512 | // Convergence is advisory - never fail the debate on it. Treat as "not |
| 513 | // converged" and continue (or stop at max rounds). |
| 514 | const message = err instanceof Error ? err.message : String(err); |
| 515 | if (err instanceof StructuredParseError) bill(config.convergenceModel, err.usage); |
| 516 | logger.warn('convergence_failed', { round, error: message }); |
| 517 | const record: ConvergenceRecord = { |
| 518 | round, |
| 519 | model: config.convergenceModel, |
| 520 | score: 0, |
| 521 | disagreements: state.lastDisagreements, |
| 522 | converged: false, |
| 523 | usage: emptyUsage(), |
| 524 | latencyMs: 0, |
| 525 | }; |
| 526 | await emit({ type: 'convergence_result', round, record }); |
| 527 | return record; |
| 528 | } |
| 529 | } |
| 530 | |
| 531 | async function synthesisPhase(): Promise<void> { |
| 532 | const chairman = { id: 'chairman', model: config.chairmanModel, displayName: displayNameForModel(config.chairmanModel) }; |
| 533 | const finalists = participants |
| 534 | .filter((p) => state.active.has(p.id)) |
| 535 | .map((p) => ({ participantId: p.id, content: state.current.get(p.id)!.content })); |
| 536 | const anon = anonymizePeers(finalists, hashSeed(opts.debateId, 'synthesis')); |
| 537 | const remaining = state.lastDisagreements.map((d) => ({ topic: d.topic, summary: d.summary })); |
| 538 | |
| 539 | await emit({ type: 'stage_started', round: state.rounds.length, stage: 'synthesis', model: chairman.model }); |
| 540 | try { |
| 541 | const out = await withTimeout( |
| 542 | (signal) => |
| 543 | structured( |
| 544 | { |
| 545 | model: chairman.model, |
| 546 | messages: buildSynthesisPrompt(config.question, anon.peers, remaining), |
| 547 | temperature: config.temperature, |
| 548 | meta: { stage: 'synthesis', round: state.rounds.length }, |
| 549 | }, |
| 550 | synthesisOutputSchema, |
| 551 | signal, |
| 552 | ), |
| 553 | config.perModelTimeoutMs, |
| 554 | opts.signal, |
| 555 | ); |
| 556 | bill(chairman.model, out.usage); |
| 557 | const record: SynthesisRecord = { |
| 558 | model: chairman.model, |
| 559 | finalAnswer: out.value.finalAnswer, |
| 560 | dissent: out.value.dissent.map((d) => ({ |
| 561 | topic: d.topic, |
| 562 | positions: d.positions |
| 563 | .map((pos) => { |
| 564 | const pid = anon.labelMap[pos.label]; |
| 565 | const part = pid ? byId.get(pid) : undefined; |
| 566 | return part ? { participantId: part.id, model: part.model, position: pos.position } : null; |
| 567 | }) |
| 568 | .filter((x): x is NonNullable<typeof x> => x !== null), |
| 569 | })), |
| 570 | usage: out.usage, |
| 571 | latencyMs: out.latencyMs, |
| 572 | }; |
| 573 | state.synthesisResult = record; |
| 574 | await emit({ type: 'synthesis_completed', record }); |
| 575 | } catch (err) { |
| 576 | // Fallback synthesis so the debate still yields a usable answer. |
| 577 | if (err instanceof StructuredParseError) bill(chairman.model, err.usage); |
| 578 | const message = err instanceof Error ? err.message : String(err); |
| 579 | logger.error('synthesis_failed', { error: message }); |
| 580 | state.failures.push({ |
| 581 | round: state.rounds.length, |
| 582 | stage: 'synthesis', |
| 583 | participantId: 'chairman', |
| 584 | model: chairman.model, |
| 585 | error: message, |
| 586 | droppedFromDebate: false, |
| 587 | }); |
| 588 | const fallback = fallbackSynthesis(); |
| 589 | state.synthesisResult = fallback; |
| 590 | await emit({ type: 'synthesis_completed', record: fallback }); |
| 591 | } |
| 592 | } |
| 593 | |
| 594 | /** |
| 595 | * Trace each claim in the final answer back to the council's final answers, |
| 596 | * flagging claims no member made (chairman additions). Runs on the cheap |
| 597 | * convergence-class model. Purely advisory: any failure is recorded and the |
| 598 | * debate completes without a provenance report. |
| 599 | */ |
| 600 | async function provenancePhase(): Promise<void> { |
| 601 | const synthesis = state.synthesisResult; |
| 602 | if (!synthesis) return; |
| 603 | const finalists = participants |
| 604 | .filter((p) => state.active.has(p.id)) |
| 605 | .map((p) => ({ participantId: p.id, content: state.current.get(p.id)!.content })); |
| 606 | if (finalists.length === 0) return; |
| 607 | |
| 608 | const anon = anonymizePeers(finalists, hashSeed(opts.debateId, 'provenance')); |
| 609 | const round = state.rounds.length; |
| 610 | await emit({ type: 'stage_started', round, stage: 'provenance', model: config.convergenceModel }); |
| 611 | try { |
| 612 | const out = await withTimeout( |
| 613 | (signal) => |
| 614 | structured( |
| 615 | { |
| 616 | model: config.convergenceModel, |
| 617 | messages: buildProvenancePrompt(config.question, synthesis.finalAnswer, anon.peers), |
| 618 | temperature: 0, |
| 619 | meta: { stage: 'provenance', round }, |
| 620 | }, |
| 621 | provenanceOutputSchema, |
| 622 | signal, |
| 623 | ), |
| 624 | config.perModelTimeoutMs, |
| 625 | opts.signal, |
| 626 | ); |
| 627 | bill(config.convergenceModel, out.usage); |
| 628 | const toMembers = (labels: string[]) => |
| 629 | labels |
| 630 | .map((label) => { |
| 631 | const pid = anon.labelMap[label]; |
| 632 | const part = pid ? byId.get(pid) : undefined; |
| 633 | return part ? { participantId: part.id, model: part.model } : null; |
| 634 | }) |
| 635 | .filter((x): x is NonNullable<typeof x> => x !== null); |
| 636 | const record: ProvenanceRecord = { |
| 637 | model: config.convergenceModel, |
| 638 | round, |
| 639 | claims: out.value.claims.map((c) => { |
| 640 | const supportedBy = toMembers(c.supportedBy); |
| 641 | return { |
| 642 | text: c.text, |
| 643 | supportedBy, |
| 644 | contestedBy: toMembers(c.contestedBy), |
| 645 | unsourced: supportedBy.length === 0, |
| 646 | }; |
| 647 | }), |
| 648 | usage: out.usage, |
| 649 | latencyMs: out.latencyMs, |
| 650 | }; |
| 651 | state.provenanceResult = record; |
| 652 | await emit({ type: 'provenance_completed', record }); |
| 653 | } catch (err) { |
| 654 | if (err instanceof StructuredParseError) bill(config.convergenceModel, err.usage); |
| 655 | const message = err instanceof Error ? err.message : String(err); |
| 656 | logger.warn('provenance_failed', { error: message }); |
| 657 | state.failures.push({ |
| 658 | round, |
| 659 | stage: 'provenance', |
| 660 | participantId: 'auditor', |
| 661 | model: config.convergenceModel, |
| 662 | error: message, |
| 663 | droppedFromDebate: false, |
| 664 | }); |
| 665 | } |
| 666 | } |
| 667 | |
| 668 | function fallbackSynthesis(): SynthesisRecord { |
| 669 | const answers = [...state.active].map((id) => state.current.get(id)!).filter(Boolean); |
| 670 | const best = answers[0]; |
| 671 | return { |
| 672 | model: config.chairmanModel, |
| 673 | finalAnswer: |
| 674 | (best?.content ?? 'No answer could be produced.') + |
| 675 | '\n\n_(Chairman synthesis was unavailable; showing the leading council answer.)_', |
| 676 | dissent: state.lastDisagreements.map((d) => ({ |
| 677 | topic: d.topic, |
| 678 | positions: d.positions |
| 679 | .filter((pos) => pos.participantId) |
| 680 | .map((pos) => ({ |
| 681 | participantId: pos.participantId!, |
| 682 | model: byId.get(pos.participantId!)?.model ?? 'unknown', |
| 683 | position: pos.stance, |
| 684 | })), |
| 685 | })), |
| 686 | usage: emptyUsage(), |
| 687 | latencyMs: 0, |
| 688 | }; |
| 689 | } |
| 690 | |
| 691 | function buildCritiqueRecord( |
| 692 | round: number, |
| 693 | reviewer: Participant, |
| 694 | reviews: Array<{ |
| 695 | label: string; |
| 696 | weaknesses: string[]; |
| 697 | strengths: string[]; |
| 698 | score: number; |
| 699 | justification: string; |
| 700 | predictedPeerMean?: number; |
| 701 | authorGuess?: { family: string; confidence: number }; |
| 702 | }>, |
| 703 | anon: AnonymizationResult, |
| 704 | usage: Usage, |
| 705 | latencyMs: number, |
| 706 | ): CritiqueRecord { |
| 707 | const mapped: PeerReview[] = reviews |
| 708 | .map((r) => { |
| 709 | const targetId = anon.labelMap[r.label]; |
| 710 | if (!targetId) return null; |
| 711 | return { |
| 712 | label: r.label, |
| 713 | targetParticipantId: targetId, |
| 714 | weaknesses: r.weaknesses, |
| 715 | strengths: r.strengths, |
| 716 | score: r.score, |
| 717 | justification: r.justification, |
| 718 | ...(r.predictedPeerMean !== undefined ? { predictedPeerMean: r.predictedPeerMean } : {}), |
| 719 | ...(r.authorGuess !== undefined ? { authorGuess: r.authorGuess } : {}), |
| 720 | } satisfies PeerReview; |
| 721 | }) |
| 722 | .filter((x): x is PeerReview => x !== null); |
| 723 | return { |
| 724 | round, |
| 725 | reviewerParticipantId: reviewer.id, |
| 726 | reviewerModel: reviewer.model, |
| 727 | reviews: mapped, |
| 728 | usage, |
| 729 | latencyMs, |
| 730 | }; |
| 731 | } |
| 732 | |
| 733 | function collectIncoming(participantId: string, critiques: CritiqueRecord[]): IncomingCritique[] { |
| 734 | const out: IncomingCritique[] = []; |
| 735 | for (const c of critiques) { |
| 736 | for (const r of c.reviews) { |
| 737 | if (r.targetParticipantId === participantId) { |
| 738 | out.push({ |
| 739 | weaknesses: r.weaknesses, |
| 740 | strengths: r.strengths, |
| 741 | score: r.score, |
| 742 | justification: r.justification, |
| 743 | }); |
| 744 | } |
| 745 | } |
| 746 | } |
| 747 | return out; |
| 748 | } |
| 749 | |
| 750 | function mapDisagreements( |
| 751 | raw: Array<{ topic: string; summary: string; positions: Array<{ label: string; stance: string }> }>, |
| 752 | anon: AnonymizationResult, |
| 753 | ): Disagreement[] { |
| 754 | return raw.map((d) => ({ |
| 755 | topic: d.topic, |
| 756 | summary: d.summary, |
| 757 | positions: d.positions.map((pos) => ({ |
| 758 | label: pos.label, |
| 759 | participantId: anon.labelMap[pos.label], |
| 760 | stance: pos.stance, |
| 761 | })), |
| 762 | })); |
| 763 | } |
| 764 | } |
| 765 | |
| 766 | // --- module-scope helpers -------------------------------------------------- |
| 767 | |
| 768 | /** |
| 769 | * Run an async task per item concurrently, isolating failures. Individual tasks |
| 770 | * are expected to handle their own errors (they emit `model_failed`); this just |
| 771 | * guarantees one thrown task can't reject the whole phase. |
| 772 | */ |
| 773 | async function runForEach<T>(items: T[], task: (item: T) => Promise<void>): Promise<void> { |
| 774 | await Promise.all( |
| 775 | items.map(async (item) => { |
| 776 | try { |
| 777 | await task(item); |
| 778 | } catch { |
| 779 | // Task-level errors are already recorded as failures by the task itself. |
| 780 | } |
| 781 | }), |
| 782 | ); |
| 783 | } |
| 784 | |
| 785 | function isAbortError(err: unknown): boolean { |
| 786 | return ( |
| 787 | (err instanceof Error && err.name === 'AbortError') || |
| 788 | (typeof err === 'object' && err !== null && 'name' in err && (err as { name?: string }).name === 'AbortError') |
| 789 | ); |
| 790 | } |
| 791 | |
| 792 | function throwIfAborted(signal?: AbortSignal): void { |
| 793 | if (signal?.aborted) { |
| 794 | const e = new Error('Aborted'); |
| 795 | e.name = 'AbortError'; |
| 796 | throw e; |
| 797 | } |
| 798 | } |
| 799 | |