profileShare

rasmusjy / roundtable

Read-only snapshot

No repository description.

main default branch 181 files Expires Sep 13, 2026, 9:06 AM

Commit

debate engine and unit tests

rasmusjy committed Jul 10, 2026, 4:56 PM

commit 50143f5

29 changed files with +2733 and −0

Jump to a changed file
  1. src/core/anonymize.test.ts +128 −0
  2. src/core/anonymize.ts +0 −0
  3. src/core/config.ts +37 −0
  4. src/core/convergence.test.ts +44 −0
  5. src/core/convergence.ts +41 −0
  6. src/core/events.ts +92 −0
  7. src/core/index.ts +23 −0
  8. src/core/json-repair.test.ts +82 −0
  9. src/core/json-repair.ts +97 −0
  10. src/core/llm-client.ts +71 −0
  11. src/core/mock-client.ts +270 −0
  12. src/core/models.ts +22 −0
  13. src/core/orchestrator.test.ts +195 −0
  14. src/core/orchestrator.ts +662 −0
  15. src/core/prompts/answer.ts +27 −0
  16. src/core/prompts/convergence.ts +48 −0
  17. src/core/prompts/critique.ts +54 −0
  18. src/core/prompts/index.ts +40 −0
  19. src/core/prompts/repair.ts +29 −0
  20. src/core/prompts/revision.ts +67 −0
  21. src/core/prompts/synthesis.ts +67 −0
  22. src/core/schemas.ts +98 −0
  23. src/core/scoring.test.ts +72 −0
  24. src/core/scoring.ts +63 −0
  25. src/core/structured.test.ts +75 −0
  26. src/core/structured.ts +71 −0
  27. src/core/timeout.ts +37 −0
  28. src/core/types.ts +204 −0
  29. src/core/usage.ts +17 −0
added src/core/anonymize.test.ts +128 −0
@@ -0,0 +1,128 @@
1 +import { describe, expect, it } from 'vitest';
2 +import {
3 + anonymizePeers,
4 + hashSeed,
5 + labelForIndex,
6 + makeRng,
7 + shuffle,
8 +} from './anonymize';
9 +
10 +describe('makeRng', () => {
11 + it('is deterministic for a given seed', () => {
12 + const a = makeRng(42);
13 + const b = makeRng(42);
14 + const seqA = [a(), a(), a(), a()];
15 + const seqB = [b(), b(), b(), b()];
16 + expect(seqA).toEqual(seqB);
17 + });
18 +
19 + it('produces values in [0, 1)', () => {
20 + const rng = makeRng(7);
21 + for (let i = 0; i < 1000; i++) {
22 + const v = rng();
23 + expect(v).toBeGreaterThanOrEqual(0);
24 + expect(v).toBeLessThan(1);
25 + }
26 + });
27 +
28 + it('produces different sequences for different seeds', () => {
29 + expect(makeRng(1)()).not.toEqual(makeRng(2)());
30 + });
31 +});
32 +
33 +describe('hashSeed', () => {
34 + it('is stable and order-sensitive', () => {
35 + expect(hashSeed('debate', 1, 'critique', 'p0')).toBe(hashSeed('debate', 1, 'critique', 'p0'));
36 + expect(hashSeed('debate', 1, 'critique', 'p0')).not.toBe(hashSeed('debate', 1, 'critique', 'p1'));
37 + expect(hashSeed('a', 'b')).not.toBe(hashSeed('b', 'a'));
38 + });
39 +
40 + it('returns an unsigned 32-bit integer', () => {
41 + const h = hashSeed('anything', 99);
42 + expect(Number.isInteger(h)).toBe(true);
43 + expect(h).toBeGreaterThanOrEqual(0);
44 + expect(h).toBeLessThanOrEqual(0xffffffff);
45 + });
46 +});
47 +
48 +describe('shuffle', () => {
49 + const rng = () => 0.5; // fixed generator
50 +
51 + it('does not mutate the input', () => {
52 + const input = [1, 2, 3, 4];
53 + const copy = [...input];
54 + shuffle(input, makeRng(1));
55 + expect(input).toEqual(copy);
56 + });
57 +
58 + it('returns a permutation (same multiset)', () => {
59 + const input = ['a', 'b', 'c', 'd', 'e'];
60 + const out = shuffle(input, makeRng(123));
61 + expect([...out].sort()).toEqual([...input].sort());
62 + expect(out).toHaveLength(input.length);
63 + });
64 +
65 + it('is deterministic for a given rng seed', () => {
66 + const input = [1, 2, 3, 4, 5, 6];
67 + expect(shuffle(input, makeRng(9))).toEqual(shuffle(input, makeRng(9)));
68 + });
69 +
70 + it('handles empty and singleton arrays', () => {
71 + expect(shuffle([], rng)).toEqual([]);
72 + expect(shuffle([42], rng)).toEqual([42]);
73 + });
74 +});
75 +
76 +describe('labelForIndex', () => {
77 + it('maps 0..25 to A..Z', () => {
78 + expect(labelForIndex(0)).toBe('A');
79 + expect(labelForIndex(1)).toBe('B');
80 + expect(labelForIndex(25)).toBe('Z');
81 + });
82 +
83 + it('continues to AA, AB for larger councils', () => {
84 + expect(labelForIndex(26)).toBe('AA');
85 + expect(labelForIndex(27)).toBe('AB');
86 + });
87 +});
88 +
89 +describe('anonymizePeers', () => {
90 + const peers = [
91 + { participantId: 'p0', content: 'answer zero' },
92 + { participantId: 'p1', content: 'answer one' },
93 + { participantId: 'p2', content: 'answer two' },
94 + ];
95 +
96 + it('assigns unique sequential labels and a consistent labelMap', () => {
97 + const { peers: out, labelMap } = anonymizePeers(peers, hashSeed('d', 1, 'p0'));
98 + const labels = out.map((p) => p.label);
99 + expect(new Set(labels).size).toBe(labels.length);
100 + expect(labels).toEqual(['A', 'B', 'C']);
101 + for (const p of out) {
102 + expect(labelMap[p.label]).toBe(p.participantId);
103 + }
104 + });
105 +
106 + it('preserves every participant exactly once (no dropping or duplication)', () => {
107 + const { peers: out } = anonymizePeers(peers, 555);
108 + expect(out.map((p) => p.participantId).sort()).toEqual(['p0', 'p1', 'p2']);
109 + });
110 +
111 + it('is deterministic for a given seed', () => {
112 + const seed = hashSeed('debate', 1, 'critique', 'reviewerA');
113 + const a1 = anonymizePeers(peers, seed).peers.map((p) => p.participantId);
114 + const a2 = anonymizePeers(peers, seed).peers.map((p) => p.participantId);
115 + expect(a1).toEqual(a2);
116 + });
117 +
118 + it('produces more than one ordering across reviewers (order is not shared)', () => {
119 + // The point of per-reviewer shuffling is that different seeds generally
120 + // yield different orderings. Sample many seeds and require real variation.
121 + const orderings = new Set<string>();
122 + for (let i = 0; i < 50; i++) {
123 + const seed = hashSeed('debate', 1, 'critique', `reviewer-${i}`);
124 + orderings.add(anonymizePeers(peers, seed).peers.map((p) => p.participantId).join(','));
125 + }
126 + expect(orderings.size).toBeGreaterThan(1);
127 + });
128 +});
added src/core/anonymize.ts +0 −0

Line changes are not available for this file.

added src/core/config.ts +37 −0
@@ -0,0 +1,37 @@
1 +/**
2 + * Turn validated API input into a fully-resolved `DebateConfig`, filling
3 + * defaults (like the convergence model) the orchestrator requires.
4 + */
5 +import { DEFAULT_CONVERGENCE_MODEL } from './models';
6 +import { debateConfigInputSchema, type DebateConfigInput } from './schemas';
7 +import { displayNameForModel, type DebateConfig, type Participant } from './types';
8 +
9 +/**
10 + * Build the stable participant list for a council. The `p{index}` id scheme is
11 + * shared by the orchestrator and the persistence layer so labels line up.
12 + */
13 +export function participantsFor(models: string[]): Participant[] {
14 + return models.map((model, i) => ({
15 + id: `p${i}`,
16 + model,
17 + displayName: displayNameForModel(model),
18 + }));
19 +}
20 +
21 +export function resolveDebateConfig(input: DebateConfigInput): DebateConfig {
22 + return {
23 + question: input.question,
24 + models: input.models,
25 + chairmanModel: input.chairmanModel,
26 + convergenceModel: input.convergenceModel ?? DEFAULT_CONVERGENCE_MODEL,
27 + maxRounds: input.maxRounds,
28 + convergenceThreshold: input.convergenceThreshold,
29 + temperature: input.temperature,
30 + perModelTimeoutMs: input.perModelTimeoutMs,
31 + };
32 +}
33 +
34 +/** Parse + resolve untrusted input in one step. Throws ZodError on bad input. */
35 +export function parseDebateConfig(raw: unknown): DebateConfig {
36 + return resolveDebateConfig(debateConfigInputSchema.parse(raw));
37 +}
added src/core/convergence.test.ts +44 −0
@@ -0,0 +1,44 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { shouldStop } from './convergence';
3 +
4 +describe('shouldStop', () => {
5 + const base = { threshold: 85, round: 1, maxRounds: 3, activeCount: 3 };
6 +
7 + it('stops when the convergence score meets the threshold', () => {
8 + expect(shouldStop({ ...base, score: 85 })).toEqual({ stop: true, reason: 'converged' });
9 + expect(shouldStop({ ...base, score: 92 })).toEqual({ stop: true, reason: 'converged' });
10 + });
11 +
12 + it('continues when below threshold and rounds remain', () => {
13 + expect(shouldStop({ ...base, score: 70 })).toEqual({ stop: false, reason: 'continue' });
14 + });
15 +
16 + it('stops at max rounds even if not converged', () => {
17 + expect(shouldStop({ ...base, score: 40, round: 3, maxRounds: 3 })).toEqual({
18 + stop: true,
19 + reason: 'max_rounds',
20 + });
21 + });
22 +
23 + it('stops with insufficient_models when fewer than two remain - even if converged', () => {
24 + expect(shouldStop({ ...base, score: 99, activeCount: 1 })).toEqual({
25 + stop: true,
26 + reason: 'insufficient_models',
27 + });
28 + expect(shouldStop({ ...base, score: 10, activeCount: 0 })).toEqual({
29 + stop: true,
30 + reason: 'insufficient_models',
31 + });
32 + });
33 +
34 + it('prioritizes insufficient_models over convergence and max_rounds', () => {
35 + // Converged AND at max rounds AND too few models -> insufficient wins.
36 + expect(shouldStop({ score: 100, threshold: 85, round: 3, maxRounds: 3, activeCount: 1 }).reason).toBe(
37 + 'insufficient_models',
38 + );
39 + });
40 +
41 + it('treats exactly two active models as enough to continue', () => {
42 + expect(shouldStop({ ...base, score: 50, activeCount: 2 })).toEqual({ stop: false, reason: 'continue' });
43 + });
44 +});
added src/core/convergence.ts +41 −0
@@ -0,0 +1,41 @@
1 +/**
2 + * The early-stop decision.
3 + *
4 + * Pure and side-effect free so it can be exhaustively unit-tested. The
5 + * orchestrator calls this after each round with the convergence score returned
6 + * by the assessor model and the current health of the council.
7 + */
8 +
9 +export type StopReason = 'converged' | 'max_rounds' | 'insufficient_models' | 'continue';
10 +
11 +export interface StopDecision {
12 + stop: boolean;
13 + reason: StopReason;
14 +}
15 +
16 +export interface StopParams {
17 + /** Convergence score for the round just completed (0-100). */
18 + score: number;
19 + /** Threshold at/above which we consider the council converged. */
20 + threshold: number;
21 + /** 1-based index of the round just completed. */
22 + round: number;
23 + /** Configured maximum number of critique→revision rounds. */
24 + maxRounds: number;
25 + /** Number of models still healthy enough to continue. */
26 + activeCount: number;
27 +}
28 +
29 +export function shouldStop({
30 + score,
31 + threshold,
32 + round,
33 + maxRounds,
34 + activeCount,
35 +}: StopParams): StopDecision {
36 + // A debate needs at least two voices; below that there is nothing to debate.
37 + if (activeCount < 2) return { stop: true, reason: 'insufficient_models' };
38 + if (score >= threshold) return { stop: true, reason: 'converged' };
39 + if (round >= maxRounds) return { stop: true, reason: 'max_rounds' };
40 + return { stop: false, reason: 'continue' };
41 +}
added src/core/events.ts +92 −0
@@ -0,0 +1,92 @@
1 +/**
2 + * The single typed event stream produced by the orchestrator.
3 + *
4 + * Both consumers subscribe to this one union:
5 + * - the SSE endpoint serializes each event to the browser
6 + * - the persistence layer reacts to `*_completed` events to write StageResults
7 + *
8 + * Keeping one source of truth means "what the UI sees" and "what we store" can
9 + * never drift apart.
10 + */
11 +import type {
12 + AnswerRecord,
13 + ConvergenceRecord,
14 + CritiqueRecord,
15 + DebateStatus,
16 + Participant,
17 + PublicDebateConfig,
18 + RevisionRecord,
19 + StageType,
20 + SynthesisRecord,
21 +} from './types';
22 +
23 +export type DebateEvent =
24 + | {
25 + type: 'debate_started';
26 + debateId: string;
27 + config: PublicDebateConfig;
28 + participants: Participant[];
29 + chairmanModel: string;
30 + /** True when chairman shares a provider family with a council member. */
31 + chairmanProviderConflict: boolean;
32 + }
33 + | { type: 'round_started'; round: number; kind: 'answers' | 'cycle' }
34 + | {
35 + type: 'stage_started';
36 + round: number;
37 + stage: StageType;
38 + participantId?: string;
39 + model?: string;
40 + }
41 + | {
42 + type: 'token_delta';
43 + round: number;
44 + stage: StageType;
45 + participantId: string;
46 + delta: string;
47 + }
48 + | { type: 'answer_completed'; round: number; record: AnswerRecord }
49 + | { type: 'critique_completed'; round: number; record: CritiqueRecord }
50 + | { type: 'revision_completed'; round: number; record: RevisionRecord }
51 + | { type: 'convergence_result'; round: number; record: ConvergenceRecord }
52 + | {
53 + type: 'model_failed';
54 + round: number;
55 + stage: StageType;
56 + participantId: string;
57 + model: string;
58 + error: string;
59 + droppedFromDebate: boolean;
60 + }
61 + | { type: 'synthesis_completed'; record: SynthesisRecord }
62 + | {
63 + type: 'cost_update';
64 + totalCostUsd: number;
65 + promptTokens: number;
66 + completionTokens: number;
67 + costByModel: Record<string, number>;
68 + }
69 + | {
70 + type: 'debate_completed';
71 + debateId: string;
72 + status: DebateStatus;
73 + totalCostUsd: number;
74 + rounds: number;
75 + durationMs: number;
76 + }
77 + | { type: 'debate_failed'; debateId: string; error: string };
78 +
79 +export type DebateEventType = DebateEvent['type'];
80 +
81 +/** Narrowing helper for consumers that only care about one event kind. */
82 +export function isEvent<T extends DebateEventType>(
83 + event: DebateEvent,
84 + type: T,
85 +): event is Extract<DebateEvent, { type: T }> {
86 + return event.type === type;
87 +}
88 +
89 +export type EmitFn = (event: DebateEvent) => void | Promise<void>;
90 +
91 +/** A no-op emitter, handy in tests that don't assert on the stream. */
92 +export const noopEmit: EmitFn = () => undefined;
added src/core/index.ts +23 −0
@@ -0,0 +1,23 @@
1 +/**
2 + * Public surface of the framework-agnostic debate engine.
3 + *
4 + * Everything a host (the Next.js route adapter, a CLI, a test) needs is
5 + * re-exported here. Nothing in this module tree imports Next.js, Prisma, React,
6 + * or a network client.
7 + */
8 +export * from './types';
9 +export * from './events';
10 +export * from './llm-client';
11 +export * from './schemas';
12 +export * from './config';
13 +export * from './models';
14 +export * from './convergence';
15 +export * from './scoring';
16 +export * from './anonymize';
17 +export * from './json-repair';
18 +export * from './usage';
19 +export { requestStructured, StructuredParseError } from './structured';
20 +export { MockLlmClient, type MockScenario } from './mock-client';
21 +export { runDebate, type OrchestratorDeps, type RunOptions, type Logger } from './orchestrator';
22 +export { PROMPT_VERSION } from './prompts';
23 +export { withTimeout, TimeoutError } from './timeout';
added src/core/json-repair.test.ts +82 −0
@@ -0,0 +1,82 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { z } from 'zod';
3 +import { extractJsonValue, lightRepair, parseStructured, stripCodeFences } from './json-repair';
4 +
5 +describe('stripCodeFences', () => {
6 + it('removes ```json fences', () => {
7 + expect(stripCodeFences('```json\n{"a":1}\n```')).toBe('{"a":1}');
8 + });
9 + it('removes bare ``` fences', () => {
10 + expect(stripCodeFences('```\n[1,2,3]\n```')).toBe('[1,2,3]');
11 + });
12 + it('leaves unfenced content untouched', () => {
13 + expect(stripCodeFences(' {"a":1} ')).toBe('{"a":1}');
14 + });
15 +});
16 +
17 +describe('extractJsonValue', () => {
18 + it('extracts an object embedded in prose', () => {
19 + expect(extractJsonValue('Sure! Here it is: {"score": 7} - hope that helps.')).toBe('{"score": 7}');
20 + });
21 +
22 + it('extracts an array', () => {
23 + expect(extractJsonValue('Result:\n[{"x":1}]')).toBe('[{"x":1}]');
24 + });
25 +
26 + it('ignores braces that appear inside string literals', () => {
27 + const raw = '{"text": "a } b { c", "n": 1}';
28 + expect(extractJsonValue(raw)).toBe(raw);
29 + });
30 +
31 + it('handles escaped quotes inside strings', () => {
32 + const raw = '{"q": "she said \\"hi\\" to me"}';
33 + expect(extractJsonValue(raw)).toBe(raw);
34 + });
35 +
36 + it('returns null when there is no JSON', () => {
37 + expect(extractJsonValue('no json here at all')).toBeNull();
38 + });
39 +});
40 +
41 +describe('lightRepair', () => {
42 + it('removes trailing commas', () => {
43 + expect(lightRepair('{"a":1,}')).toBe('{"a":1}');
44 + expect(lightRepair('[1,2,3,]')).toBe('[1,2,3]');
45 + });
46 + it('normalizes smart quotes', () => {
47 + expect(lightRepair('{“a”:1}')).toBe('{"a":1}');
48 + });
49 +});
50 +
51 +describe('parseStructured', () => {
52 + const schema = z.object({ score: z.number().min(0).max(10), notes: z.string().default('') });
53 +
54 + it('parses valid JSON', () => {
55 + const r = parseStructured('{"score": 8, "notes": "ok"}', schema);
56 + expect(r.ok).toBe(true);
57 + if (r.ok) expect(r.value.score).toBe(8);
58 + });
59 +
60 + it('parses JSON wrapped in a code fence and prose', () => {
61 + const r = parseStructured('Here you go:\n```json\n{"score": 5}\n```', schema);
62 + expect(r.ok).toBe(true);
63 + if (r.ok) expect(r.value.notes).toBe(''); // default applied
64 + });
65 +
66 + it('repairs a trailing comma', () => {
67 + const r = parseStructured('{"score": 3,}', schema);
68 + expect(r.ok).toBe(true);
69 + });
70 +
71 + it('fails with a schema message on out-of-range values', () => {
72 + const r = parseStructured('{"score": 99}', schema);
73 + expect(r.ok).toBe(false);
74 + if (!r.ok) expect(r.error).toMatch(/score/);
75 + });
76 +
77 + it('fails gracefully when there is no JSON', () => {
78 + const r = parseStructured('I refuse to answer.', schema);
79 + expect(r.ok).toBe(false);
80 + if (!r.ok) expect(r.extracted).toBeNull();
81 + });
82 +});
added src/core/json-repair.ts +97 −0
@@ -0,0 +1,97 @@
1 +/**
2 + * Defensive parsing of model-produced JSON.
3 + *
4 + * Models wrap JSON in ```json fences, prepend "Here is the JSON:", emit trailing
5 + * commas, or use smart quotes. We strip the obvious noise, extract the outermost
6 + * JSON value, and light-touch repair common syntactic slips before validating.
7 + * If parsing still fails, the caller retries once with a dedicated repair prompt
8 + * (see `structured.ts`).
9 + */
10 +import type { z } from 'zod';
11 +
12 +/** Strip Markdown code fences and surrounding prose from a JSON-ish string. */
13 +export function stripCodeFences(raw: string): string {
14 + let s = raw.trim();
15 + // ```json … ``` or ``` … ```
16 + const fence = s.match(/^```(?:json|jsonc|json5)?\s*\n?([\s\S]*?)\n?```$/i);
17 + if (fence?.[1] !== undefined) s = fence[1].trim();
18 + return s;
19 +}
20 +
21 +/**
22 + * Extract the first balanced JSON object or array from arbitrary text.
23 + * Returns null if no plausible JSON value is found. Correctly ignores braces
24 + * that appear inside string literals.
25 + */
26 +export function extractJsonValue(raw: string): string | null {
27 + const s = stripCodeFences(raw);
28 + const start = s.search(/[{[]/);
29 + if (start === -1) return null;
30 +
31 + const open = s[start]!;
32 + const close = open === '{' ? '}' : ']';
33 + let depth = 0;
34 + let inString = false;
35 + let escaped = false;
36 +
37 + for (let i = start; i < s.length; i++) {
38 + const ch = s[i]!;
39 + if (inString) {
40 + if (escaped) escaped = false;
41 + else if (ch === '\\') escaped = true;
42 + else if (ch === '"') inString = false;
43 + continue;
44 + }
45 + if (ch === '"') inString = true;
46 + else if (ch === open) depth++;
47 + else if (ch === close) {
48 + depth--;
49 + if (depth === 0) return s.slice(start, i + 1);
50 + }
51 + }
52 + return null;
53 +}
54 +
55 +/** Light syntactic repairs that are safe and common. */
56 +export function lightRepair(json: string): string {
57 + return json
58 + // smart quotes -> straight quotes
59 + .replace(/[“”]/g, '"')
60 + .replace(/[‘’]/g, "'")
61 + // trailing commas before } or ]
62 + .replace(/,\s*([}\]])/g, '$1');
63 +}
64 +
65 +export type ParseResult<T> =
66 + | { ok: true; value: T }
67 + | { ok: false; error: string; extracted: string | null };
68 +
69 +/**
70 + * Parse + validate a model response against a Zod schema, applying the full
71 + * defensive pipeline. Never throws.
72 + */
73 +export function parseStructured<T>(raw: string, schema: z.ZodType<T>): ParseResult<T> {
74 + const extracted = extractJsonValue(raw);
75 + if (extracted === null) {
76 + return { ok: false, error: 'No JSON object or array found in response', extracted: null };
77 + }
78 +
79 + for (const candidate of [extracted, lightRepair(extracted)]) {
80 + try {
81 + const parsed = JSON.parse(candidate);
82 + const result = schema.safeParse(parsed);
83 + if (result.success) return { ok: true, value: result.data };
84 + // Parsed as JSON but failed schema - report the schema error.
85 + return {
86 + ok: false,
87 + error: result.error.issues
88 + .map((i) => `${i.path.join('.') || '(root)'}: ${i.message}`)
89 + .join('; '),
90 + extracted,
91 + };
92 + } catch {
93 + // Try the next candidate (light-repaired form).
94 + }
95 + }
96 + return { ok: false, error: 'Response was not valid JSON after repair', extracted };
97 +}
added src/core/llm-client.ts +71 −0
@@ -0,0 +1,71 @@
1 +/**
2 + * The narrow LLM contract the orchestrator depends on.
3 + *
4 + * The orchestrator never imports the Vercel AI SDK, OpenRouter, or `fetch`
5 + * directly - it only knows this interface. That keeps the engine pure and
6 + * unit-testable: tests inject a deterministic mock (see `mock-client.ts`),
7 + * production injects the OpenRouter-backed client (see `lib/openrouter.ts`).
8 + */
9 +import type { StageType, Usage } from './types';
10 +
11 +export interface LlmMessage {
12 + role: 'system' | 'user' | 'assistant';
13 + content: string;
14 +}
15 +
16 +/** Opaque, provider-ignored hint used for observability and by the mock. */
17 +export interface LlmRequestMeta {
18 + stage: StageType;
19 + round: number;
20 + participantId?: string;
21 + /** Deterministic seed the mock uses to synthesize reproducible output. */
22 + seed?: string;
23 +}
24 +
25 +export interface LlmRequest {
26 + model: string;
27 + messages: LlmMessage[];
28 + temperature?: number;
29 + maxTokens?: number;
30 + /** Ask the provider for JSON output mode when supported. */
31 + json?: boolean;
32 + signal?: AbortSignal;
33 + meta?: LlmRequestMeta;
34 +}
35 +
36 +export interface LlmResult {
37 + text: string;
38 + usage: Usage;
39 + model: string;
40 + latencyMs: number;
41 +}
42 +
43 +export interface LlmStreamHandle {
44 + /** Text deltas as they arrive. Consume fully before awaiting `result`. */
45 + stream: AsyncIterable<string>;
46 + /** Resolves once the stream is exhausted, with final text + usage. */
47 + result: Promise<LlmResult>;
48 +}
49 +
50 +export interface LlmClient {
51 + /** One-shot completion (used for critique / revision / convergence / synthesis). */
52 + complete(req: LlmRequest): Promise<LlmResult>;
53 + /** Streaming completion (used for the answer + revision stages the UI watches). */
54 + streamComplete(req: LlmRequest): Promise<LlmStreamHandle>;
55 +}
56 +
57 +/**
58 + * Thrown by a client when a call fails in a way the orchestrator should treat
59 + * as a per-model failure (timeout, HTTP error, refusal). Carries whether the
60 + * error is retryable so the client's own backoff logic can decide.
61 + */
62 +export class LlmError extends Error {
63 + readonly status?: number;
64 + readonly retryable: boolean;
65 + constructor(message: string, opts?: { status?: number; retryable?: boolean; cause?: unknown }) {
66 + super(message, { cause: opts?.cause });
67 + this.name = 'LlmError';
68 + this.status = opts?.status;
69 + this.retryable = opts?.retryable ?? false;
70 + }
71 +}
added src/core/mock-client.ts +270 −0
@@ -0,0 +1,270 @@
1 +/**
2 + * Deterministic, offline LLM client.
3 + *
4 + * Powers `MOCK_LLM=1` (local dev / CI / the public demo) and the orchestrator
5 + * tests. Given the same inputs it always returns the same output - no network,
6 + * no spend, no clock or RNG dependence - which is exactly what makes the state
7 + * machine testable and the seeded demo reproducible.
8 + *
9 + * Output is synthesized from `req.meta.stage`, so the mock knows what shape to
10 + * return without brittle prompt sniffing. A small scenario object lets tests
11 + * force timeouts, failures, and JSON-repair paths.
12 + */
13 +import { hashSeed } from './anonymize';
14 +import type { LlmClient, LlmRequest, LlmResult, LlmStreamHandle } from './llm-client';
15 +import { LlmError } from './llm-client';
16 +import type { StageType, Usage } from './types';
17 +
18 +export interface MockRule {
19 + model?: string;
20 + stage?: StageType;
21 + round?: number;
22 +}
23 +
24 +export interface MockScenario {
25 + /** Calls matching any of these throw an LlmError (to test model drop-out). */
26 + fail?: MockRule[];
27 + /** First attempt returns schema-invalid JSON, forcing one repair round-trip. */
28 + malformFirst?: MockRule[];
29 + /** Convergence score = base + step * round (clamped 0-100). */
30 + convergenceBase?: number;
31 + convergenceStep?: number;
32 + /** Fixed latency reported per call (deterministic). */
33 + latencyMs?: number;
34 +}
35 +
36 +function ruleMatches(rule: MockRule, model: string, stage: StageType, round: number): boolean {
37 + if (rule.model !== undefined && rule.model !== model) return false;
38 + if (rule.stage !== undefined && rule.stage !== stage) return false;
39 + if (rule.round !== undefined && rule.round !== round) return false;
40 + return true;
41 +}
42 +
43 +const REPAIR_MARKER = 'strict JSON fixer';
44 +const isRepairCall = (req: LlmRequest) =>
45 + req.messages.some((m) => m.role === 'system' && m.content.includes(REPAIR_MARKER));
46 +
47 +/** Extract "--- Response X ---" labels from a critique/convergence/synthesis prompt. */
48 +function labelsFromPrompt(req: LlmRequest): string[] {
49 + const text = req.messages.map((m) => m.content).join('\n');
50 + const labels = [...text.matchAll(/---\s*Response\s+([A-Z]+)\s*---/g)].map((m) => m[1]!);
51 + return labels.length ? labels : ['A'];
52 +}
53 +
54 +/** On a repair call the labels live inside the echoed original response. */
55 +function labelsFromRepair(req: LlmRequest): string[] {
56 + const text = req.messages.map((m) => m.content).join('\n');
57 + const labels = [...text.matchAll(/"label"\s*:\s*"([A-Z]+)"/g)].map((m) => m[1]!);
58 + return labels.length ? [...new Set(labels)] : ['A'];
59 +}
60 +
61 +function ownAnswerFromPrompt(req: LlmRequest): string {
62 + const user = req.messages.find((m) => m.role === 'user')?.content ?? '';
63 + const match = user.match(/Your current answer:\n([\s\S]*?)\n\nCritiques of your answer:/);
64 + return match?.[1]?.trim() ?? 'Prior answer unavailable.';
65 +}
66 +
67 +function questionFromPrompt(req: LlmRequest): string {
68 + const text = req.messages.map((m) => m.content).join('\n');
69 + const match = text.match(/Question:\n([\s\S]*?)(?:\n\n|$)/);
70 + return match?.[1]?.trim() ?? 'the question';
71 +}
72 +
73 +/** Deterministic pseudo-token count (~4 chars/token). */
74 +function tokensOf(text: string): number {
75 + return Math.max(1, Math.round(text.length / 4));
76 +}
77 +
78 +function usageFor(req: LlmRequest, output: string): Usage {
79 + const promptTokens = tokensOf(req.messages.map((m) => m.content).join('\n'));
80 + const completionTokens = tokensOf(output);
81 + // Synthetic pricing: $1 / 1M prompt tokens, $3 / 1M completion tokens.
82 + const costUsd = (promptTokens * 1 + completionTokens * 3) / 1_000_000;
83 + return {
84 + promptTokens,
85 + completionTokens,
86 + totalTokens: promptTokens + completionTokens,
87 + costUsd,
88 + };
89 +}
90 +
91 +export class MockLlmClient implements LlmClient {
92 + constructor(private readonly scenario: MockScenario = {}) {}
93 +
94 + private guard(req: LlmRequest): { model: string; stage: StageType; round: number } {
95 + const model = req.model;
96 + const stage: StageType = req.meta?.stage ?? 'answer';
97 + const round = req.meta?.round ?? 0;
98 + if (req.signal?.aborted) {
99 + throw new LlmError('Request aborted', { retryable: false });
100 + }
101 + if (this.scenario.fail?.some((r) => ruleMatches(r, model, stage, round))) {
102 + throw new LlmError(`Mock forced failure for ${model} at ${stage} (round ${round})`, {
103 + status: 503,
104 + retryable: false,
105 + });
106 + }
107 + return { model, stage, round };
108 + }
109 +
110 + private render(req: LlmRequest): string {
111 + const { model, stage, round } = this.guard(req);
112 + const repair = isRepairCall(req);
113 + const forceMalform =
114 + !repair && (this.scenario.malformFirst?.some((r) => ruleMatches(r, model, stage, round)) ?? false);
115 + const seed = req.meta?.seed ?? `${model}:${stage}:${round}`;
116 +
117 + switch (stage) {
118 + case 'answer':
119 + return this.renderAnswer(model, round, questionFromPrompt(req));
120 + case 'critique':
121 + return this.renderCritique(req, repair, forceMalform, seed);
122 + case 'revision':
123 + return this.renderRevision(req, repair, forceMalform, round);
124 + case 'convergence':
125 + return this.renderConvergence(req, repair, forceMalform, round);
126 + case 'synthesis':
127 + return this.renderSynthesis(req, repair, forceMalform);
128 + default:
129 + return 'Mock response.';
130 + }
131 + }
132 +
133 + private renderAnswer(model: string, round: number, question: string): string {
134 + const stance = round === 0 ? 'initial position' : `revised position (round ${round})`;
135 + return (
136 + `**${model}** - ${stance}.\n\n` +
137 + `On the question of "${truncate(question, 120)}", the key consideration is that ` +
138 + `a well-formed answer must balance correctness against completeness. ${modelFlavor(model)} ` +
139 + `My recommendation: proceed with the approach that has the strongest evidentiary support, ` +
140 + `while explicitly noting the trade-offs involved.`
141 + );
142 + }
143 +
144 + private renderCritique(req: LlmRequest, repair: boolean, malform: boolean, seed: string): string {
145 + const labels = repair ? labelsFromRepair(req) : labelsFromPrompt(req);
146 + const reviews = labels.map((label) => {
147 + const s = hashSeed(seed, label) % 6; // 0..5
148 + const score = malform ? 42 : 4 + s; // 4..9 normally; out-of-range to trigger repair
149 + return {
150 + label,
151 + weaknesses: [`Response ${label} underspecifies the edge cases.`],
152 + strengths: [`Response ${label} states its assumptions clearly.`],
153 + score,
154 + justification: `Solid reasoning with a gap around edge cases (score ${score}).`,
155 + };
156 + });
157 + return JSON.stringify({ reviews });
158 + }
159 +
160 + private renderRevision(req: LlmRequest, repair: boolean, malform: boolean, round: number): string {
161 + const own = repair ? 'Refined answer after incorporating peer feedback.' : ownAnswerFromPrompt(req);
162 + const revised =
163 + `${own}\n\nAfter review (round ${round}): clarified the edge-case handling and ` +
164 + `tightened the recommendation.`;
165 + const payload = {
166 + // omit `answer` when malforming, to fail the schema and force a repair
167 + ...(malform ? {} : { answer: revised }),
168 + changelog: {
169 + changed: true,
170 + summary: 'Incorporated the edge-case critique and sharpened the recommendation.',
171 + bullets: ['Added edge-case handling', 'Tightened final recommendation'],
172 + },
173 + };
174 + return JSON.stringify(payload);
175 + }
176 +
177 + private renderConvergence(req: LlmRequest, repair: boolean, malform: boolean, round: number): string {
178 + const base = this.scenario.convergenceBase ?? 60;
179 + const step = this.scenario.convergenceStep ?? 18;
180 + const score = malform ? 150 : Math.max(0, Math.min(100, base + step * round));
181 + const labels = (repair ? labelsFromRepair(req) : labelsFromPrompt(req)).slice(0, 2);
182 + const disagreements =
183 + score >= 100
184 + ? []
185 + : [
186 + {
187 + topic: 'edge-case handling',
188 + summary: 'Members differ on how strictly edge cases must be handled.',
189 + positions: labels.map((label) => ({
190 + label,
191 + stance: `Response ${label} favors its own emphasis on this point.`,
192 + })),
193 + },
194 + ];
195 + return JSON.stringify({ score, disagreements });
196 + }
197 +
198 + private renderSynthesis(req: LlmRequest, repair: boolean, malform: boolean): string {
199 + const labels = (repair ? labelsFromRepair(req) : labelsFromPrompt(req)).slice(0, 2);
200 + const payload = {
201 + ...(malform ? {} : {
202 + finalAnswer:
203 + 'Synthesized answer: the council converged on prioritizing correctness, with ' +
204 + 'explicit handling of edge cases and a clearly stated recommendation and its trade-offs.',
205 + }),
206 + dissent:
207 + labels.length >= 2
208 + ? [
209 + {
210 + topic: 'strictness of edge-case handling',
211 + positions: labels.map((label) => ({
212 + label,
213 + position: `Response ${label} maintains a distinct emphasis.`,
214 + })),
215 + },
216 + ]
217 + : [],
218 + };
219 + return JSON.stringify(payload);
220 + }
221 +
222 + async complete(req: LlmRequest): Promise<LlmResult> {
223 + const text = this.render(req);
224 + return {
225 + text,
226 + usage: usageFor(req, text),
227 + model: req.model,
228 + latencyMs: this.scenario.latencyMs ?? 120 + (hashSeed(req.model, req.meta?.stage ?? '') % 400),
229 + };
230 + }
231 +
232 + async streamComplete(req: LlmRequest): Promise<LlmStreamHandle> {
233 + const text = this.render(req);
234 + const usage = usageFor(req, text);
235 + const latencyMs = this.scenario.latencyMs ?? 120 + (hashSeed(req.model, 'stream') % 400);
236 + const chunks = chunkText(text, 5);
237 +
238 + async function* gen(): AsyncGenerator<string> {
239 + for (const chunk of chunks) {
240 + if (req.signal?.aborted) throw new LlmError('Request aborted', { retryable: false });
241 + yield chunk;
242 + }
243 + }
244 +
245 + return {
246 + stream: gen(),
247 + result: Promise.resolve({ text, usage, model: req.model, latencyMs }),
248 + };
249 + }
250 +}
251 +
252 +function chunkText(text: string, parts: number): string[] {
253 + const size = Math.ceil(text.length / parts) || 1;
254 + const out: string[] = [];
255 + for (let i = 0; i < text.length; i += size) out.push(text.slice(i, i + size));
256 + return out.length ? out : [''];
257 +}
258 +
259 +function truncate(s: string, n: number): string {
260 + return s.length <= n ? s : `${s.slice(0, n)}...`;
261 +}
262 +
263 +function modelFlavor(model: string): string {
264 + const h = hashSeed(model) % 3;
265 + return h === 0
266 + ? 'This favors a conservative, evidence-first framing.'
267 + : h === 1
268 + ? 'This leans toward a pragmatic, action-oriented framing.'
269 + : 'This weighs long-term robustness above short-term simplicity.';
270 +}
added src/core/models.ts +22 −0
@@ -0,0 +1,22 @@
1 +/**
2 + * Model-identity helpers used for self-preference mitigation.
3 + *
4 + * OpenRouter slugs are "vendor/model[:variant]"; the vendor prefix is the
5 + * provider family. Two models from the same family (e.g. two OpenAI models)
6 + * share training lineage and stylistic priors, so a chairman that shares a
7 + * family with a council member is a self-preference risk worth flagging.
8 + */
9 +
10 +/** Sensible cheap/fast default for the between-rounds convergence check. */
11 +export const DEFAULT_CONVERGENCE_MODEL = 'google/gemini-2.0-flash-001';
12 +
13 +export function providerFamily(slug: string): string {
14 + const idx = slug.indexOf('/');
15 + return (idx === -1 ? slug : slug.slice(0, idx)).toLowerCase();
16 +}
17 +
18 +/** True if the chairman shares a provider family with any council member. */
19 +export function chairmanSharesProvider(chairman: string, council: readonly string[]): boolean {
20 + const fam = providerFamily(chairman);
21 + return council.some((m) => providerFamily(m) === fam);
22 +}
added src/core/orchestrator.test.ts +195 −0
@@ -0,0 +1,195 @@
1 +import { describe, expect, it } from 'vitest';
2 +import type { DebateEvent } from './events';
3 +import { MockLlmClient, type MockScenario } from './mock-client';
4 +import { runDebate } from './orchestrator';
5 +import type { DebateConfig, DebateResult } from './types';
6 +
7 +function makeConfig(overrides: Partial<DebateConfig> = {}): DebateConfig {
8 + return {
9 + question: 'What is the best sorting algorithm for nearly-sorted data?',
10 + models: ['openai/gpt-4o', 'anthropic/claude-3.5-sonnet', 'google/gemini-pro'],
11 + chairmanModel: 'x-ai/grok-2',
12 + convergenceModel: 'google/gemini-2.0-flash-001',
13 + maxRounds: 3,
14 + convergenceThreshold: 85,
15 + temperature: 0.7,
16 + perModelTimeoutMs: 5_000,
17 + ...overrides,
18 + };
19 +}
20 +
21 +interface RunHarness {
22 + result: DebateResult;
23 + events: DebateEvent[];
24 + of<T extends DebateEvent['type']>(type: T): Extract<DebateEvent, { type: T }>[];
25 +}
26 +
27 +async function run(
28 + config: DebateConfig,
29 + scenario: MockScenario = {},
30 + opts: { signal?: AbortSignal; debateId?: string } = {},
31 +): Promise<RunHarness> {
32 + const events: DebateEvent[] = [];
33 + let clock = 1_000;
34 + const result = await runDebate(
35 + config,
36 + {
37 + llm: new MockLlmClient({ latencyMs: 50, ...scenario }),
38 + emit: (e) => {
39 + events.push(e);
40 + },
41 + now: () => (clock += 10),
42 + },
43 + { debateId: opts.debateId ?? 'debate-test', signal: opts.signal },
44 + );
45 + return {
46 + result,
47 + events,
48 + of: (type) => events.filter((e) => e.type === type) as never,
49 + };
50 +}
51 +
52 +describe('runDebate - happy path', () => {
53 + it('runs answers → critique/revision rounds → synthesis and completes', async () => {
54 + const { result, of } = await run(makeConfig());
55 +
56 + expect(result.status).toBe('completed');
57 + expect(result.participants).toHaveLength(3);
58 + expect(result.initialAnswers).toHaveLength(3);
59 + expect(result.synthesis).not.toBeNull();
60 + expect(result.synthesis!.finalAnswer.length).toBeGreaterThan(0);
61 +
62 + // Default mock convergence: 60, 78, 96 → stops at round 2 by convergence.
63 + expect(result.rounds).toHaveLength(2);
64 + for (const round of result.rounds) {
65 + expect(round.critiques).toHaveLength(3);
66 + expect(round.revisions).toHaveLength(3);
67 + expect(round.convergence).not.toBeNull();
68 + }
69 + expect(result.rounds.at(-1)!.convergence!.converged).toBe(true);
70 +
71 + // Event stream sanity.
72 + expect(of('debate_started')).toHaveLength(1);
73 + expect(of('answer_completed')).toHaveLength(3);
74 + expect(of('synthesis_completed')).toHaveLength(1);
75 + expect(of('debate_completed')[0]!.status).toBe('completed');
76 +
77 + // Cost accrued and is attributed per model.
78 + expect(result.totals.costUsd).toBeGreaterThan(0);
79 + expect(Object.keys(result.totals.costByModel).length).toBeGreaterThanOrEqual(3);
80 + });
81 +
82 + it('anonymizes correctly: no reviewer scores itself, targets resolve to real peers', async () => {
83 + const { result } = await run(makeConfig());
84 + const ids = new Set(result.participants.map((p) => p.id));
85 + for (const round of result.rounds) {
86 + for (const critique of round.critiques) {
87 + for (const review of critique.reviews) {
88 + expect(review.targetParticipantId).not.toBe(critique.reviewerParticipantId);
89 + expect(ids.has(review.targetParticipantId)).toBe(true);
90 + }
91 + }
92 + }
93 + });
94 +
95 + it('streams answer token deltas to the UI', async () => {
96 + const { of } = await run(makeConfig());
97 + const deltas = of('token_delta');
98 + expect(deltas.length).toBeGreaterThan(0);
99 + expect(deltas.every((d) => d.stage === 'answer')).toBe(true);
100 + });
101 +});
102 +
103 +describe('runDebate - early stopping', () => {
104 + it('stops after one round when convergence is immediate', async () => {
105 + const { result } = await run(makeConfig(), { convergenceBase: 90, convergenceStep: 0 });
106 + expect(result.rounds).toHaveLength(1);
107 + expect(result.status).toBe('completed');
108 + });
109 +
110 + it('runs to max rounds when never converging', async () => {
111 + const { result } = await run(makeConfig({ maxRounds: 2 }), {
112 + convergenceBase: 10,
113 + convergenceStep: 0,
114 + });
115 + expect(result.rounds).toHaveLength(2);
116 + expect(result.rounds.at(-1)!.convergence!.converged).toBe(false);
117 + expect(result.synthesis).not.toBeNull();
118 + });
119 +});
120 +
121 +describe('runDebate - failure handling', () => {
122 + it('drops a model that fails its initial answer but continues with the rest', async () => {
123 + const { result, of } = await run(makeConfig(), {
124 + fail: [{ model: 'google/gemini-pro', stage: 'answer', round: 0 }],
125 + });
126 +
127 + expect(result.status).toBe('completed');
128 + expect(result.initialAnswers).toHaveLength(2);
129 + expect(result.finalAnswers.map((a) => a.model)).not.toContain('google/gemini-pro');
130 +
131 + const failed = of('model_failed');
132 + expect(failed).toHaveLength(1);
133 + expect(failed[0]!.droppedFromDebate).toBe(true);
134 + // Later rounds only involve the two survivors.
135 + expect(result.rounds[0]!.critiques).toHaveLength(2);
136 + });
137 +
138 + it('fails the debate when fewer than two models produce an initial answer', async () => {
139 + const { result } = await run(makeConfig(), {
140 + fail: [
141 + { model: 'anthropic/claude-3.5-sonnet', stage: 'answer', round: 0 },
142 + { model: 'google/gemini-pro', stage: 'answer', round: 0 },
143 + ],
144 + });
145 + expect(result.status).toBe('failed');
146 + expect(result.synthesis).toBeNull();
147 + expect(result.error).toMatch(/initial answer/i);
148 + });
149 +
150 + it('keeps a model in the debate when it only fails a single critique', async () => {
151 + const { result, of } = await run(makeConfig(), {
152 + fail: [{ model: 'google/gemini-pro', stage: 'critique', round: 1 }],
153 + });
154 + expect(result.status).toBe('completed');
155 + // The model that failed a critique is still present in the final answers.
156 + expect(result.finalAnswers.map((a) => a.model)).toContain('google/gemini-pro');
157 + const failure = of('model_failed').find((f) => f.stage === 'critique');
158 + expect(failure?.droppedFromDebate).toBe(false);
159 + });
160 +});
161 +
162 +describe('runDebate - JSON repair', () => {
163 + it('recovers from malformed critique JSON via a repair attempt (no drop)', async () => {
164 + const { result, of } = await run(makeConfig(), {
165 + malformFirst: [{ model: 'openai/gpt-4o', stage: 'critique', round: 1 }],
166 + });
167 + expect(result.status).toBe('completed');
168 + // Repaired, so no critique failure was recorded for that model.
169 + expect(of('model_failed')).toHaveLength(0);
170 + expect(result.failures).toHaveLength(0);
171 + expect(result.rounds[0]!.critiques).toHaveLength(3);
172 + });
173 +});
174 +
175 +describe('runDebate - chairman self-preference', () => {
176 + it('flags when the chairman shares a provider family with a council member', async () => {
177 + const { of } = await run(
178 + makeConfig({ chairmanModel: 'openai/gpt-4o-mini' }), // same "openai" family as a council member
179 + );
180 + expect(of('debate_started')[0]!.chairmanProviderConflict).toBe(true);
181 + });
182 +
183 + it('does not flag a chairman from an independent provider', async () => {
184 + const { of } = await run(makeConfig()); // chairman x-ai/grok-2
185 + expect(of('debate_started')[0]!.chairmanProviderConflict).toBe(false);
186 + });
187 +});
188 +
189 +describe('runDebate - cancellation', () => {
190 + it('reports aborted status when the debate signal is already aborted', async () => {
191 + const { result } = await run(makeConfig(), {}, { signal: AbortSignal.abort() });
192 + expect(result.status).toBe('aborted');
193 + expect(result.synthesis).toBeNull();
194 + });
195 +});
added src/core/orchestrator.ts +662 −0
@@ -0,0 +1,662 @@
1 +/**
2 + * The debate orchestrator - an explicit, framework-agnostic state machine.
3 + *
4 + * round 0 answers (independent, streamed)
5 + * round 1..N critique -> revision -> convergence (repeat)
6 + * final synthesis (chairman: answer + dissent)
7 + *
8 + * It depends only on an injected `LlmClient` and an `emit` callback, so it runs
9 + * identically under the mock client in a unit test and under OpenRouter behind
10 + * an SSE route. Every meaningful step is emitted as a `DebateEvent`; the return
11 + * value is the fully-materialized, replayable `DebateResult`.
12 + *
13 + * Robustness contract:
14 + * - Each model call is bounded by `perModelTimeoutMs` and cancelled on timeout.
15 + * - A model that fails a stage is dropped from that stage, not the debate,
16 + * whenever the debate can still proceed with ≥2 healthy members.
17 + * - Structured outputs get one JSON-repair retry before the model is dropped.
18 + * - Tokens spent on failed attempts are still billed.
19 + */
20 +import type { z } from 'zod';
21 +import { anonymizePeers, hashSeed, type AnonymizationResult } from './anonymize';
22 +import { participantsFor } from './config';
23 +import { shouldStop, type StopReason } from './convergence';
24 +import type { DebateEvent, EmitFn } from './events';
25 +import type { LlmClient, LlmRequest } from './llm-client';
26 +import { chairmanSharesProvider } from './models';
27 +import {
28 + buildAnswerPrompt,
29 + buildConvergencePrompt,
30 + buildCritiquePrompt,
31 + buildRevisionPrompt,
32 + buildSynthesisPrompt,
33 +} from './prompts';
34 +import type { IncomingCritique } from './prompts/revision';
35 +import {
36 + convergenceOutputSchema,
37 + critiqueOutputSchema,
38 + revisionOutputSchema,
39 + synthesisOutputSchema,
40 +} from './schemas';
41 +import { requestStructured, StructuredParseError } from './structured';
42 +import { withTimeout } from './timeout';
43 +import {
44 + displayNameForModel,
45 + emptyUsage,
46 + type AnswerRecord,
47 + type ConvergenceRecord,
48 + type CritiqueRecord,
49 + type DebateConfig,
50 + type DebateResult,
51 + type DebateStatus,
52 + type Disagreement,
53 + type FailureRecord,
54 + type Participant,
55 + type PeerReview,
56 + type RevisionRecord,
57 + type RoundRecord,
58 + type StageType,
59 + type SynthesisRecord,
60 + type Usage,
61 +} from './types';
62 +import { addUsage } from './usage';
63 +
64 +export interface Logger {
65 + info(msg: string, meta?: Record<string, unknown>): void;
66 + warn(msg: string, meta?: Record<string, unknown>): void;
67 + error(msg: string, meta?: Record<string, unknown>): void;
68 +}
69 +
70 +const noopLogger: Logger = { info: () => {}, warn: () => {}, error: () => {} };
71 +
72 +export interface OrchestratorDeps {
73 + llm: LlmClient;
74 + emit: EmitFn;
75 + now?: () => number;
76 + logger?: Logger;
77 +}
78 +
79 +export interface RunOptions {
80 + debateId: string;
81 + /** Debate-level cancellation (server shutdown / explicit user cancel). */
82 + signal?: AbortSignal;
83 +}
84 +
85 +/** Mutable run state, threaded through the phase helpers. */
86 +interface RunState {
87 + current: Map<string, AnswerRecord>;
88 + active: Set<string>;
89 + initialAnswers: AnswerRecord[];
90 + rounds: RoundRecord[];
91 + failures: FailureRecord[];
92 + totals: Usage;
93 + costByModel: Map<string, number>;
94 + lastDisagreements: Disagreement[];
95 + /** Revisions produced by the in-flight round, folded into a RoundRecord. */
96 + pendingRevisions?: RevisionRecord[];
97 + synthesisResult: SynthesisRecord | null;
98 +}
99 +
100 +export async function runDebate(
101 + config: DebateConfig,
102 + deps: OrchestratorDeps,
103 + opts: RunOptions,
104 +): Promise<DebateResult> {
105 + const now = deps.now ?? Date.now;
106 + const logger = deps.logger ?? noopLogger;
107 + const startedAt = now();
108 +
109 + const participants: Participant[] = participantsFor(config.models);
110 + const byId = new Map(participants.map((p) => [p.id, p]));
111 +
112 + const state: RunState = {
113 + current: new Map(),
114 + active: new Set(),
115 + initialAnswers: [],
116 + rounds: [],
117 + failures: [],
118 + totals: emptyUsage(),
119 + costByModel: new Map(),
120 + lastDisagreements: [],
121 + synthesisResult: null,
122 + };
123 +
124 + // --- helpers -------------------------------------------------------------
125 + const emit = (e: DebateEvent) => deps.emit(e);
126 +
127 + const bill = (model: string, usage: Usage) => {
128 + state.totals = addUsage(state.totals, usage);
129 + state.costByModel.set(model, (state.costByModel.get(model) ?? 0) + usage.costUsd);
130 + };
131 +
132 + const emitCost = async () => {
133 + await emit({
134 + type: 'cost_update',
135 + totalCostUsd: state.totals.costUsd,
136 + promptTokens: state.totals.promptTokens,
137 + completionTokens: state.totals.completionTokens,
138 + costByModel: Object.fromEntries(state.costByModel),
139 + });
140 + };
141 +
142 + const recordFailure = async (
143 + round: number,
144 + stage: StageType,
145 + p: Participant,
146 + error: unknown,
147 + droppedFromDebate: boolean,
148 + ) => {
149 + const message = error instanceof Error ? error.message : String(error);
150 + // A structured-parse failure still cost tokens - bill them.
151 + if (error instanceof StructuredParseError) bill(p.model, error.usage);
152 + state.failures.push({
153 + round,
154 + stage,
155 + participantId: p.id,
156 + model: p.model,
157 + error: message,
158 + droppedFromDebate,
159 + });
160 + if (droppedFromDebate) state.active.delete(p.id);
161 + logger.warn('model_failed', { round, stage, model: p.model, error: message, droppedFromDebate });
162 + await emit({
163 + type: 'model_failed',
164 + round,
165 + stage,
166 + participantId: p.id,
167 + model: p.model,
168 + error: message,
169 + droppedFromDebate,
170 + });
171 + };
172 +
173 + const structured = <S extends z.ZodTypeAny>(req: LlmRequest, schema: S, signal: AbortSignal) =>
174 + requestStructured(deps.llm, { ...req, signal }, schema);
175 +
176 + const finish = async (status: DebateStatus, error?: string): Promise<DebateResult> => {
177 + const finalAnswers = [...state.active]
178 + .map((id) => state.current.get(id))
179 + .filter((a): a is AnswerRecord => Boolean(a));
180 + const durationMs = now() - startedAt;
181 +
182 + await emit({
183 + type: 'debate_completed',
184 + debateId: opts.debateId,
185 + status,
186 + totalCostUsd: state.totals.costUsd,
187 + rounds: state.rounds.length,
188 + durationMs,
189 + });
190 +
191 + return {
192 + debateId: opts.debateId,
193 + config,
194 + participants,
195 + status,
196 + initialAnswers: state.initialAnswers,
197 + rounds: state.rounds,
198 + synthesis: state.synthesisResult ?? null,
199 + failures: state.failures,
200 + finalAnswers,
201 + totals: {
202 + costUsd: state.totals.costUsd,
203 + promptTokens: state.totals.promptTokens,
204 + completionTokens: state.totals.completionTokens,
205 + rounds: state.rounds.length,
206 + durationMs,
207 + costByModel: Object.fromEntries(state.costByModel),
208 + },
209 + ...(error ? { error } : {}),
210 + };
211 + };
212 +
213 + // --- start ---------------------------------------------------------------
214 + await emit({
215 + type: 'debate_started',
216 + debateId: opts.debateId,
217 + config,
218 + participants,
219 + chairmanModel: config.chairmanModel,
220 + chairmanProviderConflict: chairmanSharesProvider(config.chairmanModel, config.models),
221 + });
222 +
223 + try {
224 + // === Round 0: independent, streamed answers ===========================
225 + await emit({ type: 'round_started', round: 0, kind: 'answers' });
226 + await runForEach(participants, async (p) => {
227 + await emit({ type: 'stage_started', round: 0, stage: 'answer', participantId: p.id, model: p.model });
228 + try {
229 + const record = await streamAnswer(p, 0, config.question);
230 + state.current.set(p.id, record);
231 + state.active.add(p.id);
232 + state.initialAnswers.push(record);
233 + bill(p.model, record.usage);
234 + await emit({ type: 'answer_completed', round: 0, record });
235 + } catch (err) {
236 + // No prior answer exists → this member cannot participate at all.
237 + await recordFailure(0, 'answer', p, err, true);
238 + }
239 + });
240 + await emitCost();
241 + throwIfAborted(opts.signal);
242 +
243 + if (state.active.size < 2) {
244 + return finish('failed', 'Not enough models produced an initial answer (need ≥2).');
245 + }
246 +
247 + // === Rounds 1..maxRounds: critique -> revision -> convergence =========
248 + let stopReason: StopReason = 'max_rounds';
249 + for (let round = 1; round <= config.maxRounds; round++) {
250 + await emit({ type: 'round_started', round, kind: 'cycle' });
251 +
252 + const critiques = await critiquePhase(round);
253 + throwIfAborted(opts.signal);
254 + await revisionPhase(round, critiques);
255 + throwIfAborted(opts.signal);
256 + const convergence = await convergencePhase(round);
257 + await emitCost();
258 +
259 + const decision = shouldStop({
260 + score: convergence?.score ?? 0,
261 + threshold: config.convergenceThreshold,
262 + round,
263 + maxRounds: config.maxRounds,
264 + activeCount: state.active.size,
265 + });
266 + state.rounds.push({ round, critiques, revisions: state.pendingRevisions ?? [], convergence });
267 + state.pendingRevisions = undefined;
268 +
269 + if (decision.stop) {
270 + stopReason = decision.reason;
271 + if (decision.reason === 'insufficient_models') {
272 + return finish('failed', 'Too many models dropped out to continue the debate.');
273 + }
274 + break;
275 + }
276 + }
277 + logger.info('deliberation_complete', { stopReason, rounds: state.rounds.length });
278 + throwIfAborted(opts.signal);
279 +
280 + // === Synthesis ========================================================
281 + await synthesisPhase();
282 + await emitCost();
283 +
284 + return finish('completed');
285 + } catch (err) {
286 + if (isAbortError(err) || opts.signal?.aborted) {
287 + logger.warn('debate_aborted', { debateId: opts.debateId });
288 + return finish('aborted', 'Debate was cancelled.');
289 + }
290 + const message = err instanceof Error ? err.message : String(err);
291 + logger.error('debate_failed', { debateId: opts.debateId, error: message });
292 + await emit({ type: 'debate_failed', debateId: opts.debateId, error: message });
293 + return finish('failed', message);
294 + }
295 +
296 + // --- phase implementations (closures over state) -------------------------
297 +
298 + async function streamAnswer(p: Participant, round: number, question: string): Promise<AnswerRecord> {
299 + const t0 = now();
300 + return withTimeout(async (signal) => {
301 + const handle = await deps.llm.streamComplete({
302 + model: p.model,
303 + messages: buildAnswerPrompt(question),
304 + temperature: config.temperature,
305 + signal,
306 + meta: { stage: 'answer', round, participantId: p.id, seed: p.id },
307 + });
308 + for await (const delta of handle.stream) {
309 + await emit({ type: 'token_delta', round, stage: 'answer', participantId: p.id, delta });
310 + }
311 + const result = await handle.result;
312 + return {
313 + participantId: p.id,
314 + model: p.model,
315 + round,
316 + content: result.text,
317 + usage: result.usage,
318 + latencyMs: result.latencyMs || now() - t0,
319 + };
320 + }, config.perModelTimeoutMs, opts.signal);
321 + }
322 +
323 + async function critiquePhase(round: number): Promise<CritiqueRecord[]> {
324 + const reviewers = participants.filter((p) => state.active.has(p.id));
325 + const results: CritiqueRecord[] = [];
326 +
327 + await runForEach(reviewers, async (reviewer) => {
328 + const peers = reviewers
329 + .filter((p) => p.id !== reviewer.id)
330 + .map((p) => ({ participantId: p.id, content: state.current.get(p.id)!.content }));
331 + if (peers.length === 0) return;
332 +
333 + const anon = anonymizePeers(peers, hashSeed(opts.debateId, round, 'critique', reviewer.id));
334 + await emit({ type: 'stage_started', round, stage: 'critique', participantId: reviewer.id, model: reviewer.model });
335 + try {
336 + const out = await withTimeout(
337 + (signal) =>
338 + structured(
339 + {
340 + model: reviewer.model,
341 + messages: buildCritiquePrompt(config.question, anon.peers),
342 + temperature: config.temperature,
343 + meta: { stage: 'critique', round, participantId: reviewer.id },
344 + },
345 + critiqueOutputSchema,
346 + signal,
347 + ),
348 + config.perModelTimeoutMs,
349 + opts.signal,
350 + );
351 + bill(reviewer.model, out.usage);
352 + const record = buildCritiqueRecord(round, reviewer, out.value.reviews, anon, out.usage, out.latencyMs);
353 + results.push(record);
354 + await emit({ type: 'critique_completed', round, record });
355 + } catch (err) {
356 + // Losing a reviewer's critique doesn't remove it from the debate.
357 + await recordFailure(round, 'critique', reviewer, err, false);
358 + }
359 + });
360 +
361 + return results;
362 + }
363 +
364 + async function revisionPhase(round: number, critiques: CritiqueRecord[]): Promise<void> {
365 + const revisers = participants.filter((p) => state.active.has(p.id));
366 + const revisions: RevisionRecord[] = [];
367 +
368 + await runForEach(revisers, async (p) => {
369 + const incoming: IncomingCritique[] = collectIncoming(p.id, critiques);
370 + const ownAnswer = state.current.get(p.id)!.content;
371 + await emit({ type: 'stage_started', round, stage: 'revision', participantId: p.id, model: p.model });
372 + try {
373 + const out = await withTimeout(
374 + (signal) =>
375 + structured(
376 + {
377 + model: p.model,
378 + messages: buildRevisionPrompt(config.question, ownAnswer, incoming),
379 + temperature: config.temperature,
380 + meta: { stage: 'revision', round, participantId: p.id },
381 + },
382 + revisionOutputSchema,
383 + signal,
384 + ),
385 + config.perModelTimeoutMs,
386 + opts.signal,
387 + );
388 + bill(p.model, out.usage);
389 + const record: RevisionRecord = {
390 + round,
391 + participantId: p.id,
392 + model: p.model,
393 + content: out.value.answer,
394 + changelog: {
395 + changed: out.value.changelog.changed,
396 + summary: out.value.changelog.summary,
397 + bullets: out.value.changelog.bullets,
398 + },
399 + usage: out.usage,
400 + latencyMs: out.latencyMs,
401 + };
402 + // Adopt the revised answer as this participant's current answer.
403 + state.current.set(p.id, {
404 + participantId: p.id,
405 + model: p.model,
406 + round,
407 + content: out.value.answer,
408 + usage: out.usage,
409 + latencyMs: out.latencyMs,
410 + });
411 + revisions.push(record);
412 + await emit({ type: 'revision_completed', round, record });
413 + } catch (err) {
414 + // Keep the participant's previous answer; drop only this revision.
415 + await recordFailure(round, 'revision', p, err, false);
416 + }
417 + });
418 +
419 + state.pendingRevisions = revisions;
420 + }
421 +
422 + async function convergencePhase(round: number): Promise<ConvergenceRecord | null> {
423 + const answersList = participants
424 + .filter((p) => state.active.has(p.id))
425 + .map((p) => ({ participantId: p.id, content: state.current.get(p.id)!.content }));
426 +
427 + const anon = anonymizePeers(answersList, hashSeed(opts.debateId, round, 'convergence'));
428 + await emit({ type: 'stage_started', round, stage: 'convergence', model: config.convergenceModel });
429 + try {
430 + const out = await withTimeout(
431 + (signal) =>
432 + structured(
433 + {
434 + model: config.convergenceModel,
435 + messages: buildConvergencePrompt(config.question, anon.peers),
436 + temperature: 0,
437 + meta: { stage: 'convergence', round },
438 + },
439 + convergenceOutputSchema,
440 + signal,
441 + ),
442 + config.perModelTimeoutMs,
443 + opts.signal,
444 + );
445 + bill(config.convergenceModel, out.usage);
446 + const disagreements = mapDisagreements(out.value.disagreements, anon);
447 + state.lastDisagreements = disagreements;
448 + const record: ConvergenceRecord = {
449 + round,
450 + model: config.convergenceModel,
451 + score: out.value.score,
452 + disagreements,
453 + converged: out.value.score >= config.convergenceThreshold,
454 + usage: out.usage,
455 + latencyMs: out.latencyMs,
456 + };
457 + await emit({ type: 'convergence_result', round, record });
458 + return record;
459 + } catch (err) {
460 + // Convergence is advisory - never fail the debate on it. Treat as "not
461 + // converged" and continue (or stop at max rounds).
462 + const message = err instanceof Error ? err.message : String(err);
463 + if (err instanceof StructuredParseError) bill(config.convergenceModel, err.usage);
464 + logger.warn('convergence_failed', { round, error: message });
465 + const record: ConvergenceRecord = {
466 + round,
467 + model: config.convergenceModel,
468 + score: 0,
469 + disagreements: state.lastDisagreements,
470 + converged: false,
471 + usage: emptyUsage(),
472 + latencyMs: 0,
473 + };
474 + await emit({ type: 'convergence_result', round, record });
475 + return record;
476 + }
477 + }
478 +
479 + async function synthesisPhase(): Promise<void> {
480 + const chairman = { id: 'chairman', model: config.chairmanModel, displayName: displayNameForModel(config.chairmanModel) };
481 + const finalists = participants
482 + .filter((p) => state.active.has(p.id))
483 + .map((p) => ({ participantId: p.id, content: state.current.get(p.id)!.content }));
484 + const anon = anonymizePeers(finalists, hashSeed(opts.debateId, 'synthesis'));
485 + const remaining = state.lastDisagreements.map((d) => ({ topic: d.topic, summary: d.summary }));
486 +
487 + await emit({ type: 'stage_started', round: state.rounds.length, stage: 'synthesis', model: chairman.model });
488 + try {
489 + const out = await withTimeout(
490 + (signal) =>
491 + structured(
492 + {
493 + model: chairman.model,
494 + messages: buildSynthesisPrompt(config.question, anon.peers, remaining),
495 + temperature: config.temperature,
496 + meta: { stage: 'synthesis', round: state.rounds.length },
497 + },
498 + synthesisOutputSchema,
499 + signal,
500 + ),
501 + config.perModelTimeoutMs,
502 + opts.signal,
503 + );
504 + bill(chairman.model, out.usage);
505 + const record: SynthesisRecord = {
506 + model: chairman.model,
507 + finalAnswer: out.value.finalAnswer,
508 + dissent: out.value.dissent.map((d) => ({
509 + topic: d.topic,
510 + positions: d.positions
511 + .map((pos) => {
512 + const pid = anon.labelMap[pos.label];
513 + const part = pid ? byId.get(pid) : undefined;
514 + return part ? { participantId: part.id, model: part.model, position: pos.position } : null;
515 + })
516 + .filter((x): x is NonNullable<typeof x> => x !== null),
517 + })),
518 + usage: out.usage,
519 + latencyMs: out.latencyMs,
520 + };
521 + state.synthesisResult = record;
522 + await emit({ type: 'synthesis_completed', record });
523 + } catch (err) {
524 + // Fallback synthesis so the debate still yields a usable answer.
525 + if (err instanceof StructuredParseError) bill(chairman.model, err.usage);
526 + const message = err instanceof Error ? err.message : String(err);
527 + logger.error('synthesis_failed', { error: message });
528 + state.failures.push({
529 + round: state.rounds.length,
530 + stage: 'synthesis',
531 + participantId: 'chairman',
532 + model: chairman.model,
533 + error: message,
534 + droppedFromDebate: false,
535 + });
536 + const fallback = fallbackSynthesis();
537 + state.synthesisResult = fallback;
538 + await emit({ type: 'synthesis_completed', record: fallback });
539 + }
540 + }
541 +
542 + function fallbackSynthesis(): SynthesisRecord {
543 + const answers = [...state.active].map((id) => state.current.get(id)!).filter(Boolean);
544 + const best = answers[0];
545 + return {
546 + model: config.chairmanModel,
547 + finalAnswer:
548 + (best?.content ?? 'No answer could be produced.') +
549 + '\n\n_(Chairman synthesis was unavailable; showing the leading council answer.)_',
550 + dissent: state.lastDisagreements.map((d) => ({
551 + topic: d.topic,
552 + positions: d.positions
553 + .filter((pos) => pos.participantId)
554 + .map((pos) => ({
555 + participantId: pos.participantId!,
556 + model: byId.get(pos.participantId!)?.model ?? 'unknown',
557 + position: pos.stance,
558 + })),
559 + })),
560 + usage: emptyUsage(),
561 + latencyMs: 0,
562 + };
563 + }
564 +
565 + function buildCritiqueRecord(
566 + round: number,
567 + reviewer: Participant,
568 + reviews: Array<{ label: string; weaknesses: string[]; strengths: string[]; score: number; justification: string }>,
569 + anon: AnonymizationResult,
570 + usage: Usage,
571 + latencyMs: number,
572 + ): CritiqueRecord {
573 + const mapped: PeerReview[] = reviews
574 + .map((r) => {
575 + const targetId = anon.labelMap[r.label];
576 + if (!targetId) return null;
577 + return {
578 + label: r.label,
579 + targetParticipantId: targetId,
580 + weaknesses: r.weaknesses,
581 + strengths: r.strengths,
582 + score: r.score,
583 + justification: r.justification,
584 + } satisfies PeerReview;
585 + })
586 + .filter((x): x is PeerReview => x !== null);
587 + return {
588 + round,
589 + reviewerParticipantId: reviewer.id,
590 + reviewerModel: reviewer.model,
591 + reviews: mapped,
592 + usage,
593 + latencyMs,
594 + };
595 + }
596 +
597 + function collectIncoming(participantId: string, critiques: CritiqueRecord[]): IncomingCritique[] {
598 + const out: IncomingCritique[] = [];
599 + for (const c of critiques) {
600 + for (const r of c.reviews) {
601 + if (r.targetParticipantId === participantId) {
602 + out.push({
603 + weaknesses: r.weaknesses,
604 + strengths: r.strengths,
605 + score: r.score,
606 + justification: r.justification,
607 + });
608 + }
609 + }
610 + }
611 + return out;
612 + }
613 +
614 + function mapDisagreements(
615 + raw: Array<{ topic: string; summary: string; positions: Array<{ label: string; stance: string }> }>,
616 + anon: AnonymizationResult,
617 + ): Disagreement[] {
618 + return raw.map((d) => ({
619 + topic: d.topic,
620 + summary: d.summary,
621 + positions: d.positions.map((pos) => ({
622 + label: pos.label,
623 + participantId: anon.labelMap[pos.label],
624 + stance: pos.stance,
625 + })),
626 + }));
627 + }
628 +}
629 +
630 +// --- module-scope helpers --------------------------------------------------
631 +
632 +/**
633 + * Run an async task per item concurrently, isolating failures. Individual tasks
634 + * are expected to handle their own errors (they emit `model_failed`); this just
635 + * guarantees one thrown task can't reject the whole phase.
636 + */
637 +async function runForEach<T>(items: T[], task: (item: T) => Promise<void>): Promise<void> {
638 + await Promise.all(
639 + items.map(async (item) => {
640 + try {
641 + await task(item);
642 + } catch {
643 + // Task-level errors are already recorded as failures by the task itself.
644 + }
645 + }),
646 + );
647 +}
648 +
649 +function isAbortError(err: unknown): boolean {
650 + return (
651 + (err instanceof Error && err.name === 'AbortError') ||
652 + (typeof err === 'object' && err !== null && 'name' in err && (err as { name?: string }).name === 'AbortError')
653 + );
654 +}
655 +
656 +function throwIfAborted(signal?: AbortSignal): void {
657 + if (signal?.aborted) {
658 + const e = new Error('Aborted');
659 + e.name = 'AbortError';
660 + throw e;
661 + }
662 +}
added src/core/prompts/answer.ts +27 −0
@@ -0,0 +1,27 @@
1 +import type { LlmMessage } from '../llm-client';
2 +import { ROUNDTABLE_PREAMBLE } from './index';
3 +
4 +/**
5 + * Round 0 - independent answer.
6 + *
7 + * Each council member answers the question cold, with no knowledge of the other
8 + * members' responses. Deliberately open-ended: we want genuine diversity of
9 + * approach here, which the later rounds then reconcile.
10 + */
11 +export function buildAnswerPrompt(question: string): LlmMessage[] {
12 + return [
13 + {
14 + role: 'system',
15 + content:
16 + `${ROUNDTABLE_PREAMBLE}\n\n` +
17 + 'This is the opening round. Answer the question directly and completely on ' +
18 + 'your own. State your reasoning and note any important assumptions, caveats, ' +
19 + 'or uncertainty. Do not hedge to the point of vagueness - commit to a clear ' +
20 + 'position where the evidence supports one.',
21 + },
22 + {
23 + role: 'user',
24 + content: `Question:\n${question}`,
25 + },
26 + ];
27 +}
added src/core/prompts/convergence.ts +48 −0
@@ -0,0 +1,48 @@
1 +import type { AnonymizedPeer } from '../anonymize';
2 +import type { LlmMessage } from '../llm-client';
3 +import { jsonInstruction, ROUNDTABLE_PREAMBLE } from './index';
4 +
5 +const CONVERGENCE_SHAPE = `{
6 + "score": 72, // 0-100: how substantively converged the answers are
7 + "disagreements": [
8 + {
9 + "topic": "short label for the contested point",
10 + "summary": "what the disagreement is about",
11 + "positions": [
12 + { "label": "A", "stance": "what response A holds on this point" },
13 + { "label": "C", "stance": "what response C holds on this point" }
14 + ]
15 + }
16 + ]
17 +}`;
18 +
19 +/**
20 + * Convergence check (runs after each round, on a cheap/fast model).
21 + *
22 + * Judges *substantive* agreement, not surface wording: two answers that reach the
23 + * same conclusion by different phrasing are converged; two that share boilerplate
24 + * but differ on the key claim are not. Returns a 0-100 score plus the concrete
25 + * points still in dispute, which feed both the early-stop decision and the
26 + * chairman's dissent report.
27 + */
28 +export function buildConvergencePrompt(question: string, answers: AnonymizedPeer[]): LlmMessage[] {
29 + const block = answers.map((a) => `--- Response ${a.label} ---\n${a.content}`).join('\n\n');
30 +
31 + return [
32 + {
33 + role: 'system',
34 + content:
35 + `${ROUNDTABLE_PREAMBLE}\n\n` +
36 + 'You are the neutral convergence assessor. Read the current answers and judge ' +
37 + 'how much they substantively agree on the points that matter for the question. ' +
38 + 'Ignore stylistic differences. A score of 100 means they would give the reader ' +
39 + 'the same practical conclusion; a low score means they still disagree on ' +
40 + 'substance. List every material remaining disagreement.\n\n' +
41 + jsonInstruction(CONVERGENCE_SHAPE),
42 + },
43 + {
44 + role: 'user',
45 + content: `Question:\n${question}\n\nCurrent answers:\n\n${block}\n\nReturn your JSON assessment now.`,
46 + },
47 + ];
48 +}
added src/core/prompts/critique.ts +54 −0
@@ -0,0 +1,54 @@
1 +import type { AnonymizedPeer } from '../anonymize';
2 +import type { LlmMessage } from '../llm-client';
3 +import { jsonInstruction, ROUNDTABLE_PREAMBLE } from './index';
4 +
5 +const CRITIQUE_SHAPE = `{
6 + "reviews": [
7 + {
8 + "label": "A", // the label of the response being reviewed
9 + "weaknesses": ["specific error or gap", "..."],
10 + "strengths": ["specific strong point", "..."],
11 + "score": 7, // integer 1-10, overall quality
12 + "justification": "one or two sentences explaining the score"
13 + }
14 + // ...one entry per response shown to you
15 + ]
16 +}`;
17 +
18 +/**
19 + * Critique phase.
20 + *
21 + * The reviewer sees every *other* member's answer, fully anonymized and in a
22 + * per-reviewer randomized order (labels "Response A/B/C..."). It must return, for
23 + * each response, concrete weaknesses, concrete strengths, and a calibrated 1-10
24 + * score with justification. Anonymity + randomized order is the core bias
25 + * mitigation: a model cannot preferentially reward "its own style" if it cannot
26 + * tell which answer is whose.
27 + */
28 +export function buildCritiquePrompt(question: string, peers: AnonymizedPeer[]): LlmMessage[] {
29 + const peerBlock = peers
30 + .map((p) => `--- Response ${p.label} ---\n${p.content}`)
31 + .join('\n\n');
32 +
33 + return [
34 + {
35 + role: 'system',
36 + content:
37 + `${ROUNDTABLE_PREAMBLE}\n\n` +
38 + "You are now reviewing the other members' answers. They are anonymized and " +
39 + 'shown in a random order; you do not know who wrote which, and none of them is ' +
40 + 'your own. Judge only on merit: correctness first, then completeness, clarity, ' +
41 + 'and calibration. Be specific - cite the exact claim you think is wrong or ' +
42 + 'missing. Reward genuine strengths honestly. Scores should span the range; do ' +
43 + 'not cluster everything at 7-8.\n\n' +
44 + jsonInstruction(CRITIQUE_SHAPE),
45 + },
46 + {
47 + role: 'user',
48 + content:
49 + `Question:\n${question}\n\n` +
50 + `Answers to review (${peers.length}):\n\n${peerBlock}\n\n` +
51 + 'Return your JSON review now.',
52 + },
53 + ];
54 +}
added src/core/prompts/index.ts +40 −0
@@ -0,0 +1,40 @@
1 +/**
2 + * Versioned prompt templates for every debate stage.
3 + *
4 + * These are the *product* of Roundtable as much as the code is - the debate is
5 + * only as good as the instructions the models receive. They live here, apart
6 + * from orchestration logic, so they can be reviewed, versioned, and A/B'd
7 + * without touching the state machine.
8 + *
9 + * `PROMPT_VERSION` is bumped whenever a template changes in a way that would
10 + * alter model behavior; it is stamped onto stored results so historical debates
11 + * remain interpretable.
12 + */
13 +export const PROMPT_VERSION = '1.0.0';
14 +
15 +export { buildAnswerPrompt } from './answer';
16 +export { buildCritiquePrompt } from './critique';
17 +export { buildRevisionPrompt } from './revision';
18 +export { buildConvergencePrompt } from './convergence';
19 +export { buildSynthesisPrompt } from './synthesis';
20 +export { buildRepairPrompt } from './repair';
21 +
22 +/**
23 + * Shared system preamble establishing the ground rules of the roundtable. Every
24 + * stage builds on this so models share a consistent frame.
25 + */
26 +export const ROUNDTABLE_PREAMBLE =
27 + 'You are one member of an expert roundtable convened to answer a question as ' +
28 + 'accurately and completely as possible. Other independent experts are answering ' +
29 + 'the same question in parallel. The goal is a correct, well-reasoned answer - not ' +
30 + 'winning. Be rigorous, cite concrete reasoning, and concede points that are right.';
31 +
32 +/** Standard instruction appended to any stage that must return strict JSON. */
33 +export function jsonInstruction(shape: string): string {
34 + return (
35 + 'Respond with a SINGLE JSON object and nothing else - no prose, no Markdown ' +
36 + 'code fences, no commentary before or after. The object MUST match this shape ' +
37 + `exactly:\n\n${shape}\n\n` +
38 + 'Use plain double-quoted strings. Do not include trailing commas.'
39 + );
40 +}
added src/core/prompts/repair.ts +29 −0
@@ -0,0 +1,29 @@
1 +import type { LlmMessage } from '../llm-client';
2 +
3 +/**
4 + * JSON repair prompt.
5 + *
6 + * Sent as a single retry when a model's structured output failed to parse or
7 + * validate. We echo back the raw response and the precise validation error and
8 + * ask only for corrected JSON - cheaper and more reliable than re-running the
9 + * whole stage.
10 + */
11 +export function buildRepairPrompt(rawResponse: string, validationError: string): LlmMessage[] {
12 + return [
13 + {
14 + role: 'system',
15 + content:
16 + 'You are a strict JSON fixer. You will be given a malformed or invalid response ' +
17 + 'and the reason it failed validation. Return ONLY the corrected JSON object that ' +
18 + 'satisfies the requirement - no prose, no code fences. Preserve all of the ' +
19 + 'original content and meaning; change only what is needed to make it valid.',
20 + },
21 + {
22 + role: 'user',
23 + content:
24 + `The following response was invalid.\n\nValidation error: ${validationError}\n\n` +
25 + `Original response:\n${rawResponse}\n\n` +
26 + 'Return the corrected JSON now.',
27 + },
28 + ];
29 +}
added src/core/prompts/revision.ts +67 −0
@@ -0,0 +1,67 @@
1 +import type { LlmMessage } from '../llm-client';
2 +import { jsonInstruction, ROUNDTABLE_PREAMBLE } from './index';
3 +
4 +/** One reviewer's feedback on this model's answer, with the source hidden. */
5 +export interface IncomingCritique {
6 + weaknesses: string[];
7 + strengths: string[];
8 + score: number;
9 + justification: string;
10 +}
11 +
12 +const REVISION_SHAPE = `{
13 + "answer": "your full revised answer (or your unchanged answer if you are defending it)",
14 + "changelog": {
15 + "changed": true, // false if you are keeping your answer as-is
16 + "summary": "1-2 sentences: what you changed and why, OR why you are defending your original position",
17 + "bullets": ["concrete change 1", "concrete change 2"] // may be empty if unchanged
18 + }
19 +}`;
20 +
21 +/**
22 + * Revision phase.
23 + *
24 + * The model receives the anonymized critiques that targeted *its own* answer and
25 + * decides, point by point, whether each is right. It either revises or explicitly
26 + * defends. Crucially it must NOT cave to social pressure - a well-reasoned defense
27 + * with justification is a first-class outcome, which is why the changelog forces
28 + * an explicit `changed` boolean and reasoning either way.
29 + */
30 +export function buildRevisionPrompt(
31 + question: string,
32 + ownAnswer: string,
33 + critiques: IncomingCritique[],
34 +): LlmMessage[] {
35 + const feedbackBlock =
36 + critiques.length === 0
37 + ? 'No critiques were received this round.'
38 + : critiques
39 + .map((c, i) => {
40 + const w = c.weaknesses.length ? c.weaknesses.map((x) => ` - ${x}`).join('\n') : ' - (none)';
41 + const s = c.strengths.length ? c.strengths.map((x) => ` - ${x}`).join('\n') : ' - (none)';
42 + return `Reviewer ${i + 1} (score ${c.score}/10):\nWeaknesses:\n${w}\nStrengths:\n${s}\nComment: ${c.justification}`;
43 + })
44 + .join('\n\n');
45 +
46 + return [
47 + {
48 + role: 'system',
49 + content:
50 + `${ROUNDTABLE_PREAMBLE}\n\n` +
51 + 'You have received anonymous critiques of your answer. Weigh each on its ' +
52 + 'merits. Where a critique is correct, revise to fix it. Where it is wrong or ' +
53 + 'misguided, keep your position and say why - do not change your answer just to ' +
54 + 'appease reviewers. Aim for the most correct, complete answer, not consensus ' +
55 + 'for its own sake.\n\n' +
56 + jsonInstruction(REVISION_SHAPE),
57 + },
58 + {
59 + role: 'user',
60 + content:
61 + `Question:\n${question}\n\n` +
62 + `Your current answer:\n${ownAnswer}\n\n` +
63 + `Critiques of your answer:\n${feedbackBlock}\n\n` +
64 + 'Return your JSON revision now.',
65 + },
66 + ];
67 +}
added src/core/prompts/synthesis.ts +67 −0
@@ -0,0 +1,67 @@
1 +import type { AnonymizedPeer } from '../anonymize';
2 +import type { LlmMessage } from '../llm-client';
3 +import { jsonInstruction, ROUNDTABLE_PREAMBLE } from './index';
4 +
5 +const SYNTHESIS_SHAPE = `{
6 + "finalAnswer": "the single best answer to the question, synthesized from the council",
7 + "dissent": [
8 + {
9 + "topic": "short label for an unresolved disagreement",
10 + "positions": [
11 + { "label": "A", "position": "what response A maintains" },
12 + { "label": "B", "position": "what response B maintains" }
13 + ]
14 + }
15 + ]
16 +}`;
17 +
18 +/**
19 + * Chairman synthesis (final stage).
20 + *
21 + * The chairman receives the anonymized final answers, the outstanding
22 + * disagreements from the convergence check, and produces the authoritative
23 + * answer plus an explicit dissent report. Answers are anonymized to the chairman
24 + * as well - this is deliberate self-preference mitigation: even a chairman drawn
25 + * from the same provider family as a council member cannot favor "its own"
26 + * answer if it cannot identify it. Labels are mapped back to real models only
27 + * after the chairman has committed its judgment.
28 + */
29 +export function buildSynthesisPrompt(
30 + question: string,
31 + finalAnswers: AnonymizedPeer[],
32 + remainingDisagreements: Array<{ topic: string; summary: string }>,
33 +): LlmMessage[] {
34 + const answersBlock = finalAnswers
35 + .map((a) => `--- Response ${a.label} ---\n${a.content}`)
36 + .join('\n\n');
37 +
38 + const disagreementBlock =
39 + remainingDisagreements.length === 0
40 + ? 'The council largely converged; no major disagreements were flagged.'
41 + : remainingDisagreements.map((d) => `- ${d.topic}: ${d.summary}`).join('\n');
42 +
43 + return [
44 + {
45 + role: 'system',
46 + content:
47 + 'You are the chairman of an expert roundtable. Several anonymized experts have ' +
48 + 'answered a question and revised their answers after mutual critique. Your job ' +
49 + 'is to deliver the single best final answer and an honest dissent report.\n\n' +
50 + `${ROUNDTABLE_PREAMBLE}\n\n` +
51 + 'Synthesize - do not merely pick one answer or average them. Take the strongest, ' +
52 + 'best-supported reasoning from across the council. Where the council genuinely ' +
53 + 'disagrees and the evidence does not settle it, do NOT paper over it: record it ' +
54 + "in the dissent report with each side's position. The responses are anonymized; " +
55 + 'judge only on merit.\n\n' +
56 + jsonInstruction(SYNTHESIS_SHAPE),
57 + },
58 + {
59 + role: 'user',
60 + content:
61 + `Question:\n${question}\n\n` +
62 + `Final answers from the council:\n\n${answersBlock}\n\n` +
63 + `Points still in dispute after deliberation:\n${disagreementBlock}\n\n` +
64 + 'Return your JSON synthesis now.',
65 + },
66 + ];
67 +}
added src/core/schemas.ts +98 −0
@@ -0,0 +1,98 @@
1 +/**
2 + * Runtime schemas for everything that crosses a trust boundary:
3 + * - the structured JSON we force models to return (critique/revision/etc.)
4 + * - the debate configuration accepted from API clients
5 + *
6 + * Model output is untrusted input. Every model response that is supposed to be
7 + * JSON is parsed defensively and validated against these schemas, with one
8 + * repair attempt on failure (see `structured.ts`).
9 + */
10 +import { z } from 'zod';
11 +
12 +// ---------------------------------------------------------------------------
13 +// Structured model outputs
14 +// ---------------------------------------------------------------------------
15 +
16 +/** One reviewer's assessment of one anonymized peer answer. */
17 +export const peerReviewSchema = z.object({
18 + label: z.string().min(1).max(4),
19 + weaknesses: z.array(z.string()).default([]),
20 + strengths: z.array(z.string()).default([]),
21 + score: z.coerce.number().min(1).max(10),
22 + justification: z.string().default(''),
23 +});
24 +
25 +export const critiqueOutputSchema = z.object({
26 + reviews: z.array(peerReviewSchema).min(1),
27 +});
28 +export type CritiqueOutput = z.infer<typeof critiqueOutputSchema>;
29 +
30 +export const revisionOutputSchema = z.object({
31 + answer: z.string().min(1),
32 + changelog: z.object({
33 + changed: z.boolean(),
34 + summary: z.string().default(''),
35 + bullets: z.array(z.string()).default([]),
36 + }),
37 +});
38 +export type RevisionOutput = z.infer<typeof revisionOutputSchema>;
39 +
40 +export const convergenceOutputSchema = z.object({
41 + score: z.coerce.number().min(0).max(100),
42 + disagreements: z
43 + .array(
44 + z.object({
45 + topic: z.string(),
46 + summary: z.string().default(''),
47 + positions: z
48 + .array(z.object({ label: z.string(), stance: z.string() }))
49 + .default([]),
50 + }),
51 + )
52 + .default([]),
53 +});
54 +export type ConvergenceOutput = z.infer<typeof convergenceOutputSchema>;
55 +
56 +export const synthesisOutputSchema = z.object({
57 + finalAnswer: z.string().min(1),
58 + dissent: z
59 + .array(
60 + z.object({
61 + topic: z.string(),
62 + positions: z
63 + .array(z.object({ label: z.string(), position: z.string() }))
64 + .default([]),
65 + }),
66 + )
67 + .default([]),
68 +});
69 +export type SynthesisOutput = z.infer<typeof synthesisOutputSchema>;
70 +
71 +// ---------------------------------------------------------------------------
72 +// Debate configuration (API input)
73 +// ---------------------------------------------------------------------------
74 +
75 +/** OpenRouter model slugs look like "vendor/model[:variant]". */
76 +export const modelSlugSchema = z
77 + .string()
78 + .min(1)
79 + .max(128)
80 + .regex(/^[\w.-]+\/[\w.:-]+$/, 'Expected an OpenRouter model slug like "vendor/model"');
81 +
82 +export const debateConfigInputSchema = z
83 + .object({
84 + question: z.string().min(3).max(8000),
85 + models: z.array(modelSlugSchema).min(3).max(6),
86 + chairmanModel: modelSlugSchema,
87 + convergenceModel: modelSlugSchema.optional(),
88 + maxRounds: z.number().int().min(1).max(5).default(3),
89 + convergenceThreshold: z.number().int().min(0).max(100).default(85),
90 + temperature: z.number().min(0).max(2).default(0.7),
91 + perModelTimeoutMs: z.number().int().min(5_000).max(600_000).default(90_000),
92 + })
93 + .refine((c) => new Set(c.models).size === c.models.length, {
94 + message: 'Council models must be unique',
95 + path: ['models'],
96 + });
97 +
98 +export type DebateConfigInput = z.infer<typeof debateConfigInputSchema>;
added src/core/scoring.test.ts +72 −0
@@ -0,0 +1,72 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { buildScoreMatrix } from './scoring';
3 +import { emptyUsage, type CritiqueRecord, type Participant } from './types';
4 +
5 +const participants: Participant[] = [
6 + { id: 'p0', model: 'openai/gpt-4o', displayName: 'GPT 4o' },
7 + { id: 'p1', model: 'anthropic/claude-3.5', displayName: 'Claude 3.5' },
8 + { id: 'p2', model: 'google/gemini-pro', displayName: 'Gemini Pro' },
9 +];
10 +
11 +function critique(reviewer: string, scores: Record<string, number>): CritiqueRecord {
12 + return {
13 + round: 1,
14 + reviewerParticipantId: reviewer,
15 + reviewerModel: 'x/y',
16 + reviews: Object.entries(scores).map(([targetParticipantId, score]) => ({
17 + label: 'A',
18 + targetParticipantId,
19 + weaknesses: [],
20 + strengths: [],
21 + score,
22 + justification: '',
23 + })),
24 + usage: emptyUsage(),
25 + latencyMs: 0,
26 + };
27 +}
28 +
29 +describe('buildScoreMatrix', () => {
30 + it('places scores in reviewer×target cells with self-cells null', () => {
31 + const critiques = [
32 + critique('p0', { p1: 8, p2: 6 }),
33 + critique('p1', { p0: 7, p2: 5 }),
34 + critique('p2', { p0: 9, p1: 4 }),
35 + ];
36 + const m = buildScoreMatrix(participants, critiques);
37 +
38 + expect(m.order).toEqual(['p0', 'p1', 'p2']);
39 + expect(m.cells.p0!.p1).toBe(8);
40 + expect(m.cells.p0!.p2).toBe(6);
41 + expect(m.cells.p0!.p0).toBeNull(); // no self-review
42 + expect(m.cells.p2!.p1).toBe(4);
43 + });
44 +
45 + it('computes averages received and given', () => {
46 + const critiques = [
47 + critique('p0', { p1: 8, p2: 6 }),
48 + critique('p1', { p0: 7, p2: 5 }),
49 + critique('p2', { p0: 9, p1: 4 }),
50 + ];
51 + const m = buildScoreMatrix(participants, critiques);
52 +
53 + // p0 received 7 (from p1) and 9 (from p2) -> 8
54 + expect(m.averagesReceived.p0).toBe(8);
55 + // p1 received 8 (p0) and 4 (p2) -> 6
56 + expect(m.averagesReceived.p1).toBe(6);
57 + // p0 gave 8 and 6 -> 7
58 + expect(m.averagesGiven.p0).toBe(7);
59 + });
60 +
61 + it('returns null averages for a participant nobody scored', () => {
62 + const m = buildScoreMatrix(participants, [critique('p0', { p1: 8 })]);
63 + expect(m.averagesReceived.p2).toBeNull();
64 + expect(m.averagesGiven.p1).toBeNull();
65 + });
66 +
67 + it('ignores critiques from unknown reviewers defensively', () => {
68 + const stray = critique('ghost', { p0: 10 });
69 + const m = buildScoreMatrix(participants, [stray]);
70 + expect(m.averagesReceived.p0).toBeNull();
71 + });
72 +});
added src/core/scoring.ts +63 −0
@@ -0,0 +1,63 @@
1 +/**
2 + * Aggregate critique scores into the N×N matrix the UI renders.
3 + *
4 + * Rows are reviewers, columns are the answers being scored. A cell is null when
5 + * that reviewer did not score that target (e.g. the reviewer dropped out, or the
6 + * self-cell). Pure and deterministic - unit-tested against hand-built records.
7 + */
8 +import type { CritiqueRecord, Participant } from './types';
9 +
10 +export interface ScoreMatrix {
11 + /** participantIds in a stable order (matches `participants`). */
12 + order: string[];
13 + /** cells[reviewerId][targetId] = score or null. */
14 + cells: Record<string, Record<string, number | null>>;
15 + /** Mean score each participant received (over non-null cells). */
16 + averagesReceived: Record<string, number | null>;
17 + /** Mean score each participant handed out. */
18 + averagesGiven: Record<string, number | null>;
19 +}
20 +
21 +function mean(nums: number[]): number | null {
22 + if (nums.length === 0) return null;
23 + return nums.reduce((a, b) => a + b, 0) / nums.length;
24 +}
25 +
26 +export function buildScoreMatrix(
27 + participants: Participant[],
28 + critiques: CritiqueRecord[],
29 +): ScoreMatrix {
30 + const order = participants.map((p) => p.id);
31 + const cells: Record<string, Record<string, number | null>> = {};
32 + for (const reviewer of order) {
33 + cells[reviewer] = {};
34 + for (const target of order) cells[reviewer]![target] = null;
35 + }
36 +
37 + for (const critique of critiques) {
38 + const row = cells[critique.reviewerParticipantId];
39 + if (!row) continue; // reviewer not in participant set - ignore defensively
40 + for (const review of critique.reviews) {
41 + if (review.targetParticipantId in row) {
42 + row[review.targetParticipantId] = review.score;
43 + }
44 + }
45 + }
46 +
47 + const averagesReceived: Record<string, number | null> = {};
48 + const averagesGiven: Record<string, number | null> = {};
49 + for (const target of order) {
50 + const received: number[] = [];
51 + const given: number[] = [];
52 + for (const reviewer of order) {
53 + const toTarget = cells[reviewer]![target];
54 + if (typeof toTarget === 'number') received.push(toTarget);
55 + const fromTarget = cells[target]![reviewer];
56 + if (typeof fromTarget === 'number') given.push(fromTarget);
57 + }
58 + averagesReceived[target] = mean(received);
59 + averagesGiven[target] = mean(given);
60 + }
61 +
62 + return { order, cells, averagesReceived, averagesGiven };
63 +}
added src/core/structured.test.ts +75 −0
@@ -0,0 +1,75 @@
1 +import { describe, expect, it } from 'vitest';
2 +import { z } from 'zod';
3 +import type { LlmClient, LlmRequest, LlmResult, LlmStreamHandle } from './llm-client';
4 +import { requestStructured, StructuredParseError } from './structured';
5 +
6 +const schema = z.object({ answer: z.string(), score: z.number().min(0).max(10) });
7 +
8 +/** A client that returns a scripted sequence of raw responses. */
9 +class ScriptedClient implements LlmClient {
10 + calls: LlmRequest[] = [];
11 + constructor(private readonly responses: string[]) {}
12 + async complete(req: LlmRequest): Promise<LlmResult> {
13 + this.calls.push(req);
14 + const text = this.responses[this.calls.length - 1] ?? this.responses.at(-1) ?? '';
15 + return {
16 + text,
17 + usage: { promptTokens: 10, completionTokens: 5, totalTokens: 15, costUsd: 0.001 },
18 + model: req.model,
19 + latencyMs: 1,
20 + };
21 + }
22 + streamComplete(): Promise<LlmStreamHandle> {
23 + throw new Error('not used');
24 + }
25 +}
26 +
27 +const req: LlmRequest = { model: 'x/y', messages: [{ role: 'user', content: 'hi' }] };
28 +
29 +describe('requestStructured', () => {
30 + it('returns the parsed value on a clean first attempt', async () => {
31 + const client = new ScriptedClient(['{"answer":"ok","score":7}']);
32 + const r = await requestStructured(client, req, schema);
33 + expect(r.repaired).toBe(false);
34 + expect(r.value).toEqual({ answer: 'ok', score: 7 });
35 + expect(client.calls).toHaveLength(1);
36 + expect(r.usage.costUsd).toBeCloseTo(0.001);
37 + });
38 +
39 + it('repairs once when the first response is schema-invalid', async () => {
40 + const client = new ScriptedClient([
41 + '{"answer":"ok","score":99}', // invalid: score > 10
42 + '{"answer":"ok","score":8}', // repaired
43 + ]);
44 + const r = await requestStructured(client, req, schema);
45 + expect(r.repaired).toBe(true);
46 + expect(r.value.score).toBe(8);
47 + expect(client.calls).toHaveLength(2);
48 + // usage accumulates across both calls
49 + expect(r.usage.costUsd).toBeCloseTo(0.002);
50 + // the repair call carries the repair system prompt
51 + expect(client.calls[1]!.messages[0]!.content).toMatch(/JSON fixer/i);
52 + });
53 +
54 + it('repairs malformed (non-JSON-wrapped) output', async () => {
55 + const client = new ScriptedClient([
56 + 'I think the answer is good.', // no JSON at all
57 + '```json\n{"answer":"fixed","score":6}\n```',
58 + ]);
59 + const r = await requestStructured(client, req, schema);
60 + expect(r.repaired).toBe(true);
61 + expect(r.value.answer).toBe('fixed');
62 + });
63 +
64 + it('throws StructuredParseError (with billed usage) after a failed repair', async () => {
65 + const client = new ScriptedClient(['garbage', 'still garbage']);
66 + await expect(requestStructured(client, req, schema)).rejects.toBeInstanceOf(StructuredParseError);
67 + try {
68 + await requestStructured(client, req, schema);
69 + } catch (e) {
70 + expect(e).toBeInstanceOf(StructuredParseError);
71 + // tokens from both attempts are still billed
72 + expect((e as StructuredParseError).usage.costUsd).toBeCloseTo(0.002);
73 + }
74 + });
75 +});
added src/core/structured.ts +71 −0
@@ -0,0 +1,71 @@
1 +/**
2 + * Force a model to return schema-valid JSON, with one repair attempt.
3 + *
4 + * Pipeline: call → strip/extract/validate → on failure, re-ask with the raw
5 + * output and the exact validation error → validate again. Tokens spent on the
6 + * failed attempt are still billed (the user's key paid for them), so usage is
7 + * accumulated across both calls and surfaced even when the repair ultimately
8 + * fails.
9 + */
10 +import type { z } from 'zod';
11 +import { parseStructured } from './json-repair';
12 +import type { LlmClient, LlmRequest } from './llm-client';
13 +import { buildRepairPrompt } from './prompts';
14 +import type { Usage } from './types';
15 +import { addUsage } from './usage';
16 +
17 +export interface StructuredResult<T> {
18 + value: T;
19 + usage: Usage;
20 + latencyMs: number;
21 + raw: string;
22 + repaired: boolean;
23 +}
24 +
25 +/** Raised when JSON is still invalid after the single repair retry. */
26 +export class StructuredParseError extends Error {
27 + readonly usage: Usage;
28 + readonly latencyMs: number;
29 + constructor(message: string, opts: { usage: Usage; latencyMs: number }) {
30 + super(message);
31 + this.name = 'StructuredParseError';
32 + this.usage = opts.usage;
33 + this.latencyMs = opts.latencyMs;
34 + }
35 +}
36 +
37 +export async function requestStructured<S extends z.ZodTypeAny>(
38 + llm: LlmClient,
39 + req: LlmRequest,
40 + schema: S,
41 +): Promise<StructuredResult<z.infer<S>>> {
42 + const first = await llm.complete({ ...req, json: true });
43 + const parsed = parseStructured(first.text, schema);
44 + if (parsed.ok) {
45 + return {
46 + value: parsed.value,
47 + usage: first.usage,
48 + latencyMs: first.latencyMs,
49 + raw: first.text,
50 + repaired: false,
51 + };
52 + }
53 +
54 + // Single repair attempt: hand the model its own output + the exact error.
55 + const second = await llm.complete({
56 + ...req,
57 + json: true,
58 + messages: buildRepairPrompt(first.text, parsed.error),
59 + });
60 + const usage = addUsage(first.usage, second.usage);
61 + const latencyMs = first.latencyMs + second.latencyMs;
62 + const parsed2 = parseStructured(second.text, schema);
63 + if (parsed2.ok) {
64 + return { value: parsed2.value, usage, latencyMs, raw: second.text, repaired: true };
65 + }
66 +
67 + throw new StructuredParseError(
68 + `Model output failed schema validation after one repair attempt: ${parsed2.error}`,
69 + { usage, latencyMs },
70 + );
71 +}
added src/core/timeout.ts +37 −0
@@ -0,0 +1,37 @@
1 +/**
2 + * Race a promise-returning function against a wall-clock deadline, wiring an
3 + * AbortSignal through so the underlying LLM call is actually cancelled (not just
4 + * abandoned). Also forwards a parent signal so a client disconnect / debate
5 + * abort propagates down into in-flight model calls.
6 + */
7 +export class TimeoutError extends Error {
8 + readonly timeoutMs: number;
9 + constructor(ms: number) {
10 + super(`Operation timed out after ${ms}ms`);
11 + this.name = 'TimeoutError';
12 + this.timeoutMs = ms;
13 + }
14 +}
15 +
16 +export async function withTimeout<T>(
17 + fn: (signal: AbortSignal) => Promise<T>,
18 + ms: number,
19 + parentSignal?: AbortSignal,
20 +): Promise<T> {
21 + const controller = new AbortController();
22 + const abortFromParent = () => controller.abort(parentSignal?.reason);
23 +
24 + if (parentSignal) {
25 + if (parentSignal.aborted) controller.abort(parentSignal.reason);
26 + else parentSignal.addEventListener('abort', abortFromParent, { once: true });
27 + }
28 +
29 + const timer = setTimeout(() => controller.abort(new TimeoutError(ms)), ms);
30 +
31 + try {
32 + return await fn(controller.signal);
33 + } finally {
34 + clearTimeout(timer);
35 + parentSignal?.removeEventListener('abort', abortFromParent);
36 + }
37 +}
added src/core/types.ts +204 −0
@@ -0,0 +1,204 @@
1 +/**
2 + * Core domain types for the Roundtable debate engine.
3 + *
4 + * This module is intentionally framework-agnostic: it imports nothing from
5 + * Next.js, Prisma, or any transport. The orchestrator, the API route adapter,
6 + * and a future CLI all speak these types.
7 + */
8 +
9 +/** The five kinds of work a debate performs. Mirrors the timeline in the UI. */
10 +export type StageType = 'answer' | 'critique' | 'revision' | 'convergence' | 'synthesis';
11 +
12 +/** Anonymized reviewer-facing label, e.g. "A", "B", "C". */
13 +export type AnonLabel = string;
14 +
15 +/**
16 + * A council member. `id` is a stable, debate-local identifier (`p0`, `p1`, ...)
17 + * used everywhere internally so that the anonymized label mapping and the score
18 + * matrix stay consistent even if two members happen to use the same model.
19 + */
20 +export interface Participant {
21 + id: string;
22 + /** OpenRouter model slug, e.g. "openai/gpt-4o". */
23 + model: string;
24 + /** Human-facing display name derived from the slug. */
25 + displayName: string;
26 +}
27 +
28 +/** Fully-resolved configuration for a single debate run. */
29 +export interface DebateConfig {
30 + question: string;
31 + /** Council model slugs (3-6 enforced by the input schema). */
32 + models: string[];
33 + /** Model that writes the final synthesis + dissent report. */
34 + chairmanModel: string;
35 + /** Cheap/fast model that scores convergence between rounds. */
36 + convergenceModel: string;
37 + /** Number of critique→revision cycles (1-5). */
38 + maxRounds: number;
39 + /** Stop early once convergence score ≥ this (0-100). */
40 + convergenceThreshold: number;
41 + /** Sampling temperature applied to council answers and revisions. */
42 + temperature: number;
43 + /** Per-model, per-stage wall-clock budget in ms before the call is dropped. */
44 + perModelTimeoutMs: number;
45 +}
46 +
47 +export type DebateStatus =
48 + | 'pending'
49 + | 'running'
50 + | 'completed'
51 + | 'failed'
52 + | 'aborted';
53 +
54 +// ---------------------------------------------------------------------------
55 +// Stage records - the persisted, replayable artifacts of a debate.
56 +// ---------------------------------------------------------------------------
57 +
58 +export interface Usage {
59 + promptTokens: number;
60 + completionTokens: number;
61 + totalTokens: number;
62 + costUsd: number;
63 +}
64 +
65 +export const emptyUsage = (): Usage => ({
66 + promptTokens: 0,
67 + completionTokens: 0,
68 + totalTokens: 0,
69 + costUsd: 0,
70 +});
71 +
72 +/** One model's answer at a point in time (round 0 seed, or a later revision). */
73 +export interface AnswerRecord {
74 + participantId: string;
75 + model: string;
76 + round: number;
77 + content: string;
78 + usage: Usage;
79 + latencyMs: number;
80 +}
81 +
82 +/** A single peer review inside one reviewer's critique output. */
83 +export interface PeerReview {
84 + /** The anonymized label the reviewer saw (e.g. "B"). */
85 + label: AnonLabel;
86 + /** Resolved back to the real author (server-side only until de-anonymized). */
87 + targetParticipantId: string;
88 + weaknesses: string[];
89 + strengths: string[];
90 + /** 1-10. */
91 + score: number;
92 + justification: string;
93 +}
94 +
95 +export interface CritiqueRecord {
96 + round: number;
97 + reviewerParticipantId: string;
98 + reviewerModel: string;
99 + reviews: PeerReview[];
100 + usage: Usage;
101 + latencyMs: number;
102 +}
103 +
104 +export interface RevisionChangelog {
105 + changed: boolean;
106 + /** What changed and why, or the defense of the original position. */
107 + summary: string;
108 + bullets: string[];
109 +}
110 +
111 +export interface RevisionRecord {
112 + round: number;
113 + participantId: string;
114 + model: string;
115 + content: string;
116 + changelog: RevisionChangelog;
117 + usage: Usage;
118 + latencyMs: number;
119 +}
120 +
121 +export interface Disagreement {
122 + topic: string;
123 + summary: string;
124 + /** Per-position stance, keyed by anonymized label as the evaluator saw it. */
125 + positions: Array<{ label: AnonLabel; participantId?: string; stance: string }>;
126 +}
127 +
128 +export interface ConvergenceRecord {
129 + round: number;
130 + model: string;
131 + score: number;
132 + disagreements: Disagreement[];
133 + converged: boolean;
134 + usage: Usage;
135 + latencyMs: number;
136 +}
137 +
138 +export interface DissentEntry {
139 + topic: string;
140 + positions: Array<{ participantId: string; model: string; position: string }>;
141 +}
142 +
143 +export interface SynthesisRecord {
144 + model: string;
145 + finalAnswer: string;
146 + dissent: DissentEntry[];
147 + usage: Usage;
148 + latencyMs: number;
149 +}
150 +
151 +/** A record of a model dropping out of a stage (timeout, error, bad JSON). */
152 +export interface FailureRecord {
153 + round: number;
154 + stage: StageType;
155 + participantId: string;
156 + model: string;
157 + error: string;
158 + /** True when the participant was removed from the remainder of the debate. */
159 + droppedFromDebate: boolean;
160 +}
161 +
162 +/** One critique→revision→convergence cycle. */
163 +export interface RoundRecord {
164 + round: number;
165 + critiques: CritiqueRecord[];
166 + revisions: RevisionRecord[];
167 + convergence: ConvergenceRecord | null;
168 +}
169 +
170 +/** The full, replayable result of a debate. */
171 +export interface DebateResult {
172 + debateId: string;
173 + config: DebateConfig;
174 + participants: Participant[];
175 + status: DebateStatus;
176 + initialAnswers: AnswerRecord[];
177 + rounds: RoundRecord[];
178 + synthesis: SynthesisRecord | null;
179 + failures: FailureRecord[];
180 + /** Whichever answers were current when the debate ended. */
181 + finalAnswers: AnswerRecord[];
182 + totals: {
183 + costUsd: number;
184 + promptTokens: number;
185 + completionTokens: number;
186 + rounds: number;
187 + durationMs: number;
188 + costByModel: Record<string, number>;
189 + };
190 + error?: string;
191 +}
192 +
193 +/** Config fields that are safe to expose to the client (no secrets). */
194 +export type PublicDebateConfig = Omit<DebateConfig, never>;
195 +
196 +export function displayNameForModel(slug: string): string {
197 + // "openai/gpt-4o-mini" -> "GPT 4o Mini"
198 + const tail = slug.split('/').pop() ?? slug;
199 + return tail
200 + .replace(/[-_]/g, ' ')
201 + .replace(/\b(gpt|llm)\b/gi, (m) => m.toUpperCase())
202 + .replace(/\b\w/g, (c) => c.toUpperCase())
203 + .trim();
204 +}
added src/core/usage.ts +17 −0
@@ -0,0 +1,17 @@
1 +import { emptyUsage, type Usage } from './types';
2 +
3 +/** Sum two usage records (tokens + cost). */
4 +export function addUsage(a: Usage, b: Usage): Usage {
5 + return {
6 + promptTokens: a.promptTokens + b.promptTokens,
7 + completionTokens: a.completionTokens + b.completionTokens,
8 + totalTokens: a.totalTokens + b.totalTokens,
9 + costUsd: a.costUsd + b.costUsd,
10 + };
11 +}
12 +
13 +export function sumUsage(usages: Iterable<Usage>): Usage {
14 + let acc = emptyUsage();
15 + for (const u of usages) acc = addUsage(acc, u);
16 + return acc;
17 +}