Commit
add claim provenance audit: trace final answer claims, flag chairman additions
commit
acd5b4f
21 changed files with +432 and −4
Jump to a changed file
- prisma/schema.prisma +1 −0
- scripts/debate-cli.ts +5 −0
- src/components/debate/debate-console.tsx +1 −0
- src/components/debate/final-answer.tsx +65 −1
- src/components/debate/model-panel.tsx +1 −0
- src/core/events.ts +2 −0
- src/core/mock-client.ts +20 −0
- src/core/orchestrator.test.ts +30 −0
- src/core/orchestrator.ts +83 −0
- src/core/prompts/index.ts +2 −1
- src/core/prompts/provenance.ts +55 −0
- src/core/schemas.ts +13 −0
- src/core/types.ts +28 −2
- src/db/repositories.ts +32 −0
- src/hooks/use-debate-playback.ts +4 −0
- src/lib/debate-view.test.ts +1 −0
- src/lib/debate-view.ts +7 −0
- src/lib/demo-fixtures.data.ts +19 −0
- src/lib/demo-fixtures.ts +27 −0
- src/lib/export-markdown.test.ts +21 −0
- src/lib/export-markdown.ts +15 −0
modified prisma/schema.prisma +1 −0
| @@ -98,6 +98,7 @@enum StageType { | ||
| 98 | 98 | revision |
| 99 | 99 | convergence |
| 100 | 100 | synthesis |
| 101 | + provenance | |
| 101 | 102 | } |
| 102 | 103 | |
| 103 | 104 | model Debate { |
modified scripts/debate-cli.ts +5 −0
| @@ -55,6 +55,11 @@async function main() { | ||
| 55 | 55 | console.log('\n---- Dissent ----'); |
| 56 | 56 | for (const d of result.synthesis.dissent) console.log(` - ${d.topic}`); |
| 57 | 57 | } |
| 58 | + const unsourced = result.provenance?.claims.filter((c) => c.unsourced) ?? []; | |
| 59 | + if (unsourced.length) { | |
| 60 | + console.log('\n---- Chairman additions (claims no council member made) ----'); | |
| 61 | + for (const c of unsourced) console.log(` - ${c.text}`); | |
| 62 | + } | |
| 58 | 63 | console.log( |
| 59 | 64 | `\nRounds: ${result.totals.rounds} | Cost: $${result.totals.costUsd.toFixed(4)} | Status: ${result.status}\n`, |
| 60 | 65 | ); |
modified src/components/debate/debate-console.tsx +1 −0
| @@ -60,6 +60,7 @@export function DebateConsole({ view, showActions = true }: { view: DebateView; | ||
| 60 | 60 | synthesis={view.synthesis} |
| 61 | 61 | participants={view.participants} |
| 62 | 62 | chairmanProviderConflict={view.chairmanProviderConflict} |
| 63 | + provenance={view.provenance} | |
| 63 | 64 | /> |
| 64 | 65 | {lastConvergence && lastConvergence.disagreements.length > 0 ? ( |
| 65 | 66 | <Disagreements disagreements={lastConvergence.disagreements} participants={view.participants} /> |
modified src/components/debate/final-answer.tsx +65 −1
| @@ -9,19 +9,27 @@import { | ||
| 9 | 9 | TooltipContent, |
| 10 | 10 | TooltipTrigger, |
| 11 | 11 | } from '@/components/ui/tooltip'; |
| 12 | -import { displayNameForModel, type Participant, type SynthesisRecord } from '@/core/types'; | |
| 12 | +import { | |
| 13 | + displayNameForModel, | |
| 14 | + type Participant, | |
| 15 | + type ProvenanceRecord, | |
| 16 | + type SynthesisRecord, | |
| 17 | +} from '@/core/types'; | |
| 13 | 18 | import { participantColor, participantTag } from '@/lib/model-visuals'; |
| 14 | 19 | |
| 15 | 20 | export function FinalAnswer({ |
| 16 | 21 | synthesis, |
| 17 | 22 | participants, |
| 18 | 23 | chairmanProviderConflict, |
| 24 | + provenance, | |
| 19 | 25 | }: { |
| 20 | 26 | synthesis: SynthesisRecord; |
| 21 | 27 | participants: Participant[]; |
| 22 | 28 | chairmanProviderConflict: boolean; |
| 29 | + provenance?: ProvenanceRecord | null; | |
| 23 | 30 | }) { |
| 24 | 31 | const byId = new Map(participants.map((p, i) => [p.id, { p, i }])); |
| 32 | + const unsourcedCount = provenance?.claims.filter((c) => c.unsourced).length ?? 0; | |
| 25 | 33 | |
| 26 | 34 | return ( |
| 27 | 35 | <div className="overflow-hidden rounded-xl border-2 border-emerald-500/30 bg-gradient-to-b from-emerald-500/[0.06] to-transparent"> |
| @@ -57,6 +65,62 @@export function FinalAnswer({ | ||
| 57 | 65 | <div className="p-5"> |
| 58 | 66 | <RichText text={synthesis.finalAnswer} className="text-[15px]" /> |
| 59 | 67 | |
| 68 | + {provenance && provenance.claims.length > 0 && ( | |
| 69 | + <div className="mt-5 rounded-lg border bg-card p-4"> | |
| 70 | + <div className="mb-1 flex flex-wrap items-center gap-2"> | |
| 71 | + <span className="text-sm font-semibold">Claim check</span> | |
| 72 | + {unsourcedCount > 0 ? ( | |
| 73 | + <Badge variant="warning">{unsourcedCount} chairman addition{unsourcedCount > 1 ? 's' : ''}</Badge> | |
| 74 | + ) : ( | |
| 75 | + <Badge variant="success">all claims sourced</Badge> | |
| 76 | + )} | |
| 77 | + </div> | |
| 78 | + <p className="mb-3 text-xs text-muted-foreground"> | |
| 79 | + Each substantive claim in the final answer, traced back to the council by {provenance.model}. A claim | |
| 80 | + nobody argued is flagged: that is where synthesis hallucination hides. | |
| 81 | + </p> | |
| 82 | + <ul className="space-y-2"> | |
| 83 | + {provenance.claims.map((claim, i) => ( | |
| 84 | + <li key={i} className="flex items-start gap-2 text-xs"> | |
| 85 | + <span className="mt-1 flex shrink-0 items-center gap-0.5"> | |
| 86 | + {claim.unsourced ? ( | |
| 87 | + <span className="h-2 w-2 rounded-full bg-amber-500" title="No council member made this claim" /> | |
| 88 | + ) : ( | |
| 89 | + claim.supportedBy.map((m) => { | |
| 90 | + const entry = byId.get(m.participantId); | |
| 91 | + const idx = entry?.i ?? 0; | |
| 92 | + return ( | |
| 93 | + <span | |
| 94 | + key={m.participantId} | |
| 95 | + className="flex h-4 w-4 items-center justify-center rounded text-[9px] font-bold text-white" | |
| 96 | + style={{ backgroundColor: participantColor(idx) }} | |
| 97 | + title={`Supported by ${entry?.p.displayName ?? m.model}`} | |
| 98 | + > | |
| 99 | + {participantTag(idx)} | |
| 100 | + </span> | |
| 101 | + ); | |
| 102 | + }) | |
| 103 | + )} | |
| 104 | + </span> | |
| 105 | + <span className={claim.unsourced ? 'text-amber-600 dark:text-amber-400' : undefined}> | |
| 106 | + {claim.text} | |
| 107 | + {claim.unsourced && <span className="ml-1 font-medium">(chairman's own addition)</span>} | |
| 108 | + {claim.contestedBy.length > 0 && ( | |
| 109 | + <span className="text-muted-foreground"> | |
| 110 | + {' '} | |
| 111 | + ยท contested by{' '} | |
| 112 | + {claim.contestedBy | |
| 113 | + .map((m) => byId.get(m.participantId)?.p.displayName ?? displayNameForModel(m.model)) | |
| 114 | + .join(', ')} | |
| 115 | + </span> | |
| 116 | + )} | |
| 117 | + </span> | |
| 118 | + </li> | |
| 119 | + ))} | |
| 120 | + </ul> | |
| 121 | + </div> | |
| 122 | + )} | |
| 123 | + | |
| 60 | 124 | {synthesis.dissent.length > 0 && ( |
| 61 | 125 | <div className="mt-5 rounded-lg border bg-card p-4"> |
| 62 | 126 | <div className="mb-2 text-sm font-semibold">Dissent report</div> |
modified src/components/debate/model-panel.tsx +1 −0
| @@ -28,6 +28,7 @@const STAGE_LABEL: Record<StageType, string> = { | ||
| 28 | 28 | revision: 'Revising', |
| 29 | 29 | convergence: 'Assessing', |
| 30 | 30 | synthesis: 'Synthesizing', |
| 31 | + provenance: 'Auditing claims', | |
| 31 | 32 | }; |
| 32 | 33 | |
| 33 | 34 | export function ModelPanel({ |
modified src/core/events.ts +2 −0
| @@ -14,6 +14,7 @@import type { | ||
| 14 | 14 | CritiqueRecord, |
| 15 | 15 | DebateStatus, |
| 16 | 16 | Participant, |
| 17 | + ProvenanceRecord, | |
| 17 | 18 | PublicDebateConfig, |
| 18 | 19 | RevisionRecord, |
| 19 | 20 | StageType, |
| @@ -59,6 +60,7 @@export type DebateEvent = | ||
| 59 | 60 | droppedFromDebate: boolean; |
| 60 | 61 | } |
| 61 | 62 | | { type: 'synthesis_completed'; record: SynthesisRecord } |
| 63 | + | { type: 'provenance_completed'; record: ProvenanceRecord } | |
| 62 | 64 | | { |
| 63 | 65 | type: 'cost_update'; |
| 64 | 66 | totalCostUsd: number; |
modified src/core/mock-client.ts +20 −0
| @@ -125,6 +125,8 @@export class MockLlmClient implements LlmClient { | ||
| 125 | 125 | return this.renderConvergence(req, repair, forceMalform, round); |
| 126 | 126 | case 'synthesis': |
| 127 | 127 | return this.renderSynthesis(req, repair, forceMalform); |
| 128 | + case 'provenance': | |
| 129 | + return this.renderProvenance(req, repair, forceMalform); | |
| 128 | 130 | default: |
| 129 | 131 | return 'Mock response.'; |
| 130 | 132 | } |
| @@ -222,6 +224,24 @@export class MockLlmClient implements LlmClient { | ||
| 222 | 224 | return JSON.stringify(payload); |
| 223 | 225 | } |
| 224 | 226 | |
| 227 | + private renderProvenance(req: LlmRequest, repair: boolean, malform: boolean): string { | |
| 228 | + const labels = repair ? labelsFromRepair(req) : labelsFromPrompt(req); | |
| 229 | + const claims = [ | |
| 230 | + ...labels.map((label, i) => ({ | |
| 231 | + text: `The recommendation should weigh correctness against completeness (point ${i + 1}).`, | |
| 232 | + supportedBy: [label], | |
| 233 | + contestedBy: labels.filter((l) => l !== label).slice(0, 1), | |
| 234 | + })), | |
| 235 | + { | |
| 236 | + // Deliberately unsourced so the chairman-addition path is exercised offline. | |
| 237 | + text: 'Adoption should be revisited after one quarter of production use.', | |
| 238 | + supportedBy: [], | |
| 239 | + contestedBy: [], | |
| 240 | + }, | |
| 241 | + ]; | |
| 242 | + return JSON.stringify(malform ? { claims: [] } : { claims }); | |
| 243 | + } | |
| 244 | + | |
| 225 | 245 | async complete(req: LlmRequest): Promise<LlmResult> { |
| 226 | 246 | const text = this.render(req); |
| 227 | 247 | return { |
modified src/core/orchestrator.test.ts +30 −0
| @@ -72,6 +72,7 @@describe('runDebate - happy path', () => { | ||
| 72 | 72 | expect(of('debate_started')).toHaveLength(1); |
| 73 | 73 | expect(of('answer_completed')).toHaveLength(3); |
| 74 | 74 | expect(of('synthesis_completed')).toHaveLength(1); |
| 75 | + expect(of('provenance_completed')).toHaveLength(1); | |
| 75 | 76 | expect(of('debate_completed')[0]!.status).toBe('completed'); |
| 76 | 77 | |
| 77 | 78 | // Cost accrued and is attributed per model. |
| @@ -172,6 +173,35 @@describe('runDebate - JSON repair', () => { | ||
| 172 | 173 | }); |
| 173 | 174 | }); |
| 174 | 175 | |
| 176 | +describe('runDebate - provenance audit', () => { | |
| 177 | + it('traces final-answer claims to council members and flags chairman additions', async () => { | |
| 178 | + const { result } = await run(makeConfig()); | |
| 179 | + expect(result.provenance).not.toBeNull(); | |
| 180 | + const claims = result.provenance!.claims; | |
| 181 | + expect(claims.length).toBeGreaterThan(0); | |
| 182 | + const ids = new Set(result.participants.map((p) => p.id)); | |
| 183 | + for (const claim of claims) { | |
| 184 | + expect(claim.unsourced).toBe(claim.supportedBy.length === 0); | |
| 185 | + for (const m of [...claim.supportedBy, ...claim.contestedBy]) { | |
| 186 | + expect(ids.has(m.participantId)).toBe(true); | |
| 187 | + } | |
| 188 | + } | |
| 189 | + // The mock always emits one deliberately unsourced claim. | |
| 190 | + expect(claims.some((c) => c.unsourced)).toBe(true); | |
| 191 | + }); | |
| 192 | + | |
| 193 | + it('completes the debate even when the provenance audit fails', async () => { | |
| 194 | + const { result, of } = await run(makeConfig(), { | |
| 195 | + fail: [{ stage: 'provenance' }], | |
| 196 | + }); | |
| 197 | + expect(result.status).toBe('completed'); | |
| 198 | + expect(result.synthesis).not.toBeNull(); | |
| 199 | + expect(result.provenance).toBeNull(); | |
| 200 | + expect(of('provenance_completed')).toHaveLength(0); | |
| 201 | + expect(result.failures.some((f) => f.stage === 'provenance')).toBe(true); | |
| 202 | + }); | |
| 203 | +}); | |
| 204 | + | |
| 175 | 205 | describe('runDebate - chairman self-preference', () => { |
| 176 | 206 | it('flags when the chairman shares a provider family with a council member', async () => { |
| 177 | 207 | const { of } = await run( |
modified src/core/orchestrator.ts +83 −0
| @@ -28,13 +28,15 @@import { | ||
| 28 | 28 | buildAnswerPrompt, |
| 29 | 29 | buildConvergencePrompt, |
| 30 | 30 | buildCritiquePrompt, |
| 31 | + buildProvenancePrompt, | |
| 31 | 32 | buildRevisionPrompt, |
| 32 | 33 | buildSynthesisPrompt, |
| 33 | 34 | } from './prompts'; |
| 34 | 35 | import type { IncomingCritique } from './prompts/revision'; |
| 35 | 36 | import { |
| 36 | 37 | convergenceOutputSchema, |
| 37 | 38 | critiqueOutputSchema, |
| 39 | + provenanceOutputSchema, | |
| 38 | 40 | revisionOutputSchema, |
| 39 | 41 | synthesisOutputSchema, |
| 40 | 42 | } from './schemas'; |
| @@ -53,6 +55,7 @@import { | ||
| 53 | 55 | type FailureRecord, |
| 54 | 56 | type Participant, |
| 55 | 57 | type PeerReview, |
| 58 | + type ProvenanceRecord, | |
| 56 | 59 | type RevisionRecord, |
| 57 | 60 | type RoundRecord, |
| 58 | 61 | type StageType, |
| @@ -95,6 +98,7 @@interface RunState { | ||
| 95 | 98 | /** Revisions produced by the in-flight round, folded into a RoundRecord. */ |
| 96 | 99 | pendingRevisions?: RevisionRecord[]; |
| 97 | 100 | synthesisResult: SynthesisRecord | null; |
| 101 | + provenanceResult: ProvenanceRecord | null; | |
| 98 | 102 | } |
| 99 | 103 | |
| 100 | 104 | export async function runDebate( |
| @@ -119,6 +123,7 @@export async function runDebate( | ||
| 119 | 123 | costByModel: new Map(), |
| 120 | 124 | lastDisagreements: [], |
| 121 | 125 | synthesisResult: null, |
| 126 | + provenanceResult: null, | |
| 122 | 127 | }; |
| 123 | 128 | |
| 124 | 129 | // --- helpers ------------------------------------------------------------- |
| @@ -196,6 +201,7 @@export async function runDebate( | ||
| 196 | 201 | initialAnswers: state.initialAnswers, |
| 197 | 202 | rounds: state.rounds, |
| 198 | 203 | synthesis: state.synthesisResult ?? null, |
| 204 | + provenance: state.provenanceResult ?? null, | |
| 199 | 205 | failures: state.failures, |
| 200 | 206 | finalAnswers, |
| 201 | 207 | totals: { |
| @@ -279,6 +285,9 @@export async function runDebate( | ||
| 279 | 285 | |
| 280 | 286 | // === Synthesis ======================================================== |
| 281 | 287 | await synthesisPhase(); |
| 288 | + | |
| 289 | + // === Provenance audit (advisory) ====================================== | |
| 290 | + await provenancePhase(); | |
| 282 | 291 | await emitCost(); |
| 283 | 292 | |
| 284 | 293 | return finish('completed'); |
| @@ -539,6 +548,80 @@export async function runDebate( | ||
| 539 | 548 | } |
| 540 | 549 | } |
| 541 | 550 | |
| 551 | + /** | |
| 552 | + * Trace each claim in the final answer back to the council's final answers, | |
| 553 | + * flagging claims no member made (chairman additions). Runs on the cheap | |
| 554 | + * convergence-class model. Purely advisory: any failure is recorded and the | |
| 555 | + * debate completes without a provenance report. | |
| 556 | + */ | |
| 557 | + async function provenancePhase(): Promise<void> { | |
| 558 | + const synthesis = state.synthesisResult; | |
| 559 | + if (!synthesis) return; | |
| 560 | + const finalists = participants | |
| 561 | + .filter((p) => state.active.has(p.id)) | |
| 562 | + .map((p) => ({ participantId: p.id, content: state.current.get(p.id)!.content })); | |
| 563 | + if (finalists.length === 0) return; | |
| 564 | + | |
| 565 | + const anon = anonymizePeers(finalists, hashSeed(opts.debateId, 'provenance')); | |
| 566 | + const round = state.rounds.length; | |
| 567 | + await emit({ type: 'stage_started', round, stage: 'provenance', model: config.convergenceModel }); | |
| 568 | + try { | |
| 569 | + const out = await withTimeout( | |
| 570 | + (signal) => | |
| 571 | + structured( | |
| 572 | + { | |
| 573 | + model: config.convergenceModel, | |
| 574 | + messages: buildProvenancePrompt(config.question, synthesis.finalAnswer, anon.peers), | |
| 575 | + temperature: 0, | |
| 576 | + meta: { stage: 'provenance', round }, | |
| 577 | + }, | |
| 578 | + provenanceOutputSchema, | |
| 579 | + signal, | |
| 580 | + ), | |
| 581 | + config.perModelTimeoutMs, | |
| 582 | + opts.signal, | |
| 583 | + ); | |
| 584 | + bill(config.convergenceModel, out.usage); | |
| 585 | + const toMembers = (labels: string[]) => | |
| 586 | + labels | |
| 587 | + .map((label) => { | |
| 588 | + const pid = anon.labelMap[label]; | |
| 589 | + const part = pid ? byId.get(pid) : undefined; | |
| 590 | + return part ? { participantId: part.id, model: part.model } : null; | |
| 591 | + }) | |
| 592 | + .filter((x): x is NonNullable<typeof x> => x !== null); | |
| 593 | + const record: ProvenanceRecord = { | |
| 594 | + model: config.convergenceModel, | |
| 595 | + round, | |
| 596 | + claims: out.value.claims.map((c) => { | |
| 597 | + const supportedBy = toMembers(c.supportedBy); | |
| 598 | + return { | |
| 599 | + text: c.text, | |
| 600 | + supportedBy, | |
| 601 | + contestedBy: toMembers(c.contestedBy), | |
| 602 | + unsourced: supportedBy.length === 0, | |
| 603 | + }; | |
| 604 | + }), | |
| 605 | + usage: out.usage, | |
| 606 | + latencyMs: out.latencyMs, | |
| 607 | + }; | |
| 608 | + state.provenanceResult = record; | |
| 609 | + await emit({ type: 'provenance_completed', record }); | |
| 610 | + } catch (err) { | |
| 611 | + if (err instanceof StructuredParseError) bill(config.convergenceModel, err.usage); | |
| 612 | + const message = err instanceof Error ? err.message : String(err); | |
| 613 | + logger.warn('provenance_failed', { error: message }); | |
| 614 | + state.failures.push({ | |
| 615 | + round, | |
| 616 | + stage: 'provenance', | |
| 617 | + participantId: 'auditor', | |
| 618 | + model: config.convergenceModel, | |
| 619 | + error: message, | |
| 620 | + droppedFromDebate: false, | |
| 621 | + }); | |
| 622 | + } | |
| 623 | + } | |
| 624 | + | |
| 542 | 625 | function fallbackSynthesis(): SynthesisRecord { |
| 543 | 626 | const answers = [...state.active].map((id) => state.current.get(id)!).filter(Boolean); |
| 544 | 627 | const best = answers[0]; |
modified src/core/prompts/index.ts +2 −1
| @@ -10,13 +10,14 @@ | ||
| 10 | 10 | * alter model behavior; it is stamped onto stored results so historical debates |
| 11 | 11 | * remain interpretable. |
| 12 | 12 | */ |
| 13 | -export const PROMPT_VERSION = '1.1.0'; | |
| 13 | +export const PROMPT_VERSION = '1.2.0'; | |
| 14 | 14 | |
| 15 | 15 | export { buildAnswerPrompt } from './answer'; |
| 16 | 16 | export { buildCritiquePrompt } from './critique'; |
| 17 | 17 | export { buildRevisionPrompt } from './revision'; |
| 18 | 18 | export { buildConvergencePrompt } from './convergence'; |
| 19 | 19 | export { buildSynthesisPrompt } from './synthesis'; |
| 20 | +export { buildProvenancePrompt } from './provenance'; | |
| 20 | 21 | export { buildRepairPrompt } from './repair'; |
| 21 | 22 | |
| 22 | 23 | /** |
added src/core/prompts/provenance.ts +55 −0
| @@ -0,0 +1,55 @@ | ||
| 1 | +import type { AnonymizedPeer } from '../anonymize'; | |
| 2 | +import type { LlmMessage } from '../llm-client'; | |
| 3 | +import { jsonInstruction } from './index'; | |
| 4 | + | |
| 5 | +const PROVENANCE_SHAPE = `{ | |
| 6 | + "claims": [ | |
| 7 | + { | |
| 8 | + "text": "one substantive claim made by the final answer, in its own words", | |
| 9 | + "supportedBy": ["A", "C"], // labels of responses that state or clearly imply this claim | |
| 10 | + "contestedBy": ["B"] // labels of responses that dispute it (often empty) | |
| 11 | + } | |
| 12 | + // ...one entry per substantive claim, typically 4-10 | |
| 13 | + ] | |
| 14 | +}`; | |
| 15 | + | |
| 16 | +/** | |
| 17 | + * Provenance audit (advisory, post-synthesis). | |
| 18 | + * | |
| 19 | + * A cheap model decomposes the chairman's final answer into its substantive | |
| 20 | + * claims and traces each back to the anonymized council answers. A claim with | |
| 21 | + * an empty `supportedBy` is unsourced: content the chairman introduced that no | |
| 22 | + * council member argued - exactly where synthesis hallucination lives. Run by | |
| 23 | + * the convergence-class model; failures never fail the debate. | |
| 24 | + */ | |
| 25 | +export function buildProvenancePrompt( | |
| 26 | + question: string, | |
| 27 | + finalAnswer: string, | |
| 28 | + peers: AnonymizedPeer[], | |
| 29 | +): LlmMessage[] { | |
| 30 | + const peerBlock = peers.map((p) => `--- Response ${p.label} ---\n${p.content}`).join('\n\n'); | |
| 31 | + | |
| 32 | + return [ | |
| 33 | + { | |
| 34 | + role: 'system', | |
| 35 | + content: | |
| 36 | + 'You are an audit assistant. A panel answered a question and a chairman wrote a ' + | |
| 37 | + 'final synthesized answer from their responses. Decompose the final answer into its ' + | |
| 38 | + 'substantive claims (recommendations, factual assertions, caveats - not filler), and ' + | |
| 39 | + 'for each claim report which panel responses support it and which dispute it. A ' + | |
| 40 | + 'response supports a claim only if it states it or clearly implies it; do not ' + | |
| 41 | + 'stretch. If NO response contains a claim, return it with an empty "supportedBy" ' + | |
| 42 | + 'list - identifying such chairman-added content is the entire point of this audit, ' + | |
| 43 | + 'so never invent support.\n\n' + | |
| 44 | + jsonInstruction(PROVENANCE_SHAPE), | |
| 45 | + }, | |
| 46 | + { | |
| 47 | + role: 'user', | |
| 48 | + content: | |
| 49 | + `Question:\n${question}\n\n` + | |
| 50 | + `Final synthesized answer:\n${finalAnswer}\n\n` + | |
| 51 | + `Panel responses (${peers.length}):\n\n${peerBlock}\n\n` + | |
| 52 | + 'Return your JSON audit now.', | |
| 53 | + }, | |
| 54 | + ]; | |
| 55 | +} |
modified src/core/schemas.ts +13 −0
| @@ -55,6 +55,19 @@export const convergenceOutputSchema = z.object({ | ||
| 55 | 55 | }); |
| 56 | 56 | export type ConvergenceOutput = z.infer<typeof convergenceOutputSchema>; |
| 57 | 57 | |
| 58 | +export const provenanceOutputSchema = z.object({ | |
| 59 | + claims: z | |
| 60 | + .array( | |
| 61 | + z.object({ | |
| 62 | + text: z.string().min(1), | |
| 63 | + supportedBy: z.array(z.string()).default([]), | |
| 64 | + contestedBy: z.array(z.string()).default([]), | |
| 65 | + }), | |
| 66 | + ) | |
| 67 | + .min(1), | |
| 68 | +}); | |
| 69 | +export type ProvenanceOutput = z.infer<typeof provenanceOutputSchema>; | |
| 70 | + | |
| 58 | 71 | export const synthesisOutputSchema = z.object({ |
| 59 | 72 | finalAnswer: z.string().min(1), |
| 60 | 73 | dissent: z |
modified src/core/types.ts +28 −2
| @@ -6,8 +6,8 @@ | ||
| 6 | 6 | * and a future CLI all speak these types. |
| 7 | 7 | */ |
| 8 | 8 | |
| 9 | -/** The five kinds of work a debate performs. Mirrors the timeline in the UI. */ | |
| 10 | -export type StageType = 'answer' | 'critique' | 'revision' | 'convergence' | 'synthesis'; | |
| 9 | +/** The kinds of work a debate performs. Mirrors the timeline in the UI. */ | |
| 10 | +export type StageType = 'answer' | 'critique' | 'revision' | 'convergence' | 'synthesis' | 'provenance'; | |
| 11 | 11 | |
| 12 | 12 | /** Anonymized reviewer-facing label, e.g. "A", "B", "C". */ |
| 13 | 13 | export type AnonLabel = string; |
| @@ -156,6 +156,31 @@export interface SynthesisRecord { | ||
| 156 | 156 | latencyMs: number; |
| 157 | 157 | } |
| 158 | 158 | |
| 159 | +/** One claim from the final answer, traced back to the deliberation. */ | |
| 160 | +export interface ProvenanceClaim { | |
| 161 | + text: string; | |
| 162 | + /** Council members whose final answers state or clearly imply the claim. */ | |
| 163 | + supportedBy: Array<{ participantId: string; model: string }>; | |
| 164 | + /** Council members whose final answers dispute the claim. */ | |
| 165 | + contestedBy: Array<{ participantId: string; model: string }>; | |
| 166 | + /** True when no council answer contains the claim - the chairman's own addition. */ | |
| 167 | + unsourced: boolean; | |
| 168 | +} | |
| 169 | + | |
| 170 | +/** | |
| 171 | + * Advisory audit of the chairman's synthesis: which claims in the final answer | |
| 172 | + * are actually grounded in the council's final answers. Unsourced claims are | |
| 173 | + * where synthesis hallucination lives. Produced by the cheap convergence-class | |
| 174 | + * model; a failure never fails the debate. | |
| 175 | + */ | |
| 176 | +export interface ProvenanceRecord { | |
| 177 | + model: string; | |
| 178 | + round: number; | |
| 179 | + claims: ProvenanceClaim[]; | |
| 180 | + usage: Usage; | |
| 181 | + latencyMs: number; | |
| 182 | +} | |
| 183 | + | |
| 159 | 184 | /** A record of a model dropping out of a stage (timeout, error, bad JSON). */ |
| 160 | 185 | export interface FailureRecord { |
| 161 | 186 | round: number; |
| @@ -184,6 +209,7 @@export interface DebateResult { | ||
| 184 | 209 | initialAnswers: AnswerRecord[]; |
| 185 | 210 | rounds: RoundRecord[]; |
| 186 | 211 | synthesis: SynthesisRecord | null; |
| 212 | + provenance: ProvenanceRecord | null; | |
| 187 | 213 | failures: FailureRecord[]; |
| 188 | 214 | /** Whichever answers were current when the debate ended. */ |
| 189 | 215 | finalAnswers: AnswerRecord[]; |
modified src/db/repositories.ts +32 −0
| @@ -20,6 +20,8 @@import type { | ||
| 20 | 20 | FailureRecord, |
| 21 | 21 | Participant, |
| 22 | 22 | PeerReview, |
| 23 | + ProvenanceClaim, | |
| 24 | + ProvenanceRecord, | |
| 23 | 25 | RevisionChangelog, |
| 24 | 26 | RevisionRecord, |
| 25 | 27 | RoundRecord, |
| @@ -162,6 +164,24 @@export async function persistEvent(debateId: string, event: DebateEvent): Promis | ||
| 162 | 164 | }); |
| 163 | 165 | return; |
| 164 | 166 | } |
| 167 | + case 'provenance_completed': { | |
| 168 | + const r = event.record; | |
| 169 | + await prisma.stageResult.create({ | |
| 170 | + data: { | |
| 171 | + debateId, | |
| 172 | + round: r.round, | |
| 173 | + stage: 'provenance', | |
| 174 | + model: r.model, | |
| 175 | + content: '', | |
| 176 | + data: { claims: r.claims } as unknown as Prisma.InputJsonValue, | |
| 177 | + promptTokens: r.usage.promptTokens, | |
| 178 | + completionTokens: r.usage.completionTokens, | |
| 179 | + costUsd: r.usage.costUsd, | |
| 180 | + latencyMs: r.latencyMs, | |
| 181 | + }, | |
| 182 | + }); | |
| 183 | + return; | |
| 184 | + } | |
| 165 | 185 | case 'synthesis_completed': { |
| 166 | 186 | const r = event.record; |
| 167 | 187 | await prisma.synthesisResult.upsert({ |
| @@ -332,6 +352,17 @@function toResult(debate: DebateWithRelations): DebateResult { | ||
| 332 | 352 | } |
| 333 | 353 | : null; |
| 334 | 354 | |
| 355 | + const provRow = debate.stageResults.find((s) => s.stage === 'provenance' && !s.error); | |
| 356 | + const provenance: ProvenanceRecord | null = provRow | |
| 357 | + ? { | |
| 358 | + model: provRow.model, | |
| 359 | + round: provRow.round, | |
| 360 | + claims: readJson<{ claims: ProvenanceClaim[] }>(provRow.data)?.claims ?? [], | |
| 361 | + usage: usageFromRow(provRow), | |
| 362 | + latencyMs: provRow.latencyMs, | |
| 363 | + } | |
| 364 | + : null; | |
| 365 | + | |
| 335 | 366 | const costByModel: Record<string, number> = {}; |
| 336 | 367 | for (const s of debate.stageResults) { |
| 337 | 368 | costByModel[s.model] = (costByModel[s.model] ?? 0) + s.costUsd; |
| @@ -345,6 +376,7 @@function toResult(debate: DebateWithRelations): DebateResult { | ||
| 345 | 376 | initialAnswers, |
| 346 | 377 | rounds, |
| 347 | 378 | synthesis, |
| 379 | + provenance, | |
| 348 | 380 | failures, |
| 349 | 381 | finalAnswers, |
| 350 | 382 | totals: { |
modified src/hooks/use-debate-playback.ts +4 −0
| @@ -59,6 +59,10 @@function buildTimeline(result: DebateResult): TimedEvent[] { | ||
| 59 | 59 | emit({ type: 'stage_started', round: result.rounds.length, stage: 'synthesis', model: result.synthesis.model }, 300); |
| 60 | 60 | emit({ type: 'synthesis_completed', record: result.synthesis }, 400); |
| 61 | 61 | } |
| 62 | + if (result.provenance) { | |
| 63 | + emit({ type: 'stage_started', round: result.provenance.round, stage: 'provenance', model: result.provenance.model }, 120); | |
| 64 | + emit({ type: 'provenance_completed', record: result.provenance }, 260); | |
| 65 | + } | |
| 62 | 66 | emit( |
| 63 | 67 | { |
| 64 | 68 | type: 'debate_completed', |
modified src/lib/debate-view.test.ts +1 −0
| @@ -160,6 +160,7 @@describe('fromResult', () => { | ||
| 160 | 160 | initialAnswers: [answer('p0', 0, 'a0'), answer('p1', 0, 'a1')], |
| 161 | 161 | rounds: [], |
| 162 | 162 | synthesis: { model: 'x-ai/grok-2', finalAnswer: 'done', dissent: [], usage: emptyUsage(), latencyMs: 0 }, |
| 163 | + provenance: null, | |
| 163 | 164 | failures: [], |
| 164 | 165 | finalAnswers: [answer('p0', 0, 'a0'), answer('p1', 0, 'a1')], |
| 165 | 166 | totals: { costUsd: 1, promptTokens: 5, completionTokens: 5, rounds: 0, durationMs: 100, costByModel: {} }, |
modified src/lib/debate-view.ts +7 −0
| @@ -18,6 +18,7 @@import type { | ||
| 18 | 18 | DebateStatus, |
| 19 | 19 | FailureRecord, |
| 20 | 20 | Participant, |
| 21 | + ProvenanceRecord, | |
| 21 | 22 | RevisionRecord, |
| 22 | 23 | RoundRecord, |
| 23 | 24 | StageType, |
| @@ -36,6 +37,7 @@export interface DebateView { | ||
| 36 | 37 | initialAnswers: Record<string, AnswerRecord>; |
| 37 | 38 | rounds: RoundRecord[]; |
| 38 | 39 | synthesis: SynthesisRecord | null; |
| 40 | + provenance: ProvenanceRecord | null; | |
| 39 | 41 | failures: FailureRecord[]; |
| 40 | 42 | totals: { |
| 41 | 43 | costUsd: number; |
| @@ -65,6 +67,7 @@export function initialDebateView(question = ''): DebateView { | ||
| 65 | 67 | initialAnswers: {}, |
| 66 | 68 | rounds: [], |
| 67 | 69 | synthesis: null, |
| 70 | + provenance: null, | |
| 68 | 71 | failures: [], |
| 69 | 72 | totals: { costUsd: 0, promptTokens: 0, completionTokens: 0, costByModel: {} }, |
| 70 | 73 | streaming: {}, |
| @@ -200,6 +203,9 @@export function applyEvent(prev: DebateView, event: DebateEvent): DebateView { | ||
| 200 | 203 | case 'synthesis_completed': |
| 201 | 204 | return { ...prev, synthesis: event.record, activeStage: null }; |
| 202 | 205 | |
| 206 | + case 'provenance_completed': | |
| 207 | + return { ...prev, provenance: event.record, activeStage: null }; | |
| 208 | + | |
| 203 | 209 | case 'debate_completed': |
| 204 | 210 | return { |
| 205 | 211 | ...prev, |
| @@ -233,6 +239,7 @@export function fromResult(result: DebateResult): DebateView { | ||
| 233 | 239 | initialAnswers, |
| 234 | 240 | rounds: result.rounds, |
| 235 | 241 | synthesis: result.synthesis, |
| 242 | + provenance: result.provenance, | |
| 236 | 243 | failures: result.failures, |
| 237 | 244 | totals: { |
| 238 | 245 | costUsd: result.totals.costUsd, |
modified src/lib/demo-fixtures.data.ts +19 −0
| @@ -189,6 +189,25 @@Extract a context into its own service only when you hit a concrete, nameable tr | ||
| 189 | 189 | ], |
| 190 | 190 | }, |
| 191 | 191 | ], |
| 192 | + provenance: [ | |
| 193 | + { | |
| 194 | + text: 'Start with one deployable application, split into bounded contexts (billing, identity, core domain) behind explicit interfaces.', | |
| 195 | + supportedBy: [0, 1], | |
| 196 | + contestedBy: [2], | |
| 197 | + }, | |
| 198 | + { | |
| 199 | + text: 'Enforce module boundaries automatically with an architecture test in CI that fails on cross-context imports or shared tables.', | |
| 200 | + supportedBy: [0, 1], | |
| 201 | + }, | |
| 202 | + { | |
| 203 | + text: 'Extract a context into its own service only on a concrete trigger: independent scaling, a different runtime, a compliance boundary, or dedicated ownership.', | |
| 204 | + supportedBy: [0, 1, 2], | |
| 205 | + }, | |
| 206 | + { | |
| 207 | + text: 'Because the boundaries were enforced from the start, a later extraction is mechanical rather than a rewrite.', | |
| 208 | + supportedBy: [], | |
| 209 | + }, | |
| 210 | + ], | |
| 192 | 211 | }; |
| 193 | 212 | |
| 194 | 213 | const restVsGraphql: FixtureSpec = { |
modified src/lib/demo-fixtures.ts +27 −0
| @@ -23,6 +23,7 @@import { | ||
| 23 | 23 | type DebateResult, |
| 24 | 24 | type Participant, |
| 25 | 25 | type PeerReview, |
| 26 | + type ProvenanceRecord, | |
| 26 | 27 | type RevisionRecord, |
| 27 | 28 | type RoundRecord, |
| 28 | 29 | type SynthesisRecord, |
| @@ -69,6 +70,8 @@export interface FixtureSpec { | ||
| 69 | 70 | rounds: FixtureRound[]; |
| 70 | 71 | finalAnswer: string; |
| 71 | 72 | dissent: { topic: string; positions: { p: number; text: string }[] }[]; |
| 73 | + /** Claim-provenance audit of the final answer; empty supportedBy = chairman addition. */ | |
| 74 | + provenance?: { text: string; supportedBy: number[]; contestedBy?: number[] }[]; | |
| 72 | 75 | } |
| 73 | 76 | |
| 74 | 77 | // --- cost model ------------------------------------------------------------ |
| @@ -227,6 +230,26 @@export function buildDebateResult(spec: FixtureSpec): DebateResult { | ||
| 227 | 230 | latencyMs: 0, |
| 228 | 231 | })); |
| 229 | 232 | |
| 233 | + const member = (i: number) => ({ participantId: `p${i}`, model: models[i]! }); | |
| 234 | + const provenance: ProvenanceRecord | null = spec.provenance | |
| 235 | + ? { | |
| 236 | + model: spec.convergenceModel, | |
| 237 | + round: rounds.length, | |
| 238 | + claims: spec.provenance.map((c) => ({ | |
| 239 | + text: c.text, | |
| 240 | + supportedBy: c.supportedBy.map(member), | |
| 241 | + contestedBy: (c.contestedBy ?? []).map(member), | |
| 242 | + unsourced: c.supportedBy.length === 0, | |
| 243 | + })), | |
| 244 | + usage: usageFor( | |
| 245 | + spec.convergenceModel, | |
| 246 | + spec.finalAnswer.length + current.reduce((s, c) => s + c.length, 0) + 300, | |
| 247 | + spec.provenance.reduce((s, c) => s + c.text.length + 30, 0), | |
| 248 | + ), | |
| 249 | + latencyMs: 950, | |
| 250 | + } | |
| 251 | + : null; | |
| 252 | + | |
| 230 | 253 | const allUsage: Usage[] = [ |
| 231 | 254 | ...initialAnswers.map((a) => a.usage), |
| 232 | 255 | ...rounds.flatMap((r) => [ |
| @@ -235,6 +258,7 @@export function buildDebateResult(spec: FixtureSpec): DebateResult { | ||
| 235 | 258 | ...(r.convergence ? [r.convergence.usage] : []), |
| 236 | 259 | ]), |
| 237 | 260 | synthesis.usage, |
| 261 | + ...(provenance ? [provenance.usage] : []), | |
| 238 | 262 | ]; |
| 239 | 263 | const totals = sumUsage(allUsage); |
| 240 | 264 | const costByModel: Record<string, number> = {}; |
| @@ -248,6 +272,7 @@export function buildDebateResult(spec: FixtureSpec): DebateResult { | ||
| 248 | 272 | if (r.convergence) add(r.convergence.model, r.convergence.usage); |
| 249 | 273 | }); |
| 250 | 274 | add(synthesis.model, synthesis.usage); |
| 275 | + if (provenance) add(provenance.model, provenance.usage); | |
| 251 | 276 | |
| 252 | 277 | return { |
| 253 | 278 | debateId: '', |
| @@ -257,6 +282,7 @@export function buildDebateResult(spec: FixtureSpec): DebateResult { | ||
| 257 | 282 | initialAnswers, |
| 258 | 283 | rounds, |
| 259 | 284 | synthesis, |
| 285 | + provenance, | |
| 260 | 286 | failures: [], |
| 261 | 287 | finalAnswers, |
| 262 | 288 | totals: { |
| @@ -280,6 +306,7 @@export function resultToStageEvents(result: DebateResult): DebateEvent[] { | ||
| 280 | 306 | if (round.convergence) events.push({ type: 'convergence_result', round: round.round, record: round.convergence }); |
| 281 | 307 | } |
| 282 | 308 | if (result.synthesis) events.push({ type: 'synthesis_completed', record: result.synthesis }); |
| 309 | + if (result.provenance) events.push({ type: 'provenance_completed', record: result.provenance }); | |
| 283 | 310 | return events; |
| 284 | 311 | } |
| 285 | 312 |
modified src/lib/export-markdown.test.ts +21 −0
| @@ -60,6 +60,21 @@function makeResult(): DebateResult { | ||
| 60 | 60 | usage: emptyUsage(), |
| 61 | 61 | latencyMs: 0, |
| 62 | 62 | }, |
| 63 | + provenance: { | |
| 64 | + model: 'google/gemini-2.0-flash-001', | |
| 65 | + round: 1, | |
| 66 | + claims: [ | |
| 67 | + { | |
| 68 | + text: 'Start with a monolith.', | |
| 69 | + supportedBy: [{ participantId: 'p0', model: 'openai/gpt-4o' }], | |
| 70 | + contestedBy: [], | |
| 71 | + unsourced: false, | |
| 72 | + }, | |
| 73 | + { text: 'Revisit in a year.', supportedBy: [], contestedBy: [], unsourced: true }, | |
| 74 | + ], | |
| 75 | + usage: emptyUsage(), | |
| 76 | + latencyMs: 0, | |
| 77 | + }, | |
| 63 | 78 | failures: [], |
| 64 | 79 | finalAnswers: [], |
| 65 | 80 | totals: { costUsd: 0.02, promptTokens: 100, completionTokens: 200, rounds: 1, durationMs: 5000, costByModel: {} }, |
| @@ -91,6 +106,12 @@describe('debateToMarkdown', () => { | ||
| 91 | 106 | expect(md).toContain('Convergence score: 88/100'); |
| 92 | 107 | }); |
| 93 | 108 | |
| 109 | + it('renders the claim check with sourced and unsourced claims', () => { | |
| 110 | + expect(md).toContain('### Claim Check'); | |
| 111 | + expect(md).toContain('Start with a monolith. (supported by GPT 4o)'); | |
| 112 | + expect(md).toContain("Revisit in a year. (**chairman's own addition"); | |
| 113 | + }); | |
| 114 | + | |
| 94 | 115 | it('renders round 0 answers', () => { |
| 95 | 116 | expect(md).toContain('## Round 0'); |
| 96 | 117 | expect(md).toContain('Monolith.'); |
modified src/lib/export-markdown.ts +15 −0
| @@ -42,6 +42,21 @@export function debateToMarkdown(result: DebateResult): string { | ||
| 42 | 42 | } |
| 43 | 43 | push(); |
| 44 | 44 | } |
| 45 | + if (result.provenance && result.provenance.claims.length) { | |
| 46 | + push(`### Claim Check`); | |
| 47 | + push(`_Each substantive claim traced back to the council by \`${result.provenance.model}\`._`); | |
| 48 | + push(); | |
| 49 | + for (const claim of result.provenance.claims) { | |
| 50 | + const source = claim.unsourced | |
| 51 | + ? "**chairman's own addition - no council member made this claim**" | |
| 52 | + : `supported by ${claim.supportedBy.map((m) => nameOf(m.participantId)).join(', ')}` + | |
| 53 | + (claim.contestedBy.length | |
| 54 | + ? `; contested by ${claim.contestedBy.map((m) => nameOf(m.participantId)).join(', ')}` | |
| 55 | + : ''); | |
| 56 | + push(`- ${claim.text} (${source})`); | |
| 57 | + } | |
| 58 | + push(); | |
| 59 | + } | |
| 45 | 60 | } |
| 46 | 61 | |
| 47 | 62 | push(`## Round 0 - Independent Answers`); |