provenance.ts
2,267 bytes
| 1 | import type { AnonymizedPeer } from '../anonymize'; |
|---|---|
| 2 | import type { LlmMessage } from '../llm-client'; |
| 3 | import { jsonInstruction } from './index'; |
| 4 | |
| 5 | const PROVENANCE_SHAPE = `{ |
| 6 | "claims": [ |
| 7 | { |
| 8 | "text": "one substantive claim made by the final answer, in its own words", |
| 9 | "supportedBy": ["A", "C"], // labels of responses that state or clearly imply this claim |
| 10 | "contestedBy": ["B"] // labels of responses that dispute it (often empty) |
| 11 | } |
| 12 | // ...one entry per substantive claim, typically 4-10 |
| 13 | ] |
| 14 | }`; |
| 15 | |
| 16 | /** |
| 17 | * Provenance audit (advisory, post-synthesis). |
| 18 | * |
| 19 | * A cheap model decomposes the chairman's final answer into its substantive |
| 20 | * claims and traces each back to the anonymized council answers. A claim with |
| 21 | * an empty `supportedBy` is unsourced: content the chairman introduced that no |
| 22 | * council member argued - exactly where synthesis hallucination lives. Run by |
| 23 | * the convergence-class model; failures never fail the debate. |
| 24 | */ |
| 25 | export function buildProvenancePrompt( |
| 26 | question: string, |
| 27 | finalAnswer: string, |
| 28 | peers: AnonymizedPeer[], |
| 29 | ): LlmMessage[] { |
| 30 | const peerBlock = peers.map((p) => `--- Response ${p.label} ---\n${p.content}`).join('\n\n'); |
| 31 | |
| 32 | return [ |
| 33 | { |
| 34 | role: 'system', |
| 35 | content: |
| 36 | 'You are an audit assistant. A panel answered a question and a chairman wrote a ' + |
| 37 | 'final synthesized answer from their responses. Decompose the final answer into its ' + |
| 38 | 'substantive claims (recommendations, factual assertions, caveats - not filler), and ' + |
| 39 | 'for each claim report which panel responses support it and which dispute it. A ' + |
| 40 | 'response supports a claim only if it states it or clearly implies it; do not ' + |
| 41 | 'stretch. If NO response contains a claim, return it with an empty "supportedBy" ' + |
| 42 | 'list - identifying such chairman-added content is the entire point of this audit, ' + |
| 43 | 'so never invent support.\n\n' + |
| 44 | jsonInstruction(PROVENANCE_SHAPE), |
| 45 | }, |
| 46 | { |
| 47 | role: 'user', |
| 48 | content: |
| 49 | `Question:\n${question}\n\n` + |
| 50 | `Final synthesized answer:\n${finalAnswer}\n\n` + |
| 51 | `Panel responses (${peers.length}):\n\n${peerBlock}\n\n` + |
| 52 | 'Return your JSON audit now.', |
| 53 | }, |
| 54 | ]; |
| 55 | } |
| 56 | |