critique.ts
3,136 bytes
| 1 | import type { AnonymizedPeer } from '../anonymize'; |
|---|---|
| 2 | import type { LlmMessage } from '../llm-client'; |
| 3 | import { jsonInstruction, ROUNDTABLE_PREAMBLE } from './index'; |
| 4 | |
| 5 | const CRITIQUE_SHAPE = `{ |
| 6 | "reviews": [ |
| 7 | { |
| 8 | "label": "A", // the label of the response being reviewed |
| 9 | "weaknesses": ["specific error or gap", "..."], |
| 10 | "strengths": ["specific strong point", "..."], |
| 11 | "score": 7, // integer 1-10, overall quality |
| 12 | "justification": "one or two sentences explaining the score", |
| 13 | "predictedPeerMean": 6.5, // 1-10: the average score you expect the OTHER reviewers to give this response |
| 14 | "authorGuess": { // optional: your honest guess at who wrote it |
| 15 | "family": "anthropic", // one of: openai, anthropic, google, meta, mistral, x-ai, deepseek, other |
| 16 | "confidence": 0.3 // 0-1 |
| 17 | } |
| 18 | } |
| 19 | // ...one entry per response shown to you |
| 20 | ] |
| 21 | }`; |
| 22 | |
| 23 | /** |
| 24 | * Critique phase. |
| 25 | * |
| 26 | * The reviewer sees every *other* member's answer, fully anonymized and in a |
| 27 | * per-reviewer randomized order (labels "Response A/B/C..."). It must return, for |
| 28 | * each response, concrete weaknesses, concrete strengths, and a calibrated 1-10 |
| 29 | * score with justification. Anonymity + randomized order is the core bias |
| 30 | * mitigation: a model cannot preferentially reward "its own style" if it cannot |
| 31 | * tell which answer is whose. |
| 32 | */ |
| 33 | export function buildCritiquePrompt(question: string, peers: AnonymizedPeer[]): LlmMessage[] { |
| 34 | const peerBlock = peers |
| 35 | .map((p) => `--- Response ${p.label} ---\n${p.content}`) |
| 36 | .join('\n\n'); |
| 37 | |
| 38 | return [ |
| 39 | { |
| 40 | role: 'system', |
| 41 | content: |
| 42 | `${ROUNDTABLE_PREAMBLE}\n\n` + |
| 43 | "You are now reviewing the other members' answers. They are anonymized and " + |
| 44 | 'shown in a random order; you do not know who wrote which, and none of them is ' + |
| 45 | 'your own. Judge only on merit: correctness first, then completeness, clarity, ' + |
| 46 | 'and calibration. Be specific - cite the exact claim you think is wrong or ' + |
| 47 | 'missing. Reward genuine strengths honestly. Scores should span the range; do ' + |
| 48 | 'not cluster everything at 7-8. For each response, also predict the average ' + |
| 49 | 'score the other reviewers will give it. Your prediction is checked against ' + |
| 50 | 'their actual average, so report what you expect THEM to conclude - it may ' + |
| 51 | 'legitimately differ from your own score when you see a flaw or strength you ' + |
| 52 | 'suspect others will miss. Finally, give your honest guess at which model ' + |
| 53 | 'family wrote each response, with a confidence between 0 and 1. This guess is ' + |
| 54 | 'scored later to audit whether the anonymization is working; it has no effect ' + |
| 55 | 'on the debate, so do not let it color your review.\n\n' + |
| 56 | jsonInstruction(CRITIQUE_SHAPE), |
| 57 | }, |
| 58 | { |
| 59 | role: 'user', |
| 60 | content: |
| 61 | `Question:\n${question}\n\n` + |
| 62 | `Answers to review (${peers.length}):\n\n${peerBlock}\n\n` + |
| 63 | 'Return your JSON review now.', |
| 64 | }, |
| 65 | ]; |
| 66 | } |
| 67 | |