revision.ts
2,599 bytes
| 1 | import type { LlmMessage } from '../llm-client'; |
|---|---|
| 2 | import { jsonInstruction, ROUNDTABLE_PREAMBLE } from './index'; |
| 3 | |
| 4 | /** One reviewer's feedback on this model's answer, with the source hidden. */ |
| 5 | export interface IncomingCritique { |
| 6 | weaknesses: string[]; |
| 7 | strengths: string[]; |
| 8 | score: number; |
| 9 | justification: string; |
| 10 | } |
| 11 | |
| 12 | const REVISION_SHAPE = `{ |
| 13 | "answer": "your full revised answer (or your unchanged answer if you are defending it)", |
| 14 | "changelog": { |
| 15 | "changed": true, // false if you are keeping your answer as-is |
| 16 | "summary": "1-2 sentences: what you changed and why, OR why you are defending your original position", |
| 17 | "bullets": ["concrete change 1", "concrete change 2"] // may be empty if unchanged |
| 18 | } |
| 19 | }`; |
| 20 | |
| 21 | /** |
| 22 | * Revision phase. |
| 23 | * |
| 24 | * The model receives the anonymized critiques that targeted *its own* answer and |
| 25 | * decides, point by point, whether each is right. It either revises or explicitly |
| 26 | * defends. Crucially it must NOT cave to social pressure - a well-reasoned defense |
| 27 | * with justification is a first-class outcome, which is why the changelog forces |
| 28 | * an explicit `changed` boolean and reasoning either way. |
| 29 | */ |
| 30 | export function buildRevisionPrompt( |
| 31 | question: string, |
| 32 | ownAnswer: string, |
| 33 | critiques: IncomingCritique[], |
| 34 | ): LlmMessage[] { |
| 35 | const feedbackBlock = |
| 36 | critiques.length === 0 |
| 37 | ? 'No critiques were received this round.' |
| 38 | : critiques |
| 39 | .map((c, i) => { |
| 40 | const w = c.weaknesses.length ? c.weaknesses.map((x) => ` - ${x}`).join('\n') : ' - (none)'; |
| 41 | const s = c.strengths.length ? c.strengths.map((x) => ` - ${x}`).join('\n') : ' - (none)'; |
| 42 | return `Reviewer ${i + 1} (score ${c.score}/10):\nWeaknesses:\n${w}\nStrengths:\n${s}\nComment: ${c.justification}`; |
| 43 | }) |
| 44 | .join('\n\n'); |
| 45 | |
| 46 | return [ |
| 47 | { |
| 48 | role: 'system', |
| 49 | content: |
| 50 | `${ROUNDTABLE_PREAMBLE}\n\n` + |
| 51 | 'You have received anonymous critiques of your answer. Weigh each on its ' + |
| 52 | 'merits. Where a critique is correct, revise to fix it. Where it is wrong or ' + |
| 53 | 'misguided, keep your position and say why - do not change your answer just to ' + |
| 54 | 'appease reviewers. Aim for the most correct, complete answer, not consensus ' + |
| 55 | 'for its own sake.\n\n' + |
| 56 | jsonInstruction(REVISION_SHAPE), |
| 57 | }, |
| 58 | { |
| 59 | role: 'user', |
| 60 | content: |
| 61 | `Question:\n${question}\n\n` + |
| 62 | `Your current answer:\n${ownAnswer}\n\n` + |
| 63 | `Critiques of your answer:\n${feedbackBlock}\n\n` + |
| 64 | 'Return your JSON revision now.', |
| 65 | }, |
| 66 | ]; |
| 67 | } |
| 68 | |