roundtable | profileShare

modified src/components/debate/critique-matrix.tsx +59 −2

		@@ -4,14 +4,24 @@import { Eye, EyeOff } from 'lucide-react';
4	4	import { useMemo, useState } from 'react';
5	5	import { Button } from '@/components/ui/button';
6	6	import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip';
7		-import { buildScoreMatrix } from '@/core/scoring';
	7	+import {
	8	+ buildPredictionCells,
	9	+ buildScoreMatrix,
	10	+ classifyPrediction,
	11	+ predictionCalibration,
	12	+ type PredictionCell,
	13	+ type PredictionReading,
	14	+} from '@/core/scoring';
8	15	import type { CritiqueRecord, Participant, PeerReview } from '@/core/types';
9	16	import { participantColor, participantTag, scoreColor } from '@/lib/model-visuals';
10	17
11	18	/**
12	19	* N×N critique grid: rows are reviewers, columns are the answers scored.
13	20	* Anonymized by default (identities hidden as the models saw them); a toggle
14	21	* de-anonymizes to real model names. Hovering a cell reveals the full critique.
	22	+ * Cells where the reviewer disagreed with the council are outlined: violet when
	23	+ * it predicted the council's view accurately (informed contrarian), amber when
	24	+ * it thought everyone would agree with it (miscalibrated).
15	25	*/
16	26	export function CritiqueMatrix({
17	27	participants,
		@@ -23,6 +33,12 @@export function CritiqueMatrix({
23	33	const [deanon, setDeanon] = useState(false);
24	34	const matrix = useMemo(() => buildScoreMatrix(participants, critiques), [participants, critiques]);
25	35	const byId = useMemo(() => new Map(participants.map((p, i) => [p.id, { p, i }])), [participants]);
	36	+ const predictions = useMemo(() => {
	37	+ const cells = buildPredictionCells(participants, critiques);
	38	+ const byPair = new Map<string, PredictionCell>();
	39	+ for (const c of cells) byPair.set(`${c.reviewerId}:${c.targetId}`, c);
	40	+ return { byPair, calibration: predictionCalibration(participants, cells), any: cells.length > 0 };
	41	+ }, [participants, critiques]);
26	42
27	43	const reviewLookup = useMemo(() => {
28	44	const map = new Map<string, PeerReview>();
		@@ -47,6 +63,13 @@export function CritiqueMatrix({
47	63	<div className="flex items-center justify-between">
48	64	<p className="text-xs text-muted-foreground">
49	65	Rows critique columns · scores 1-10 · hover a cell for the full review
	66	+ {predictions.any && (
	67	+ <>
	68	+ {' '}
	69	+ · <span className="text-violet-500">outlined violet</span> = informed contrarian ·{' '}
	70	+ <span className="text-amber-500">outlined amber</span> = miscalibrated
	71	+ </>
	72	+ )}
50	73	</p>
51	74	<Button variant="outline" size="sm" className="gap-1.5" onClick={() => setDeanon((v) => !v)}>
52	75	{deanon ? <EyeOff className="h-3.5 w-3.5" /> : <Eye className="h-3.5 w-3.5" />}
		@@ -65,6 +88,7 @@export function CritiqueMatrix({
65	88	</th>
66	89	))}
67	90	<th className="p-1 text-xs font-medium text-muted-foreground">avg recv</th>
	91	+ {predictions.any && <th className="p-1 text-xs font-medium text-muted-foreground">council read</th>}
68	92	</tr>
69	93	</thead>
70	94	<tbody>
		@@ -76,6 +100,8 @@export function CritiqueMatrix({
76	100	{matrix.order.map((targetId) => {
77	101	const score = matrix.cells[reviewerId]?.[targetId] ?? null;
78	102	const review = reviewLookup.get(`${reviewerId}:${targetId}`);
	103	+ const prediction = predictions.byPair.get(`${reviewerId}:${targetId}`);
	104	+ const reading = prediction ? classifyPrediction(prediction) : null;
79	105	return (
80	106	<td key={targetId} className="p-0.5 text-center">
81	107	{score === null ? (
		@@ -86,7 +112,7 @@export function CritiqueMatrix({
86	112	<Tooltip>
87	113	<TooltipTrigger asChild>
88	114	<div
89		- className="flex h-11 w-14 cursor-help items-center justify-center rounded-md font-semibold text-white transition-transform hover:scale-105"
	115	+ className={`flex h-11 w-14 cursor-help items-center justify-center rounded-md font-semibold text-white transition-transform hover:scale-105 ${readingOutline(reading)}`}
90	116	style={{ backgroundColor: scoreColor(score) }}
91	117	>
92	118	{score}
		@@ -108,6 +134,16 @@export function CritiqueMatrix({
108	134	</div>
109	135	)}
110	136	<div className="text-[11px] text-muted-foreground">{review.justification}</div>
	137	+ {prediction && (
	138	+ <div className="text-[11px]">
	139	+ <span className="text-sky-400">Predicted council avg:</span>{' '}
	140	+ {prediction.predicted.toFixed(1)}
	141	+ {prediction.actualPeerMean !== null &&
	142	+ ` · actual ${prediction.actualPeerMean.toFixed(1)}`}
	143	+ {reading === 'informed_contrarian' && ' · informed contrarian'}
	144	+ {reading === 'miscalibrated' && ' · miscalibrated'}
	145	+ </div>
	146	+ )}
111	147	</TooltipContent>
112	148	)}
113	149	</Tooltip>
		@@ -118,6 +154,11 @@export function CritiqueMatrix({
118	154	<td className="p-1 text-center text-xs font-medium">
119	155	{fmtAvg(matrix.averagesReceived[reviewerId])}
120	156	</td>
	157	+ {predictions.any && (
	158	+ <td className="p-1 text-center text-xs text-muted-foreground">
	159	+ {fmtCalibration(predictions.calibration[reviewerId])}
	160	+ </td>
	161	+ )}
121	162	</tr>
122	163	))}
123	164	</tbody>
		@@ -144,3 +185,19 @@function ColHeader({ index, name }: { index: number; name: string }) {
144	185	function fmtAvg(v: number \| null \| undefined): string {
145	186	return typeof v === 'number' ? v.toFixed(1) : '-';
146	187	}
	188	+
	189	+/** Mean absolute error of this reviewer's council predictions; lower reads better. */
	190	+function fmtCalibration(v: number \| null \| undefined): string {
	191	+ return typeof v === 'number' ? `±${v.toFixed(1)}` : '-';
	192	+}
	193	+
	194	+function readingOutline(reading: PredictionReading \| null): string {
	195	+ switch (reading) {
	196	+ case 'informed_contrarian':
	197	+ return 'ring-2 ring-violet-500';
	198	+ case 'miscalibrated':
	199	+ return 'ring-2 ring-amber-500';
	200	+ default:
	201	+ return '';
	202	+ }
	203	+}

modified src/core/mock-client.ts +3 −0

		@@ -146,12 +146,15 @@export class MockLlmClient implements LlmClient {
146	146	const reviews = labels.map((label) => {
147	147	const s = hashSeed(seed, label) % 6; // 0..5
148	148	const score = malform ? 42 : 4 + s; // 4..9 normally; out-of-range to trigger repair
	149	+ // Deterministic council-mean prediction near (but not equal to) the score.
	150	+ const predictedPeerMean = Math.min(10, Math.max(1, score + (hashSeed(seed, label, 'pred') % 3) - 1));
149	151	return {
150	152	label,
151	153	weaknesses: [`Response ${label} underspecifies the edge cases.`],
152	154	strengths: [`Response ${label} states its assumptions clearly.`],
153	155	score,
154	156	justification: `Solid reasoning with a gap around edge cases (score ${score}).`,
	157	+ predictedPeerMean,
155	158	};
156	159	});
157	160	return JSON.stringify({ reviews });

modified src/core/orchestrator.ts +9 −1

		@@ -565,7 +565,14 @@export async function runDebate(
565	565	function buildCritiqueRecord(
566	566	round: number,
567	567	reviewer: Participant,
568		- reviews: Array<{ label: string; weaknesses: string[]; strengths: string[]; score: number; justification: string }>,
	568	+ reviews: Array<{
	569	+ label: string;
	570	+ weaknesses: string[];
	571	+ strengths: string[];
	572	+ score: number;
	573	+ justification: string;
	574	+ predictedPeerMean?: number;
	575	+ }>,
569	576	anon: AnonymizationResult,
570	577	usage: Usage,
571	578	latencyMs: number,
		@@ -581,6 +588,7 @@export async function runDebate(
581	588	strengths: r.strengths,
582	589	score: r.score,
583	590	justification: r.justification,
	591	+ ...(r.predictedPeerMean !== undefined ? { predictedPeerMean: r.predictedPeerMean } : {}),
584	592	} satisfies PeerReview;
585	593	})
586	594	.filter((x): x is PeerReview => x !== null);

modified src/core/prompts/critique.ts +7 −2

		@@ -9,7 +9,8 @@const CRITIQUE_SHAPE = `{
9	9	"weaknesses": ["specific error or gap", "..."],
10	10	"strengths": ["specific strong point", "..."],
11	11	"score": 7, // integer 1-10, overall quality
12		- "justification": "one or two sentences explaining the score"
	12	+ "justification": "one or two sentences explaining the score",
	13	+ "predictedPeerMean": 6.5 // 1-10: the average score you expect the OTHER reviewers to give this response
13	14	}
14	15	// ...one entry per response shown to you
15	16	]
		@@ -40,7 +41,11 @@export function buildCritiquePrompt(question: string, peers: AnonymizedPeer[]):
40	41	'your own. Judge only on merit: correctness first, then completeness, clarity, ' +
41	42	'and calibration. Be specific - cite the exact claim you think is wrong or ' +
42	43	'missing. Reward genuine strengths honestly. Scores should span the range; do ' +
43		- 'not cluster everything at 7-8.\n\n' +
	44	+ 'not cluster everything at 7-8. For each response, also predict the average ' +
	45	+ 'score the other reviewers will give it. Your prediction is checked against ' +
	46	+ 'their actual average, so report what you expect THEM to conclude - it may ' +
	47	+ 'legitimately differ from your own score when you see a flaw or strength you ' +
	48	+ 'suspect others will miss.\n\n' +
44	49	jsonInstruction(CRITIQUE_SHAPE),
45	50	},
46	51	{

modified src/core/prompts/index.ts +1 −1

		@@ -10,7 +10,7 @@
10	10	* alter model behavior; it is stamped onto stored results so historical debates
11	11	* remain interpretable.
12	12	*/
13		-export const PROMPT_VERSION = '1.0.0';
	13	+export const PROMPT_VERSION = '1.1.0';
14	14
15	15	export { buildAnswerPrompt } from './answer';
16	16	export { buildCritiquePrompt } from './critique';

modified src/core/schemas.ts +2 −0

		@@ -20,6 +20,8 @@export const peerReviewSchema = z.object({
20	20	strengths: z.array(z.string()).default([]),
21	21	score: z.coerce.number().min(1).max(10),
22	22	justification: z.string().default(''),
	23	+ /** The reviewer's prediction of the other reviewers' average score (1-10). */
	24	+ predictedPeerMean: z.coerce.number().min(1).max(10).optional(),
23	25	});
24	26
25	27	export const critiqueOutputSchema = z.object({

modified src/core/scoring.test.ts +94 −1

		@@ -1,5 +1,11 @@
1	1	import { describe, expect, it } from 'vitest';
2		-import { buildScoreMatrix } from './scoring';
	2	+import {
	3	+ buildPredictionCells,
	4	+ buildScoreMatrix,
	5	+ classifyPrediction,
	6	+ predictionCalibration,
	7	+ type PredictionCell,
	8	+} from './scoring';
3	9	import { emptyUsage, type CritiqueRecord, type Participant } from './types';
4	10
5	11	const participants: Participant[] = [
		@@ -70,3 +76,90 @@describe('buildScoreMatrix', () => {
70	76	expect(m.averagesReceived.p0).toBeNull();
71	77	});
72	78	});
	79	+
	80	+function critiqueWithPredictions(
	81	+ reviewer: string,
	82	+ reviews: Record<string, { score: number; predicted?: number }>,
	83	+): CritiqueRecord {
	84	+ return {
	85	+ round: 1,
	86	+ reviewerParticipantId: reviewer,
	87	+ reviewerModel: 'x/y',
	88	+ reviews: Object.entries(reviews).map(([targetParticipantId, r]) => ({
	89	+ label: 'A',
	90	+ targetParticipantId,
	91	+ weaknesses: [],
	92	+ strengths: [],
	93	+ score: r.score,
	94	+ justification: '',
	95	+ ...(r.predicted !== undefined ? { predictedPeerMean: r.predicted } : {}),
	96	+ })),
	97	+ usage: emptyUsage(),
	98	+ latencyMs: 0,
	99	+ };
	100	+}
	101	+
	102	+describe('buildPredictionCells', () => {
	103	+ it('pairs each prediction with the mean of the other reviewers scores', () => {
	104	+ const critiques = [
	105	+ critiqueWithPredictions('p0', { p2: { score: 3, predicted: 7 } }),
	106	+ critiqueWithPredictions('p1', { p2: { score: 8 } }),
	107	+ critiqueWithPredictions('p2', { p0: { score: 6 } }),
	108	+ ];
	109	+ const cells = buildPredictionCells(participants, critiques);
	110	+ expect(cells).toHaveLength(1);
	111	+ // p0 predicted 7 for p2; the only other reviewer of p2 (p1) gave 8.
	112	+ expect(cells[0]).toMatchObject({ reviewerId: 'p0', targetId: 'p2', ownScore: 3, predicted: 7, actualPeerMean: 8 });
	113	+ });
	114	+
	115	+ it('yields a null actual mean when no other reviewer scored the target', () => {
	116	+ const critiques = [critiqueWithPredictions('p0', { p1: { score: 5, predicted: 6 } })];
	117	+ const cells = buildPredictionCells(participants, critiques);
	118	+ expect(cells[0]!.actualPeerMean).toBeNull();
	119	+ });
	120	+
	121	+ it('skips reviews without predictions', () => {
	122	+ const critiques = [critiqueWithPredictions('p0', { p1: { score: 5 } })];
	123	+ expect(buildPredictionCells(participants, critiques)).toHaveLength(0);
	124	+ });
	125	+});
	126	+
	127	+describe('classifyPrediction', () => {
	128	+ const cell = (ownScore: number, predicted: number, actualPeerMean: number \| null): PredictionCell => ({
	129	+ reviewerId: 'p0',
	130	+ targetId: 'p1',
	131	+ ownScore,
	132	+ predicted,
	133	+ actualPeerMean,
	134	+ });
	135	+
	136	+ it('is aligned when the reviewer roughly agrees with the council', () => {
	137	+ expect(classifyPrediction(cell(7, 7, 7.5))).toBe('aligned');
	138	+ });
	139	+
	140	+ it('is an informed contrarian when disagreeing but predicting the council accurately', () => {
	141	+ expect(classifyPrediction(cell(3, 7.5, 8))).toBe('informed_contrarian');
	142	+ });
	143	+
	144	+ it('is miscalibrated when disagreeing and mispredicting the council', () => {
	145	+ expect(classifyPrediction(cell(3, 3.5, 8))).toBe('miscalibrated');
	146	+ });
	147	+
	148	+ it('is null when there is no actual peer mean to compare against', () => {
	149	+ expect(classifyPrediction(cell(3, 7, null))).toBeNull();
	150	+ });
	151	+});
	152	+
	153	+describe('predictionCalibration', () => {
	154	+ it('averages absolute prediction error per reviewer and nulls the rest', () => {
	155	+ const cells: PredictionCell[] = [
	156	+ { reviewerId: 'p0', targetId: 'p1', ownScore: 5, predicted: 6, actualPeerMean: 8 },
	157	+ { reviewerId: 'p0', targetId: 'p2', ownScore: 5, predicted: 7, actualPeerMean: 6 },
	158	+ { reviewerId: 'p1', targetId: 'p2', ownScore: 5, predicted: 5, actualPeerMean: null },
	159	+ ];
	160	+ const calib = predictionCalibration(participants, cells);
	161	+ expect(calib.p0).toBeCloseTo(1.5); // \|6-8\|=2, \|7-6\|=1
	162	+ expect(calib.p1).toBeNull(); // its only cell has no actual mean
	163	+ expect(calib.p2).toBeNull();
	164	+ });
	165	+});

modified src/core/scoring.ts +84 −1

		@@ -1,9 +1,17 @@
1	1	/**
2		- * Aggregate critique scores into the N×N matrix the UI renders.
	2	+ * Aggregate critique scores into the N×N matrix the UI renders, plus
	3	+ * peer-prediction analytics.
3	4	*
4	5	* Rows are reviewers, columns are the answers being scored. A cell is null when
5	6	* that reviewer did not score that target (e.g. the reviewer dropped out, or the
6	7	* self-cell). Pure and deterministic - unit-tested against hand-built records.
	8	+ *
	9	+ * Peer prediction: alongside its score, a reviewer may predict the average the
	10	+ * OTHER reviewers will give the same answer. Comparing prediction to reality
	11	+ * splits disagreement into two signals: an informed contrarian scores low while
	12	+ * correctly predicting the council will score high (their dissent is deliberate
	13	+ * and worth attention); a miscalibrated reviewer scores low believing everyone
	14	+ * else will too.
7	15	*/
8	16	import type { CritiqueRecord, Participant } from './types';
9	17
		@@ -61,3 +69,78 @@export function buildScoreMatrix(
61	69
62	70	return { order, cells, averagesReceived, averagesGiven };
63	71	}
	72	+
	73	+// ---------------------------------------------------------------------------
	74	+// Peer-prediction analytics
	75	+// ---------------------------------------------------------------------------
	76	+
	77	+/** Own score must differ from the peer mean by at least this to count as disagreement. */
	78	+export const CONTRARIAN_GAP = 2;
	79	+/** A prediction within this distance of the actual peer mean counts as accurate. */
	80	+export const PREDICTION_TOLERANCE = 1.5;
	81	+
	82	+export interface PredictionCell {
	83	+ reviewerId: string;
	84	+ targetId: string;
	85	+ ownScore: number;
	86	+ predicted: number;
	87	+ /** Mean score the OTHER reviewers gave the target; null if nobody else scored it. */
	88	+ actualPeerMean: number \| null;
	89	+}
	90	+
	91	+export type PredictionReading = 'aligned' \| 'informed_contrarian' \| 'miscalibrated';
	92	+
	93	+export function classifyPrediction(cell: PredictionCell): PredictionReading \| null {
	94	+ if (cell.actualPeerMean === null) return null;
	95	+ if (Math.abs(cell.ownScore - cell.actualPeerMean) < CONTRARIAN_GAP) return 'aligned';
	96	+ return Math.abs(cell.predicted - cell.actualPeerMean) <= PREDICTION_TOLERANCE
	97	+ ? 'informed_contrarian'
	98	+ : 'miscalibrated';
	99	+}
	100	+
	101	+/** One cell per review that carried a `predictedPeerMean`. */
	102	+export function buildPredictionCells(
	103	+ participants: Participant[],
	104	+ critiques: CritiqueRecord[],
	105	+): PredictionCell[] {
	106	+ const matrix = buildScoreMatrix(participants, critiques);
	107	+ const cells: PredictionCell[] = [];
	108	+ for (const critique of critiques) {
	109	+ const reviewerId = critique.reviewerParticipantId;
	110	+ if (!matrix.cells[reviewerId]) continue;
	111	+ for (const review of critique.reviews) {
	112	+ if (review.predictedPeerMean === undefined) continue;
	113	+ const targetId = review.targetParticipantId;
	114	+ const others = matrix.order
	115	+ .filter((id) => id !== reviewerId)
	116	+ .map((id) => matrix.cells[id]![targetId])
	117	+ .filter((v): v is number => typeof v === 'number');
	118	+ cells.push({
	119	+ reviewerId,
	120	+ targetId,
	121	+ ownScore: review.score,
	122	+ predicted: review.predictedPeerMean,
	123	+ actualPeerMean: mean(others),
	124	+ });
	125	+ }
	126	+ }
	127	+ return cells;
	128	+}
	129	+
	130	+/**
	131	+ * Per-reviewer mean absolute prediction error ("how well does this model read
	132	+ * the council"). Null for reviewers with no scoreable predictions.
	133	+ */
	134	+export function predictionCalibration(
	135	+ participants: Participant[],
	136	+ cells: PredictionCell[],
	137	+): Record<string, number \| null> {
	138	+ const out: Record<string, number \| null> = {};
	139	+ for (const p of participants) {
	140	+ const errors = cells
	141	+ .filter((c) => c.reviewerId === p.id && c.actualPeerMean !== null)
	142	+ .map((c) => Math.abs(c.predicted - c.actualPeerMean!));
	143	+ out[p.id] = mean(errors);
	144	+ }
	145	+ return out;
	146	+}

modified src/core/types.ts +8 −0

		@@ -90,6 +90,14 @@export interface PeerReview {
90	90	/** 1-10. */
91	91	score: number;
92	92	justification: string;
	93	+ /**
	94	+ * The reviewer's prediction of the average score the OTHER reviewers will
	95	+ * give this answer (1-10). Comparing it to the actual peer mean separates
	96	+ * informed contrarians (low score, accurate prediction) from miscalibrated
	97	+ * reviewers (low score, wrong prediction). Optional: older debates and
	98	+ * models that omit it degrade gracefully.
	99	+ */
	100	+ predictedPeerMean?: number;
93	101	}
94	102
95	103	export interface CritiqueRecord {

rasmusjy / roundtable

add peer-prediction to critiques: contrarian vs miscalibrated reviewers