roundtable | profileShare

modified README.md +1 −0

		@@ -37,6 +37,7 @@Everything a reviewer needs to _trust_ the answer is on screen: the critique mat
37	37	\| Pressure response ("spine") \| Per model, per round, cross the critique it received with whether it revised: _revised_, _defended_, _caved_ (changed a well-scored answer), _stonewalled_ (kept a poorly-scored one). Surfaces sycophancy and stubbornness with zero extra LLM calls. \|
38	38	\| Peer prediction \| Each reviewer also predicts the council's average score for an answer. Comparing prediction to reality splits dissent into informed contrarians (low score, accurate read) and miscalibrated reviewers (low score, wrong read), with a per-reviewer "council read" calibration column. \|
39	39	\| Claim provenance \| An advisory post-synthesis audit traces every substantive claim in the final answer back to the council's answers and flags unsourced "chairman additions" - synthesis hallucination detection - run on the cheap convergence model. \|
	40	+\| Masquerade audit \| Reviewers also guess who wrote each anonymized answer; the guesses are scored against the real mapping. A hit rate near chance proves the masks are holding; well above flags style leakage - the app continuously audits its own bias mitigation. \|
40	41	\| Re-run & key recovery \| Re-run any past debate (`/debate?from=<id>` prefills its question and council). A start that fails for a missing key drops you into the key dialog with the composed debate intact, not a dead end. \|
41	42	\| BYOK, two modes \| Bring your own OpenRouter key - saved (AES-256-GCM encrypted at rest) or session-only (encrypted HttpOnly cookie). No server-paid key. \|
42	43	\| Portfolio demo mode \| Public, no-login page that replays real recorded debates through the full UI with simulated streaming. \|

modified src/components/debate/debate-console.tsx +3 −1

		@@ -7,6 +7,7 @@import { CritiqueMatrix } from '@/components/debate/critique-matrix';
7	7	import { DebateActions } from '@/components/debate/debate-actions';
8	8	import { Disagreements } from '@/components/debate/disagreements';
9	9	import { FinalAnswer } from '@/components/debate/final-answer';
	10	+import { MasqueradeNote } from '@/components/debate/masquerade-note';
10	11	import { ModelPanel } from '@/components/debate/model-panel';
11	12	import { RevisionDiff } from '@/components/debate/revision-diff';
12	13	import { SpinePanel } from '@/components/debate/spine-panel';
		@@ -15,7 +16,7 @@import { Badge } from '@/components/ui/badge';
15	16	import { Progress } from '@/components/ui/progress';
16	17	import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs';
17	18	import type { AnswerRecord } from '@/core/types';
18		-import { currentAnswers, type DebateView } from '@/lib/debate-view';
	19	+import { allCritiques, currentAnswers, type DebateView } from '@/lib/debate-view';
19	20	import { formatUsd } from '@/lib/utils';
20	21
21	22	export function DebateConsole({ view, showActions = true }: { view: DebateView; showActions?: boolean }) {
		@@ -136,6 +137,7 @@export function DebateConsole({ view, showActions = true }: { view: DebateView;
136	137	<div className="rounded-xl border bg-card p-4">
137	138	<SpinePanel participants={view.participants} rounds={view.rounds} />
138	139	</div>
	140	+ {finished && <MasqueradeNote participants={view.participants} critiques={allCritiques(view)} />}
139	141	</section>
140	142	)}
141	143

added src/components/debate/masquerade-note.tsx +42 −0

		@@ -0,0 +1,42 @@
	1	+'use client';
	2	+
	3	+import { VenetianMask } from 'lucide-react';
	4	+import { useMemo } from 'react';
	5	+import { buildMasqueradeReport } from '@/core/masquerade';
	6	+import type { CritiqueRecord, Participant } from '@/core/types';
	7	+
	8	+/**
	9	+ * One-line anonymity audit: how often reviewers correctly guessed who wrote
	10	+ * an answer, versus chance. Near chance means the masks are holding; well
	11	+ * above means models are stylistically fingerprintable in this debate.
	12	+ */
	13	+export function MasqueradeNote({
	14	+ participants,
	15	+ critiques,
	16	+}: {
	17	+ participants: Participant[];
	18	+ critiques: CritiqueRecord[];
	19	+}) {
	20	+ const report = useMemo(() => buildMasqueradeReport(participants, critiques), [participants, critiques]);
	21	+ if (report.verdict === 'insufficient' \|\| report.hitRate === null \|\| report.chanceRate === null) return null;
	22	+
	23	+ const leaking = report.verdict === 'leaking';
	24	+ return (
	25	+ <p
	26	+ className={`flex items-center gap-1.5 text-xs ${
	27	+ leaking ? 'text-amber-600 dark:text-amber-400' : 'text-muted-foreground'
	28	+ }`}
	29	+ >
	30	+ <VenetianMask className="h-3.5 w-3.5 shrink-0" />
	31	+ Anonymity check: reviewers identified authors in {pct(report.hitRate)} of {report.guesses} guesses vs{' '}
	32	+ {pct(report.chanceRate)} chance.{' '}
	33	+ {leaking
	34	+ ? 'Writing style is leaking authorship in this debate; treat critique scores with extra care.'
	35	+ : 'The masks held.'}
	36	+ </p>
	37	+ );
	38	+}
	39	+
	40	+function pct(v: number): string {
	41	+ return `${Math.round(v * 100)}%`;
	42	+}

added src/core/masquerade.test.ts +94 −0

		@@ -0,0 +1,94 @@
	1	+import { describe, expect, it } from 'vitest';
	2	+import { buildMasqueradeReport, normalizeFamilyGuess } from './masquerade';
	3	+import { emptyUsage, type CritiqueRecord, type Participant } from './types';
	4	+
	5	+const participants: Participant[] = [
	6	+ { id: 'p0', model: 'openai/gpt-4o', displayName: 'GPT 4o' },
	7	+ { id: 'p1', model: 'anthropic/claude-3.5-sonnet', displayName: 'Claude 3.5' },
	8	+ { id: 'p2', model: 'google/gemini-pro-1.5', displayName: 'Gemini Pro' },
	9	+];
	10	+
	11	+function critique(
	12	+ reviewer: string,
	13	+ guesses: Record<string, { family: string; confidence: number } \| null>,
	14	+): CritiqueRecord {
	15	+ return {
	16	+ round: 1,
	17	+ reviewerParticipantId: reviewer,
	18	+ reviewerModel: 'x/y',
	19	+ reviews: Object.entries(guesses).map(([targetParticipantId, guess]) => ({
	20	+ label: 'A',
	21	+ targetParticipantId,
	22	+ weaknesses: [],
	23	+ strengths: [],
	24	+ score: 5,
	25	+ justification: '',
	26	+ ...(guess ? { authorGuess: guess } : {}),
	27	+ })),
	28	+ usage: emptyUsage(),
	29	+ latencyMs: 0,
	30	+ };
	31	+}
	32	+
	33	+describe('normalizeFamilyGuess', () => {
	34	+ it('maps informal names to provider prefixes', () => {
	35	+ expect(normalizeFamilyGuess('ChatGPT / GPT-4 style')).toBe('openai');
	36	+ expect(normalizeFamilyGuess('Claude')).toBe('anthropic');
	37	+ expect(normalizeFamilyGuess('gemini')).toBe('google');
	38	+ expect(normalizeFamilyGuess('Grok (xAI)')).toBe('x-ai');
	39	+ expect(normalizeFamilyGuess('llama-3')).toBe('meta-llama');
	40	+ });
	41	+
	42	+ it('passes through unknown families lowercased', () => {
	43	+ expect(normalizeFamilyGuess('Cohere')).toBe('cohere');
	44	+ });
	45	+});
	46	+
	47	+describe('buildMasqueradeReport', () => {
	48	+ it('scores guesses against real authorship', () => {
	49	+ const critiques = [
	50	+ critique('p0', {
	51	+ p1: { family: 'anthropic', confidence: 0.8 }, // correct
	52	+ p2: { family: 'openai', confidence: 0.4 }, // wrong
	53	+ }),
	54	+ critique('p1', {
	55	+ p0: { family: 'gpt', confidence: 0.6 }, // correct via keyword mapping
	56	+ p2: { family: 'gemini', confidence: 0.5 }, // correct
	57	+ }),
	58	+ ];
	59	+ const report = buildMasqueradeReport(participants, critiques);
	60	+ expect(report.guesses).toBe(4);
	61	+ expect(report.correct).toBe(3);
	62	+ expect(report.hitRate).toBeCloseTo(0.75);
	63	+ // Each reviewer saw 2 distinct peer families -> chance 0.5 per guess.
	64	+ expect(report.chanceRate).toBeCloseTo(0.5);
	65	+ expect(report.verdict).toBe('leaking');
	66	+ expect(report.meanConfidenceCorrect).toBeCloseTo((0.8 + 0.6 + 0.5) / 3);
	67	+ expect(report.meanConfidenceIncorrect).toBeCloseTo(0.4);
	68	+ });
	69	+
	70	+ it('reports holding when hits stay near chance', () => {
	71	+ const critiques = [
	72	+ critique('p0', { p1: { family: 'google', confidence: 0.3 }, p2: { family: 'anthropic', confidence: 0.3 } }),
	73	+ critique('p1', { p0: { family: 'google', confidence: 0.3 }, p2: { family: 'openai', confidence: 0.3 } }),
	74	+ ];
	75	+ const report = buildMasqueradeReport(participants, critiques);
	76	+ expect(report.correct).toBe(0);
	77	+ expect(report.verdict).toBe('holding');
	78	+ });
	79	+
	80	+ it('is insufficient below the minimum sample', () => {
	81	+ const critiques = [critique('p0', { p1: { family: 'anthropic', confidence: 0.9 } })];
	82	+ const report = buildMasqueradeReport(participants, critiques);
	83	+ expect(report.guesses).toBe(1);
	84	+ expect(report.verdict).toBe('insufficient');
	85	+ });
	86	+
	87	+ it('ignores reviews without guesses and unknown participants', () => {
	88	+ const critiques = [critique('p0', { p1: null, ghost: { family: 'openai', confidence: 1 } })];
	89	+ const report = buildMasqueradeReport(participants, critiques);
	90	+ expect(report.guesses).toBe(0);
	91	+ expect(report.hitRate).toBeNull();
	92	+ expect(report.verdict).toBe('insufficient');
	93	+ });
	94	+});

added src/core/masquerade.ts +118 −0

		@@ -0,0 +1,118 @@
	1	+/**
	2	+ * Masquerade audit: does the anonymization actually hold?
	3	+ *
	4	+ * Roundtable's core trust claim is that reviewers cannot tell whose answer is
	5	+ * whose. During critique each reviewer optionally guesses the model family
	6	+ * behind every response; here those guesses are scored against the real
	7	+ * authorship. A hit rate near chance means the masks are working; a rate well
	8	+ * above chance means models are stylistically fingerprintable and the bias
	9	+ * mitigation is weaker than assumed. Pure and deterministic - zero LLM calls.
	10	+ */
	11	+import { providerFamily } from './models';
	12	+import type { CritiqueRecord, Participant } from './types';
	13	+
	14	+/** Hit rate this far above chance is reported as a leak. */
	15	+export const LEAK_MARGIN = 0.15;
	16	+/** Below this many guesses the sample says nothing. */
	17	+export const MIN_GUESSES = 4;
	18	+
	19	+export type MasqueradeVerdict = 'holding' \| 'leaking' \| 'insufficient';
	20	+
	21	+export interface MasqueradeReport {
	22	+ guesses: number;
	23	+ correct: number;
	24	+ /** Fraction of guesses that named the right family, null with no guesses. */
	25	+ hitRate: number \| null;
	26	+ /** Expected hit rate from uniform guessing over the families each reviewer saw. */
	27	+ chanceRate: number \| null;
	28	+ /** Mean confidence on correct vs incorrect guesses (calibration signal). */
	29	+ meanConfidenceCorrect: number \| null;
	30	+ meanConfidenceIncorrect: number \| null;
	31	+ verdict: MasqueradeVerdict;
	32	+}
	33	+
	34	+/** Normalize a free-text family guess to a provider prefix. */
	35	+const FAMILY_KEYWORDS: Array<[RegExp, string]> = [
	36	+ [/gpt\|openai\|o[1-9]/, 'openai'],
	37	+ [/claude\|anthropic/, 'anthropic'],
	38	+ [/gemini\|google\|palm/, 'google'],
	39	+ [/llama\|meta/, 'meta-llama'],
	40	+ [/mistral\|mixtral/, 'mistralai'],
	41	+ [/grok\|x-?ai/, 'x-ai'],
	42	+ [/deepseek/, 'deepseek'],
	43	+ [/qwen\|alibaba/, 'qwen'],
	44	+];
	45	+
	46	+export function normalizeFamilyGuess(guess: string): string {
	47	+ const g = guess.trim().toLowerCase();
	48	+ for (const [pattern, family] of FAMILY_KEYWORDS) {
	49	+ if (pattern.test(g)) return family;
	50	+ }
	51	+ return g;
	52	+}
	53	+
	54	+/** Family of a participant's slug, normalized through the same keyword map. */
	55	+function familyOf(model: string): string {
	56	+ return normalizeFamilyGuess(providerFamily(model));
	57	+}
	58	+
	59	+export function buildMasqueradeReport(
	60	+ participants: Participant[],
	61	+ critiques: CritiqueRecord[],
	62	+): MasqueradeReport {
	63	+ const byId = new Map(participants.map((p) => [p.id, p]));
	64	+
	65	+ let guesses = 0;
	66	+ let correct = 0;
	67	+ let chanceSum = 0;
	68	+ const confCorrect: number[] = [];
	69	+ const confIncorrect: number[] = [];
	70	+
	71	+ for (const critique of critiques) {
	72	+ const reviewer = byId.get(critique.reviewerParticipantId);
	73	+ // Families this reviewer could have picked among (its peers in this debate).
	74	+ const peerFamilies = new Set(
	75	+ participants.filter((p) => p.id !== critique.reviewerParticipantId).map((p) => familyOf(p.model)),
	76	+ );
	77	+ if (!reviewer \|\| peerFamilies.size === 0) continue;
	78	+
	79	+ for (const review of critique.reviews) {
	80	+ if (!review.authorGuess) continue;
	81	+ const target = byId.get(review.targetParticipantId);
	82	+ if (!target) continue;
	83	+ guesses++;
	84	+ chanceSum += 1 / peerFamilies.size;
	85	+ const hit = normalizeFamilyGuess(review.authorGuess.family) === familyOf(target.model);
	86	+ if (hit) {
	87	+ correct++;
	88	+ confCorrect.push(review.authorGuess.confidence);
	89	+ } else {
	90	+ confIncorrect.push(review.authorGuess.confidence);
	91	+ }
	92	+ }
	93	+ }
	94	+
	95	+ const hitRate = guesses === 0 ? null : correct / guesses;
	96	+ const chanceRate = guesses === 0 ? null : chanceSum / guesses;
	97	+ const verdict: MasqueradeVerdict =
	98	+ guesses < MIN_GUESSES \|\| hitRate === null \|\| chanceRate === null
	99	+ ? 'insufficient'
	100	+ : hitRate > chanceRate + LEAK_MARGIN
	101	+ ? 'leaking'
	102	+ : 'holding';
	103	+
	104	+ return {
	105	+ guesses,
	106	+ correct,
	107	+ hitRate,
	108	+ chanceRate,
	109	+ meanConfidenceCorrect: mean(confCorrect),
	110	+ meanConfidenceIncorrect: mean(confIncorrect),
	111	+ verdict,
	112	+ };
	113	+}
	114	+
	115	+function mean(nums: number[]): number \| null {
	116	+ if (nums.length === 0) return null;
	117	+ return nums.reduce((a, b) => a + b, 0) / nums.length;
	118	+}

modified src/core/mock-client.ts +6 −0

		@@ -150,13 +150,19 @@export class MockLlmClient implements LlmClient {
150	150	const score = malform ? 42 : 4 + s; // 4..9 normally; out-of-range to trigger repair
151	151	// Deterministic council-mean prediction near (but not equal to) the score.
152	152	const predictedPeerMean = Math.min(10, Math.max(1, score + (hashSeed(seed, label, 'pred') % 3) - 1));
	153	+ const families = ['openai', 'anthropic', 'google'];
	154	+ const authorGuess = {
	155	+ family: families[hashSeed(seed, label, 'guess') % families.length]!,
	156	+ confidence: 0.2 + (hashSeed(seed, label, 'conf') % 5) / 10,
	157	+ };
153	158	return {
154	159	label,
155	160	weaknesses: [`Response ${label} underspecifies the edge cases.`],
156	161	strengths: [`Response ${label} states its assumptions clearly.`],
157	162	score,
158	163	justification: `Solid reasoning with a gap around edge cases (score ${score}).`,
159	164	predictedPeerMean,
	165	+ authorGuess,
160	166	};
161	167	});
162	168	return JSON.stringify({ reviews });

modified src/core/orchestrator.ts +2 −0

		@@ -698,6 +698,7 @@export async function runDebate(
698	698	score: number;
699	699	justification: string;
700	700	predictedPeerMean?: number;
	701	+ authorGuess?: { family: string; confidence: number };
701	702	}>,
702	703	anon: AnonymizationResult,
703	704	usage: Usage,
		@@ -715,6 +716,7 @@export async function runDebate(
715	716	score: r.score,
716	717	justification: r.justification,
717	718	...(r.predictedPeerMean !== undefined ? { predictedPeerMean: r.predictedPeerMean } : {}),
	719	+ ...(r.authorGuess !== undefined ? { authorGuess: r.authorGuess } : {}),
718	720	} satisfies PeerReview;
719	721	})
720	722	.filter((x): x is PeerReview => x !== null);

modified src/core/prompts/critique.ts +9 −2

		@@ -10,7 +10,11 @@const CRITIQUE_SHAPE = `{
10	10	"strengths": ["specific strong point", "..."],
11	11	"score": 7, // integer 1-10, overall quality
12	12	"justification": "one or two sentences explaining the score",
13		- "predictedPeerMean": 6.5 // 1-10: the average score you expect the OTHER reviewers to give this response
	13	+ "predictedPeerMean": 6.5, // 1-10: the average score you expect the OTHER reviewers to give this response
	14	+ "authorGuess": { // optional: your honest guess at who wrote it
	15	+ "family": "anthropic", // one of: openai, anthropic, google, meta, mistral, x-ai, deepseek, other
	16	+ "confidence": 0.3 // 0-1
	17	+ }
14	18	}
15	19	// ...one entry per response shown to you
16	20	]
		@@ -45,7 +49,10 @@export function buildCritiquePrompt(question: string, peers: AnonymizedPeer[]):
45	49	'score the other reviewers will give it. Your prediction is checked against ' +
46	50	'their actual average, so report what you expect THEM to conclude - it may ' +
47	51	'legitimately differ from your own score when you see a flaw or strength you ' +
48		- 'suspect others will miss.\n\n' +
	52	+ 'suspect others will miss. Finally, give your honest guess at which model ' +
	53	+ 'family wrote each response, with a confidence between 0 and 1. This guess is ' +
	54	+ 'scored later to audit whether the anonymization is working; it has no effect ' +
	55	+ 'on the debate, so do not let it color your review.\n\n' +
49	56	jsonInstruction(CRITIQUE_SHAPE),
50	57	},
51	58	{

modified src/core/prompts/index.ts +1 −1

		@@ -10,7 +10,7 @@
10	10	* alter model behavior; it is stamped onto stored results so historical debates
11	11	* remain interpretable.
12	12	*/
13		-export const PROMPT_VERSION = '1.2.0';
	13	+export const PROMPT_VERSION = '1.3.0';
14	14
15	15	export { buildAnswerPrompt } from './answer';
16	16	export { buildCritiquePrompt } from './critique';

modified src/core/schemas.ts +7 −0

		@@ -22,6 +22,13 @@export const peerReviewSchema = z.object({
22	22	justification: z.string().default(''),
23	23	/** The reviewer's prediction of the other reviewers' average score (1-10). */
24	24	predictedPeerMean: z.coerce.number().min(1).max(10).optional(),
	25	+ /** The reviewer's guess at which model family wrote the answer (anonymity audit). */
	26	+ authorGuess: z
	27	+ .object({
	28	+ family: z.string().min(1).max(40),
	29	+ confidence: z.coerce.number().min(0).max(1),
	30	+ })
	31	+ .optional(),
25	32	});
26	33
27	34	export const critiqueOutputSchema = z.object({

modified src/core/types.ts +6 −0

		@@ -104,6 +104,12 @@export interface PeerReview {
104	104	* models that omit it degrade gracefully.
105	105	*/
106	106	predictedPeerMean?: number;
	107	+ /**
	108	+ * The reviewer's guess at which model family authored the answer. Scored
	109	+ * against the real mapping to audit whether anonymization actually holds
	110	+ * (see `masquerade.ts`). Optional and advisory.
	111	+ */
	112	+ authorGuess?: { family: string; confidence: number };
107	113	}
108	114
109	115	export interface CritiqueRecord {

modified src/lib/export-markdown.ts +15 −0

		@@ -2,6 +2,7 @@
2	2	* Render a `DebateResult` as a clean, self-contained Markdown deliberation
3	3	* report - the downloadable counterpart to the shareable web page.
4	4	*/
	5	+import { buildMasqueradeReport } from '@/core/masquerade';
5	6	import { buildScoreMatrix } from '@/core/scoring';
6	7	import { buildSpineProfiles } from '@/core/spine';
7	8	import type { DebateResult, Participant } from '@/core/types';
		@@ -117,6 +118,20 @@export function debateToMarkdown(result: DebateResult): string {
117	118	push();
118	119	}
119	120
	121	+ const masquerade = buildMasqueradeReport(
	122	+ result.participants,
	123	+ result.rounds.flatMap((r) => r.critiques),
	124	+ );
	125	+ if (masquerade.verdict !== 'insufficient' && masquerade.hitRate !== null && masquerade.chanceRate !== null) {
	126	+ push(`## Anonymity Check`);
	127	+ push(
	128	+ `Reviewers identified authors in ${Math.round(masquerade.hitRate * 100)}% of ${masquerade.guesses} guesses ` +
	129	+ `vs ${Math.round(masquerade.chanceRate * 100)}% chance - ` +
	130	+ (masquerade.verdict === 'holding' ? 'the masks held.' : 'writing style leaked authorship in this debate.'),
	131	+ );
	132	+ push();
	133	+ }
	134	+
120	135	push(`---`);
121	136	push(`_Generated by Roundtable._`);
122	137	return lines.join('\n');

rasmusjy / roundtable

add masquerade audit: score author guesses to verify anonymity holds