masquerade.ts
4,032 bytes
| 1 | /** |
|---|---|
| 2 | * Masquerade audit: does the anonymization actually hold? |
| 3 | * |
| 4 | * Roundtable's core trust claim is that reviewers cannot tell whose answer is |
| 5 | * whose. During critique each reviewer optionally guesses the model family |
| 6 | * behind every response; here those guesses are scored against the real |
| 7 | * authorship. A hit rate near chance means the masks are working; a rate well |
| 8 | * above chance means models are stylistically fingerprintable and the bias |
| 9 | * mitigation is weaker than assumed. Pure and deterministic - zero LLM calls. |
| 10 | */ |
| 11 | import { providerFamily } from './models'; |
| 12 | import type { CritiqueRecord, Participant } from './types'; |
| 13 | |
| 14 | /** Hit rate this far above chance is reported as a leak. */ |
| 15 | export const LEAK_MARGIN = 0.15; |
| 16 | /** Below this many guesses the sample says nothing. */ |
| 17 | export const MIN_GUESSES = 4; |
| 18 | |
| 19 | export type MasqueradeVerdict = 'holding' | 'leaking' | 'insufficient'; |
| 20 | |
| 21 | export interface MasqueradeReport { |
| 22 | guesses: number; |
| 23 | correct: number; |
| 24 | /** Fraction of guesses that named the right family, null with no guesses. */ |
| 25 | hitRate: number | null; |
| 26 | /** Expected hit rate from uniform guessing over the families each reviewer saw. */ |
| 27 | chanceRate: number | null; |
| 28 | /** Mean confidence on correct vs incorrect guesses (calibration signal). */ |
| 29 | meanConfidenceCorrect: number | null; |
| 30 | meanConfidenceIncorrect: number | null; |
| 31 | verdict: MasqueradeVerdict; |
| 32 | } |
| 33 | |
| 34 | /** Normalize a free-text family guess to a provider prefix. */ |
| 35 | const FAMILY_KEYWORDS: Array<[RegExp, string]> = [ |
| 36 | [/gpt|openai|o[1-9]/, 'openai'], |
| 37 | [/claude|anthropic/, 'anthropic'], |
| 38 | [/gemini|google|palm/, 'google'], |
| 39 | [/llama|meta/, 'meta-llama'], |
| 40 | [/mistral|mixtral/, 'mistralai'], |
| 41 | [/grok|x-?ai/, 'x-ai'], |
| 42 | [/deepseek/, 'deepseek'], |
| 43 | [/qwen|alibaba/, 'qwen'], |
| 44 | ]; |
| 45 | |
| 46 | export function normalizeFamilyGuess(guess: string): string { |
| 47 | const g = guess.trim().toLowerCase(); |
| 48 | for (const [pattern, family] of FAMILY_KEYWORDS) { |
| 49 | if (pattern.test(g)) return family; |
| 50 | } |
| 51 | return g; |
| 52 | } |
| 53 | |
| 54 | /** Family of a participant's slug, normalized through the same keyword map. */ |
| 55 | function familyOf(model: string): string { |
| 56 | return normalizeFamilyGuess(providerFamily(model)); |
| 57 | } |
| 58 | |
| 59 | export function buildMasqueradeReport( |
| 60 | participants: Participant[], |
| 61 | critiques: CritiqueRecord[], |
| 62 | ): MasqueradeReport { |
| 63 | const byId = new Map(participants.map((p) => [p.id, p])); |
| 64 | |
| 65 | let guesses = 0; |
| 66 | let correct = 0; |
| 67 | let chanceSum = 0; |
| 68 | const confCorrect: number[] = []; |
| 69 | const confIncorrect: number[] = []; |
| 70 | |
| 71 | for (const critique of critiques) { |
| 72 | const reviewer = byId.get(critique.reviewerParticipantId); |
| 73 | // Families this reviewer could have picked among (its peers in this debate). |
| 74 | const peerFamilies = new Set( |
| 75 | participants.filter((p) => p.id !== critique.reviewerParticipantId).map((p) => familyOf(p.model)), |
| 76 | ); |
| 77 | if (!reviewer || peerFamilies.size === 0) continue; |
| 78 | |
| 79 | for (const review of critique.reviews) { |
| 80 | if (!review.authorGuess) continue; |
| 81 | const target = byId.get(review.targetParticipantId); |
| 82 | if (!target) continue; |
| 83 | guesses++; |
| 84 | chanceSum += 1 / peerFamilies.size; |
| 85 | const hit = normalizeFamilyGuess(review.authorGuess.family) === familyOf(target.model); |
| 86 | if (hit) { |
| 87 | correct++; |
| 88 | confCorrect.push(review.authorGuess.confidence); |
| 89 | } else { |
| 90 | confIncorrect.push(review.authorGuess.confidence); |
| 91 | } |
| 92 | } |
| 93 | } |
| 94 | |
| 95 | const hitRate = guesses === 0 ? null : correct / guesses; |
| 96 | const chanceRate = guesses === 0 ? null : chanceSum / guesses; |
| 97 | const verdict: MasqueradeVerdict = |
| 98 | guesses < MIN_GUESSES || hitRate === null || chanceRate === null |
| 99 | ? 'insufficient' |
| 100 | : hitRate > chanceRate + LEAK_MARGIN |
| 101 | ? 'leaking' |
| 102 | : 'holding'; |
| 103 | |
| 104 | return { |
| 105 | guesses, |
| 106 | correct, |
| 107 | hitRate, |
| 108 | chanceRate, |
| 109 | meanConfidenceCorrect: mean(confCorrect), |
| 110 | meanConfidenceIncorrect: mean(confIncorrect), |
| 111 | verdict, |
| 112 | }; |
| 113 | } |
| 114 | |
| 115 | function mean(nums: number[]): number | null { |
| 116 | if (nums.length === 0) return null; |
| 117 | return nums.reduce((a, b) => a + b, 0) / nums.length; |
| 118 | } |
| 119 | |