masquerade.test.ts
3,658 bytes
| 1 | import { describe, expect, it } from 'vitest'; |
|---|---|
| 2 | import { buildMasqueradeReport, normalizeFamilyGuess } from './masquerade'; |
| 3 | import { emptyUsage, type CritiqueRecord, type Participant } from './types'; |
| 4 | |
| 5 | const participants: Participant[] = [ |
| 6 | { id: 'p0', model: 'openai/gpt-4o', displayName: 'GPT 4o' }, |
| 7 | { id: 'p1', model: 'anthropic/claude-3.5-sonnet', displayName: 'Claude 3.5' }, |
| 8 | { id: 'p2', model: 'google/gemini-pro-1.5', displayName: 'Gemini Pro' }, |
| 9 | ]; |
| 10 | |
| 11 | function critique( |
| 12 | reviewer: string, |
| 13 | guesses: Record<string, { family: string; confidence: number } | null>, |
| 14 | ): CritiqueRecord { |
| 15 | return { |
| 16 | round: 1, |
| 17 | reviewerParticipantId: reviewer, |
| 18 | reviewerModel: 'x/y', |
| 19 | reviews: Object.entries(guesses).map(([targetParticipantId, guess]) => ({ |
| 20 | label: 'A', |
| 21 | targetParticipantId, |
| 22 | weaknesses: [], |
| 23 | strengths: [], |
| 24 | score: 5, |
| 25 | justification: '', |
| 26 | ...(guess ? { authorGuess: guess } : {}), |
| 27 | })), |
| 28 | usage: emptyUsage(), |
| 29 | latencyMs: 0, |
| 30 | }; |
| 31 | } |
| 32 | |
| 33 | describe('normalizeFamilyGuess', () => { |
| 34 | it('maps informal names to provider prefixes', () => { |
| 35 | expect(normalizeFamilyGuess('ChatGPT / GPT-4 style')).toBe('openai'); |
| 36 | expect(normalizeFamilyGuess('Claude')).toBe('anthropic'); |
| 37 | expect(normalizeFamilyGuess('gemini')).toBe('google'); |
| 38 | expect(normalizeFamilyGuess('Grok (xAI)')).toBe('x-ai'); |
| 39 | expect(normalizeFamilyGuess('llama-3')).toBe('meta-llama'); |
| 40 | }); |
| 41 | |
| 42 | it('passes through unknown families lowercased', () => { |
| 43 | expect(normalizeFamilyGuess('Cohere')).toBe('cohere'); |
| 44 | }); |
| 45 | }); |
| 46 | |
| 47 | describe('buildMasqueradeReport', () => { |
| 48 | it('scores guesses against real authorship', () => { |
| 49 | const critiques = [ |
| 50 | critique('p0', { |
| 51 | p1: { family: 'anthropic', confidence: 0.8 }, // correct |
| 52 | p2: { family: 'openai', confidence: 0.4 }, // wrong |
| 53 | }), |
| 54 | critique('p1', { |
| 55 | p0: { family: 'gpt', confidence: 0.6 }, // correct via keyword mapping |
| 56 | p2: { family: 'gemini', confidence: 0.5 }, // correct |
| 57 | }), |
| 58 | ]; |
| 59 | const report = buildMasqueradeReport(participants, critiques); |
| 60 | expect(report.guesses).toBe(4); |
| 61 | expect(report.correct).toBe(3); |
| 62 | expect(report.hitRate).toBeCloseTo(0.75); |
| 63 | // Each reviewer saw 2 distinct peer families -> chance 0.5 per guess. |
| 64 | expect(report.chanceRate).toBeCloseTo(0.5); |
| 65 | expect(report.verdict).toBe('leaking'); |
| 66 | expect(report.meanConfidenceCorrect).toBeCloseTo((0.8 + 0.6 + 0.5) / 3); |
| 67 | expect(report.meanConfidenceIncorrect).toBeCloseTo(0.4); |
| 68 | }); |
| 69 | |
| 70 | it('reports holding when hits stay near chance', () => { |
| 71 | const critiques = [ |
| 72 | critique('p0', { p1: { family: 'google', confidence: 0.3 }, p2: { family: 'anthropic', confidence: 0.3 } }), |
| 73 | critique('p1', { p0: { family: 'google', confidence: 0.3 }, p2: { family: 'openai', confidence: 0.3 } }), |
| 74 | ]; |
| 75 | const report = buildMasqueradeReport(participants, critiques); |
| 76 | expect(report.correct).toBe(0); |
| 77 | expect(report.verdict).toBe('holding'); |
| 78 | }); |
| 79 | |
| 80 | it('is insufficient below the minimum sample', () => { |
| 81 | const critiques = [critique('p0', { p1: { family: 'anthropic', confidence: 0.9 } })]; |
| 82 | const report = buildMasqueradeReport(participants, critiques); |
| 83 | expect(report.guesses).toBe(1); |
| 84 | expect(report.verdict).toBe('insufficient'); |
| 85 | }); |
| 86 | |
| 87 | it('ignores reviews without guesses and unknown participants', () => { |
| 88 | const critiques = [critique('p0', { p1: null, ghost: { family: 'openai', confidence: 1 } })]; |
| 89 | const report = buildMasqueradeReport(participants, critiques); |
| 90 | expect(report.guesses).toBe(0); |
| 91 | expect(report.hitRate).toBeNull(); |
| 92 | expect(report.verdict).toBe('insufficient'); |
| 93 | }); |
| 94 | }); |
| 95 | |