spine.test.ts
4,269 bytes
| 1 | import { describe, expect, it } from 'vitest'; |
|---|---|
| 2 | import { buildSpineProfiles } from './spine'; |
| 3 | import { |
| 4 | emptyUsage, |
| 5 | type CritiqueRecord, |
| 6 | type Participant, |
| 7 | type RevisionRecord, |
| 8 | type RoundRecord, |
| 9 | } from './types'; |
| 10 | |
| 11 | const participants: Participant[] = [ |
| 12 | { id: 'p0', model: 'openai/gpt-4o', displayName: 'GPT 4o' }, |
| 13 | { id: 'p1', model: 'anthropic/claude-3.5', displayName: 'Claude 3.5' }, |
| 14 | ]; |
| 15 | |
| 16 | function critique(round: number, reviewer: string, scores: Record<string, number>): CritiqueRecord { |
| 17 | return { |
| 18 | round, |
| 19 | reviewerParticipantId: reviewer, |
| 20 | reviewerModel: 'x/y', |
| 21 | reviews: Object.entries(scores).map(([targetParticipantId, score]) => ({ |
| 22 | label: 'A', |
| 23 | targetParticipantId, |
| 24 | weaknesses: [], |
| 25 | strengths: [], |
| 26 | score, |
| 27 | justification: '', |
| 28 | })), |
| 29 | usage: emptyUsage(), |
| 30 | latencyMs: 0, |
| 31 | }; |
| 32 | } |
| 33 | |
| 34 | function revision(round: number, participantId: string, changed: boolean): RevisionRecord { |
| 35 | return { |
| 36 | round, |
| 37 | participantId, |
| 38 | model: 'x/y', |
| 39 | content: 'answer', |
| 40 | changelog: { changed, summary: '', bullets: [] }, |
| 41 | usage: emptyUsage(), |
| 42 | latencyMs: 0, |
| 43 | }; |
| 44 | } |
| 45 | |
| 46 | function round(n: number, critiques: CritiqueRecord[], revisions: RevisionRecord[]): RoundRecord { |
| 47 | return { round: n, critiques, revisions, convergence: null }; |
| 48 | } |
| 49 | |
| 50 | describe('buildSpineProfiles', () => { |
| 51 | it('classifies a change under harsh criticism as revised', () => { |
| 52 | const rounds = [round(1, [critique(1, 'p1', { p0: 3 })], [revision(1, 'p0', true)])]; |
| 53 | const [p0] = buildSpineProfiles(participants, rounds); |
| 54 | expect(p0!.entries).toEqual([{ round: 1, meanIncomingScore: 3, changed: true, verdict: 'revised' }]); |
| 55 | }); |
| 56 | |
| 57 | it('classifies keeping the answer under mild criticism as defended', () => { |
| 58 | const rounds = [round(1, [critique(1, 'p1', { p0: 6 })], [revision(1, 'p0', false)])]; |
| 59 | const [p0] = buildSpineProfiles(participants, rounds); |
| 60 | expect(p0!.entries[0]!.verdict).toBe('defended'); |
| 61 | }); |
| 62 | |
| 63 | it('flags changing a highly-scored answer as caved', () => { |
| 64 | const rounds = [round(1, [critique(1, 'p1', { p0: 9 })], [revision(1, 'p0', true)])]; |
| 65 | const [p0] = buildSpineProfiles(participants, rounds); |
| 66 | expect(p0!.entries[0]!.verdict).toBe('caved'); |
| 67 | }); |
| 68 | |
| 69 | it('flags keeping a poorly-scored answer as stonewalled', () => { |
| 70 | const rounds = [round(1, [critique(1, 'p1', { p0: 2 })], [revision(1, 'p0', false)])]; |
| 71 | const [p0] = buildSpineProfiles(participants, rounds); |
| 72 | expect(p0!.entries[0]!.verdict).toBe('stonewalled'); |
| 73 | }); |
| 74 | |
| 75 | it('averages incoming scores from multiple reviewers', () => { |
| 76 | const three: Participant[] = [...participants, { id: 'p2', model: 'g/g', displayName: 'G' }]; |
| 77 | const rounds = [ |
| 78 | round(1, [critique(1, 'p1', { p0: 4 }), critique(1, 'p2', { p0: 8 })], [revision(1, 'p0', false)]), |
| 79 | ]; |
| 80 | const [p0] = buildSpineProfiles(three, rounds); |
| 81 | expect(p0!.entries[0]!.meanIncomingScore).toBe(6); |
| 82 | expect(p0!.entries[0]!.verdict).toBe('defended'); |
| 83 | }); |
| 84 | |
| 85 | it('falls back to revised/defended when nobody scored the participant', () => { |
| 86 | const rounds = [round(1, [], [revision(1, 'p0', true), revision(1, 'p1', false)])]; |
| 87 | const [p0, p1] = buildSpineProfiles(participants, rounds); |
| 88 | expect(p0!.entries[0]).toMatchObject({ meanIncomingScore: null, verdict: 'revised' }); |
| 89 | expect(p1!.entries[0]).toMatchObject({ meanIncomingScore: null, verdict: 'defended' }); |
| 90 | }); |
| 91 | |
| 92 | it('skips rounds where the participant produced no revision', () => { |
| 93 | const rounds = [round(1, [critique(1, 'p1', { p0: 3 })], [revision(1, 'p1', false)])]; |
| 94 | const [p0, p1] = buildSpineProfiles(participants, rounds); |
| 95 | expect(p0!.entries).toHaveLength(0); |
| 96 | expect(p0!.changeRate).toBeNull(); |
| 97 | expect(p1!.entries).toHaveLength(1); |
| 98 | }); |
| 99 | |
| 100 | it('aggregates counts and change rate across rounds', () => { |
| 101 | const rounds = [ |
| 102 | round(1, [critique(1, 'p1', { p0: 3 })], [revision(1, 'p0', true)]), |
| 103 | round(2, [critique(2, 'p1', { p0: 9 })], [revision(2, 'p0', true)]), |
| 104 | round(3, [critique(3, 'p1', { p0: 6 })], [revision(3, 'p0', false)]), |
| 105 | ]; |
| 106 | const [p0] = buildSpineProfiles(participants, rounds); |
| 107 | expect(p0!.counts).toEqual({ revised: 1, defended: 1, caved: 1, stonewalled: 0 }); |
| 108 | expect(p0!.changeRate).toBeCloseTo(2 / 3); |
| 109 | }); |
| 110 | }); |
| 111 | |