Commit
add spine analytics: per-model pressure response from existing records
commit
d34f9f0
5 changed files with +307 and −0
Jump to a changed file
- src/components/debate/debate-console.tsx +11 −0
- src/components/debate/spine-panel.tsx +88 −0
- src/core/spine.test.ts +110 −0
- src/core/spine.ts +81 −0
- src/lib/export-markdown.ts +17 −0
modified src/components/debate/debate-console.tsx +11 −0
| @@ -9,6 +9,7 @@import { Disagreements } from '@/components/debate/disagreements'; | ||
| 9 | 9 | import { FinalAnswer } from '@/components/debate/final-answer'; |
| 10 | 10 | import { ModelPanel } from '@/components/debate/model-panel'; |
| 11 | 11 | import { RevisionDiff } from '@/components/debate/revision-diff'; |
| 12 | +import { SpinePanel } from '@/components/debate/spine-panel'; | |
| 12 | 13 | import { StageTimeline } from '@/components/debate/stage-timeline'; |
| 13 | 14 | import { Badge } from '@/components/ui/badge'; |
| 14 | 15 | import { Progress } from '@/components/ui/progress'; |
| @@ -105,6 +106,16 @@export function DebateConsole({ view, showActions = true }: { view: DebateView; | ||
| 105 | 106 | /> |
| 106 | 107 | ))} |
| 107 | 108 | |
| 109 | + {/* Pressure response */} | |
| 110 | + {view.rounds.some((r) => r.revisions.length > 0) && ( | |
| 111 | + <section id="section-spine" className="space-y-3"> | |
| 112 | + <h2 className="text-sm font-semibold text-muted-foreground">Under pressure</h2> | |
| 113 | + <div className="rounded-xl border bg-card p-4"> | |
| 114 | + <SpinePanel participants={view.participants} rounds={view.rounds} /> | |
| 115 | + </div> | |
| 116 | + </section> | |
| 117 | + )} | |
| 118 | + | |
| 108 | 119 | {/* Cost breakdown */} |
| 109 | 120 | {finished && ( |
| 110 | 121 | <section id="section-cost" className="space-y-3"> |
added src/components/debate/spine-panel.tsx +88 −0
| @@ -0,0 +1,88 @@ | ||
| 1 | +'use client'; | |
| 2 | + | |
| 3 | +import { useMemo } from 'react'; | |
| 4 | +import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip'; | |
| 5 | +import { buildSpineProfiles, type SpineEntry, type SpineVerdict } from '@/core/spine'; | |
| 6 | +import type { Participant, RoundRecord } from '@/core/types'; | |
| 7 | +import { participantColor, participantTag } from '@/lib/model-visuals'; | |
| 8 | + | |
| 9 | +/** | |
| 10 | + * Per-model pressure response: for each round, did the model revise, defend, | |
| 11 | + * cave (change a well-scored answer) or stonewall (keep a poorly-scored one)? | |
| 12 | + * Computed client-side from records the debate already produced. | |
| 13 | + */ | |
| 14 | +export function SpinePanel({ participants, rounds }: { participants: Participant[]; rounds: RoundRecord[] }) { | |
| 15 | + const profiles = useMemo(() => buildSpineProfiles(participants, rounds), [participants, rounds]); | |
| 16 | + const withEntries = profiles.filter((p) => p.entries.length > 0); | |
| 17 | + if (withEntries.length === 0) return null; | |
| 18 | + | |
| 19 | + return ( | |
| 20 | + <div className="space-y-3"> | |
| 21 | + <p className="text-xs text-muted-foreground"> | |
| 22 | + How each model handled peer pressure. Caved: changed an answer the council scored highly. | |
| 23 | + Stonewalled: kept an answer the council scored poorly. | |
| 24 | + </p> | |
| 25 | + <div className="space-y-2"> | |
| 26 | + {withEntries.map((profile) => { | |
| 27 | + const idx = participants.findIndex((p) => p.id === profile.participantId); | |
| 28 | + const participant = participants[idx]; | |
| 29 | + return ( | |
| 30 | + <div key={profile.participantId} className="flex flex-wrap items-center gap-x-3 gap-y-1.5"> | |
| 31 | + <div className="flex w-44 min-w-0 items-center gap-1.5"> | |
| 32 | + <span | |
| 33 | + className="flex h-5 w-5 shrink-0 items-center justify-center rounded text-[10px] font-bold text-white" | |
| 34 | + style={{ backgroundColor: participantColor(idx < 0 ? 0 : idx) }} | |
| 35 | + > | |
| 36 | + {participantTag(idx < 0 ? 0 : idx)} | |
| 37 | + </span> | |
| 38 | + <span className="truncate text-xs font-medium">{participant?.displayName ?? profile.model}</span> | |
| 39 | + </div> | |
| 40 | + <div className="flex flex-wrap items-center gap-1.5"> | |
| 41 | + {profile.entries.map((entry) => ( | |
| 42 | + <VerdictChip key={entry.round} entry={entry} /> | |
| 43 | + ))} | |
| 44 | + </div> | |
| 45 | + <span className="text-[11px] text-muted-foreground">{summarize(profile.counts, profile.changeRate)}</span> | |
| 46 | + </div> | |
| 47 | + ); | |
| 48 | + })} | |
| 49 | + </div> | |
| 50 | + </div> | |
| 51 | + ); | |
| 52 | +} | |
| 53 | + | |
| 54 | +const VERDICT_STYLE: Record<SpineVerdict, string> = { | |
| 55 | + revised: 'bg-sky-500/10 text-sky-600 dark:text-sky-400 border-sky-500/30', | |
| 56 | + defended: 'bg-muted/60 text-muted-foreground border-border', | |
| 57 | + caved: 'bg-amber-500/10 text-amber-600 dark:text-amber-400 border-amber-500/30', | |
| 58 | + stonewalled: 'bg-violet-500/10 text-violet-600 dark:text-violet-400 border-violet-500/30', | |
| 59 | +}; | |
| 60 | + | |
| 61 | +function VerdictChip({ entry }: { entry: SpineEntry }) { | |
| 62 | + return ( | |
| 63 | + <Tooltip> | |
| 64 | + <TooltipTrigger asChild> | |
| 65 | + <span | |
| 66 | + className={`inline-flex cursor-help items-center gap-1 rounded-full border px-2 py-0.5 text-[11px] font-medium ${VERDICT_STYLE[entry.verdict]}`} | |
| 67 | + > | |
| 68 | + R{entry.round} {entry.verdict} | |
| 69 | + </span> | |
| 70 | + </TooltipTrigger> | |
| 71 | + <TooltipContent side="top" className="text-xs"> | |
| 72 | + Round {entry.round}: {entry.changed ? 'changed its answer' : 'kept its answer'} | |
| 73 | + {entry.meanIncomingScore === null | |
| 74 | + ? ', no peer scores received' | |
| 75 | + : `, peers scored it ${entry.meanIncomingScore.toFixed(1)}/10`} | |
| 76 | + . | |
| 77 | + </TooltipContent> | |
| 78 | + </Tooltip> | |
| 79 | + ); | |
| 80 | +} | |
| 81 | + | |
| 82 | +function summarize(counts: Record<SpineVerdict, number>, changeRate: number | null): string { | |
| 83 | + const flags: string[] = []; | |
| 84 | + if (counts.caved > 0) flags.push(`caved ${counts.caved}x`); | |
| 85 | + if (counts.stonewalled > 0) flags.push(`stonewalled ${counts.stonewalled}x`); | |
| 86 | + const rate = changeRate === null ? '' : `changed ${Math.round(changeRate * 100)}% of rounds`; | |
| 87 | + return [rate, ...flags].filter(Boolean).join(' ยท '); | |
| 88 | +} |
added src/core/spine.test.ts +110 −0
| @@ -0,0 +1,110 @@ | ||
| 1 | +import { describe, expect, it } from 'vitest'; | |
| 2 | +import { buildSpineProfiles } from './spine'; | |
| 3 | +import { | |
| 4 | + emptyUsage, | |
| 5 | + type CritiqueRecord, | |
| 6 | + type Participant, | |
| 7 | + type RevisionRecord, | |
| 8 | + type RoundRecord, | |
| 9 | +} from './types'; | |
| 10 | + | |
| 11 | +const participants: Participant[] = [ | |
| 12 | + { id: 'p0', model: 'openai/gpt-4o', displayName: 'GPT 4o' }, | |
| 13 | + { id: 'p1', model: 'anthropic/claude-3.5', displayName: 'Claude 3.5' }, | |
| 14 | +]; | |
| 15 | + | |
| 16 | +function critique(round: number, reviewer: string, scores: Record<string, number>): CritiqueRecord { | |
| 17 | + return { | |
| 18 | + round, | |
| 19 | + reviewerParticipantId: reviewer, | |
| 20 | + reviewerModel: 'x/y', | |
| 21 | + reviews: Object.entries(scores).map(([targetParticipantId, score]) => ({ | |
| 22 | + label: 'A', | |
| 23 | + targetParticipantId, | |
| 24 | + weaknesses: [], | |
| 25 | + strengths: [], | |
| 26 | + score, | |
| 27 | + justification: '', | |
| 28 | + })), | |
| 29 | + usage: emptyUsage(), | |
| 30 | + latencyMs: 0, | |
| 31 | + }; | |
| 32 | +} | |
| 33 | + | |
| 34 | +function revision(round: number, participantId: string, changed: boolean): RevisionRecord { | |
| 35 | + return { | |
| 36 | + round, | |
| 37 | + participantId, | |
| 38 | + model: 'x/y', | |
| 39 | + content: 'answer', | |
| 40 | + changelog: { changed, summary: '', bullets: [] }, | |
| 41 | + usage: emptyUsage(), | |
| 42 | + latencyMs: 0, | |
| 43 | + }; | |
| 44 | +} | |
| 45 | + | |
| 46 | +function round(n: number, critiques: CritiqueRecord[], revisions: RevisionRecord[]): RoundRecord { | |
| 47 | + return { round: n, critiques, revisions, convergence: null }; | |
| 48 | +} | |
| 49 | + | |
| 50 | +describe('buildSpineProfiles', () => { | |
| 51 | + it('classifies a change under harsh criticism as revised', () => { | |
| 52 | + const rounds = [round(1, [critique(1, 'p1', { p0: 3 })], [revision(1, 'p0', true)])]; | |
| 53 | + const [p0] = buildSpineProfiles(participants, rounds); | |
| 54 | + expect(p0!.entries).toEqual([{ round: 1, meanIncomingScore: 3, changed: true, verdict: 'revised' }]); | |
| 55 | + }); | |
| 56 | + | |
| 57 | + it('classifies keeping the answer under mild criticism as defended', () => { | |
| 58 | + const rounds = [round(1, [critique(1, 'p1', { p0: 6 })], [revision(1, 'p0', false)])]; | |
| 59 | + const [p0] = buildSpineProfiles(participants, rounds); | |
| 60 | + expect(p0!.entries[0]!.verdict).toBe('defended'); | |
| 61 | + }); | |
| 62 | + | |
| 63 | + it('flags changing a highly-scored answer as caved', () => { | |
| 64 | + const rounds = [round(1, [critique(1, 'p1', { p0: 9 })], [revision(1, 'p0', true)])]; | |
| 65 | + const [p0] = buildSpineProfiles(participants, rounds); | |
| 66 | + expect(p0!.entries[0]!.verdict).toBe('caved'); | |
| 67 | + }); | |
| 68 | + | |
| 69 | + it('flags keeping a poorly-scored answer as stonewalled', () => { | |
| 70 | + const rounds = [round(1, [critique(1, 'p1', { p0: 2 })], [revision(1, 'p0', false)])]; | |
| 71 | + const [p0] = buildSpineProfiles(participants, rounds); | |
| 72 | + expect(p0!.entries[0]!.verdict).toBe('stonewalled'); | |
| 73 | + }); | |
| 74 | + | |
| 75 | + it('averages incoming scores from multiple reviewers', () => { | |
| 76 | + const three: Participant[] = [...participants, { id: 'p2', model: 'g/g', displayName: 'G' }]; | |
| 77 | + const rounds = [ | |
| 78 | + round(1, [critique(1, 'p1', { p0: 4 }), critique(1, 'p2', { p0: 8 })], [revision(1, 'p0', false)]), | |
| 79 | + ]; | |
| 80 | + const [p0] = buildSpineProfiles(three, rounds); | |
| 81 | + expect(p0!.entries[0]!.meanIncomingScore).toBe(6); | |
| 82 | + expect(p0!.entries[0]!.verdict).toBe('defended'); | |
| 83 | + }); | |
| 84 | + | |
| 85 | + it('falls back to revised/defended when nobody scored the participant', () => { | |
| 86 | + const rounds = [round(1, [], [revision(1, 'p0', true), revision(1, 'p1', false)])]; | |
| 87 | + const [p0, p1] = buildSpineProfiles(participants, rounds); | |
| 88 | + expect(p0!.entries[0]).toMatchObject({ meanIncomingScore: null, verdict: 'revised' }); | |
| 89 | + expect(p1!.entries[0]).toMatchObject({ meanIncomingScore: null, verdict: 'defended' }); | |
| 90 | + }); | |
| 91 | + | |
| 92 | + it('skips rounds where the participant produced no revision', () => { | |
| 93 | + const rounds = [round(1, [critique(1, 'p1', { p0: 3 })], [revision(1, 'p1', false)])]; | |
| 94 | + const [p0, p1] = buildSpineProfiles(participants, rounds); | |
| 95 | + expect(p0!.entries).toHaveLength(0); | |
| 96 | + expect(p0!.changeRate).toBeNull(); | |
| 97 | + expect(p1!.entries).toHaveLength(1); | |
| 98 | + }); | |
| 99 | + | |
| 100 | + it('aggregates counts and change rate across rounds', () => { | |
| 101 | + const rounds = [ | |
| 102 | + round(1, [critique(1, 'p1', { p0: 3 })], [revision(1, 'p0', true)]), | |
| 103 | + round(2, [critique(2, 'p1', { p0: 9 })], [revision(2, 'p0', true)]), | |
| 104 | + round(3, [critique(3, 'p1', { p0: 6 })], [revision(3, 'p0', false)]), | |
| 105 | + ]; | |
| 106 | + const [p0] = buildSpineProfiles(participants, rounds); | |
| 107 | + expect(p0!.counts).toEqual({ revised: 1, defended: 1, caved: 1, stonewalled: 0 }); | |
| 108 | + expect(p0!.changeRate).toBeCloseTo(2 / 3); | |
| 109 | + }); | |
| 110 | +}); |
added src/core/spine.ts +81 −0
| @@ -0,0 +1,81 @@ | ||
| 1 | +/** | |
| 2 | + * Spine analytics: how each council member responds to peer pressure. | |
| 3 | + * | |
| 4 | + * Every round already records the critique scores a participant received and | |
| 5 | + * whether its revision changed (`RevisionChangelog.changed`). Crossing the two | |
| 6 | + * classifies each decision: | |
| 7 | + * | |
| 8 | + * revised changed under genuine criticism - the intended behavior | |
| 9 | + * defended kept the answer absent strong criticism - also fine | |
| 10 | + * caved changed an answer the council scored highly (sycophancy) | |
| 11 | + * stonewalled kept an answer the council scored poorly (stubbornness) | |
| 12 | + * | |
| 13 | + * Pure and deterministic - computed from persisted records, zero LLM calls. | |
| 14 | + */ | |
| 15 | +import type { Participant, RoundRecord } from './types'; | |
| 16 | + | |
| 17 | +/** Mean incoming score at or below this counts as harsh criticism. */ | |
| 18 | +export const PRESSURE_MAX = 5; | |
| 19 | +/** Mean incoming score at or above this counts as clear approval. */ | |
| 20 | +export const PRAISE_MIN = 7.5; | |
| 21 | + | |
| 22 | +export type SpineVerdict = 'revised' | 'defended' | 'caved' | 'stonewalled'; | |
| 23 | + | |
| 24 | +export interface SpineEntry { | |
| 25 | + round: number; | |
| 26 | + /** Mean critique score received this round, null if nobody scored them. */ | |
| 27 | + meanIncomingScore: number | null; | |
| 28 | + changed: boolean; | |
| 29 | + verdict: SpineVerdict; | |
| 30 | +} | |
| 31 | + | |
| 32 | +export interface SpineProfile { | |
| 33 | + participantId: string; | |
| 34 | + model: string; | |
| 35 | + entries: SpineEntry[]; | |
| 36 | + counts: Record<SpineVerdict, number>; | |
| 37 | + /** Fraction of revision decisions where the answer changed, null if none. */ | |
| 38 | + changeRate: number | null; | |
| 39 | +} | |
| 40 | + | |
| 41 | +function classify(changed: boolean, meanIncomingScore: number | null): SpineVerdict { | |
| 42 | + if (meanIncomingScore !== null) { | |
| 43 | + if (changed && meanIncomingScore >= PRAISE_MIN) return 'caved'; | |
| 44 | + if (!changed && meanIncomingScore <= PRESSURE_MAX) return 'stonewalled'; | |
| 45 | + } | |
| 46 | + return changed ? 'revised' : 'defended'; | |
| 47 | +} | |
| 48 | + | |
| 49 | +function mean(nums: number[]): number | null { | |
| 50 | + if (nums.length === 0) return null; | |
| 51 | + return nums.reduce((a, b) => a + b, 0) / nums.length; | |
| 52 | +} | |
| 53 | + | |
| 54 | +export function buildSpineProfiles(participants: Participant[], rounds: RoundRecord[]): SpineProfile[] { | |
| 55 | + return participants.map((p) => { | |
| 56 | + const entries: SpineEntry[] = []; | |
| 57 | + for (const round of rounds) { | |
| 58 | + const revision = round.revisions.find((r) => r.participantId === p.id); | |
| 59 | + if (!revision) continue; // dropped out or revision failed - no decision made | |
| 60 | + const incoming = round.critiques | |
| 61 | + .flatMap((c) => c.reviews) | |
| 62 | + .filter((r) => r.targetParticipantId === p.id) | |
| 63 | + .map((r) => r.score); | |
| 64 | + const meanIncomingScore = mean(incoming); | |
| 65 | + const changed = revision.changelog.changed; | |
| 66 | + entries.push({ round: round.round, meanIncomingScore, changed, verdict: classify(changed, meanIncomingScore) }); | |
| 67 | + } | |
| 68 | + | |
| 69 | + const counts: Record<SpineVerdict, number> = { revised: 0, defended: 0, caved: 0, stonewalled: 0 }; | |
| 70 | + for (const e of entries) counts[e.verdict]++; | |
| 71 | + const changedCount = entries.filter((e) => e.changed).length; | |
| 72 | + | |
| 73 | + return { | |
| 74 | + participantId: p.id, | |
| 75 | + model: p.model, | |
| 76 | + entries, | |
| 77 | + counts, | |
| 78 | + changeRate: entries.length === 0 ? null : changedCount / entries.length, | |
| 79 | + }; | |
| 80 | + }); | |
| 81 | +} |
modified src/lib/export-markdown.ts +17 −0
| @@ -3,6 +3,7 @@ | ||
| 3 | 3 | * report - the downloadable counterpart to the shareable web page. |
| 4 | 4 | */ |
| 5 | 5 | import { buildScoreMatrix } from '@/core/scoring'; |
| 6 | +import { buildSpineProfiles } from '@/core/spine'; | |
| 6 | 7 | import type { DebateResult, Participant } from '@/core/types'; |
| 7 | 8 | |
| 8 | 9 | export function debateToMarkdown(result: DebateResult): string { |
| @@ -85,6 +86,22 @@export function debateToMarkdown(result: DebateResult): string { | ||
| 85 | 86 | } |
| 86 | 87 | } |
| 87 | 88 | |
| 89 | + const spine = buildSpineProfiles(result.participants, result.rounds).filter((p) => p.entries.length > 0); | |
| 90 | + if (spine.length) { | |
| 91 | + push(`## Pressure Response`); | |
| 92 | + push(`_How each model handled peer critique: caved = changed a well-scored answer, stonewalled = kept a poorly-scored one._`); | |
| 93 | + push(); | |
| 94 | + push(`| Model | Round | Peers scored it | Response |`); | |
| 95 | + push(`| --- | --- | --- | --- |`); | |
| 96 | + for (const profile of spine) { | |
| 97 | + for (const entry of profile.entries) { | |
| 98 | + const score = entry.meanIncomingScore === null ? '-' : `${entry.meanIncomingScore.toFixed(1)}/10`; | |
| 99 | + push(`| ${nameOf(profile.participantId)} | ${entry.round} | ${score} | ${entry.verdict} |`); | |
| 100 | + } | |
| 101 | + } | |
| 102 | + push(); | |
| 103 | + } | |
| 104 | + | |
| 88 | 105 | push(`---`); |
| 89 | 106 | push(`_Generated by Roundtable._`); |
| 90 | 107 | return lines.join('\n'); |