scoring.ts
5,904 bytes
| 1 | /** |
|---|---|
| 2 | * Aggregate critique scores into the N×N matrix the UI renders, plus |
| 3 | * peer-prediction analytics. |
| 4 | * |
| 5 | * Rows are reviewers, columns are the answers being scored. A cell is null when |
| 6 | * that reviewer did not score that target (e.g. the reviewer dropped out, or the |
| 7 | * self-cell). Pure and deterministic - unit-tested against hand-built records. |
| 8 | * |
| 9 | * Peer prediction: alongside its score, a reviewer may predict the average the |
| 10 | * OTHER reviewers will give the same answer. Comparing prediction to reality |
| 11 | * splits disagreement into two signals: an informed contrarian scores low while |
| 12 | * correctly predicting the council will score high (their dissent is deliberate |
| 13 | * and worth attention); a miscalibrated reviewer scores low believing everyone |
| 14 | * else will too. |
| 15 | */ |
| 16 | import type { CritiqueRecord, Participant } from './types'; |
| 17 | |
| 18 | export interface ScoreMatrix { |
| 19 | /** participantIds in a stable order (matches `participants`). */ |
| 20 | order: string[]; |
| 21 | /** cells[reviewerId][targetId] = score or null. */ |
| 22 | cells: Record<string, Record<string, number | null>>; |
| 23 | /** Mean score each participant received (over non-null cells). */ |
| 24 | averagesReceived: Record<string, number | null>; |
| 25 | /** Mean score each participant handed out. */ |
| 26 | averagesGiven: Record<string, number | null>; |
| 27 | } |
| 28 | |
| 29 | function mean(nums: number[]): number | null { |
| 30 | if (nums.length === 0) return null; |
| 31 | return nums.reduce((a, b) => a + b, 0) / nums.length; |
| 32 | } |
| 33 | |
| 34 | export function buildScoreMatrix( |
| 35 | participants: Participant[], |
| 36 | critiques: CritiqueRecord[], |
| 37 | ): ScoreMatrix { |
| 38 | const order = participants.map((p) => p.id); |
| 39 | const cells: Record<string, Record<string, number | null>> = {}; |
| 40 | for (const reviewer of order) { |
| 41 | cells[reviewer] = {}; |
| 42 | for (const target of order) cells[reviewer]![target] = null; |
| 43 | } |
| 44 | |
| 45 | for (const critique of critiques) { |
| 46 | const row = cells[critique.reviewerParticipantId]; |
| 47 | if (!row) continue; // reviewer not in participant set - ignore defensively |
| 48 | for (const review of critique.reviews) { |
| 49 | if (review.targetParticipantId in row) { |
| 50 | row[review.targetParticipantId] = review.score; |
| 51 | } |
| 52 | } |
| 53 | } |
| 54 | |
| 55 | const averagesReceived: Record<string, number | null> = {}; |
| 56 | const averagesGiven: Record<string, number | null> = {}; |
| 57 | for (const target of order) { |
| 58 | const received: number[] = []; |
| 59 | const given: number[] = []; |
| 60 | for (const reviewer of order) { |
| 61 | const toTarget = cells[reviewer]![target]; |
| 62 | if (typeof toTarget === 'number') received.push(toTarget); |
| 63 | const fromTarget = cells[target]![reviewer]; |
| 64 | if (typeof fromTarget === 'number') given.push(fromTarget); |
| 65 | } |
| 66 | averagesReceived[target] = mean(received); |
| 67 | averagesGiven[target] = mean(given); |
| 68 | } |
| 69 | |
| 70 | return { order, cells, averagesReceived, averagesGiven }; |
| 71 | } |
| 72 | |
| 73 | // --------------------------------------------------------------------------- |
| 74 | // Peer-prediction analytics |
| 75 | // --------------------------------------------------------------------------- |
| 76 | |
| 77 | /** Own score must differ from the peer mean by at least this to count as disagreement. */ |
| 78 | export const CONTRARIAN_GAP = 2; |
| 79 | /** A prediction within this distance of the actual peer mean counts as accurate. */ |
| 80 | export const PREDICTION_TOLERANCE = 1.5; |
| 81 | |
| 82 | export interface PredictionCell { |
| 83 | reviewerId: string; |
| 84 | targetId: string; |
| 85 | ownScore: number; |
| 86 | predicted: number; |
| 87 | /** Mean score the OTHER reviewers gave the target; null if nobody else scored it. */ |
| 88 | actualPeerMean: number | null; |
| 89 | } |
| 90 | |
| 91 | export type PredictionReading = 'aligned' | 'informed_contrarian' | 'miscalibrated'; |
| 92 | |
| 93 | export function classifyPrediction(cell: PredictionCell): PredictionReading | null { |
| 94 | if (cell.actualPeerMean === null) return null; |
| 95 | if (Math.abs(cell.ownScore - cell.actualPeerMean) < CONTRARIAN_GAP) return 'aligned'; |
| 96 | return Math.abs(cell.predicted - cell.actualPeerMean) <= PREDICTION_TOLERANCE |
| 97 | ? 'informed_contrarian' |
| 98 | : 'miscalibrated'; |
| 99 | } |
| 100 | |
| 101 | /** |
| 102 | * One cell per review that carried a `predictedPeerMean`. Predictions are |
| 103 | * compared against the scores from the SAME round only - mixing rounds would |
| 104 | * judge a round-1 prediction against a round-2 consensus. |
| 105 | */ |
| 106 | export function buildPredictionCells( |
| 107 | participants: Participant[], |
| 108 | critiques: CritiqueRecord[], |
| 109 | ): PredictionCell[] { |
| 110 | const matrixByRound = new Map<number, ScoreMatrix>(); |
| 111 | const matrixFor = (round: number): ScoreMatrix => { |
| 112 | let m = matrixByRound.get(round); |
| 113 | if (!m) { |
| 114 | m = buildScoreMatrix(participants, critiques.filter((c) => c.round === round)); |
| 115 | matrixByRound.set(round, m); |
| 116 | } |
| 117 | return m; |
| 118 | }; |
| 119 | |
| 120 | const cells: PredictionCell[] = []; |
| 121 | for (const critique of critiques) { |
| 122 | const reviewerId = critique.reviewerParticipantId; |
| 123 | const matrix = matrixFor(critique.round); |
| 124 | if (!matrix.cells[reviewerId]) continue; |
| 125 | for (const review of critique.reviews) { |
| 126 | if (review.predictedPeerMean === undefined) continue; |
| 127 | const targetId = review.targetParticipantId; |
| 128 | const others = matrix.order |
| 129 | .filter((id) => id !== reviewerId) |
| 130 | .map((id) => matrix.cells[id]![targetId]) |
| 131 | .filter((v): v is number => typeof v === 'number'); |
| 132 | cells.push({ |
| 133 | reviewerId, |
| 134 | targetId, |
| 135 | ownScore: review.score, |
| 136 | predicted: review.predictedPeerMean, |
| 137 | actualPeerMean: mean(others), |
| 138 | }); |
| 139 | } |
| 140 | } |
| 141 | return cells; |
| 142 | } |
| 143 | |
| 144 | /** |
| 145 | * Per-reviewer mean absolute prediction error ("how well does this model read |
| 146 | * the council"). Null for reviewers with no scoreable predictions. |
| 147 | */ |
| 148 | export function predictionCalibration( |
| 149 | participants: Participant[], |
| 150 | cells: PredictionCell[], |
| 151 | ): Record<string, number | null> { |
| 152 | const out: Record<string, number | null> = {}; |
| 153 | for (const p of participants) { |
| 154 | const errors = cells |
| 155 | .filter((c) => c.reviewerId === p.id && c.actualPeerMean !== null) |
| 156 | .map((c) => Math.abs(c.predicted - c.actualPeerMean!)); |
| 157 | out[p.id] = mean(errors); |
| 158 | } |
| 159 | return out; |
| 160 | } |
| 161 | |