mock-client.ts
11,865 bytes
| 1 | /** |
|---|---|
| 2 | * Deterministic, offline LLM client. |
| 3 | * |
| 4 | * Powers `MOCK_LLM=1` (local dev / CI / the public demo) and the orchestrator |
| 5 | * tests. Given the same inputs it always returns the same output - no network, |
| 6 | * no spend, no clock or RNG dependence - which is exactly what makes the state |
| 7 | * machine testable and the seeded demo reproducible. |
| 8 | * |
| 9 | * Output is synthesized from `req.meta.stage`, so the mock knows what shape to |
| 10 | * return without brittle prompt sniffing. A small scenario object lets tests |
| 11 | * force timeouts, failures, and JSON-repair paths. |
| 12 | */ |
| 13 | import { hashSeed } from './anonymize'; |
| 14 | import type { LlmClient, LlmRequest, LlmResult, LlmStreamHandle } from './llm-client'; |
| 15 | import { LlmError } from './llm-client'; |
| 16 | import type { StageType, Usage } from './types'; |
| 17 | |
| 18 | export interface MockRule { |
| 19 | model?: string; |
| 20 | stage?: StageType; |
| 21 | round?: number; |
| 22 | } |
| 23 | |
| 24 | export interface MockScenario { |
| 25 | /** Calls matching any of these throw an LlmError (to test model drop-out). */ |
| 26 | fail?: MockRule[]; |
| 27 | /** First attempt returns schema-invalid JSON, forcing one repair round-trip. */ |
| 28 | malformFirst?: MockRule[]; |
| 29 | /** Convergence score = base + step * round (clamped 0-100). */ |
| 30 | convergenceBase?: number; |
| 31 | convergenceStep?: number; |
| 32 | /** Fixed latency reported per call (deterministic). */ |
| 33 | latencyMs?: number; |
| 34 | } |
| 35 | |
| 36 | function ruleMatches(rule: MockRule, model: string, stage: StageType, round: number): boolean { |
| 37 | if (rule.model !== undefined && rule.model !== model) return false; |
| 38 | if (rule.stage !== undefined && rule.stage !== stage) return false; |
| 39 | if (rule.round !== undefined && rule.round !== round) return false; |
| 40 | return true; |
| 41 | } |
| 42 | |
| 43 | const REPAIR_MARKER = 'strict JSON fixer'; |
| 44 | const isRepairCall = (req: LlmRequest) => |
| 45 | req.messages.some((m) => m.role === 'system' && m.content.includes(REPAIR_MARKER)); |
| 46 | |
| 47 | /** Extract "--- Response X ---" labels from a critique/convergence/synthesis prompt. */ |
| 48 | function labelsFromPrompt(req: LlmRequest): string[] { |
| 49 | const text = req.messages.map((m) => m.content).join('\n'); |
| 50 | const labels = [...text.matchAll(/---\s*Response\s+([A-Z]+)\s*---/g)].map((m) => m[1]!); |
| 51 | return labels.length ? labels : ['A']; |
| 52 | } |
| 53 | |
| 54 | /** On a repair call the labels live inside the echoed original response. */ |
| 55 | function labelsFromRepair(req: LlmRequest): string[] { |
| 56 | const text = req.messages.map((m) => m.content).join('\n'); |
| 57 | const labels = [...text.matchAll(/"label"\s*:\s*"([A-Z]+)"/g)].map((m) => m[1]!); |
| 58 | return labels.length ? [...new Set(labels)] : ['A']; |
| 59 | } |
| 60 | |
| 61 | function ownAnswerFromPrompt(req: LlmRequest): string { |
| 62 | const user = req.messages.find((m) => m.role === 'user')?.content ?? ''; |
| 63 | const match = user.match(/Your current answer:\n([\s\S]*?)\n\nCritiques of your answer:/); |
| 64 | return match?.[1]?.trim() ?? 'Prior answer unavailable.'; |
| 65 | } |
| 66 | |
| 67 | function questionFromPrompt(req: LlmRequest): string { |
| 68 | const text = req.messages.map((m) => m.content).join('\n'); |
| 69 | const match = text.match(/Question:\n([\s\S]*?)(?:\n\n|$)/); |
| 70 | return match?.[1]?.trim() ?? 'the question'; |
| 71 | } |
| 72 | |
| 73 | /** Deterministic pseudo-token count (~4 chars/token). */ |
| 74 | function tokensOf(text: string): number { |
| 75 | return Math.max(1, Math.round(text.length / 4)); |
| 76 | } |
| 77 | |
| 78 | function usageFor(req: LlmRequest, output: string): Usage { |
| 79 | const promptTokens = tokensOf(req.messages.map((m) => m.content).join('\n')); |
| 80 | const completionTokens = tokensOf(output); |
| 81 | // Synthetic pricing: $1 / 1M prompt tokens, $3 / 1M completion tokens. |
| 82 | const costUsd = (promptTokens * 1 + completionTokens * 3) / 1_000_000; |
| 83 | return { |
| 84 | promptTokens, |
| 85 | completionTokens, |
| 86 | totalTokens: promptTokens + completionTokens, |
| 87 | costUsd, |
| 88 | }; |
| 89 | } |
| 90 | |
| 91 | export class MockLlmClient implements LlmClient { |
| 92 | constructor(private readonly scenario: MockScenario = {}) {} |
| 93 | |
| 94 | private guard(req: LlmRequest): { model: string; stage: StageType; round: number } { |
| 95 | const model = req.model; |
| 96 | const stage: StageType = req.meta?.stage ?? 'answer'; |
| 97 | const round = req.meta?.round ?? 0; |
| 98 | if (req.signal?.aborted) { |
| 99 | throw new LlmError('Request aborted', { retryable: false }); |
| 100 | } |
| 101 | if (this.scenario.fail?.some((r) => ruleMatches(r, model, stage, round))) { |
| 102 | throw new LlmError(`Mock forced failure for ${model} at ${stage} (round ${round})`, { |
| 103 | status: 503, |
| 104 | retryable: false, |
| 105 | }); |
| 106 | } |
| 107 | return { model, stage, round }; |
| 108 | } |
| 109 | |
| 110 | private render(req: LlmRequest): string { |
| 111 | const { model, stage, round } = this.guard(req); |
| 112 | const repair = isRepairCall(req); |
| 113 | const forceMalform = |
| 114 | !repair && (this.scenario.malformFirst?.some((r) => ruleMatches(r, model, stage, round)) ?? false); |
| 115 | const seed = req.meta?.seed ?? `${model}:${stage}:${round}`; |
| 116 | |
| 117 | switch (stage) { |
| 118 | case 'answer': |
| 119 | return this.renderAnswer(model, round, questionFromPrompt(req)); |
| 120 | case 'critique': |
| 121 | return this.renderCritique(req, repair, forceMalform, seed); |
| 122 | case 'revision': |
| 123 | return this.renderRevision(req, repair, forceMalform, round); |
| 124 | case 'convergence': |
| 125 | return this.renderConvergence(req, repair, forceMalform, round); |
| 126 | case 'synthesis': |
| 127 | return this.renderSynthesis(req, repair, forceMalform); |
| 128 | case 'provenance': |
| 129 | return this.renderProvenance(req, repair, forceMalform); |
| 130 | default: |
| 131 | return 'Mock response.'; |
| 132 | } |
| 133 | } |
| 134 | |
| 135 | private renderAnswer(model: string, round: number, question: string): string { |
| 136 | const stance = round === 0 ? 'initial position' : `revised position (round ${round})`; |
| 137 | return ( |
| 138 | `**${model}** - ${stance}.\n\n` + |
| 139 | `On the question of "${truncate(question, 120)}", the key consideration is that ` + |
| 140 | `a well-formed answer must balance correctness against completeness. ${modelFlavor(model)} ` + |
| 141 | `My recommendation: proceed with the approach that has the strongest evidentiary support, ` + |
| 142 | `while explicitly noting the trade-offs involved.` |
| 143 | ); |
| 144 | } |
| 145 | |
| 146 | private renderCritique(req: LlmRequest, repair: boolean, malform: boolean, seed: string): string { |
| 147 | const labels = repair ? labelsFromRepair(req) : labelsFromPrompt(req); |
| 148 | const reviews = labels.map((label) => { |
| 149 | const s = hashSeed(seed, label) % 6; // 0..5 |
| 150 | const score = malform ? 42 : 4 + s; // 4..9 normally; out-of-range to trigger repair |
| 151 | // Deterministic council-mean prediction near (but not equal to) the score. |
| 152 | const predictedPeerMean = Math.min(10, Math.max(1, score + (hashSeed(seed, label, 'pred') % 3) - 1)); |
| 153 | const families = ['openai', 'anthropic', 'google']; |
| 154 | const authorGuess = { |
| 155 | family: families[hashSeed(seed, label, 'guess') % families.length]!, |
| 156 | confidence: 0.2 + (hashSeed(seed, label, 'conf') % 5) / 10, |
| 157 | }; |
| 158 | return { |
| 159 | label, |
| 160 | weaknesses: [`Response ${label} underspecifies the edge cases.`], |
| 161 | strengths: [`Response ${label} states its assumptions clearly.`], |
| 162 | score, |
| 163 | justification: `Solid reasoning with a gap around edge cases (score ${score}).`, |
| 164 | predictedPeerMean, |
| 165 | authorGuess, |
| 166 | }; |
| 167 | }); |
| 168 | return JSON.stringify({ reviews }); |
| 169 | } |
| 170 | |
| 171 | private renderRevision(req: LlmRequest, repair: boolean, malform: boolean, round: number): string { |
| 172 | const own = repair ? 'Refined answer after incorporating peer feedback.' : ownAnswerFromPrompt(req); |
| 173 | const revised = |
| 174 | `${own}\n\nAfter review (round ${round}): clarified the edge-case handling and ` + |
| 175 | `tightened the recommendation.`; |
| 176 | const payload = { |
| 177 | // omit `answer` when malforming, to fail the schema and force a repair |
| 178 | ...(malform ? {} : { answer: revised }), |
| 179 | changelog: { |
| 180 | changed: true, |
| 181 | summary: 'Incorporated the edge-case critique and sharpened the recommendation.', |
| 182 | bullets: ['Added edge-case handling', 'Tightened final recommendation'], |
| 183 | }, |
| 184 | }; |
| 185 | return JSON.stringify(payload); |
| 186 | } |
| 187 | |
| 188 | private renderConvergence(req: LlmRequest, repair: boolean, malform: boolean, round: number): string { |
| 189 | const base = this.scenario.convergenceBase ?? 60; |
| 190 | const step = this.scenario.convergenceStep ?? 18; |
| 191 | const score = malform ? 150 : Math.max(0, Math.min(100, base + step * round)); |
| 192 | const labels = (repair ? labelsFromRepair(req) : labelsFromPrompt(req)).slice(0, 2); |
| 193 | const disagreements = |
| 194 | score >= 100 |
| 195 | ? [] |
| 196 | : [ |
| 197 | { |
| 198 | topic: 'edge-case handling', |
| 199 | summary: 'Members differ on how strictly edge cases must be handled.', |
| 200 | positions: labels.map((label) => ({ |
| 201 | label, |
| 202 | stance: `Response ${label} favors its own emphasis on this point.`, |
| 203 | })), |
| 204 | }, |
| 205 | ]; |
| 206 | return JSON.stringify({ score, disagreements }); |
| 207 | } |
| 208 | |
| 209 | private renderSynthesis(req: LlmRequest, repair: boolean, malform: boolean): string { |
| 210 | const labels = (repair ? labelsFromRepair(req) : labelsFromPrompt(req)).slice(0, 2); |
| 211 | const payload = { |
| 212 | ...(malform ? {} : { |
| 213 | finalAnswer: |
| 214 | 'Synthesized answer: the council converged on prioritizing correctness, with ' + |
| 215 | 'explicit handling of edge cases and a clearly stated recommendation and its trade-offs.', |
| 216 | }), |
| 217 | dissent: |
| 218 | labels.length >= 2 |
| 219 | ? [ |
| 220 | { |
| 221 | topic: 'strictness of edge-case handling', |
| 222 | positions: labels.map((label) => ({ |
| 223 | label, |
| 224 | position: `Response ${label} maintains a distinct emphasis.`, |
| 225 | })), |
| 226 | }, |
| 227 | ] |
| 228 | : [], |
| 229 | }; |
| 230 | return JSON.stringify(payload); |
| 231 | } |
| 232 | |
| 233 | private renderProvenance(req: LlmRequest, repair: boolean, malform: boolean): string { |
| 234 | const labels = repair ? labelsFromRepair(req) : labelsFromPrompt(req); |
| 235 | const claims = [ |
| 236 | ...labels.map((label, i) => ({ |
| 237 | text: `The recommendation should weigh correctness against completeness (point ${i + 1}).`, |
| 238 | supportedBy: [label], |
| 239 | contestedBy: labels.filter((l) => l !== label).slice(0, 1), |
| 240 | })), |
| 241 | { |
| 242 | // Deliberately unsourced so the chairman-addition path is exercised offline. |
| 243 | text: 'Adoption should be revisited after one quarter of production use.', |
| 244 | supportedBy: [], |
| 245 | contestedBy: [], |
| 246 | }, |
| 247 | ]; |
| 248 | return JSON.stringify(malform ? { claims: [] } : { claims }); |
| 249 | } |
| 250 | |
| 251 | async complete(req: LlmRequest): Promise<LlmResult> { |
| 252 | const text = this.render(req); |
| 253 | return { |
| 254 | text, |
| 255 | usage: usageFor(req, text), |
| 256 | model: req.model, |
| 257 | latencyMs: this.scenario.latencyMs ?? 120 + (hashSeed(req.model, req.meta?.stage ?? '') % 400), |
| 258 | }; |
| 259 | } |
| 260 | |
| 261 | async streamComplete(req: LlmRequest): Promise<LlmStreamHandle> { |
| 262 | const text = this.render(req); |
| 263 | const usage = usageFor(req, text); |
| 264 | const latencyMs = this.scenario.latencyMs ?? 120 + (hashSeed(req.model, 'stream') % 400); |
| 265 | const chunks = chunkText(text, 5); |
| 266 | |
| 267 | async function* gen(): AsyncGenerator<string> { |
| 268 | for (const chunk of chunks) { |
| 269 | if (req.signal?.aborted) throw new LlmError('Request aborted', { retryable: false }); |
| 270 | yield chunk; |
| 271 | } |
| 272 | } |
| 273 | |
| 274 | return { |
| 275 | stream: gen(), |
| 276 | result: Promise.resolve({ text, usage, model: req.model, latencyMs }), |
| 277 | }; |
| 278 | } |
| 279 | } |
| 280 | |
| 281 | function chunkText(text: string, parts: number): string[] { |
| 282 | const size = Math.ceil(text.length / parts) || 1; |
| 283 | const out: string[] = []; |
| 284 | for (let i = 0; i < text.length; i += size) out.push(text.slice(i, i + size)); |
| 285 | return out.length ? out : ['']; |
| 286 | } |
| 287 | |
| 288 | function truncate(s: string, n: number): string { |
| 289 | return s.length <= n ? s : `${s.slice(0, n)}...`; |
| 290 | } |
| 291 | |
| 292 | function modelFlavor(model: string): string { |
| 293 | const h = hashSeed(model) % 3; |
| 294 | return h === 0 |
| 295 | ? 'This favors a conservative, evidence-first framing.' |
| 296 | : h === 1 |
| 297 | ? 'This leans toward a pragmatic, action-oriented framing.' |
| 298 | : 'This weighs long-term robustness above short-term simplicity.'; |
| 299 | } |
| 300 | |