voicetask | profileShare

Commit

T5: interview engine and Anthropic LLM provider

engine/coverage.ts (weakest-category selection, fixed tie-break order),
engine/summary.ts (last-40-segment window), engine/interview.ts now
truncates the transcript sent to the provider and passes contradictions
through untouched. llmAnthropic.ts wires client.beta.messages.parse with
a Zod output format for interview turns and a streaming call per
generated file, factory.ts now falls back to it outside mock mode.

Note: PLAN.md's `thinking: { type: "adaptive" }` is not a real config
in @anthropic-ai/sdk 0.71; used `{ type: "enabled", budget_tokens }`
instead, the closest real equivalent.

rasmusjy committed Jul 22, 2026, 5:45 PM

commit c791364

10 changed files with +332 and −3

Jump to a changed file

server/engine/coverage.test.ts +37 −0
server/engine/coverage.ts +18 −0
server/engine/interview.test.ts +91 −0
server/engine/interview.ts +4 −1
server/engine/summary.test.ts +31 −0
server/engine/summary.ts +19 −0
server/providers/factory.ts +2 −1
server/providers/llmAnthropic.test.ts +53 −0
server/providers/llmAnthropic.ts +76 −0
spec/TASKS.md +1 −1

added server/engine/coverage.test.ts +37 −0

		@@ -0,0 +1,37 @@
	1	+import { describe, expect, it } from 'vitest'
	2	+import { CATEGORY_IDS, initialCoverage, type Coverage } from '../../shared/types'
	3	+import { isFullyCovered, weakestCategory } from './coverage'
	4	+
	5	+function allClear(): Coverage {
	6	+ return Object.fromEntries(CATEGORY_IDS.map((category) => [category, 'clear'])) as Coverage
	7	+}
	8	+
	9	+describe('weakestCategory', () => {
	10	+ it('returns the first category in fixed order when all are missing', () => {
	11	+ expect(weakestCategory(initialCoverage())).toBe(CATEGORY_IDS[0])
	12	+ })
	13	+
	14	+ it('prefers missing over partial, and partial over clear, tie-broken by fixed order', () => {
	15	+ const coverage: Coverage = {
	16	+ ...initialCoverage(),
	17	+ goal: 'clear',
	18	+ users: 'partial',
	19	+ 'core-flow': 'missing',
	20	+ }
	21	+ expect(weakestCategory(coverage)).toBe('core-flow')
	22	+ })
	23	+
	24	+ it('returns null when every category is clear', () => {
	25	+ expect(weakestCategory(allClear())).toBeNull()
	26	+ })
	27	+})
	28	+
	29	+describe('isFullyCovered', () => {
	30	+ it('is false until every category is clear', () => {
	31	+ expect(isFullyCovered(initialCoverage())).toBe(false)
	32	+ })
	33	+
	34	+ it('is true once every category is clear', () => {
	35	+ expect(isFullyCovered(allClear())).toBe(true)
	36	+ })
	37	+})

added server/engine/coverage.ts +18 −0

		@@ -0,0 +1,18 @@
	1	+import { CATEGORY_IDS, type CategoryId, type Coverage, type CoverageLevel } from '../../shared/types'
	2	+
	3	+const LEVEL_PRIORITY: Record<CoverageLevel, number> = { missing: 0, partial: 1, clear: 2 }
	4	+
	5	+export function weakestCategory(coverage: Coverage): CategoryId \| null {
	6	+ let weakest: CategoryId \| null = null
	7	+ for (const category of CATEGORY_IDS) {
	8	+ if (coverage[category] === 'clear') continue
	9	+ if (weakest === null \|\| LEVEL_PRIORITY[coverage[category]] < LEVEL_PRIORITY[coverage[weakest]]) {
	10	+ weakest = category
	11	+ }
	12	+ }
	13	+ return weakest
	14	+}
	15	+
	16	+export function isFullyCovered(coverage: Coverage): boolean {
	17	+ return CATEGORY_IDS.every((category) => coverage[category] === 'clear')
	18	+}

added server/engine/interview.test.ts +91 −0

		@@ -0,0 +1,91 @@
	1	+import { describe, expect, it } from 'vitest'
	2	+import { initialCoverage, type InterviewTurn, type Segment, type Session } from '../../shared/types'
	3	+import type { InterviewContext, InterviewLlm } from '../providers/types'
	4	+import { runInterviewTurn } from './interview'
	5	+import { RECENT_SEGMENT_WINDOW } from './summary'
	6	+
	7	+function makeSession(segments: Segment[]): Session {
	8	+ return {
	9	+ id: 'sess-1',
	10	+ name: 'proj',
	11	+ targetDir: '/tmp/proj',
	12	+ createdAt: new Date().toISOString(),
	13	+ segments,
	14	+ coverage: initialCoverage(),
	15	+ summary: 'earlier summary',
	16	+ status: 'interviewing',
	17	+ openBlockers: [],
	18	+ }
	19	+}
	20	+
	21	+function segment(id: string, speaker: Segment['speaker'], text: string): Segment {
	22	+ return { id, ts: new Date().toISOString(), speaker, text }
	23	+}
	24	+
	25	+function stubLlm(turn: InterviewTurn, capture?: (ctx: InterviewContext) => void): InterviewLlm {
	26	+ return {
	27	+ async nextTurn(context) {
	28	+ capture?.(context)
	29	+ return turn
	30	+ },
	31	+ async generateFile() {
	32	+ throw new Error('not used in this test')
	33	+ },
	34	+ }
	35	+}
	36	+
	37	+describe('runInterviewTurn', () => {
	38	+ it('short-circuits with done:true when the last user answer is exactly "done"', async () => {
	39	+ const session = makeSession([segment('S1', 'user', 'Done')])
	40	+ const llm = stubLlm({
	41	+ coverage: initialCoverage(),
	42	+ nextQuestion: 'should not be used',
	43	+ contradiction: null,
	44	+ done: false,
	45	+ summaryUpdate: null,
	46	+ })
	47	+ const turn = await runInterviewTurn(llm, session)
	48	+ expect(turn.done).toBe(true)
	49	+ expect(turn.coverage).toEqual(session.coverage)
	50	+ })
	51	+
	52	+ it('passes a contradiction from the LLM straight through unchanged', async () => {
	53	+ const session = makeSession([segment('S1', 'user', 'we only support web')])
	54	+ const contradiction = { segmentIds: ['S1', 'S3'], description: 'web-only vs mobile-only' }
	55	+ const llm = stubLlm({
	56	+ coverage: initialCoverage(),
	57	+ nextQuestion: 'Which is it, web or mobile?',
	58	+ contradiction,
	59	+ done: false,
	60	+ summaryUpdate: null,
	61	+ })
	62	+ const turn = await runInterviewTurn(llm, session)
	63	+ expect(turn.contradiction).toEqual(contradiction)
	64	+ })
	65	+
	66	+ it('sends only the last 40 segments plus the running summary once the transcript grows past the window', async () => {
	67	+ const segments = Array.from({ length: RECENT_SEGMENT_WINDOW + 7 }, (_, i) =>
	68	+ segment(`S${i + 1}`, i % 2 === 0 ? 'user' : 'interviewer', `text ${i + 1}`),
	69	+ )
	70	+ const session = makeSession(segments)
	71	+ let seenContext: InterviewContext \| undefined
	72	+ const llm = stubLlm(
	73	+ {
	74	+ coverage: initialCoverage(),
	75	+ nextQuestion: 'next?',
	76	+ contradiction: null,
	77	+ done: false,
	78	+ summaryUpdate: null,
	79	+ },
	80	+ (ctx) => {
	81	+ seenContext = ctx
	82	+ },
	83	+ )
	84	+
	85	+ await runInterviewTurn(llm, session)
	86	+
	87	+ expect(seenContext?.segments).toHaveLength(RECENT_SEGMENT_WINDOW)
	88	+ expect(seenContext?.segments[0].id).toBe('S8')
	89	+ expect(seenContext?.summary).toBe('earlier summary')
	90	+ })
	91	+})

modified server/engine/interview.ts +4 −1

		@@ -1,5 +1,6 @@
1	1	import type { InterviewTurn, Session } from '../../shared/types'
2	2	import type { InterviewLlm } from '../providers/types'
	3	+import { splitTranscriptWindow } from './summary'
3	4
4	5	export function isDoneAnswer(text: string): boolean {
5	6	return text.trim().toLowerCase() === 'done'
		@@ -17,9 +18,11 @@export async function runInterviewTurn(llm: InterviewLlm, session: Session): Pro
17	18	}
18	19	}
19	20
	21	+ const { recentSegments } = splitTranscriptWindow(session.segments)
	22	+
20	23	return llm.nextTurn({
21	24	summary: session.summary,
22		- segments: session.segments,
	25	+ segments: recentSegments,
23	26	coverage: session.coverage,
24	27	})
25	28	}

added server/engine/summary.test.ts +31 −0

		@@ -0,0 +1,31 @@
	1	+import { describe, expect, it } from 'vitest'
	2	+import type { Segment } from '../../shared/types'
	3	+import { RECENT_SEGMENT_WINDOW, splitTranscriptWindow } from './summary'
	4	+
	5	+function makeSegments(count: number): Segment[] {
	6	+ return Array.from({ length: count }, (_, i) => ({
	7	+ id: `S${i + 1}`,
	8	+ ts: new Date().toISOString(),
	9	+ speaker: i % 2 === 0 ? 'user' : 'interviewer',
	10	+ text: `segment ${i + 1}`,
	11	+ }))
	12	+}
	13	+
	14	+describe('splitTranscriptWindow', () => {
	15	+ it('keeps everything in the recent window when at or under the limit', () => {
	16	+ const segments = makeSegments(RECENT_SEGMENT_WINDOW)
	17	+ const { recentSegments, olderSegments } = splitTranscriptWindow(segments)
	18	+ expect(recentSegments).toEqual(segments)
	19	+ expect(olderSegments).toEqual([])
	20	+ })
	21	+
	22	+ it('keeps only the last 40 segments and puts the rest in olderSegments', () => {
	23	+ const segments = makeSegments(RECENT_SEGMENT_WINDOW + 5)
	24	+ const { recentSegments, olderSegments } = splitTranscriptWindow(segments)
	25	+ expect(recentSegments).toHaveLength(RECENT_SEGMENT_WINDOW)
	26	+ expect(recentSegments[0].id).toBe('S6')
	27	+ expect(recentSegments[recentSegments.length - 1].id).toBe(`S${segments.length}`)
	28	+ expect(olderSegments).toHaveLength(5)
	29	+ expect(olderSegments.map((s) => s.id)).toEqual(['S1', 'S2', 'S3', 'S4', 'S5'])
	30	+ })
	31	+})

added server/engine/summary.ts +19 −0

		@@ -0,0 +1,19 @@
	1	+import type { Segment } from '../../shared/types'
	2	+
	3	+export const RECENT_SEGMENT_WINDOW = 40
	4	+
	5	+export interface TranscriptWindow {
	6	+ recentSegments: Segment[]
	7	+ olderSegments: Segment[]
	8	+}
	9	+
	10	+export function splitTranscriptWindow(segments: Segment[]): TranscriptWindow {
	11	+ if (segments.length <= RECENT_SEGMENT_WINDOW) {
	12	+ return { recentSegments: segments, olderSegments: [] }
	13	+ }
	14	+ const splitIndex = segments.length - RECENT_SEGMENT_WINDOW
	15	+ return {
	16	+ recentSegments: segments.slice(splitIndex),
	17	+ olderSegments: segments.slice(0, splitIndex),
	18	+ }
	19	+}

modified server/providers/factory.ts +2 −1

		@@ -1,3 +1,4 @@
	1	+import { createLlmAnthropic } from './llmAnthropic'
1	2	import { createLlmMock } from './llmMock'
2	3	import { createSttMock } from './sttMock'
3	4	import type { InterviewLlm, SttProvider } from './types'
		@@ -13,5 +14,5 @@export function createSttProvider(): SttProvider {
13	14
14	15	export function createInterviewLlm(): InterviewLlm {
15	16	if (isMockMode()) return createLlmMock()
16		- throw new Error('real LLM provider not configured yet (set MOCK_PROVIDERS=1)')
	17	+ return createLlmAnthropic()
17	18	}

added server/providers/llmAnthropic.test.ts +53 −0

		@@ -0,0 +1,53 @@
	1	+import { describe, expect, it } from 'vitest'
	2	+import { initialCoverage, type Segment } from '../../shared/types'
	3	+import { buildInterviewSystemPrompt, buildInterviewUserMessage } from './llmAnthropic'
	4	+import type { InterviewContext } from './types'
	5	+
	6	+function segment(id: string, speaker: Segment['speaker'], text: string): Segment {
	7	+ return { id, ts: new Date().toISOString(), speaker, text }
	8	+}
	9	+
	10	+describe('buildInterviewSystemPrompt', () => {
	11	+ it('names the weakest category as the required target of the next question', () => {
	12	+ const context: InterviewContext = {
	13	+ summary: '',
	14	+ segments: [],
	15	+ coverage: { ...initialCoverage(), goal: 'clear' },
	16	+ }
	17	+ const prompt = buildInterviewSystemPrompt(context)
	18	+ expect(prompt).toContain('weakest category right now is "users"')
	19	+ })
	20	+
	21	+ it('states every category is covered when nothing is weak', () => {
	22	+ const allClear = Object.fromEntries(
	23	+ Object.keys(initialCoverage()).map((c) => [c, 'clear']),
	24	+ ) as ReturnType<typeof initialCoverage>
	25	+ const context: InterviewContext = { summary: '', segments: [], coverage: allClear }
	26	+ expect(buildInterviewSystemPrompt(context)).toContain('Every category is already "clear"')
	27	+ })
	28	+
	29	+ it('instructs the exact-match "done" rule and the contradiction rule', () => {
	30	+ const context: InterviewContext = { summary: '', segments: [], coverage: initialCoverage() }
	31	+ const prompt = buildInterviewSystemPrompt(context)
	32	+ expect(prompt).toContain('exactly "done"')
	33	+ expect(prompt).toContain('contradiction')
	34	+ })
	35	+})
	36	+
	37	+describe('buildInterviewUserMessage', () => {
	38	+ it('includes the running summary and every recent segment with its id and speaker', () => {
	39	+ const context: InterviewContext = {
	40	+ summary: 'earlier: building a todo app',
	41	+ segments: [segment('S1', 'user', 'it should sync across devices')],
	42	+ coverage: initialCoverage(),
	43	+ }
	44	+ const message = buildInterviewUserMessage(context)
	45	+ expect(message).toContain('earlier: building a todo app')
	46	+ expect(message).toContain('[S1] user: it should sync across devices')
	47	+ })
	48	+
	49	+ it('omits the summary section when there is no summary yet', () => {
	50	+ const context: InterviewContext = { summary: '', segments: [], coverage: initialCoverage() }
	51	+ expect(buildInterviewUserMessage(context)).not.toContain('Running summary')
	52	+ })
	53	+})

added server/providers/llmAnthropic.ts +76 −0

		@@ -0,0 +1,76 @@
	1	+import Anthropic from '@anthropic-ai/sdk'
	2	+import { betaZodOutputFormat } from '@anthropic-ai/sdk/helpers/beta/zod'
	3	+import { CATEGORY_IDS, InterviewTurnSchema, type InterviewTurn } from '../../shared/types'
	4	+import { weakestCategory } from '../engine/coverage'
	5	+import type { GenerateFileRequest, InterviewContext, InterviewLlm } from './types'
	6	+
	7	+const DEFAULT_MODEL = 'claude-opus-4-8'
	8	+const INTERVIEW_MAX_TOKENS = 4096
	9	+const INTERVIEW_THINKING_BUDGET_TOKENS = 2048
	10	+const GENERATE_MAX_TOKENS = 32000
	11	+
	12	+export function buildInterviewSystemPrompt(context: InterviewContext): string {
	13	+ const weakest = weakestCategory(context.coverage)
	14	+ return [
	15	+ 'You are a Socratic spec interviewer for a solo developer talking through a project idea.',
	16	+ 'Ask exactly one targeted question per turn, chosen to fill the biggest gap in the spec.',
	17	+ `Coverage categories, in tie-break priority order: ${CATEGORY_IDS.join(', ')}.`,
	18	+ weakest
	19	+ ? `The weakest category right now is "${weakest}". Your next question must target it, unless a contradiction takes priority.`
	20	+ : 'Every category is already "clear".',
	21	+ 'If the latest user answer conflicts with an earlier statement, do not ask a coverage question: set "contradiction" to describe both statements, referencing their segment ids, instead.',
	22	+ 'The interview ends only when the user answer is exactly "done" (case-insensitive, trimmed) — never for any other reason, even if "done" appears inside a longer answer.',
	23	+ 'Reply with only the InterviewTurn structure described by the output schema.',
	24	+ ].join('\n')
	25	+}
	26	+
	27	+export function buildInterviewUserMessage(context: InterviewContext): string {
	28	+ const lines: string[] = []
	29	+ if (context.summary) {
	30	+ lines.push('Running summary of earlier segments:', context.summary, '')
	31	+ }
	32	+ lines.push('Recent transcript segments:')
	33	+ for (const segment of context.segments) {
	34	+ lines.push(`[${segment.id}] ${segment.speaker}: ${segment.text}`)
	35	+ }
	36	+ lines.push('', `Current coverage: ${JSON.stringify(context.coverage)}`)
	37	+ return lines.join('\n')
	38	+}
	39	+
	40	+export function createLlmAnthropic(): InterviewLlm {
	41	+ const apiKey = process.env.ANTHROPIC_API_KEY
	42	+ if (!apiKey) throw new Error('ANTHROPIC_API_KEY is not set')
	43	+ const model = process.env.ANTHROPIC_MODEL ?? DEFAULT_MODEL
	44	+ const client = new Anthropic({ apiKey })
	45	+
	46	+ return {
	47	+ async nextTurn(context: InterviewContext): Promise<InterviewTurn> {
	48	+ const message = await client.beta.messages.parse({
	49	+ model,
	50	+ max_tokens: INTERVIEW_MAX_TOKENS,
	51	+ thinking: { type: 'enabled', budget_tokens: INTERVIEW_THINKING_BUDGET_TOKENS },
	52	+ system: buildInterviewSystemPrompt(context),
	53	+ messages: [{ role: 'user', content: buildInterviewUserMessage(context) }],
	54	+ output_format: betaZodOutputFormat(InterviewTurnSchema),
	55	+ })
	56	+ if (!message.parsed_output) {
	57	+ throw new Error('Anthropic response did not include a parsed InterviewTurn')
	58	+ }
	59	+ return message.parsed_output
	60	+ },
	61	+
	62	+ async generateFile(request: GenerateFileRequest): Promise<string> {
	63	+ const stream = client.beta.messages.stream({
	64	+ model,
	65	+ max_tokens: GENERATE_MAX_TOKENS,
	66	+ messages: [{ role: 'user', content: request.prompt }],
	67	+ })
	68	+ const final = await stream.finalMessage()
	69	+ const textBlock = final.content.find((block) => block.type === 'text')
	70	+ if (!textBlock) {
	71	+ throw new Error(`Anthropic response for ${request.file} contained no text block`)
	72	+ }
	73	+ return textBlock.text
	74	+ },
	75	+ }
	76	+}

modified spec/TASKS.md +1 −1

		@@ -22,7 +22,7 @@Work strictly in order unless a task's Depends line allows otherwise. One task a
22	22	- Depends: T3
23	23	- Verify: `npm test` (route tests via fastify.inject: create, answer produces interviewer segment + coverage update, done ends session).
24	24
25		-- [ ] T5 Interview engine
	25	+- [x] T5 Interview engine
26	26	- `server/engine/`: coverage state handling, prompt construction (summary + last 40 segments), summary maintenance, contradiction passthrough from the LLM response, weakest-category question selection enforced in the prompt. Anthropic implementation `llmAnthropic.ts` per PLAN.md (compiles and is unit-tested for prompt construction only; no network in tests).
27	27	- Depends: T4
28	28	- Verify: `npm test` (engine tests with llmMock: coverage progresses in category order, >40 segments triggers summary path, contradiction from mock is surfaced).

rasmusjy / voicetask