voicetask | profileShare

Commit

T6: audio upload route and OpenAI STT provider

POST /api/sessions/:id/audio parses a multipart upload, rejects a
zero-byte file before calling STT, and maps EmptyTranscriptError /
generic provider failures to 400 / 502. sttOpenai.ts posts to OpenAI's
transcriptions endpoint via fetch and throws EmptyTranscriptError on a
blank result. Extracted submitAnswer() (engine/turn.ts) so the answer
and audio routes share the same turn-processing logic instead of
duplicating it.

rasmusjy committed Jul 22, 2026, 5:48 PM

commit d51024b

9 changed files with +243 and −30

Jump to a changed file

server/app.ts +8 −2
server/engine/turn.ts +32 −0
server/providers/factory.ts +2 −1
server/providers/sttOpenai.ts +33 −0
server/providers/types.ts +6 −0
server/routes/audio.test.ts +106 −0
server/routes/audio.ts +44 −0
server/routes/sessions.ts +11 −26
spec/TASKS.md +1 −1

modified server/app.ts +8 −2

		@@ -1,26 +1,32 @@
1	1	import cors from '@fastify/cors'
	2	+import multipart from '@fastify/multipart'
2	3	import Fastify, { type FastifyInstance } from 'fastify'
3		-import { createInterviewLlm } from './providers/factory'
4		-import type { InterviewLlm } from './providers/types'
	4	+import { createInterviewLlm, createSttProvider } from './providers/factory'
	5	+import type { InterviewLlm, SttProvider } from './providers/types'
	6	+import { registerAudioRoutes } from './routes/audio'
5	7	import { registerSessionRoutes } from './routes/sessions'
6	8	import { createDefaultSessionStore, SessionStore } from './store/sessionStore'
7	9
8	10	export interface AppDeps {
9	11	store: SessionStore
10	12	llm: InterviewLlm
	13	+ stt: SttProvider
11	14	}
12	15
13	16	export function buildApp(deps: Partial<AppDeps> = {}): FastifyInstance {
14	17	const resolved: AppDeps = {
15	18	store: deps.store ?? createDefaultSessionStore(),
16	19	llm: deps.llm ?? createInterviewLlm(),
	20	+ stt: deps.stt ?? createSttProvider(),
17	21	}
18	22
19	23	const app = Fastify({ logger: false })
20	24	void app.register(cors, { origin: true })
	25	+ void app.register(multipart)
21	26
22	27	app.get('/api/health', async () => ({ ok: true }))
23	28	registerSessionRoutes(app, resolved)
	29	+ registerAudioRoutes(app, resolved)
24	30
25	31	return app
26	32	}

added server/engine/turn.ts +32 −0

		@@ -0,0 +1,32 @@
	1	+import type { AnswerResponse } from '../../shared/types'
	2	+import type { InterviewLlm } from '../providers/types'
	3	+import type { SessionStore } from '../store/sessionStore'
	4	+import { runInterviewTurn } from './interview'
	5	+
	6	+export class SessionNotFoundError extends Error {
	7	+ constructor(sessionId: string) {
	8	+ super(`session not found: ${sessionId}`)
	9	+ }
	10	+}
	11	+
	12	+export async function submitAnswer(
	13	+ store: SessionStore,
	14	+ llm: InterviewLlm,
	15	+ sessionId: string,
	16	+ text: string,
	17	+): Promise<AnswerResponse> {
	18	+ const existing = await store.getSession(sessionId)
	19	+ if (!existing) throw new SessionNotFoundError(sessionId)
	20	+
	21	+ const { session: afterAnswer, segment } = await store.appendSegment(sessionId, 'user', text)
	22	+ const turn = await runInterviewTurn(llm, afterAnswer)
	23	+
	24	+ await store.appendSegment(sessionId, 'interviewer', turn.nextQuestion)
	25	+ await store.updateTurn(sessionId, {
	26	+ coverage: turn.coverage,
	27	+ summary: turn.summaryUpdate ?? afterAnswer.summary,
	28	+ status: turn.done ? 'done' : 'interviewing',
	29	+ })
	30	+
	31	+ return { segment, turn }
	32	+}

modified server/providers/factory.ts +2 −1

		@@ -1,6 +1,7 @@
1	1	import { createLlmAnthropic } from './llmAnthropic'
2	2	import { createLlmMock } from './llmMock'
3	3	import { createSttMock } from './sttMock'
	4	+import { createSttOpenai } from './sttOpenai'
4	5	import type { InterviewLlm, SttProvider } from './types'
5	6
6	7	function isMockMode(): boolean {
		@@ -9,7 +10,7 @@function isMockMode(): boolean {
9	10
10	11	export function createSttProvider(): SttProvider {
11	12	if (isMockMode()) return createSttMock()
12		- throw new Error('real STT provider not configured yet (set MOCK_PROVIDERS=1)')
	13	+ return createSttOpenai()
13	14	}
14	15
15	16	export function createInterviewLlm(): InterviewLlm {

added server/providers/sttOpenai.ts +33 −0

		@@ -0,0 +1,33 @@
	1	+import { EmptyTranscriptError, type SttProvider } from './types'
	2	+
	3	+const DEFAULT_STT_MODEL = 'gpt-4o-mini-transcribe'
	4	+
	5	+export function createSttOpenai(): SttProvider {
	6	+ const apiKey = process.env.OPENAI_API_KEY
	7	+ if (!apiKey) throw new Error('OPENAI_API_KEY is not set')
	8	+ const model = process.env.STT_MODEL ?? DEFAULT_STT_MODEL
	9	+
	10	+ return {
	11	+ async transcribe(audio: Buffer, mimeType: string): Promise<string> {
	12	+ const form = new FormData()
	13	+ form.append('model', model)
	14	+ form.append('file', new Blob([audio], { type: mimeType }), 'audio.webm')
	15	+
	16	+ const res = await fetch('https://api.openai.com/v1/audio/transcriptions', {
	17	+ method: 'POST',
	18	+ headers: { Authorization: `Bearer ${apiKey}` },
	19	+ body: form,
	20	+ })
	21	+
	22	+ if (!res.ok) {
	23	+ const body = await res.text()
	24	+ throw new Error(`OpenAI STT request failed (${res.status}): ${body}`)
	25	+ }
	26	+
	27	+ const data = (await res.json()) as { text?: string }
	28	+ const text = (data.text ?? '').trim()
	29	+ if (text.length === 0) throw new EmptyTranscriptError()
	30	+ return text
	31	+ },
	32	+ }
	33	+}

modified server/providers/types.ts +6 −0

		@@ -1,5 +1,11 @@
1	1	import type { Coverage, InterviewTurn, Segment } from '../../shared/types'
2	2
	3	+export class EmptyTranscriptError extends Error {
	4	+ constructor() {
	5	+ super('transcript is empty after trimming')
	6	+ }
	7	+}
	8	+
3	9	export interface SttProvider {
4	10	transcribe(audio: Buffer, mimeType: string): Promise<string>
5	11	}

added server/routes/audio.test.ts +106 −0

		@@ -0,0 +1,106 @@
	1	+import { mkdtemp, rm } from 'node:fs/promises'
	2	+import { tmpdir } from 'node:os'
	3	+import path from 'node:path'
	4	+import type { FastifyInstance } from 'fastify'
	5	+import { afterEach, beforeEach, describe, expect, it } from 'vitest'
	6	+import { CATEGORY_IDS, type AnswerResponse, type Session } from '../../shared/types'
	7	+import { buildApp } from '../app'
	8	+import { createLlmMock } from '../providers/llmMock'
	9	+import { createSttMock } from '../providers/sttMock'
	10	+import { SessionStore } from '../store/sessionStore'
	11	+
	12	+function buildMultipartPayload(
	13	+ fileBuffer: Buffer,
	14	+ filename: string,
	15	+ mimeType: string,
	16	+): { body: Buffer; contentType: string } {
	17	+ const boundary = '----voicetaskTestBoundary1234567890'
	18	+ const parts = [
	19	+ Buffer.from(`--${boundary}\r\n`),
	20	+ Buffer.from(`Content-Disposition: form-data; name="audio"; filename="${filename}"\r\n`),
	21	+ Buffer.from(`Content-Type: ${mimeType}\r\n\r\n`),
	22	+ fileBuffer,
	23	+ Buffer.from(`\r\n--${boundary}--\r\n`),
	24	+ ]
	25	+ return { body: Buffer.concat(parts), contentType: `multipart/form-data; boundary=${boundary}` }
	26	+}
	27	+
	28	+describe('audio route', () => {
	29	+ let baseDir: string
	30	+ let app: FastifyInstance
	31	+
	32	+ beforeEach(async () => {
	33	+ baseDir = await mkdtemp(path.join(tmpdir(), 'voicetask-audio-'))
	34	+ app = buildApp({ store: new SessionStore(baseDir), llm: createLlmMock(), stt: createSttMock() })
	35	+ })
	36	+
	37	+ afterEach(async () => {
	38	+ await app.close()
	39	+ await rm(baseDir, { recursive: true, force: true })
	40	+ })
	41	+
	42	+ async function createSession(): Promise<Session> {
	43	+ return app
	44	+ .inject({ method: 'POST', url: '/api/sessions', payload: { name: 'p', targetDir: '/tmp/p' } })
	45	+ .then((r) => r.json<Session>())
	46	+ }
	47	+
	48	+ it('transcribes an uploaded clip, stores a segment, and triggers a turn', async () => {
	49	+ const session = await createSession()
	50	+ const { body, contentType } = buildMultipartPayload(
	51	+ Buffer.from('we want push notifications', 'utf8'),
	52	+ 'clip.webm',
	53	+ 'text/plain',
	54	+ )
	55	+
	56	+ const res = await app.inject({
	57	+ method: 'POST',
	58	+ url: `/api/sessions/${session.id}/audio`,
	59	+ headers: { 'content-type': contentType },
	60	+ payload: body,
	61	+ })
	62	+
	63	+ expect(res.statusCode).toBe(200)
	64	+ const responseBody = res.json<AnswerResponse>()
	65	+ expect(responseBody.segment.speaker).toBe('user')
	66	+ expect(responseBody.segment.text).toBe('we want push notifications')
	67	+ expect(responseBody.turn.coverage[CATEGORY_IDS[0]]).toBe('clear')
	68	+
	69	+ const reloaded = await app
	70	+ .inject({ method: 'GET', url: `/api/sessions/${session.id}` })
	71	+ .then((r) => r.json<Session>())
	72	+ expect(reloaded.segments).toHaveLength(2)
	73	+ expect(reloaded.segments[0]).toMatchObject({ speaker: 'user', text: 'we want push notifications' })
	74	+ })
	75	+
	76	+ it('rejects an empty audio upload with a 4xx and stores nothing', async () => {
	77	+ const session = await createSession()
	78	+ const { body, contentType } = buildMultipartPayload(Buffer.alloc(0), 'clip.webm', 'audio/webm')
	79	+
	80	+ const res = await app.inject({
	81	+ method: 'POST',
	82	+ url: `/api/sessions/${session.id}/audio`,
	83	+ headers: { 'content-type': contentType },
	84	+ payload: body,
	85	+ })
	86	+
	87	+ expect(res.statusCode).toBeGreaterThanOrEqual(400)
	88	+ expect(res.statusCode).toBeLessThan(500)
	89	+
	90	+ const reloaded = await app
	91	+ .inject({ method: 'GET', url: `/api/sessions/${session.id}` })
	92	+ .then((r) => r.json<Session>())
	93	+ expect(reloaded.segments).toHaveLength(0)
	94	+ })
	95	+
	96	+ it('returns 404 for an unknown session', async () => {
	97	+ const { body, contentType } = buildMultipartPayload(Buffer.from('hello'), 'clip.webm', 'text/plain')
	98	+ const res = await app.inject({
	99	+ method: 'POST',
	100	+ url: '/api/sessions/nonexistent/audio',
	101	+ headers: { 'content-type': contentType },
	102	+ payload: body,
	103	+ })
	104	+ expect(res.statusCode).toBe(404)
	105	+ })
	106	+})

added server/routes/audio.ts +44 −0

		@@ -0,0 +1,44 @@
	1	+import type { FastifyInstance } from 'fastify'
	2	+import { SessionNotFoundError, submitAnswer } from '../engine/turn'
	3	+import { EmptyTranscriptError, type InterviewLlm, type SttProvider } from '../providers/types'
	4	+import type { SessionStore } from '../store/sessionStore'
	5	+
	6	+export interface AudioRouteDeps {
	7	+ store: SessionStore
	8	+ llm: InterviewLlm
	9	+ stt: SttProvider
	10	+}
	11	+
	12	+export function registerAudioRoutes(app: FastifyInstance, deps: AudioRouteDeps): void {
	13	+ const { store, llm, stt } = deps
	14	+
	15	+ app.post<{ Params: { id: string } }>('/api/sessions/:id/audio', async (request, reply) => {
	16	+ const file = await request.file()
	17	+ if (!file) return reply.code(400).send({ error: 'no audio file uploaded' })
	18	+
	19	+ const buffer = await file.toBuffer()
	20	+ if (buffer.length === 0) {
	21	+ return reply.code(400).send({ error: 'empty audio upload' })
	22	+ }
	23	+
	24	+ let text: string
	25	+ try {
	26	+ text = await stt.transcribe(buffer, file.mimetype)
	27	+ } catch (err) {
	28	+ if (err instanceof EmptyTranscriptError) {
	29	+ return reply.code(400).send({ error: 'empty transcript' })
	30	+ }
	31	+ return reply.code(502).send({ error: err instanceof Error ? err.message : 'STT provider error' })
	32	+ }
	33	+
	34	+ try {
	35	+ const response = await submitAnswer(store, llm, request.params.id, text)
	36	+ return reply.send(response)
	37	+ } catch (err) {
	38	+ if (err instanceof SessionNotFoundError) {
	39	+ return reply.code(404).send({ error: 'session not found' })
	40	+ }
	41	+ throw err
	42	+ }
	43	+ })
	44	+}

modified server/routes/sessions.ts +11 −26

		@@ -1,10 +1,6 @@
1	1	import type { FastifyInstance } from 'fastify'
2		-import {
3		- AnswerRequestSchema,
4		- CreateSessionRequestSchema,
5		- type AnswerResponse,
6		-} from '../../shared/types'
7		-import { runInterviewTurn } from '../engine/interview'
	2	+import { AnswerRequestSchema, CreateSessionRequestSchema } from '../../shared/types'
	3	+import { SessionNotFoundError, submitAnswer } from '../engine/turn'
8	4	import type { InterviewLlm } from '../providers/types'
9	5	import type { SessionStore } from '../store/sessionStore'
10	6
		@@ -41,25 +37,14 @@export function registerSessionRoutes(app: FastifyInstance, deps: SessionRouteDe
41	37	return reply.code(400).send({ error: parsed.error.message })
42	38	}
43	39
44		- const existing = await store.getSession(request.params.id)
45		- if (!existing) return reply.code(404).send({ error: 'session not found' })
46		-
47		- const { session: afterAnswer, segment } = await store.appendSegment(
48		- request.params.id,
49		- 'user',
50		- parsed.data.text,
51		- )
52		-
53		- const turn = await runInterviewTurn(llm, afterAnswer)
54		-
55		- await store.appendSegment(request.params.id, 'interviewer', turn.nextQuestion)
56		- await store.updateTurn(request.params.id, {
57		- coverage: turn.coverage,
58		- summary: turn.summaryUpdate ?? afterAnswer.summary,
59		- status: turn.done ? 'done' : 'interviewing',
60		- })
61		-
62		- const response: AnswerResponse = { segment, turn }
63		- return reply.send(response)
	40	+ try {
	41	+ const response = await submitAnswer(store, llm, request.params.id, parsed.data.text)
	42	+ return reply.send(response)
	43	+ } catch (err) {
	44	+ if (err instanceof SessionNotFoundError) {
	45	+ return reply.code(404).send({ error: 'session not found' })
	46	+ }
	47	+ throw err
	48	+ }
64	49	})
65	50	}

modified spec/TASKS.md +1 −1

		@@ -27,7 +27,7 @@Work strictly in order unless a task's Depends line allows otherwise. One task a
27	27	- Depends: T4
28	28	- Verify: `npm test` (engine tests with llmMock: coverage progresses in category order, >40 segments triggers summary path, contradiction from mock is surfaced).
29	29
30		-- [ ] T6 Audio route and OpenAI STT
	30	+- [x] T6 Audio route and OpenAI STT
31	31	- `@fastify/multipart` upload route `POST /api/sessions/:id/audio`, `sttOpenai.ts` per PLAN.md, empty-transcript rejection, provider errors mapped to 502 with `{error}`.
32	32	- Depends: T4
33	33	- Verify: `npm test` (audio route with sttMock: uploaded text buffer becomes a segment and triggers a turn; empty buffer returns 4xx and stores nothing).

rasmusjy / voicetask