Commit
Deduplicate model catalog requests
commit
6c901e5
2 changed files with +79 and −8
Jump to a changed file
- src/lib/model-cache.test.ts +65 −0
- src/lib/model-cache.ts +14 −8
added src/lib/model-cache.test.ts +65 −0
| @@ -0,0 +1,65 @@ | ||
| 1 | +import { beforeEach, describe, expect, it, vi } from 'vitest'; | |
| 2 | +import type { OpenRouterModel } from './openrouter'; | |
| 3 | + | |
| 4 | +const mocks = vi.hoisted(() => ({ | |
| 5 | + fetchOpenRouterModels: vi.fn(), | |
| 6 | + pricingFromModels: vi.fn(), | |
| 7 | +})); | |
| 8 | + | |
| 9 | +vi.mock('./env', () => ({ env: { MOCK_LLM: false } })); | |
| 10 | +vi.mock('./openrouter', () => mocks); | |
| 11 | + | |
| 12 | +const catalog: OpenRouterModel[] = [ | |
| 13 | + { | |
| 14 | + id: 'vendor/model', | |
| 15 | + name: 'Model', | |
| 16 | + context_length: 32_000, | |
| 17 | + pricing: { prompt: '0.000001', completion: '0.000002' }, | |
| 18 | + }, | |
| 19 | +]; | |
| 20 | + | |
| 21 | +describe('getModels', () => { | |
| 22 | + beforeEach(() => { | |
| 23 | + vi.resetModules(); | |
| 24 | + mocks.fetchOpenRouterModels.mockReset(); | |
| 25 | + mocks.pricingFromModels.mockReset(); | |
| 26 | + }); | |
| 27 | + | |
| 28 | + it('shares one catalog request between concurrent callers', async () => { | |
| 29 | + let resolveCatalog: ((models: OpenRouterModel[]) => void) | undefined; | |
| 30 | + mocks.fetchOpenRouterModels.mockReturnValue( | |
| 31 | + new Promise<OpenRouterModel[]>((resolve) => { | |
| 32 | + resolveCatalog = resolve; | |
| 33 | + }), | |
| 34 | + ); | |
| 35 | + const { getModels } = await import('./model-cache'); | |
| 36 | + | |
| 37 | + const first = getModels(); | |
| 38 | + const second = getModels(); | |
| 39 | + | |
| 40 | + expect(mocks.fetchOpenRouterModels).toHaveBeenCalledTimes(1); | |
| 41 | + resolveCatalog?.(catalog); | |
| 42 | + await expect(Promise.all([first, second])).resolves.toEqual([catalog, catalog]); | |
| 43 | + }); | |
| 44 | + | |
| 45 | + it('reuses the cached catalog after a request completes', async () => { | |
| 46 | + mocks.fetchOpenRouterModels.mockResolvedValue(catalog); | |
| 47 | + const { getModels } = await import('./model-cache'); | |
| 48 | + | |
| 49 | + await getModels(); | |
| 50 | + await getModels(); | |
| 51 | + | |
| 52 | + expect(mocks.fetchOpenRouterModels).toHaveBeenCalledTimes(1); | |
| 53 | + }); | |
| 54 | + | |
| 55 | + it('shares the fallback catalog when a request fails', async () => { | |
| 56 | + mocks.fetchOpenRouterModels.mockRejectedValue(new Error('offline')); | |
| 57 | + const { getModels } = await import('./model-cache'); | |
| 58 | + | |
| 59 | + const [first, second] = await Promise.all([getModels(), getModels()]); | |
| 60 | + | |
| 61 | + expect(mocks.fetchOpenRouterModels).toHaveBeenCalledTimes(1); | |
| 62 | + expect(first).toBe(second); | |
| 63 | + expect(first.length).toBeGreaterThan(0); | |
| 64 | + }); | |
| 65 | +}); |
modified src/lib/model-cache.ts +14 −8
| @@ -16,6 +16,7 @@import { | ||
| 16 | 16 | |
| 17 | 17 | const TTL_MS = 10 * 60 * 1000; |
| 18 | 18 | let cache: { at: number; models: OpenRouterModel[] } | null = null; |
| 19 | +let pending: Promise<OpenRouterModel[]> | null = null; | |
| 19 | 20 | |
| 20 | 21 | const MOCK_CATALOG: OpenRouterModel[] = [ |
| 21 | 22 | mock('openai/gpt-4o', 'GPT-4o', 2.5e-6, 1e-5, 128000), |
| @@ -43,15 +44,20 @@function mock(id: string, name: string, prompt: number, completion: number, ctx: | ||
| 43 | 44 | export async function getModels(): Promise<OpenRouterModel[]> { |
| 44 | 45 | if (env.MOCK_LLM) return MOCK_CATALOG; |
| 45 | 46 | if (cache && Date.now() - cache.at < TTL_MS) return cache.models; |
| 46 | - try { | |
| 47 | - const models = await fetchOpenRouterModels(); | |
| 48 | - cache = { at: Date.now(), models }; | |
| 49 | - return models; | |
| 50 | - } catch { | |
| 51 | - // Fall back to the last good cache, else the mock catalog, so the picker | |
| 52 | - // never hard-fails. | |
| 53 | - return cache?.models ?? MOCK_CATALOG; | |
| 47 | + | |
| 48 | + if (!pending) { | |
| 49 | + pending = fetchOpenRouterModels() | |
| 50 | + .then((models) => { | |
| 51 | + cache = { at: Date.now(), models }; | |
| 52 | + return models; | |
| 53 | + }) | |
| 54 | + .catch(() => cache?.models ?? MOCK_CATALOG) | |
| 55 | + .finally(() => { | |
| 56 | + pending = null; | |
| 57 | + }); | |
| 54 | 58 | } |
| 59 | + | |
| 60 | + return pending; | |
| 55 | 61 | } |
| 56 | 62 | |
| 57 | 63 | export async function getPricing(): Promise<PricingMap> { |