model-cache.ts
2,287 bytes
| 1 | /** |
|---|---|
| 2 | * In-memory, TTL-cached OpenRouter model catalog. |
| 3 | * |
| 4 | * The `/models` endpoint is public (no key needed) and changes slowly, so we |
| 5 | * cache it process-wide for 10 minutes. Pricing for cost attribution is derived |
| 6 | * from the same catalog. In `MOCK_LLM` mode we serve a small static catalog so |
| 7 | * the picker and demo work fully offline. |
| 8 | */ |
| 9 | import { env } from './env'; |
| 10 | import { |
| 11 | fetchOpenRouterModels, |
| 12 | pricingFromModels, |
| 13 | type OpenRouterModel, |
| 14 | type PricingMap, |
| 15 | } from './openrouter'; |
| 16 | |
| 17 | const TTL_MS = 10 * 60 * 1000; |
| 18 | let cache: { at: number; models: OpenRouterModel[] } | null = null; |
| 19 | let pending: Promise<OpenRouterModel[]> | null = null; |
| 20 | |
| 21 | const MOCK_CATALOG: OpenRouterModel[] = [ |
| 22 | mock('openai/gpt-4o', 'GPT-4o', 2.5e-6, 1e-5, 128000), |
| 23 | mock('openai/gpt-4o-mini', 'GPT-4o Mini', 1.5e-7, 6e-7, 128000), |
| 24 | mock('anthropic/claude-3.5-sonnet', 'Claude 3.5 Sonnet', 3e-6, 1.5e-5, 200000), |
| 25 | mock('anthropic/claude-3.5-haiku', 'Claude 3.5 Haiku', 8e-7, 4e-6, 200000), |
| 26 | mock('google/gemini-2.0-flash-001', 'Gemini 2.0 Flash', 1e-7, 4e-7, 1000000), |
| 27 | mock('google/gemini-pro-1.5', 'Gemini Pro 1.5', 1.25e-6, 5e-6, 2000000), |
| 28 | mock('meta-llama/llama-3.3-70b-instruct', 'Llama 3.3 70B', 1.2e-7, 3e-7, 131072), |
| 29 | mock('mistralai/mistral-large', 'Mistral Large', 2e-6, 6e-6, 128000), |
| 30 | mock('x-ai/grok-2-1212', 'Grok 2', 2e-6, 1e-5, 131072), |
| 31 | mock('deepseek/deepseek-chat', 'DeepSeek Chat', 1.4e-7, 2.8e-7, 64000), |
| 32 | ]; |
| 33 | |
| 34 | function mock(id: string, name: string, prompt: number, completion: number, ctx: number): OpenRouterModel { |
| 35 | return { |
| 36 | id, |
| 37 | name, |
| 38 | description: `${name} (mock catalog entry)`, |
| 39 | context_length: ctx, |
| 40 | pricing: { prompt: String(prompt), completion: String(completion) }, |
| 41 | }; |
| 42 | } |
| 43 | |
| 44 | export async function getModels(): Promise<OpenRouterModel[]> { |
| 45 | if (env.MOCK_LLM) return MOCK_CATALOG; |
| 46 | if (cache && Date.now() - cache.at < TTL_MS) return cache.models; |
| 47 | |
| 48 | if (!pending) { |
| 49 | pending = fetchOpenRouterModels() |
| 50 | .then((models) => { |
| 51 | cache = { at: Date.now(), models }; |
| 52 | return models; |
| 53 | }) |
| 54 | .catch(() => cache?.models ?? MOCK_CATALOG) |
| 55 | .finally(() => { |
| 56 | pending = null; |
| 57 | }); |
| 58 | } |
| 59 | |
| 60 | return pending; |
| 61 | } |
| 62 | |
| 63 | export async function getPricing(): Promise<PricingMap> { |
| 64 | return pricingFromModels(await getModels()); |
| 65 | } |
| 66 | |