Freeway: self-hosted OpenAI-compatible gateway for free-tier providers

One endpoint in front of every free LLM tier, with automatic failover and no
provider key ever leaving the box.

Providers are data, not code: adding one is a JSON file plus an env var, never
a TypeScript change. Ships 12 providers pre-written; `verify-provider --apply`
derives the live model list from your own key.

- core: registry, bucketed quota meter (rolling + UTC-calendar windows),
  key-pool store, 6-step model router with diagnostic rejection reasons
- gateway: node:http, failover that commits to a stream only after 200 headers,
  SSE passthrough with usage sniffing, virtual keys with admin/inference roles
- context engine: sqlite sessions and a 5-rung refit ladder, so a 128k thread
  keeps working when it fails over to an 8k model
- cache: exact (sha256, safe-by-default) + opt-in semantic (flat Float32Array)
- guard: inbound limits, quota forecasting, limit calibration from upstream
  headers, key health, secret/PII/injection scanning
- dashboard: one self-contained HTML file, live SSE request log, token usage
- sdk: zero-dep typed client, drop-in for the OpenAI SDK

Zero runtime dependencies. 326 tests, 27-check smoke suite against a scripted
mock upstream so routing is testable without spending quota.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

packages/core/test/cache.test.ts

@@ -0,0 +1,264 @@
+import { strict as assert } from 'node:assert';
+import { describe, it } from 'node:test';
+
+import { ExactCache, cacheKey, isCacheable, replayAsStream } from '../src/cache/exact.ts';
+import { SemanticCache, queryTextOf, systemPromptOf } from '../src/cache/semantic.ts';
+
+const opts = { file: ':memory:', mode: 'safe' as const, ttlMs: 3_600_000, maxEntries: 100 };
+const body = (over: Record<string, unknown> = {}) => ({
+  model: 'auto',
+  messages: [{ role: 'user', content: 'what is the capital of France?' }],
+  temperature: 0,
+  ...over,
+});
+
+describe('cacheKey — what counts as the same request', () => {
+  it('is stable regardless of key order', () => {
+    const a = cacheKey('auto', { temperature: 0, messages: [{ role: 'user', content: 'hi' }] });
+    const b = cacheKey('auto', { messages: [{ role: 'user', content: 'hi' }], temperature: 0 });
+    assert.equal(a, b);
+  });
+
+  it('changes when anything that affects the answer changes', () => {
+    const base = cacheKey('auto', body());
+    assert.notEqual(base, cacheKey('auto', body({ temperature: 0.5 })));
+    assert.notEqual(base, cacheKey('auto', body({ max_tokens: 10 })));
+    assert.notEqual(base, cacheKey('auto', body({ messages: [{ role: 'user', content: 'different' }] })));
+    assert.notEqual(base, cacheKey('auto', body({ response_format: { type: 'json_object' } })));
+  });
+
+  it('ignores fields that cannot change the answer', () => {
+    assert.equal(cacheKey('auto', body()), cacheKey('auto', body({ user: 'alice', metadata: { trace: 'x' } })));
+  });
+
+  it('keys on the resolution, so two aliases for the same models share an entry', () => {
+    assert.equal(cacheKey('alias:fast', body()), cacheKey('alias:fast', body()));
+    assert.notEqual(cacheKey('alias:fast', body()), cacheKey('pin:groq/llama', body()));
+  });
+});
+
+describe('isCacheable — safe mode only caches deterministic calls', () => {
+  it('caches temperature 0', () => {
+    assert.equal(isCacheable('safe', body({ temperature: 0 })), true);
+    assert.equal(isCacheable('safe', body({ temperature: 0.1 })), true);
+  });
+
+  it('refuses a creative call', () => {
+    // Replaying a temperature 0.9 answer turns a creative endpoint into a
+    // broken record, which is worse than not caching at all.
+    assert.equal(isCacheable('safe', body({ temperature: 0.9 })), false);
+    assert.equal(isCacheable('safe', body({ temperature: undefined })), false, 'the provider default is not deterministic');
+  });
+
+  it('caches anything with an explicit seed', () => {
+    assert.equal(isCacheable('safe', body({ temperature: 1, seed: 42 })), true);
+  });
+
+  it('aggressive caches everything, off caches nothing', () => {
+    assert.equal(isCacheable('aggressive', body({ temperature: 1.5 })), true);
+    assert.equal(isCacheable('off', body({ temperature: 0 })), false);
+  });
+
+  it('leaves streams alone unless asked', () => {
+    assert.equal(isCacheable('safe', body({ stream: true })), false);
+    assert.equal(isCacheable('aggressive', body({ stream: true })), true);
+  });
+});
+
+describe('ExactCache', () => {
+  it('returns what it stored', () => {
+    const cache = new ExactCache(opts);
+    cache.put('k1', '{"answer":"Paris"}', 'mistral', 'mistral-small', 10, 5);
+
+    const hit = cache.get('k1');
+    assert.equal(hit?.body, '{"answer":"Paris"}');
+    assert.equal(hit?.providerId, 'mistral');
+    assert.equal(hit?.tokensOut, 5);
+    cache.close();
+  });
+
+  it('misses on an unknown key', () => {
+    const cache = new ExactCache(opts);
+    assert.equal(cache.get('nope'), null);
+    assert.equal(cache.stats().misses, 1);
+    cache.close();
+  });
+
+  it('expires entries past their TTL', () => {
+    const cache = new ExactCache({ ...opts, ttlMs: 1000 });
+    const past = Date.now() - 5000;
+    cache.put('k1', '{}', 'p', 'm', 1, 1, past);
+    assert.equal(cache.get('k1'), null, 'a stale answer is worse than none');
+    cache.close();
+  });
+
+  it('evicts least-recently-used once full', () => {
+    const cache = new ExactCache({ ...opts, maxEntries: 3 });
+    for (let i = 0; i < 3; i++) cache.put(`k${i}`, '{}', 'p', 'm', 1, 1);
+    cache.get('k0'); // k0 becomes most recent, k1 the oldest
+    cache.put('k3', '{}', 'p', 'm', 1, 1);
+
+    assert.ok(cache.get('k0'), 'the recently used entry survived');
+    assert.equal(cache.get('k1'), null, 'the least recently used was evicted');
+    cache.close();
+  });
+
+  it('tracks the tokens it saved', () => {
+    const cache = new ExactCache(opts);
+    cache.put('k1', '{}', 'p', 'm', 100, 50);
+    cache.get('k1');
+    cache.get('k1');
+
+    const stats = cache.stats();
+    assert.equal(stats.hits, 2);
+    assert.equal(stats.tokensSaved, 300);
+    assert.equal(stats.hitRate, 1);
+    cache.close();
+  });
+
+  it('stores nothing at all when off', () => {
+    const cache = new ExactCache({ ...opts, mode: 'off' });
+    cache.put('k1', '{}', 'p', 'm', 1, 1);
+    assert.equal(cache.get('k1'), null);
+    cache.close();
+  });
+});
+
+describe('replayAsStream — a hit must be indistinguishable from a live call', () => {
+  const completion = JSON.stringify({
+    id: 'chatcmpl-1',
+    model: 'mistral-small',
+    choices: [{ index: 0, message: { role: 'assistant', content: 'Paris is the capital of France.' } }],
+    usage: { prompt_tokens: 10, completion_tokens: 8, total_tokens: 18 },
+  });
+
+  it('emits well-formed SSE ending in [DONE]', () => {
+    const sse = replayAsStream(completion);
+    assert.match(sse, /^data: /m);
+    assert.ok(sse.trimEnd().endsWith('data: [DONE]'));
+  });
+
+  it('reassembles to exactly the cached text', () => {
+    const content = [...replayAsStream(completion).matchAll(/^data: (.+)$/gm)]
+      .map((m) => m[1])
+      .filter((p): p is string => p !== undefined && p !== '[DONE]')
+      .map((p) => JSON.parse(p) as { choices?: { delta?: { content?: string } }[] })
+      .flatMap((o) => o.choices ?? [])
+      .map((c) => c.delta?.content ?? '')
+      .join('');
+    assert.equal(content, 'Paris is the capital of France.');
+  });
+
+  it('carries the usage block through', () => {
+    assert.match(replayAsStream(completion), /"usage":\{"prompt_tokens":10/);
+  });
+
+  it('degrades to a bare [DONE] on unparseable input', () => {
+    assert.equal(replayAsStream('not json'), 'data: [DONE]\n\n');
+  });
+});
+
+// ---------------------------------------------------------------------------
+
+/** Deterministic unit-ish vector, so similarity is predictable in tests. */
+function vec(seed: number, dims = 8): number[] {
+  return Array.from({ length: dims }, (_, i) => Math.sin(seed + i * 0.7));
+}
+
+describe('SemanticCache', () => {
+  const make = (over = {}) => new SemanticCache({ enabled: true, threshold: 0.92, maxEntries: 100, dims: 8, ...over });
+
+  it('matches a near-identical vector above the threshold', () => {
+    const cache = make();
+    const v = vec(1);
+    cache.add(v, 'key-1', 'scope-a', 'what is the capital of France?');
+
+    const hit = cache.search(v, 'scope-a');
+    assert.ok(hit, 'an identical vector must hit');
+    assert.equal(hit.entry.cacheKey, 'key-1');
+    assert.ok(hit.score > 0.99);
+  });
+
+  it('misses when similarity falls below the threshold', () => {
+    const cache = make();
+    cache.add(vec(1), 'key-1', 'scope-a', 'q1');
+    assert.equal(cache.search(vec(50), 'scope-a'), null, 'an unrelated question must not hit');
+  });
+
+  it('never crosses scopes', () => {
+    // A different system prompt can make the same question have a different
+    // correct answer, so entries are partitioned rather than compared globally.
+    const cache = make();
+    const v = vec(1);
+    cache.add(v, 'key-1', 'pirate-prompt', 'who are you?');
+    assert.equal(cache.search(v, 'lawyer-prompt'), null);
+    assert.ok(cache.search(v, 'pirate-prompt'));
+  });
+
+  it('derives a scope from resolution and system prompt', () => {
+    const a = SemanticCache.scopeOf('alias:fast', 'You are a pirate.');
+    const b = SemanticCache.scopeOf('alias:fast', 'You are a lawyer.');
+    const c = SemanticCache.scopeOf('alias:fast', 'You are a pirate.');
+    assert.notEqual(a, b);
+    assert.equal(a, c, 'the same inputs must give the same scope');
+  });
+
+  it('respects the TTL', () => {
+    const cache = make({ ttlMs: 1000 });
+    const v = vec(1);
+    cache.add(v, 'key-1', 'scope-a', 'q', Date.now() - 5000);
+    assert.equal(cache.search(v, 'scope-a'), null);
+  });
+
+  it('does nothing when disabled', () => {
+    const cache = new SemanticCache({ enabled: false, threshold: 0.9, maxEntries: 10, dims: 8 });
+    cache.add(vec(1), 'k', 's', 'q');
+    assert.equal(cache.search(vec(1), 's'), null);
+  });
+
+  it('remembers a recent miss so the same text is not re-embedded', () => {
+    // Every miss costs an embedding call against real quota.
+    const cache = make();
+    assert.equal(cache.recentlyMissed('s', 'q'), false);
+    cache.noteMiss('s', 'q');
+    assert.equal(cache.recentlyMissed('s', 'q'), true);
+    assert.equal(cache.recentlyMissed('s', 'other'), false);
+  });
+
+  it('stays within its memory budget', () => {
+    const cache = new SemanticCache({ enabled: true, threshold: 0.92, maxEntries: 5000, dims: 1024 });
+    const stats = cache.stats();
+    assert.equal(stats.bytes, 5000 * 1024 * 4);
+    assert.ok(stats.bytes < 25 * 1024 * 1024, 'roughly 20MB, no vector database required');
+  });
+
+  it('adopts the provider’s embedding width on first use', () => {
+    // bge-large is 1024, bge-small is 384; guessing wrong wastes the slab.
+    const cache = new SemanticCache({ enabled: true, threshold: 0.9, maxEntries: 10, dims: 1024 });
+    cache.add(vec(1, 384), 'k', 's', 'q');
+    assert.equal(cache.stats().dims, 384);
+    assert.ok(cache.search(vec(1, 384), 's'));
+  });
+});
+
+describe('semantic cache helpers', () => {
+  it('keys on the newest user turn', () => {
+    assert.equal(
+      queryTextOf([
+        { role: 'system', content: 'sys' },
+        { role: 'user', content: 'first' },
+        { role: 'assistant', content: 'answer' },
+        { role: 'user', content: 'second' },
+      ]),
+      'second',
+    );
+  });
+
+  it('flattens a multi-part user turn', () => {
+    assert.equal(queryTextOf([{ role: 'user', content: [{ type: 'text', text: 'describe this' }] }]), 'describe this');
+  });
+
+  it('collects the system prompt', () => {
+    assert.equal(systemPromptOf([{ role: 'system', content: 'be terse' }, { role: 'user', content: 'hi' }]), 'be terse');
+  });
+});