#!/usr/bin/env node /** * Exercise every route against mock upstreams. No real quota is spent. * * npm run smoke * * This is the thing to run after adding a provider or touching routing. It * boots two fake providers and a real gateway, then walks the whole surface — * routing, failover, streaming, sessions, cache, guard, probing, dashboard API. * * Exit code 0 if everything passed. */ import { once } from 'node:events'; import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { ContextStore, ExactCache, Registry, Router, SemanticCache, Store, Tokenizer, parseConfig, silentLogger, } from '../packages/core/src/index.ts'; import { createGateway } from '../packages/gateway/src/server.ts'; import { createMockUpstream } from './mock-upstream.js'; const C = process.stdout.isTTY ? { g: '\x1b[32m', r: '\x1b[31m', d: '\x1b[2m', B: '\x1b[1m', x: '\x1b[0m' } : { g: '', r: '', d: '', B: '', x: '' }; let passed = 0; let failed = 0; const failures = []; async function check(name, fn) { try { const detail = await fn(); passed += 1; console.log(` ${C.g}✓${C.x} ${name}${detail ? ` ${C.d}${detail}${C.x}` : ''}`); } catch (err) { failed += 1; failures.push({ name, message: err instanceof Error ? err.message : String(err) }); console.log(` ${C.r}✗${C.x} ${name}\n ${C.r}${err instanceof Error ? err.message : err}${C.x}`); } } function assert(condition, message) { if (!condition) throw new Error(message); } function eq(actual, expected, what) { if (actual !== expected) throw new Error(`${what}: expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`); } // --------------------------------------------------------------------------- const alpha = await createMockUpstream({ name: 'alpha' }); const beta = await createMockUpstream({ name: 'beta' }); const dir = mkdtempSync(join(tmpdir(), 'freeway-smoke-')); writeFileSync(join(dir, 'alpha.json'), JSON.stringify({ id: 'alpha', label: 'Alpha (mock)', priority: 10, baseUrl: alpha.url, auth: { type: 'bearer', envKeys: ['ALPHA_KEY'] }, limits: { rpm: 500 }, models: [ { id: 'alpha-chat', alias: ['fast', 'chat'], context: 128000, caps: ['chat', 'tools', 'json'] }, { id: 'alpha-embed', alias: ['embed'], context: 8000, caps: ['embed'] }, ], })); writeFileSync(join(dir, 'beta.json'), JSON.stringify({ id: 'beta', label: 'Beta (mock)', priority: 20, baseUrl: beta.url, auth: { type: 'bearer', envKeys: ['BETA_KEY'] }, limits: { rpm: 500 }, models: [{ id: '@cf/meta/llama-3.1-8b-instruct', alias: ['fast'], context: 8192, caps: ['chat'] }], })); const registry = new Registry({ dir, env: { ALPHA_KEY: 'ak-smoke-0001,ak-smoke-0002', BETA_KEY: 'bk-smoke-0001' }, logger: silentLogger }); const store = new Store({ file: null, logger: silentLogger }); const { config } = parseConfig({}, {}); config.cache.mode = 'safe'; const router = new Router(registry, store, config); const context = new ContextStore(':memory:'); const cache = new ExactCache({ file: ':memory:', mode: 'safe', ttlMs: 3_600_000, maxEntries: 100 }); const server = createGateway({ registry, store, router, config, logger: silentLogger, contextStore: context, tokenizer: new Tokenizer(), cache, semanticCache: new SemanticCache({ enabled: false, threshold: 0.92, maxEntries: 10 }), }); server.listen(0, '127.0.0.1'); await once(server, 'listening'); const B = `http://127.0.0.1:${server.address().port}`; const post = (path, body, headers = {}) => fetch(`${B}${path}`, { method: 'POST', headers: { 'content-type': 'application/json', ...headers }, body: JSON.stringify(body) }); const chat = (over = {}) => post('/v1/chat/completions', { model: 'auto', messages: [{ role: 'user', content: 'hello' }], ...over }); console.log(`\n${C.B}Freeway smoke test${C.x} ${C.d}${B}${C.x}\n`); // --------------------------------------------------------------------------- console.log(`${C.B}core routes${C.x}`); await check('GET /healthz', async () => { const body = await (await fetch(`${B}/healthz`)).json(); eq(body.ok, true, 'ok'); eq(body.providers.usable, 2, 'usable providers'); return `${body.providers.usable}/${body.providers.total} providers usable`; }); await check('GET / serves the dashboard', async () => { const res = await fetch(`${B}/`); eq(res.status, 200, 'status'); const html = await res.text(); assert(html.includes('Freeway'), 'no title'); assert(!/\b(?:src|href)\s*=\s*["'](?:https?:)?\/\//i.test(html), 'dashboard references an external asset'); return 'self-contained, no CDN'; }); await check('GET /v1/models', async () => { const body = await (await fetch(`${B}/v1/models`)).json(); const ids = body.data.map((d) => d.id); assert(ids.includes('alpha/alpha-chat'), 'missing provider/model id'); assert(ids.includes('beta/@cf/meta/llama-3.1-8b-instruct'), 'slashy id mangled'); assert(ids.includes('auto'), 'missing virtual model'); return `${ids.length} entries`; }); await check('POST /v1/chat/completions', async () => { const res = await chat(); eq(res.status, 200, 'status'); eq(res.headers.get('x-freeway-provider'), 'alpha', 'provider'); const body = await res.json(); assert(body.choices[0].message.content.includes('[alpha]'), 'wrong upstream answered'); return `served by ${res.headers.get('x-freeway-provider')} in ${res.headers.get('x-freeway-ms')}ms`; }); await check('POST /v1/embeddings', async () => { const res = await post('/v1/embeddings', { model: 'embed', input: ['a', 'b'] }); eq(res.status, 200, 'status'); eq(res.headers.get('x-freeway-model'), 'alpha-embed', 'model'); const body = await res.json(); eq(body.data.length, 2, 'vector count'); return '2 vectors'; }); await check('streaming', async () => { const res = await chat({ model: 'alpha/alpha-chat', stream: true }); eq(res.status, 200, 'status'); assert((res.headers.get('content-type') ?? '').includes('text/event-stream'), 'not SSE'); const text = await res.text(); assert(text.includes('data: [DONE]'), 'stream did not terminate'); return 'SSE proxied intact'; }); // --------------------------------------------------------------------------- console.log(`\n${C.B}routing and failover${C.x}`); await check('alias resolution', async () => { const res = await chat({ model: 'fast' }); eq(res.headers.get('x-freeway-route'), 'alias:fast', 'resolution'); return 'alias:fast'; }); await check('provider/model pin survives slashes', async () => { const res = await chat({ model: 'beta/@cf/meta/llama-3.1-8b-instruct' }); eq(res.status, 200, 'status'); eq(res.headers.get('x-freeway-model'), '@cf/meta/llama-3.1-8b-instruct', 'model'); return 'first-slash split'; }); await check('429 on alpha fails over to beta', async () => { alpha.script([{ status: 429, repeat: 99 }]); const res = await chat({ model: 'fast' }); eq(res.status, 200, 'status'); eq(res.headers.get('x-freeway-provider'), 'beta', 'provider'); alpha.script([]); store.provider('alpha').cooldownUntil = 0; for (const k of store.provider('alpha').keys.values()) k.cooldownUntil = 0; return `failed over after ${res.headers.get('x-freeway-attempts')} attempts`; }); await check('400 returns immediately without retrying', async () => { alpha.script([{ status: 400, body: { error: { message: 'bad param' } }, repeat: 1 }, { status: 200, repeat: 99 }]); const before = beta.callsTo('/chat/completions').length; const res = await chat({ model: 'alpha/alpha-chat' }); eq(res.status, 400, 'status'); eq(beta.callsTo('/chat/completions').length, before, 'beta should not have been tried'); alpha.script([]); return 'no wasted retry'; }); await check('unknown model returns 404 with suggestions', async () => { const res = await chat({ model: 'gpt-9-ultra' }); eq(res.status, 404, 'status'); const body = await res.json(); eq(body.error.code, 'model_not_found', 'code'); return `${body.freeway.suggestions.length} suggestions`; }); await check('exhausted quota returns 429 with diagnostics', async () => { const saved = registry.get('alpha').spec.limits.rpm; registry.get('alpha').spec.limits.rpm = 1; registry.get('beta').spec.limits.rpm = 1; for (let i = 0; i < 3; i++) { store.recordSuccess('alpha', 'alpha#0', 1); store.recordSuccess('beta', 'beta#0', 1); } const res = await chat({ model: 'fast' }); eq(res.status, 429, 'status'); const body = await res.json(); eq(body.error.code, 'all_providers_blocked', 'code'); assert(body.freeway.blocked.length > 0, 'no blocked reasons'); assert(/rpm limit reached/.test(body.freeway.blocked[0].reason), `reason not diagnostic: ${body.freeway.blocked[0].reason}`); registry.get('alpha').spec.limits.rpm = saved; registry.get('beta').spec.limits.rpm = saved; return body.freeway.blocked[0].reason; }); // --------------------------------------------------------------------------- console.log(`\n${C.B}context engine${C.x}`); await check('session rebuilds the thread', async () => { await post('/v1/chat/completions', { model: 'fast', session: 'smoke', messages: [{ role: 'user', content: 'my name is Ada' }] }); const res = await post('/v1/chat/completions', { model: 'fast', session: 'smoke', messages: [{ role: 'user', content: 'what is my name?' }] }); eq(res.status, 200, 'status'); eq(res.headers.get('x-freeway-session'), 'smoke', 'session header'); const stored = context.messages('smoke'); assert(stored.length >= 3, `expected stored history, got ${stored.length}`); return `${stored.length} messages stored`; }); await check('refit reshapes history for a smaller model', async () => { const turn = 'padding '.repeat(300); const messages = [{ role: 'system', content: 'be helpful' }]; for (let i = 0; i < 20; i++) messages.push({ role: i % 2 ? 'assistant' : 'user', content: `${i}: ${turn}` }); messages.push({ role: 'user', content: 'short question' }); alpha.script([{ status: 429, repeat: 99 }]); const res = await post('/v1/chat/completions', { model: 'fast', messages }); eq(res.status, 200, 'status'); eq(res.headers.get('x-freeway-provider'), 'beta', 'provider'); const note = res.headers.get('x-freeway-context'); assert(note, 'no x-freeway-context header'); alpha.script([]); store.provider('alpha').cooldownUntil = 0; for (const k of store.provider('alpha').keys.values()) k.cooldownUntil = 0; return note; }); await check('GET /api/sessions', async () => { const body = await (await fetch(`${B}/api/sessions`)).json(); assert(body.sessions.some((s) => s.id === 'smoke'), 'session missing'); return `${body.sessions.length} sessions`; }); // --------------------------------------------------------------------------- console.log(`\n${C.B}cache${C.x}`); await check('identical request hits the cache', async () => { const q = { model: 'alpha/alpha-chat', temperature: 0, messages: [{ role: 'user', content: 'cache me' }] }; await post('/v1/chat/completions', q); const before = alpha.callsTo('/chat/completions').length; const res = await post('/v1/chat/completions', q); eq(res.headers.get('x-freeway-cache'), 'exact', 'cache header'); eq(alpha.callsTo('/chat/completions').length, before, 'upstream was called on a hit'); return 'no upstream call, no quota'; }); await check('GET /api/cache', async () => { const body = await (await fetch(`${B}/api/cache`)).json(); eq(body.enabled, true, 'enabled'); assert(body.exact.hits >= 1, 'no hits recorded'); return `${body.exact.entries} entries, ${body.exact.hits} hits, ${body.exact.tokensSaved} tokens saved`; }); // --------------------------------------------------------------------------- console.log(`\n${C.B}guard and probing${C.x}`); await check('GET /api/alerts', async () => { const body = await (await fetch(`${B}/api/alerts`)).json(); assert(Array.isArray(body.alerts), 'no alerts array'); assert(Array.isArray(body.forecasts), 'no forecasts'); return `${body.alerts.length} alerts, ${body.forecasts.length} forecasts`; }); await check('POST /api/providers/:id/probe discovers models', async () => { const body = await (await post('/api/providers/alpha/probe', { validate: false })).json(); const result = body.results[0]; eq(result.auth, 'ok', 'auth'); assert(result.discovered.length > 0, 'discovered nothing'); return `${result.discovered.length} models discovered`; }); await check('content scanning flags a planted key', async () => { const res = await chat({ model: 'alpha/alpha-chat', messages: [{ role: 'user', content: 'deploy with sk-abcdefghijklmnopqrstuvwxyz12' }] }); eq(res.status, 200, 'flag mode should not block'); const log = store.logs(1)[0]; return `logged, request still served (mode=${config.guard.scan.mode})`; }); // --------------------------------------------------------------------------- console.log(`\n${C.B}dashboard API${C.x}`); await check('GET /api/providers masks every key', async () => { const body = await (await fetch(`${B}/api/providers`)).json(); const raw = JSON.stringify(body); assert(!raw.includes('ak-smoke-0001'), 'a raw provider key leaked into /api/providers'); assert(!raw.includes('bk-smoke-0001'), 'a raw provider key leaked into /api/providers'); const alphaCard = body.providers.find((p) => p.id === 'alpha'); eq(alphaCard.keys.length, 2, 'key pool size'); return `keys shown as ${alphaCard.keys[0].masked}`; }); await check('GET /api/stats', async () => { const body = await (await fetch(`${B}/api/stats`)).json(); eq(body.providers.total, 2, 'provider count'); return `${body.requests.ok} ok / ${body.requests.fail} failed`; }); await check('GET /api/logs', async () => { const body = await (await fetch(`${B}/api/logs?limit=10`)).json(); assert(body.logs.length > 0, 'log is empty'); return `${body.logs.length} entries`; }); await check('GET /api/events streams live', async () => { const controller = new AbortController(); const res = await fetch(`${B}/api/events`, { signal: controller.signal, headers: { accept: 'text/event-stream' } }); eq(res.status, 200, 'status'); const reader = res.body.getReader(); const first = await reader.read(); const text = new TextDecoder().decode(first.value); assert(text.includes('event: hello'), 'no hello frame'); controller.abort(); return 'SSE connected'; }); await check('POST /api/providers/:id/enabled toggles', async () => { await post('/api/providers/alpha/enabled', { enabled: false }); const res = await chat({ model: 'fast' }); eq(res.headers.get('x-freeway-provider'), 'beta', 'should route around the disabled provider'); await post('/api/providers/alpha/enabled', { enabled: true }); return 'routed around, then restored'; }); await check('POST /api/reload', async () => { const body = await (await post('/api/reload', {})).json(); eq(body.ok, true, 'ok'); return `${body.providers} providers reloaded`; }); // --------------------------------------------------------------------------- console.log(`\n${C.B}shipped provider catalog${C.x}`); await check('every providers/*.json parses', async () => { const real = new Registry({ dir: 'providers', env: {}, logger: silentLogger }); const errors = real.issues().filter((i) => i.level === 'error'); assert(errors.length === 0, errors.map((e) => `${e.file}: ${e.message}`).join('; ')); return `${real.all().length} providers, ${real.allModels().length} models`; }); // --------------------------------------------------------------------------- server.close(); context.close(); cache.close(); await alpha.close(); await beta.close(); rmSync(dir, { recursive: true, force: true }); const colour = failed === 0 ? C.g : C.r; console.log(`\n${colour}${C.B}${failed === 0 ? 'PASS' : 'FAIL'}${C.x} — ${passed} passed, ${failed} failed\n`); if (failed > 0) { for (const f of failures) console.log(` ${C.r}${f.name}${C.x}: ${f.message}`); } process.exit(failed === 0 ? 0 : 1);