Freeway: self-hosted OpenAI-compatible gateway for free-tier providers
One endpoint in front of every free LLM tier, with automatic failover and no provider key ever leaving the box. Providers are data, not code: adding one is a JSON file plus an env var, never a TypeScript change. Ships 12 providers pre-written; `verify-provider --apply` derives the live model list from your own key. - core: registry, bucketed quota meter (rolling + UTC-calendar windows), key-pool store, 6-step model router with diagnostic rejection reasons - gateway: node:http, failover that commits to a stream only after 200 headers, SSE passthrough with usage sniffing, virtual keys with admin/inference roles - context engine: sqlite sessions and a 5-rung refit ladder, so a 128k thread keeps working when it fails over to an 8k model - cache: exact (sha256, safe-by-default) + opt-in semantic (flat Float32Array) - guard: inbound limits, quota forecasting, limit calibration from upstream headers, key health, secret/PII/injection scanning - dashboard: one self-contained HTML file, live SSE request log, token usage - sdk: zero-dep typed client, drop-in for the OpenAI SDK Zero runtime dependencies. 326 tests, 27-check smoke suite against a scripted mock upstream so routing is testable without spending quota. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Aryan SinghandClaude Opus 5 committed
commit
cb985dcf9c
86 files changed
+16350
No files matched your search
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"name": "@freeway/sdk",
|
||||
"version": "0.1.0",
|
||||
"type": "module",
|
||||
"main": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts",
|
||||
"exports": {
|
||||
".": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
}
|
||||
},
|
||||
"files": [
|
||||
"dist"
|
||||
],
|
||||
"sideEffects": false,
|
||||
"scripts": {
|
||||
"test": "node --test test/*.test.ts"
|
||||
},
|
||||
"private": true
|
||||
}
|
||||
@@ -0,0 +1,521 @@
|
||||
/**
|
||||
* @freeway/sdk — a small typed client for a Freeway gateway.
|
||||
*
|
||||
* Zero dependencies, runs in Node and the browser. Typed against the parts of
|
||||
* the OpenAI shapes that actually get used rather than the whole surface, and
|
||||
* every result carries a `.route` telling you which provider served it.
|
||||
*/
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// OpenAI-ish types (only the parts we use)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type Role = 'system' | 'user' | 'assistant' | 'tool';
|
||||
|
||||
export interface TextPart {
|
||||
type: 'text';
|
||||
text: string;
|
||||
}
|
||||
|
||||
export interface ImagePart {
|
||||
type: 'image_url';
|
||||
image_url: { url: string; detail?: 'auto' | 'low' | 'high' };
|
||||
}
|
||||
|
||||
export type ContentPart = TextPart | ImagePart;
|
||||
|
||||
export interface Message {
|
||||
role: Role;
|
||||
content: string | ContentPart[] | null;
|
||||
name?: string;
|
||||
tool_call_id?: string;
|
||||
tool_calls?: ToolCall[];
|
||||
}
|
||||
|
||||
export interface ToolCall {
|
||||
id: string;
|
||||
type: 'function';
|
||||
function: { name: string; arguments: string };
|
||||
}
|
||||
|
||||
export interface Tool {
|
||||
type: 'function';
|
||||
function: { name: string; description?: string; parameters?: Record<string, unknown> };
|
||||
}
|
||||
|
||||
export interface Usage {
|
||||
prompt_tokens: number;
|
||||
completion_tokens: number;
|
||||
total_tokens: number;
|
||||
}
|
||||
|
||||
export interface ChatRequest {
|
||||
model: string;
|
||||
messages: Message[];
|
||||
temperature?: number;
|
||||
top_p?: number;
|
||||
max_tokens?: number;
|
||||
stop?: string | string[];
|
||||
stream?: boolean;
|
||||
tools?: Tool[];
|
||||
tool_choice?: 'auto' | 'none' | 'required' | { type: 'function'; function: { name: string } };
|
||||
response_format?: { type: 'text' | 'json_object' | 'json_schema'; json_schema?: Record<string, unknown> };
|
||||
/**
|
||||
* Freeway extension: server-side conversation id. Send only the new message
|
||||
* and the gateway rehydrates the thread, refitting it to whichever model ends
|
||||
* up serving the call.
|
||||
*/
|
||||
session?: string;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export interface Choice {
|
||||
index: number;
|
||||
message: { role: 'assistant'; content: string | null; tool_calls?: ToolCall[] };
|
||||
finish_reason: string | null;
|
||||
}
|
||||
|
||||
export interface ChatCompletion {
|
||||
id: string;
|
||||
object: string;
|
||||
created: number;
|
||||
model: string;
|
||||
choices: Choice[];
|
||||
usage?: Usage;
|
||||
}
|
||||
|
||||
export interface ChunkChoice {
|
||||
index: number;
|
||||
delta: { role?: Role; content?: string; tool_calls?: Partial<ToolCall>[] };
|
||||
finish_reason: string | null;
|
||||
}
|
||||
|
||||
export interface ChatChunk {
|
||||
id: string;
|
||||
object: string;
|
||||
created: number;
|
||||
model: string;
|
||||
choices: ChunkChoice[];
|
||||
usage?: Usage;
|
||||
}
|
||||
|
||||
export interface EmbeddingResponse {
|
||||
object: string;
|
||||
model: string;
|
||||
data: { object: string; index: number; embedding: number[] }[];
|
||||
usage?: Usage;
|
||||
}
|
||||
|
||||
export interface ModelEntry {
|
||||
id: string;
|
||||
object: string;
|
||||
owned_by: string;
|
||||
freeway?: {
|
||||
virtual?: boolean;
|
||||
provider?: string;
|
||||
model?: string;
|
||||
aliases?: string[];
|
||||
caps?: string[];
|
||||
context?: number | null;
|
||||
enabled?: boolean;
|
||||
configured?: boolean;
|
||||
resolvesTo?: string[];
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Freeway-specific
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** What the `x-freeway-*` response headers said about this call. */
|
||||
export interface RouteInfo {
|
||||
provider: string | null;
|
||||
model: string | null;
|
||||
/** How the model string was interpreted, e.g. `alias:fast`. */
|
||||
route: string | null;
|
||||
attempts: number;
|
||||
ms: number;
|
||||
/** Whether token counts were estimated rather than reported by the provider. */
|
||||
estimated: boolean;
|
||||
/** Cache tier that served it, when the gateway's cache is enabled. */
|
||||
cache: string | null;
|
||||
/** How the context engine reshaped the history, when sessions are in use. */
|
||||
context: string | null;
|
||||
}
|
||||
|
||||
export interface BlockedProvider {
|
||||
provider: string;
|
||||
model: string;
|
||||
reason: string;
|
||||
retryAfterMs: number | null;
|
||||
}
|
||||
|
||||
/** Thrown for any non-2xx reply, carrying the gateway's diagnostics intact. */
|
||||
export class FreewayError extends Error {
|
||||
readonly status: number;
|
||||
readonly code: string | null;
|
||||
readonly type: string | null;
|
||||
readonly route: RouteInfo;
|
||||
/** Per-provider rejection reasons — the whole point of a 429 from Freeway. */
|
||||
readonly blocked: BlockedProvider[];
|
||||
readonly suggestions: string[];
|
||||
readonly retryAfterMs: number | null;
|
||||
readonly body: unknown;
|
||||
|
||||
constructor(init: {
|
||||
message: string;
|
||||
status: number;
|
||||
code?: string | null;
|
||||
type?: string | null;
|
||||
route: RouteInfo;
|
||||
blocked?: BlockedProvider[];
|
||||
suggestions?: string[];
|
||||
retryAfterMs?: number | null;
|
||||
body?: unknown;
|
||||
}) {
|
||||
super(init.message);
|
||||
this.name = 'FreewayError';
|
||||
this.status = init.status;
|
||||
this.code = init.code ?? null;
|
||||
this.type = init.type ?? null;
|
||||
this.route = init.route;
|
||||
this.blocked = init.blocked ?? [];
|
||||
this.suggestions = init.suggestions ?? [];
|
||||
this.retryAfterMs = init.retryAfterMs ?? null;
|
||||
this.body = init.body;
|
||||
}
|
||||
}
|
||||
|
||||
export type Routed<T> = T & { route: RouteInfo };
|
||||
|
||||
export interface FreewayOptions {
|
||||
/** e.g. `http://localhost:8787/v1`. A trailing `/v1` is added if missing. */
|
||||
baseUrl?: string;
|
||||
apiKey?: string;
|
||||
/** Total attempts per call, including the first. Default 3. */
|
||||
maxRetries?: number;
|
||||
/** Per-request timeout. Default 60s (300s for streams). */
|
||||
timeoutMs?: number;
|
||||
streamTimeoutMs?: number;
|
||||
headers?: Record<string, string>;
|
||||
fetch?: typeof globalThis.fetch;
|
||||
}
|
||||
|
||||
export interface RequestOptions {
|
||||
signal?: AbortSignal;
|
||||
timeoutMs?: number;
|
||||
headers?: Record<string, string>;
|
||||
maxRetries?: number;
|
||||
}
|
||||
|
||||
export interface ChatOptions extends RequestOptions {
|
||||
model?: string;
|
||||
system?: string;
|
||||
temperature?: number;
|
||||
max_tokens?: number;
|
||||
session?: string;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Client
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface ChatFn {
|
||||
/** Prompt in, assistant text out. */
|
||||
(prompt: string, options?: ChatOptions): Promise<string>;
|
||||
/** The full completion, plus `.route`. */
|
||||
raw(body: ChatRequest, options?: RequestOptions): Promise<Routed<ChatCompletion>>;
|
||||
}
|
||||
|
||||
export interface FreewayClient {
|
||||
chat: ChatFn;
|
||||
stream(body: ChatRequest, options?: RequestOptions): AsyncGenerator<ChatChunk, RouteInfo, undefined>;
|
||||
/** Streamed text only, for when the chunks themselves are not interesting. */
|
||||
streamText(body: ChatRequest, options?: RequestOptions): AsyncGenerator<string, RouteInfo, undefined>;
|
||||
embed(input: string | string[], options?: RequestOptions & { model?: string }): Promise<Routed<number[][]>>;
|
||||
models(options?: RequestOptions): Promise<ModelEntry[]>;
|
||||
health(options?: RequestOptions): Promise<{ ok: boolean; providers: { total: number; usable: number } }>;
|
||||
/**
|
||||
* Drop-in config for the official OpenAI SDK:
|
||||
* `new OpenAI(fw.openai())` — existing apps keep working unchanged.
|
||||
*/
|
||||
openai(): { baseURL: string; apiKey: string; defaultHeaders?: Record<string, string> };
|
||||
readonly baseUrl: string;
|
||||
readonly apiKey: string;
|
||||
}
|
||||
|
||||
const DEFAULT_BASE = 'http://localhost:8787/v1';
|
||||
|
||||
export function createFreeway(options: FreewayOptions = {}): FreewayClient {
|
||||
const baseUrl = normalizeBase(options.baseUrl ?? DEFAULT_BASE);
|
||||
const apiKey = options.apiKey ?? '';
|
||||
const maxRetries = Math.max(1, options.maxRetries ?? 3);
|
||||
const timeoutMs = options.timeoutMs ?? 60_000;
|
||||
const streamTimeoutMs = options.streamTimeoutMs ?? 300_000;
|
||||
const doFetch = options.fetch ?? globalThis.fetch.bind(globalThis);
|
||||
|
||||
function headersFor(extra?: Record<string, string>): Record<string, string> {
|
||||
const h: Record<string, string> = { 'content-type': 'application/json', ...options.headers, ...extra };
|
||||
if (apiKey) h['authorization'] = `Bearer ${apiKey}`;
|
||||
return h;
|
||||
}
|
||||
|
||||
/**
|
||||
* One HTTP attempt. Retries live in `send` so streaming can share them: a
|
||||
* stream that fails before its first byte is still safe to retry.
|
||||
*/
|
||||
async function attempt(path: string, init: RequestInit, opts: RequestOptions | undefined, deadlineMs: number): Promise<Response> {
|
||||
const controller = new AbortController();
|
||||
// A signal that is *already* aborted never fires an `abort` event, so
|
||||
// listening alone would silently let the request through.
|
||||
if (opts?.signal?.aborted) controller.abort(opts.signal.reason);
|
||||
const timer = setTimeout(() => controller.abort(), opts?.timeoutMs ?? deadlineMs);
|
||||
const onAbort = (): void => controller.abort(opts?.signal?.reason);
|
||||
opts?.signal?.addEventListener('abort', onAbort, { once: true });
|
||||
try {
|
||||
return await doFetch(`${baseUrl}${path}`, { ...init, signal: controller.signal });
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
opts?.signal?.removeEventListener('abort', onAbort);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The gateway has already failed over across every provider it has, so a
|
||||
* retry here is a last resort for a transient gateway or network blip —
|
||||
* hence full jittered exponential backoff rather than a tight loop.
|
||||
*/
|
||||
async function send(path: string, init: RequestInit, opts: RequestOptions | undefined, deadlineMs: number): Promise<Response> {
|
||||
const attempts = Math.max(1, opts?.maxRetries ?? maxRetries);
|
||||
let lastError: unknown;
|
||||
|
||||
for (let i = 0; i < attempts; i++) {
|
||||
if (opts?.signal?.aborted) throw abortError(opts.signal.reason);
|
||||
let res: Response;
|
||||
try {
|
||||
res = await attempt(path, init, opts, deadlineMs);
|
||||
} catch (err) {
|
||||
lastError = err;
|
||||
if (opts?.signal?.aborted) throw err;
|
||||
if (i === attempts - 1) break;
|
||||
await sleep(backoffMs(i, null));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (res.ok || !isRetryable(res.status) || i === attempts - 1) return res;
|
||||
|
||||
// Honour the gateway's own Retry-After — it knows when the upstream
|
||||
// window actually reopens, which is better than any local guess.
|
||||
const retryAfter = parseRetryAfter(res.headers.get('retry-after'));
|
||||
res.body?.cancel().catch(() => {});
|
||||
await sleep(backoffMs(i, retryAfter));
|
||||
}
|
||||
|
||||
throw lastError instanceof Error
|
||||
? lastError
|
||||
: new FreewayError({ message: String(lastError ?? 'request failed'), status: 0, route: emptyRoute() });
|
||||
}
|
||||
|
||||
async function json<T>(path: string, body: unknown, opts?: RequestOptions): Promise<{ data: T; route: RouteInfo }> {
|
||||
const res = await send(path, { method: 'POST', headers: headersFor(opts?.headers), body: JSON.stringify(body) }, opts, timeoutMs);
|
||||
const route = routeFrom(res.headers);
|
||||
const parsed: unknown = await res.json().catch(() => null);
|
||||
if (!res.ok) throw errorFrom(res, parsed, route);
|
||||
return { data: parsed as T, route };
|
||||
}
|
||||
|
||||
const chat = (async (prompt: string, opts: ChatOptions = {}): Promise<string> => {
|
||||
const messages: Message[] = [];
|
||||
if (opts.system) messages.push({ role: 'system', content: opts.system });
|
||||
messages.push({ role: 'user', content: prompt });
|
||||
|
||||
const body: ChatRequest = { model: opts.model ?? 'auto', messages };
|
||||
if (opts.temperature !== undefined) body.temperature = opts.temperature;
|
||||
if (opts.max_tokens !== undefined) body.max_tokens = opts.max_tokens;
|
||||
if (opts.session !== undefined) body.session = opts.session;
|
||||
|
||||
const { data } = await json<ChatCompletion>('/chat/completions', body, opts);
|
||||
return data.choices[0]?.message.content ?? '';
|
||||
}) as ChatFn;
|
||||
|
||||
chat.raw = async (body: ChatRequest, opts?: RequestOptions): Promise<Routed<ChatCompletion>> => {
|
||||
const { data, route } = await json<ChatCompletion>('/chat/completions', { ...body, stream: false }, opts);
|
||||
return Object.assign(data, { route });
|
||||
};
|
||||
|
||||
async function* stream(body: ChatRequest, opts?: RequestOptions): AsyncGenerator<ChatChunk, RouteInfo, undefined> {
|
||||
const res = await send(
|
||||
'/chat/completions',
|
||||
{ method: 'POST', headers: headersFor({ accept: 'text/event-stream', ...opts?.headers }), body: JSON.stringify({ ...body, stream: true }) },
|
||||
opts,
|
||||
streamTimeoutMs,
|
||||
);
|
||||
const route = routeFrom(res.headers);
|
||||
if (!res.ok) throw errorFrom(res, await res.json().catch(() => null), route);
|
||||
if (!res.body) return route;
|
||||
|
||||
for await (const payload of sseLines(res.body)) {
|
||||
if (payload === '[DONE]') break;
|
||||
let chunk: ChatChunk;
|
||||
try {
|
||||
chunk = JSON.parse(payload) as ChatChunk;
|
||||
} catch {
|
||||
continue; // a keep-alive or comment frame, not a chunk
|
||||
}
|
||||
yield chunk;
|
||||
}
|
||||
return route;
|
||||
}
|
||||
|
||||
async function* streamText(body: ChatRequest, opts?: RequestOptions): AsyncGenerator<string, RouteInfo, undefined> {
|
||||
const iter = stream(body, opts);
|
||||
while (true) {
|
||||
const next = await iter.next();
|
||||
if (next.done) return next.value;
|
||||
const text = next.value.choices[0]?.delta.content;
|
||||
if (text) yield text;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
chat,
|
||||
stream,
|
||||
streamText,
|
||||
|
||||
async embed(input, opts) {
|
||||
const { data, route } = await json<EmbeddingResponse>(
|
||||
'/embeddings',
|
||||
{ model: opts?.model ?? 'auto', input: Array.isArray(input) ? input : [input] },
|
||||
opts,
|
||||
);
|
||||
const vectors = [...data.data].sort((a, b) => a.index - b.index).map((d) => d.embedding);
|
||||
return Object.assign(vectors, { route });
|
||||
},
|
||||
|
||||
async models(opts) {
|
||||
const res = await send('/models', { method: 'GET', headers: headersFor(opts?.headers) }, opts, timeoutMs);
|
||||
const route = routeFrom(res.headers);
|
||||
const parsed: unknown = await res.json().catch(() => null);
|
||||
if (!res.ok) throw errorFrom(res, parsed, route);
|
||||
return ((parsed as { data?: ModelEntry[] } | null)?.data ?? []);
|
||||
},
|
||||
|
||||
async health(opts) {
|
||||
// `/healthz` sits outside `/v1`, and outside the virtual-key gate.
|
||||
const res = await doFetch(`${baseUrl.replace(/\/v1$/, '')}/healthz`, { headers: headersFor(opts?.headers) });
|
||||
if (!res.ok) throw new FreewayError({ message: `health check failed (${res.status})`, status: res.status, route: emptyRoute() });
|
||||
return (await res.json()) as { ok: boolean; providers: { total: number; usable: number } };
|
||||
},
|
||||
|
||||
openai() {
|
||||
const cfg: { baseURL: string; apiKey: string; defaultHeaders?: Record<string, string> } = {
|
||||
baseURL: baseUrl,
|
||||
// The OpenAI SDK rejects an empty key, and an open gateway ignores it.
|
||||
apiKey: apiKey || 'fw-unused',
|
||||
};
|
||||
if (options.headers) cfg.defaultHeaders = options.headers;
|
||||
return cfg;
|
||||
},
|
||||
|
||||
baseUrl,
|
||||
apiKey,
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function normalizeBase(url: string): string {
|
||||
const trimmed = url.replace(/\/+$/, '');
|
||||
return /\/v\d+$/.test(trimmed) ? trimmed : `${trimmed}/v1`;
|
||||
}
|
||||
|
||||
function emptyRoute(): RouteInfo {
|
||||
return { provider: null, model: null, route: null, attempts: 0, ms: 0, estimated: false, cache: null, context: null };
|
||||
}
|
||||
|
||||
function routeFrom(headers: Headers): RouteInfo {
|
||||
return {
|
||||
provider: headers.get('x-freeway-provider'),
|
||||
model: headers.get('x-freeway-model'),
|
||||
route: headers.get('x-freeway-route'),
|
||||
attempts: Number(headers.get('x-freeway-attempts') ?? '0') || 0,
|
||||
ms: Number(headers.get('x-freeway-ms') ?? '0') || 0,
|
||||
estimated: headers.get('x-freeway-usage') === 'estimated',
|
||||
cache: headers.get('x-freeway-cache'),
|
||||
context: headers.get('x-freeway-context'),
|
||||
};
|
||||
}
|
||||
|
||||
function errorFrom(res: Response, body: unknown, route: RouteInfo): FreewayError {
|
||||
const b = (body ?? {}) as { error?: { message?: string; code?: string; type?: string }; freeway?: Record<string, unknown> };
|
||||
const fw = b.freeway ?? {};
|
||||
return new FreewayError({
|
||||
message: b.error?.message ?? `request failed with ${res.status}`,
|
||||
status: res.status,
|
||||
code: b.error?.code ?? null,
|
||||
type: b.error?.type ?? null,
|
||||
route,
|
||||
blocked: Array.isArray(fw['blocked']) ? (fw['blocked'] as BlockedProvider[]) : [],
|
||||
suggestions: Array.isArray(fw['suggestions']) ? (fw['suggestions'] as string[]) : [],
|
||||
retryAfterMs: typeof fw['retryAfterMs'] === 'number' ? fw['retryAfterMs'] : parseRetryAfter(res.headers.get('retry-after')),
|
||||
body,
|
||||
});
|
||||
}
|
||||
|
||||
function isRetryable(status: number): boolean {
|
||||
return status === 408 || status === 429 || status >= 500;
|
||||
}
|
||||
|
||||
/** Full jitter: retrying in lockstep is how a thundering herd starts. */
|
||||
function backoffMs(attempt: number, retryAfterMs: number | null): number {
|
||||
if (retryAfterMs !== null) return Math.min(retryAfterMs, 30_000);
|
||||
const capped = Math.min(500 * 2 ** attempt, 8000);
|
||||
return Math.random() * capped;
|
||||
}
|
||||
|
||||
function parseRetryAfter(raw: string | null): number | null {
|
||||
if (!raw) return null;
|
||||
const seconds = Number(raw);
|
||||
if (Number.isFinite(seconds)) return seconds * 1000;
|
||||
const date = Date.parse(raw);
|
||||
return Number.isFinite(date) ? Math.max(0, date - Date.now()) : null;
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((r) => setTimeout(r, ms));
|
||||
}
|
||||
|
||||
function abortError(reason: unknown): Error {
|
||||
if (reason instanceof Error) return reason;
|
||||
const err = new Error('the request was aborted');
|
||||
err.name = 'AbortError';
|
||||
return err;
|
||||
}
|
||||
|
||||
/** Yield the payload of each `data:` frame, handling chunk-split lines. */
|
||||
async function* sseLines(body: ReadableStream<Uint8Array>): AsyncGenerator<string, void, undefined> {
|
||||
const reader = body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = '';
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
|
||||
let newline: number;
|
||||
while ((newline = buffer.indexOf('\n')) >= 0) {
|
||||
const line = buffer.slice(0, newline).trim();
|
||||
buffer = buffer.slice(newline + 1);
|
||||
if (line.startsWith('data:')) yield line.slice(5).trim();
|
||||
}
|
||||
}
|
||||
const tail = buffer.trim();
|
||||
if (tail.startsWith('data:')) yield tail.slice(5).trim();
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,309 @@
|
||||
/**
|
||||
* SDK tests drive a real gateway in front of real mock upstreams — the same
|
||||
* stack an app would hit, so nothing here is mocked at the fetch layer.
|
||||
*/
|
||||
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { once } from 'node:events';
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import type { Server } from 'node:http';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { after, describe, it } from 'node:test';
|
||||
|
||||
import { Registry, Router, Store, parseConfig, silentLogger, type FreewayConfig } from '@freeway/core';
|
||||
import { createGateway } from '@freeway/gateway';
|
||||
|
||||
import { FreewayError, createFreeway, type FreewayClient } from '../src/index.ts';
|
||||
import { createMockUpstream } from '../../../scripts/mock-upstream.js';
|
||||
|
||||
const cleanups: (() => unknown)[] = [];
|
||||
after(async () => {
|
||||
for (const c of cleanups.reverse()) await c();
|
||||
});
|
||||
|
||||
interface Mock {
|
||||
url: string;
|
||||
script(steps: unknown[]): void;
|
||||
callsTo(path: string): unknown[];
|
||||
close(): Promise<unknown>;
|
||||
}
|
||||
|
||||
interface Harness {
|
||||
fw: FreewayClient;
|
||||
base: string;
|
||||
alpha: Mock;
|
||||
beta: Mock;
|
||||
store: Store;
|
||||
}
|
||||
|
||||
async function harness(overrides: Partial<FreewayConfig> = {}, clientKey = ''): Promise<Harness> {
|
||||
const alpha = (await createMockUpstream({ name: 'alpha' })) as Mock;
|
||||
const beta = (await createMockUpstream({ name: 'beta' })) as Mock;
|
||||
cleanups.push(() => alpha.close(), () => beta.close());
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), 'freeway-sdk-'));
|
||||
cleanups.push(() => rmSync(dir, { recursive: true, force: true }));
|
||||
|
||||
writeFileSync(join(dir, 'alpha.json'), JSON.stringify({
|
||||
id: 'alpha', priority: 10, baseUrl: alpha.url,
|
||||
auth: { type: 'bearer', envKeys: ['ALPHA_KEY'] }, limits: { rpm: 100 },
|
||||
models: [
|
||||
{ id: 'alpha-chat', alias: ['fast'], context: 128000, caps: ['chat', 'tools'] },
|
||||
{ id: 'alpha-embed', alias: ['embed'], context: 8000, caps: ['embed'] },
|
||||
],
|
||||
}));
|
||||
writeFileSync(join(dir, 'beta.json'), JSON.stringify({
|
||||
id: 'beta', priority: 20, baseUrl: beta.url,
|
||||
auth: { type: 'bearer', envKeys: ['BETA_KEY'] }, limits: { rpm: 100 },
|
||||
models: [{ id: 'beta-chat', alias: ['fast'], context: 8192, caps: ['chat'] }],
|
||||
}));
|
||||
|
||||
const registry = new Registry({ dir, env: { ALPHA_KEY: 'ak-00000000001', BETA_KEY: 'bk-00000000001' }, logger: silentLogger });
|
||||
const store = new Store({ file: null, logger: silentLogger });
|
||||
const { config } = parseConfig({}, {});
|
||||
Object.assign(config, overrides);
|
||||
const router = new Router(registry, store, config);
|
||||
|
||||
const server: Server = createGateway({ registry, store, router, config, logger: silentLogger });
|
||||
server.listen(0, '127.0.0.1');
|
||||
await once(server, 'listening');
|
||||
const addr = server.address();
|
||||
const port = typeof addr === 'object' && addr !== null ? addr.port : 0;
|
||||
cleanups.push(() => new Promise((r) => server.close(() => r(undefined))));
|
||||
|
||||
const base = `http://127.0.0.1:${port}`;
|
||||
return { fw: createFreeway({ baseUrl: `${base}/v1`, apiKey: clientKey, maxRetries: 2 }), base, alpha, beta, store };
|
||||
}
|
||||
|
||||
describe('SDK — chat', () => {
|
||||
it('prompt in, string out', async () => {
|
||||
const { fw } = await harness();
|
||||
const text = await fw.chat('summarize this');
|
||||
assert.equal(typeof text, 'string');
|
||||
assert.match(text, /\[alpha\]/);
|
||||
});
|
||||
|
||||
it('accepts a system prompt and options', async () => {
|
||||
const { fw, alpha } = await harness();
|
||||
await fw.chat('hello', { model: 'fast', system: 'be terse', temperature: 0.1, max_tokens: 64 });
|
||||
|
||||
const call = alpha.callsTo('/chat/completions')[0] as { body: { messages: { role: string }[]; temperature: number; max_tokens: number } };
|
||||
assert.deepEqual(call.body.messages.map((m) => m.role), ['system', 'user']);
|
||||
assert.equal(call.body.temperature, 0.1);
|
||||
assert.equal(call.body.max_tokens, 64);
|
||||
});
|
||||
|
||||
it('chat.raw exposes the full completion plus the route', async () => {
|
||||
const { fw } = await harness();
|
||||
const res = await fw.chat.raw({ model: 'auto', messages: [{ role: 'user', content: 'hi' }] });
|
||||
|
||||
assert.ok(Array.isArray(res.choices));
|
||||
assert.equal(res.route.provider, 'alpha');
|
||||
assert.equal(res.route.model, 'alpha-chat');
|
||||
assert.equal(res.route.route, 'auto');
|
||||
assert.equal(res.route.attempts, 1);
|
||||
assert.ok(res.route.ms >= 0);
|
||||
});
|
||||
|
||||
it('surfaces which provider actually served a failed-over call', async () => {
|
||||
const { fw, alpha } = await harness();
|
||||
alpha.script([{ status: 429, repeat: Infinity }]);
|
||||
|
||||
const res = await fw.chat.raw({ model: 'fast', messages: [{ role: 'user', content: 'hi' }] });
|
||||
assert.equal(res.route.provider, 'beta', 'the app can see the failover from inside');
|
||||
assert.equal(res.route.attempts, 2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('SDK — streaming', () => {
|
||||
it('yields chunks and returns the route when done', async () => {
|
||||
const { fw } = await harness();
|
||||
const iter = fw.stream({ model: 'alpha/alpha-chat', messages: [{ role: 'user', content: 'stream please' }] });
|
||||
|
||||
let text = '';
|
||||
let result = await iter.next();
|
||||
while (!result.done) {
|
||||
text += result.value.choices[0]?.delta.content ?? '';
|
||||
result = await iter.next();
|
||||
}
|
||||
|
||||
assert.match(text, /\[alpha\]/);
|
||||
assert.equal(result.value.provider, 'alpha');
|
||||
});
|
||||
|
||||
it('streamText yields only the text', async () => {
|
||||
const { fw } = await harness();
|
||||
const parts: string[] = [];
|
||||
for await (const piece of fw.streamText({ model: 'fast', messages: [{ role: 'user', content: 'hi' }] })) {
|
||||
parts.push(piece);
|
||||
}
|
||||
assert.ok(parts.length > 1, 'arrived in several chunks');
|
||||
assert.match(parts.join(''), /\[alpha\]/);
|
||||
});
|
||||
|
||||
it('reassembles a frame split across network chunks', async () => {
|
||||
// The mock deliberately chunks mid-word; a naive parser drops those.
|
||||
const { fw } = await harness();
|
||||
let text = '';
|
||||
for await (const piece of fw.streamText({ model: 'fast', messages: [{ role: 'user', content: 'x'.repeat(300) }] })) {
|
||||
text += piece;
|
||||
}
|
||||
assert.ok(text.length > 50);
|
||||
});
|
||||
});
|
||||
|
||||
describe('SDK — embeddings and models', () => {
|
||||
it('returns vectors in input order', async () => {
|
||||
const { fw } = await harness();
|
||||
const vectors = await fw.embed(['alpha', 'beta', 'gamma']);
|
||||
assert.equal(vectors.length, 3);
|
||||
assert.ok(Array.isArray(vectors[0]));
|
||||
assert.equal(vectors.route.model, 'alpha-embed');
|
||||
});
|
||||
|
||||
it('accepts a bare string', async () => {
|
||||
const { fw } = await harness();
|
||||
const vectors = await fw.embed('just one');
|
||||
assert.equal(vectors.length, 1);
|
||||
});
|
||||
|
||||
it('lists the catalog including virtual aliases', async () => {
|
||||
const { fw } = await harness();
|
||||
const models = await fw.models();
|
||||
const ids = models.map((m) => m.id);
|
||||
assert.ok(ids.includes('alpha/alpha-chat'));
|
||||
assert.ok(ids.includes('auto'));
|
||||
assert.ok(models.find((m) => m.id === 'auto')?.freeway?.virtual);
|
||||
});
|
||||
|
||||
it('reads health without a key', async () => {
|
||||
const { fw } = await harness();
|
||||
const health = await fw.health();
|
||||
assert.equal(health.ok, true);
|
||||
assert.equal(health.providers.usable, 2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('SDK — errors', () => {
|
||||
it('throws FreewayError carrying the blocked list', async () => {
|
||||
const { fw, store } = await harness();
|
||||
for (let i = 0; i < 100; i++) {
|
||||
store.recordSuccess('alpha', 'alpha#0', 1);
|
||||
store.recordSuccess('beta', 'beta#0', 1);
|
||||
}
|
||||
|
||||
await assert.rejects(
|
||||
() => fw.chat('hi', { model: 'fast', maxRetries: 1 }),
|
||||
(err: unknown) => {
|
||||
assert.ok(err instanceof FreewayError);
|
||||
assert.equal(err.status, 429);
|
||||
assert.equal(err.code, 'all_providers_blocked');
|
||||
assert.equal(err.blocked.length, 2);
|
||||
assert.match(err.blocked[0]?.reason ?? '', /rpm limit reached/);
|
||||
assert.ok((err.retryAfterMs ?? 0) > 0);
|
||||
return true;
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it('throws with suggestions for an unknown model', async () => {
|
||||
const { fw } = await harness();
|
||||
await assert.rejects(
|
||||
() => fw.chat('hi', { model: 'gpt-9-ultra', maxRetries: 1 }),
|
||||
(err: unknown) => {
|
||||
assert.ok(err instanceof FreewayError);
|
||||
assert.equal(err.status, 404);
|
||||
assert.equal(err.code, 'model_not_found');
|
||||
return true;
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it('does not retry a 400', async () => {
|
||||
const { fw, alpha } = await harness();
|
||||
alpha.script([{ status: 400, body: { error: { message: 'bad param' } }, repeat: Infinity }]);
|
||||
|
||||
await assert.rejects(() => fw.chat('hi', { model: 'alpha/alpha-chat' }));
|
||||
assert.equal(alpha.callsTo('/chat/completions').length, 1, 'a malformed request is not worth repeating');
|
||||
});
|
||||
|
||||
it('honours the gateway’s Retry-After and succeeds once the bench expires', async () => {
|
||||
// A 5xx benches the provider for `cooldownSeconds`, so a useful retry has to
|
||||
// wait that long — which is exactly what Retry-After tells the client.
|
||||
const { fw, alpha, beta } = await harness({ maxAttempts: 1, cooldownSeconds: 1 });
|
||||
alpha.script([{ status: 500, repeat: 1 }, { status: 200, repeat: Infinity }]);
|
||||
beta.script([{ status: 500, repeat: 1 }, { status: 200, repeat: Infinity }]);
|
||||
|
||||
const started = Date.now();
|
||||
const text = await fw.chat('hi', { model: 'fast', maxRetries: 4 });
|
||||
assert.match(text, /\[(alpha|beta)\]/);
|
||||
assert.ok(Date.now() - started >= 900, 'it waited for the window rather than hammering');
|
||||
});
|
||||
});
|
||||
|
||||
describe('SDK — auth and configuration', () => {
|
||||
it('sends the virtual key', async () => {
|
||||
const { fw } = await harness({ keys: ['fw-secret'] }, 'fw-secret');
|
||||
assert.match(await fw.chat('hi'), /\[alpha\]/);
|
||||
});
|
||||
|
||||
it('fails cleanly with a wrong key', async () => {
|
||||
const { fw } = await harness({ keys: ['fw-secret'] }, 'fw-wrong');
|
||||
await assert.rejects(
|
||||
() => fw.chat('hi', { maxRetries: 1 }),
|
||||
(err: unknown) => err instanceof FreewayError && err.status === 401,
|
||||
);
|
||||
});
|
||||
|
||||
it('normalises a base URL without /v1', () => {
|
||||
assert.equal(createFreeway({ baseUrl: 'http://localhost:8787' }).baseUrl, 'http://localhost:8787/v1');
|
||||
assert.equal(createFreeway({ baseUrl: 'http://localhost:8787/v1/' }).baseUrl, 'http://localhost:8787/v1');
|
||||
});
|
||||
|
||||
it('exports a drop-in config for the OpenAI SDK', async () => {
|
||||
const { fw, base } = await harness({}, 'fw-key');
|
||||
const cfg = fw.openai();
|
||||
assert.equal(cfg.baseURL, `${base}/v1`);
|
||||
assert.equal(cfg.apiKey, 'fw-key');
|
||||
|
||||
// Prove the shape actually works by calling it the way that SDK would.
|
||||
const res = await fetch(`${cfg.baseURL}/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json', authorization: `Bearer ${cfg.apiKey}` },
|
||||
body: JSON.stringify({ model: 'auto', messages: [{ role: 'user', content: 'hi' }] }),
|
||||
});
|
||||
assert.equal(res.status, 200);
|
||||
});
|
||||
|
||||
it('substitutes a placeholder key when the gateway is open', () => {
|
||||
// The OpenAI SDK refuses to construct with an empty apiKey.
|
||||
assert.equal(createFreeway({ baseUrl: 'http://x/v1' }).openai().apiKey, 'fw-unused');
|
||||
});
|
||||
|
||||
it('puts a session id in the request body', async () => {
|
||||
// Asserted against the wire the SDK writes, not against what a provider
|
||||
// eventually sees — the gateway consumes `session` once Phase 8 lands.
|
||||
let sent: Record<string, unknown> | null = null;
|
||||
const client = createFreeway({
|
||||
baseUrl: 'http://example.invalid/v1',
|
||||
fetch: async (_url, init) => {
|
||||
sent = JSON.parse(String((init as RequestInit).body)) as Record<string, unknown>;
|
||||
return new Response(JSON.stringify({ choices: [{ message: { content: 'ok' } }] }), {
|
||||
status: 200,
|
||||
headers: { 'content-type': 'application/json' },
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
await client.chat('hi', { session: 'thread-42' });
|
||||
assert.equal((sent as unknown as Record<string, unknown>)['session'], 'thread-42');
|
||||
});
|
||||
|
||||
it('respects an external AbortSignal', async () => {
|
||||
const { fw } = await harness();
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
await assert.rejects(() => fw.chat('hi', { signal: controller.signal, maxRetries: 1 }));
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"extends": "../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "./src",
|
||||
"outDir": "./dist",
|
||||
"tsBuildInfoFile": "./dist/.tsbuildinfo",
|
||||
"lib": ["ES2023", "DOM"]
|
||||
},
|
||||
"include": ["src/**/*.ts"]
|
||||
}
|
||||
Reference in new issue
Block a user