Something went wrong. Try again.
Something went wrong. Try again.
4.8 kB · 128 lines
TypeScript
at main
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129import assert from "node:assert/strict"import fs from "node:fs/promises"import os from "node:os"import path from "node:path"import test from "node:test"import { createServer } from "./server"import { closeTokenUsageDatabases } from "./token-usage"
test("cache warming travels over HTTP through Chat and Responses without changing the prompt", async () => { const root = await fs.mkdtemp(path.join(os.tmpdir(), "bridge-cache-warm-")) const previousStateDir = process.env.BRIDGE_STATE_DIR process.env.BRIDGE_STATE_DIR = root const authPath = path.join(root, "auth.json") await fs.writeFile( authPath, JSON.stringify({ tokens: { access_token: "test.payload.signature", account_id: "acct_test" } }), ) const sent: Record<string, any>[] = [] let rejectPrewarm = false const app = createServer( { port: 0, host: "127.0.0.1", models: { "gpt-6-sol": { id: "gpt-6-sol", name: "GPT 6 Sol", provider: "codex", apiKey: "", preferWebSockets: false }, }, providers: { codex: { auth_path: authPath } }, }, new Map(), { fetchImpl: (async (_url: unknown, init?: RequestInit) => { const body = JSON.parse(String(init?.body)) sent.push(body) if (rejectPrewarm) return Response.json( { error: { message: "prewarm is unavailable", type: "invalid_request_error" } }, { status: 400 }, ) const response = { id: `resp_${sent.length}`, model: "gpt-6-sol", status: "completed", output: [], usage: { input_tokens: 2048, output_tokens: 0, total_tokens: 2048, input_tokens_details: { cached_tokens: 2048 }, }, } return new Response(`data: ${JSON.stringify({ type: "response.completed", response })}\n\ndata: [DONE]\n\n`, { headers: { "content-type": "text/event-stream" }, }) }) as typeof fetch, }, ) try { const base = await app.listen({ host: "127.0.0.1", port: 0 }) const prompt = { model: "gpt-6-sol", messages: [ { role: "developer", content: "stable instructions" }, { role: "user", content: "stable context" }, ], tools: [ { type: "function", function: { name: "lookup", description: "Look up", parameters: { type: "object", properties: {} } }, }, ], reasoning_effort: "high", tool_choice: "auto", } const post = (route: string, body: unknown) => fetch(`${base}/v1/${route}`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body), }) const ordinary = await post("chat/completions", prompt) assert.equal(ordinary.status, 200) const warm = await post("chat/completions", { ...prompt, prompt_cache_options: { prewarm: true, ttl: "30m" } }) assert.equal(warm.status, 200) const result = (await warm.json()) as any assert.equal(result.choices[0].message.content, null) assert.equal(result.usage.completion_tokens, 0) assert.equal(result.usage.prompt_tokens_details.cached_tokens, 2048) assert.deepEqual(sent[1].prompt_cache_options, { prewarm: true, ttl: "30m" }) for (const field of ["input", "instructions", "tools", "reasoning", "text", "tool_choice"]) assert.deepEqual(sent[1][field], sent[0][field], field) assert.equal(sent[0].prompt_cache_options, undefined)
const native = await post("responses", { model: "gpt-6-sol", input: [{ role: "user", content: "stable context" }], prompt_cache_options: { prewarm: true, ttl: "30m" }, }) assert.equal(native.status, 200) assert.deepEqual(sent[2].prompt_cache_options, { prewarm: true, ttl: "30m" }) rejectPrewarm = true const failed = await post("chat/completions", { ...prompt, prompt_cache_options: { prewarm: true } }) assert.equal(failed.status, 400) assert.match(await failed.text(), /prewarm is unavailable/) assert.equal(sent.length, 4, "failed warming must not retry as ordinary generation") await fs.writeFile( "/tmp/llm-bridge-cache-warming-e2e.json", JSON.stringify( { simulatedUpstream: true, outputTokens: 0, identicalPrompt: true, chatOptions: sent[1].prompt_cache_options, responsesOptions: sent[2].prompt_cache_options, unsupportedStatus: failed.status, upstreamRequests: sent.length, }, null, 2, ), ) } finally { await app.close() closeTokenUsageDatabases() if (previousStateDir === undefined) delete process.env.BRIDGE_STATE_DIR else process.env.BRIDGE_STATE_DIR = previousStateDir await fs.rm(root, { recursive: true, force: true }) }})