diff --git a/apps/server/src/providers/catalog.ts b/apps/server/src/providers/catalog.ts index 8164084..5da79cb 100644 --- a/apps/server/src/providers/catalog.ts +++ b/apps/server/src/providers/catalog.ts @@ -31,6 +31,8 @@ export type ResolvedModel = { provider: ModelProvider model: LanguageModelV4 capabilities: Capabilities + /** Whether the call runs on the user's own key or the admin's. */ + source: 'admin' | 'user' } export type CatalogDeps = { @@ -151,7 +153,7 @@ export class ModelCatalog { fetch: endpoint && !provider.allowPrivateNetworks ? this.deps.guardedFetch : undefined, session, }) - return { ref, provider, model, capabilities: own.info.capabilities } + return { ref, provider, model, capabilities: own.info.capabilities, source: 'user' } } const admin = (await this.adminModelsFor(did)).find( (m) => m.provider === ref.provider && m.id === ref.id, @@ -165,6 +167,7 @@ export class ModelCatalog { provider, model: provider.createModel({ modelId: ref.id, session }), capabilities: admin.capabilities, + source: 'admin', } } } diff --git a/apps/server/src/providers/generate-text.ts b/apps/server/src/providers/generate-text.ts index 3e6884d..b9f7848 100644 --- a/apps/server/src/providers/generate-text.ts +++ b/apps/server/src/providers/generate-text.ts @@ -3,18 +3,20 @@ import { generateText } from 'ai' import type { Logger } from '../logger.ts' import type { ModelCatalog } from './catalog.ts' import { mapEffort } from './effort.ts' +import { PROVIDER_RETRIES, withProviderRetries } from './retries.ts' /** Make one-off model calls for plugins, resolving the model the same way as a turn. */ export function createGenerateText(catalog: ModelCatalog, logger: Logger) { return async (request: GenerateTextRequest): Promise<{ text: string; finishReason: string }> => { const resolved = await catalog.resolve(request.user, request.model, request.conversation) const result = await generateText({ - model: resolved.model, + model: withProviderRetries(resolved.model), instructions: request.system, prompt: request.prompt, maxOutputTokens: request.maxOutputTokens, reasoning: resolved.capabilities.reasoning ? mapEffort(request.effort, logger) : undefined, abortSignal: request.signal, + maxRetries: PROVIDER_RETRIES, providerOptions: resolved.provider.providerOptions as never, }) return { text: result.text, finishReason: result.finishReason } diff --git a/apps/server/src/providers/retries.ts b/apps/server/src/providers/retries.ts new file mode 100644 index 0000000..a24081a --- /dev/null +++ b/apps/server/src/providers/retries.ts @@ -0,0 +1,82 @@ +import type { LanguageModelV4 } from '@scn-chat/plugin-api' +import { APICallError, RetryError, wrapLanguageModel } from 'ai' +import { safeErrorMessage } from '../safe-error.ts' +import type { ResolvedModel } from './catalog.ts' + +/** Retry a busy provider for about a minute, at 2, 4, 8, 16, and 32 seconds, unless it asks for other waits. */ +export const PROVIDER_RETRIES = 5 + +/** OpenAI's out of credit, and LiteLLM's key or proxy over its budget. */ +const QUOTA_ERRORS = new Set(['insufficient_quota', 'budget_exceeded']) + +const lastErrorOf = (err: unknown) => (RetryError.isInstance(err) ? err.lastError : err) + +/** Has the key run out of credit or budget, rather than the provider asking us to slow down? */ +export function isQuotaExhausted(err: unknown): boolean { + const last = lastErrorOf(err) + if (!APICallError.isInstance(last) || !last.responseBody) return false + let body: { error?: { type?: unknown; code?: unknown } } | null + try { + body = JSON.parse(last.responseBody) + } catch (parseErr) { + if (parseErr instanceof SyntaxError) return false + throw parseErr + } + return QUOTA_ERRORS.has(String(body?.error?.type)) || QUOTA_ERRORS.has(String(body?.error?.code)) +} + +/** + * The model with a quota error made final, since waiting won't bring credit back, and `onRetry` + * called whenever the provider turns away a request that will be tried again. + */ +export function withProviderRetries( + model: LanguageModelV4, + onRetry: () => void = () => {}, +): LanguageModelV4 { + const settle = async (call: () => PromiseLike): Promise => { + try { + return await call() + } catch (err) { + if (!APICallError.isInstance(err) || !err.isRetryable) throw err + if (isQuotaExhausted(err)) { + throw new APICallError({ + message: err.message, + url: err.url, + requestBodyValues: err.requestBodyValues, + statusCode: err.statusCode, + responseHeaders: err.responseHeaders, + responseBody: err.responseBody, + data: err.data, + cause: err, + isRetryable: false, + }) + } + onRetry() + throw err + } + } + return wrapLanguageModel({ + model, + middleware: { + wrapGenerate: ({ doGenerate }) => settle(doGenerate), + wrapStream: ({ doStream }) => settle(doStream), + }, + }) +} + +/** A model call's error as the user reads it, saying plainly when a key is spent or rate limited. */ +export function providerErrorMessage( + err: unknown, + model: Pick | undefined, +): string { + if (!model) return safeErrorMessage(err) + if (isQuotaExhausted(err)) { + if (model.source === 'admin') + return "This app's shared key has reached its usage limit. Add your own API key under Settings > API keys to keep chatting." + return 'Your API key has run out of credit or reached its spending limit. Add credit to the account it belongs to, or use a different key.' + } + const last = lastErrorOf(err) + if (APICallError.isInstance(last) && last.statusCode === 429) + return 'The model provider is limiting requests right now. Try again in a minute.' + return safeErrorMessage(err) +} diff --git a/apps/server/src/turns/runner.ts b/apps/server/src/turns/runner.ts index 556b443..8696f54 100644 --- a/apps/server/src/turns/runner.ts +++ b/apps/server/src/turns/runner.ts @@ -13,7 +13,13 @@ import type { Logger } from '../logger.ts' import type { PluginHost } from '../plugins/host.ts' import type { PluginRuntime, RuntimeLease } from '../plugins/runtime.ts' import { isToolEnabled, readToolChoices } from '../plugins/user-tools.ts' +import type { ResolvedModel } from '../providers/catalog.ts' import { mapEffort } from '../providers/effort.ts' +import { + PROVIDER_RETRIES, + providerErrorMessage, + withProviderRetries, +} from '../providers/retries.ts' import { safeErrorMessage } from '../safe-error.ts' import type { ChatService } from '../storage/chat-service.ts' import { InvalidStoredRecord, RecordExists } from '../storage/record-store.ts' @@ -480,10 +486,11 @@ export class TurnRunner { } let status: 'complete' | 'error' | 'cancelled' = 'complete' let error: string | undefined + let resolved: ResolvedModel | undefined try { if (!model) throw new Error('No model selected. Choose a model or set a default in your preferences.') - const resolved = await catalog.resolve(did, model, conversationUri) + resolved = await catalog.resolve(did, model, conversationUri) const branch = branchTo(loaded.messages, userMessage.rkey) const requested = generation.tools if (requested?.length && !resolved.capabilities.tools) @@ -519,7 +526,10 @@ export class TurnRunner { isUntrusted: (name) => this.isUntrusted(host, name), }) const result = streamText({ - model: resolved.model, + model: withProviderRetries(resolved.model, () => + this.deps.hub.publish(key, { type: 'retrying' }), + ), + maxRetries: PROVIDER_RETRIES, instructions: prompt.instructions || undefined, messages, tools, @@ -536,7 +546,7 @@ export class TurnRunner { const failure = accumulator.push(chunk) if (failure && status !== 'cancelled') { status = 'error' - error = safeErrorMessage(failure) + error = providerErrorMessage(failure, resolved) } } if (controller.signal.aborted && status === 'complete') status = 'cancelled' @@ -544,7 +554,7 @@ export class TurnRunner { if (controller.signal.aborted) status = 'cancelled' else { status = 'error' - error = safeErrorMessage(err) + error = providerErrorMessage(err, resolved) this.deps.logger.warn({ err, did, skey, replyRkey }, 'turn ended with an error') } } finally { diff --git a/apps/server/src/turns/stream-hub.ts b/apps/server/src/turns/stream-hub.ts index fcb3c94..45306b3 100644 --- a/apps/server/src/turns/stream-hub.ts +++ b/apps/server/src/turns/stream-hub.ts @@ -3,6 +3,7 @@ import type { StreamEvent } from './accumulator.ts' export type HubEvent = | StreamEvent | { type: 'queued' } + | { type: 'retrying' } | { type: 'status'; status: string; error?: string } type Channel = { events: HubEvent[]; listeners: Set<(event: HubEvent) => void>; done: boolean } diff --git a/apps/server/tests/helpers/turns.ts b/apps/server/tests/helpers/turns.ts index 2c8eb5a..1536c97 100644 --- a/apps/server/tests/helpers/turns.ts +++ b/apps/server/tests/helpers/turns.ts @@ -28,13 +28,19 @@ export function textReply(text: string, options: { whole?: boolean } = {}) { ] } -/** A model that streams the given parts, recording each prompt it was sent. */ -export function scriptedModel(parts: unknown[], options: { delayMs?: number; fail?: Error } = {}) { +/** + * A model that streams the given parts, recording each prompt it was sent. With `fail`, its calls + * throw that error, only the first `failures` of them if given. + */ +export function scriptedModel( + parts: unknown[], + options: { delayMs?: number; fail?: Error; failures?: number } = {}, +) { const prompts: unknown[] = [] const model = new MockLanguageModelV4({ doStream: async (call) => { prompts.push(call.prompt) - if (options.fail) throw options.fail + if (options.fail && prompts.length <= (options.failures ?? Infinity)) throw options.fail return { stream: simulateReadableStream({ chunks: parts as never[], @@ -118,9 +124,10 @@ export async function turnsHarness( 'did:plc:admin', ) } + const box = new SecretBox(Buffer.alloc(32, 8)) const catalog = new ModelCatalog({ db: h.db, - box: new SecretBox(Buffer.alloc(32, 8)), + box, providers, rolesOf: async () => ['user'], guardedFetch: fetch, @@ -167,6 +174,8 @@ export async function turnsHarness( runner, hub, catalog, + box, + provider, plugins, settings, created, diff --git a/apps/server/tests/providers/generate-text.test.ts b/apps/server/tests/providers/generate-text.test.ts index b8ac4d7..28145a3 100644 --- a/apps/server/tests/providers/generate-text.test.ts +++ b/apps/server/tests/providers/generate-text.test.ts @@ -1,23 +1,26 @@ +import { APICallError } from 'ai' import { MockLanguageModelV4 } from 'ai/test' import { describe, expect, it, vi } from 'vitest' import type { ModelCatalog } from '../../src/providers/catalog.ts' import { createGenerateText } from '../../src/providers/generate-text.ts' -function setup(reasoning: boolean) { - const model = new MockLanguageModelV4({ - doGenerate: async () => - ({ - content: [{ type: 'text', text: 'A short title' }], - finishReason: { unified: 'stop', raw: 'stop' }, - usage: { inputTokens: { total: 1 }, outputTokens: { total: 3 } }, - warnings: [], - }) as never, - }) +const titled = async () => + ({ + content: [{ type: 'text', text: 'A short title' }], + finishReason: { unified: 'stop', raw: 'stop' }, + usage: { inputTokens: { total: 1 }, outputTokens: { total: 3 } }, + warnings: [], + }) as never + +function setup(reasoning: boolean, doGenerate: () => Promise = titled) { + const model = new MockLanguageModelV4({ doGenerate }) const catalog = { resolve: vi.fn(async () => ({ + ref: { provider: 'fake', id: 'm' }, model, - provider: { providerOptions: undefined }, - capabilities: { reasoning }, + provider: { id: 'fake', name: 'Fake', providerOptions: undefined }, + capabilities: { vision: false, reasoning, tools: false }, + source: 'admin', })), } as unknown as ModelCatalog return { model, catalog, generate: createGenerateText(catalog, { warn: vi.fn() } as never) } @@ -61,4 +64,22 @@ describe('createGenerateText', () => { await generate({ ...request, effort: 'none' }) expect(model.doGenerateCalls[0]?.reasoning).toBeUndefined() }) + + it('gives up at once on a key that is out of credit', async () => { + const message = 'You exceeded your current quota, please check your plan and billing details.' + const { model, generate } = setup(false, async () => { + throw new APICallError({ + message, + url: 'https://provider.test/v1/chat/completions', + requestBodyValues: {}, + statusCode: 429, + responseHeaders: { 'retry-after-ms': '0' }, + responseBody: JSON.stringify({ + error: { message, type: 'insufficient_quota', param: null, code: 'insufficient_quota' }, + }), + }) + }) + await expect(generate(request)).rejects.toBeInstanceOf(APICallError) + expect(model.doGenerateCalls).toHaveLength(1) + }) }) diff --git a/apps/server/tests/providers/retries.test.ts b/apps/server/tests/providers/retries.test.ts new file mode 100644 index 0000000..4d7f7bf --- /dev/null +++ b/apps/server/tests/providers/retries.test.ts @@ -0,0 +1,39 @@ +import { APICallError, RetryError } from 'ai' +import { describe, expect, it } from 'vitest' +import { isQuotaExhausted } from '../../src/providers/retries.ts' + +const failure = (responseBody: string | undefined) => + new APICallError({ + message: 'Provider error', + url: 'https://provider.test/v1/chat/completions', + requestBodyValues: {}, + statusCode: 429, + responseBody, + }) + +const outOfCredit = JSON.stringify({ + error: { + message: 'You exceeded your current quota, please check your plan and billing details.', + type: 'insufficient_quota', + param: null, + code: 'insufficient_quota', + }, +}) + +describe('isQuotaExhausted', () => { + it('finds a spent key behind the retries that came before it', () => { + const err = new RetryError({ + message: 'Failed after 2 attempts', + reason: 'maxRetriesExceeded', + errors: [failure('{}'), failure(outOfCredit)], + }) + expect(isQuotaExhausted(err)).toBe(true) + }) + + it.each([ + ['a body that is not JSON', 'Too Many Requests'], + ['a JSON body that is not an object', 'null'], + ])('treats a rate limit with %s as a rate limit, not a spent key', (_name, body) => { + expect(isQuotaExhausted(failure(body))).toBe(false) + }) +}) diff --git a/apps/server/tests/turns/runner.test.ts b/apps/server/tests/turns/runner.test.ts index 682a898..3ff2c87 100644 --- a/apps/server/tests/turns/runner.test.ts +++ b/apps/server/tests/turns/runner.test.ts @@ -6,11 +6,13 @@ import { type ToolSource, type TurnContext, } from '@scn-chat/plugin-api' +import { APICallError } from 'ai' import pino from 'pino' import { describe, expect, it, vi } from 'vitest' import { z } from 'zod' import { loadPlugins } from '../../src/plugins/host.ts' import { writeToolChoice } from '../../src/plugins/user-tools.ts' +import { saveCredential } from '../../src/providers/user-credentials.ts' import { safeErrorMessage } from '../../src/safe-error.ts' import { LocalRecordStore } from '../../src/storage/local-record-store.ts' import { ALICE, textContent, userMessage } from '../helpers/spaces.ts' @@ -617,6 +619,107 @@ describe('TurnRunner.start', () => { expect(reply).toMatchObject({ status: 'error', error: expect.stringContaining('ECONNREFUSED') }) }) + // Provider errors as they arrive, asking for no wait so any retries run at once. + const providerError = (statusCode: number, error: Record) => () => + new APICallError({ + message: String(error.message), + url: 'https://provider.test/v1/chat/completions', + requestBodyValues: {}, + statusCode, + responseHeaders: { 'retry-after-ms': '0' }, + responseBody: JSON.stringify({ error }), + }) + const rateLimited = providerError(429, { + message: 'Rate limit reached for requests', + type: 'requests', + param: null, + code: 'rate_limit_exceeded', + }) + const outOfCredit = providerError(429, { + message: 'You exceeded your current quota, please check your plan and billing details.', + type: 'insufficient_quota', + param: null, + code: 'insufficient_quota', + }) + const overBudget = providerError(429, { + message: 'Budget has been exceeded! Current cost: 5.02, Max budget: 5.0', + type: 'budget_exceeded', + param: null, + code: '429', + }) + + /** Run one turn against a provider failing its first `failures` calls, on the admin key unless `ownKey`. */ + async function failingTurn( + fail: () => APICallError, + options: { failures?: number; ownKey?: boolean } = {}, + ) { + let calls: unknown[] = [] + const h = await turnsHarness({ + model: () => { + const scripted = scriptedModel(textReply('Hello'), { + fail: fail(), + failures: options.failures, + }) + calls = scripted.prompts + return scripted.model + }, + }) + if (options.ownKey) { + await saveCredential(h.db, h.box, ALICE, h.provider, { + apiKey: 'sk-own-key', + models: [ + { + id: 'default-model', + name: 'Default', + capabilities: { vision: true, reasoning: true, tools: true }, + }, + ], + }) + } + const { skey } = await h.chats.createConversation() + await sendUser(h, skey, '3uuuuuuuuuuu1') + const events: string[] = [] + h.hub.subscribe(h.runner.streamKey(ALICE, skey, '3uuuuuuuuuuu1.r0'), (event) => + events.push(event.type), + ) + await h.runner.start(ALICE, skey, '3uuuuuuuuuuu1') + await h.runner.idle() + return { reply: (await h.messages(skey)).get('3uuuuuuuuuuu1.r0'), events, calls: calls.length } + } + + it('retries a rate limited provider past the AI SDK default until the reply comes through', async () => { + const { reply } = await failingTurn(rateLimited, { failures: 4 }) + expect(reply?.status).toBe('complete') + expect(partsOf(reply)).toBe('Hello') + }) + + it('tells the stream before each retry of a rate limited provider', async () => { + const { events } = await failingTurn(rateLimited, { failures: 4 }) + expect(events.slice(0, events.indexOf('part-start'))).toEqual([ + 'retrying', + 'retrying', + 'retrying', + 'retrying', + ]) + }) + + it('gives up at once on a key that is out of credit, without telling the stream to wait', async () => { + const { calls, events } = await failingTurn(outOfCredit) + expect(calls).toBe(1) + expect(events).not.toContain('retrying') + }) + + it('sends users to add their own key when the shared key is over its budget', async () => { + const { reply } = await failingTurn(overBudget) + expect(reply).toMatchObject({ status: 'error', error: expect.stringContaining('API keys') }) + }) + + it('does not send users to add a key when their own key is out of credit', async () => { + const { reply } = await failingTurn(outOfCredit, { ownKey: true }) + expect(reply?.status).toBe('error') + expect(reply?.error).not.toContain('API keys') + }) + it('writes an error reply when the finished reply cannot be written but the error can', async () => { const h = await turnsHarness() const { skey } = await h.chats.createConversation() diff --git a/apps/web/src/features/conversation/components/StreamingReply.tsx b/apps/web/src/features/conversation/components/StreamingReply.tsx index 00d7360..969beb8 100644 --- a/apps/web/src/features/conversation/components/StreamingReply.tsx +++ b/apps/web/src/features/conversation/components/StreamingReply.tsx @@ -15,7 +15,8 @@ const streaming = { role: 'assistant', status: 'pending' } /** * The reply streamed so far, step by step, shown until the reply's record arrives. Before anything - * arrives the reply's header shows it waiting, so this only explains a wait for the rate limit. + * arrives the reply's header shows it waiting, so this only explains a wait for the rate limit or + * a busy provider. */ export function StreamingReply(props: Props) { if (props.reply.queued) @@ -23,12 +24,19 @@ export function StreamingReply(props: Props) {

In line: you have started the most replies allowed this minute.

) return ( - + <> + + {props.reply.retrying && ( +

+ The model's provider is busy. Trying again… +

+ )} + ) } diff --git a/apps/web/src/features/conversation/lib/reply-stream.ts b/apps/web/src/features/conversation/lib/reply-stream.ts index 3cf86fa..54a71fc 100644 --- a/apps/web/src/features/conversation/lib/reply-stream.ts +++ b/apps/web/src/features/conversation/lib/reply-stream.ts @@ -47,7 +47,7 @@ export async function* replyEvents(url: string, signal: AbortSignal): AsyncGener done = true controller.close() } - for (const type of ['part-start', 'delta', 'part', 'queued', 'status'] as const) + for (const type of ['part-start', 'delta', 'part', 'queued', 'retrying', 'status'] as const) source.addEventListener(type, (event) => { if (done) return try { @@ -86,11 +86,16 @@ export async function* replyEvents(url: string, signal: AbortSignal): AsyncGener /** * A reply streamed so far: its parts in the shape of the final record, in the order the model - * produced them, whether it waits for the rate limit, and the final status once sent. + * produced them, whether it waits for the rate limit or a busy provider, and the final status once sent. */ -export type ReplyParts = { parts: Part[]; queued: boolean; status: string | null } +export type ReplyParts = { + parts: Part[] + queued: boolean + retrying: boolean + status: string | null +} -export const noParts: ReplyParts = { parts: [], queued: false, status: null } +export const noParts: ReplyParts = { parts: [], queued: false, retrying: false, status: null } export function addEvent(reply: ReplyParts, event: ReplyEvent): ReplyParts { switch (event.type) { @@ -98,6 +103,8 @@ export function addEvent(reply: ReplyParts, event: ReplyEvent): ReplyParts { return { ...reply, status: event.status } case 'queued': return { ...reply, queued: true } + case 'retrying': + return { ...reply, retrying: true } case 'part-start': return setPart(reply, event.index, { $type: `${nsid.defs}#${event.partType}`, text: '' }) case 'part': @@ -117,9 +124,9 @@ export function addEvent(reply: ReplyParts, event: ReplyEvent): ReplyParts { export const addEvents = (reply: ReplyParts, events: ReplyEvent[]): ReplyParts => events.reduce(addEvent, reply) -/** The reply with one part replaced. A part arriving means the turn has left the queue. */ +/** The reply with one part replaced. A part arriving means the turn is no longer waiting. */ function setPart(reply: ReplyParts, index: number, part: Part): ReplyParts { const parts = [...reply.parts] parts[index] = part - return { ...reply, parts, queued: false } + return { ...reply, parts, queued: false, retrying: false } } diff --git a/apps/web/tests/features/conversation/components/MessageView.test.tsx b/apps/web/tests/features/conversation/components/MessageView.test.tsx index 6690d0f..d022460 100644 --- a/apps/web/tests/features/conversation/components/MessageView.test.tsx +++ b/apps/web/tests/features/conversation/components/MessageView.test.tsx @@ -372,6 +372,17 @@ describe('MessageView', () => { expect(container).toBeEmptyDOMElement() }) + it('announces a wait on a busy provider below what the reply streamed so far', () => { + const text = { $type: d('textPart'), text: 'Here' } + const { rerender } = render( + , + ) + expect(screen.getByText('Here')).toBeVisible() + expect(screen.getByRole('status')).toBeInTheDocument() + rerender() + expect(screen.queryByRole('status')).not.toBeInTheDocument() + }) + it('shows a pending reply as its model waiting in the header', () => { const { container } = render( { ) }) + it('marks a turn waiting on its provider until the next part arrives', () => { + const started = addEvent(noParts, { type: 'part-start', index: 0, partType: 'textPart' }) + const retrying = addEvent(started, { type: 'retrying' }) + expect(retrying.retrying).toBe(true) + expect( + addEvent(retrying, { type: 'part-start', index: 1, partType: 'textPart' }).retrying, + ).toBe(false) + }) + it('keeps the final status', () => { expect(addEvent(noParts, { type: 'status', status: 'error' }).status).toBe('error') }) diff --git a/specs/chat-turns.md b/specs/chat-turns.md index 91dabfb..9a2de77 100644 --- a/specs/chat-turns.md +++ b/specs/chat-turns.md @@ -77,7 +77,7 @@ Other parts are ignored. When a stream part carries `providerMetadata`, it is JS ### Ending states - **Complete.** The stream finished normally. Status `complete`. -- **Error.** The stream yielded an `error` part, or anything in the turn threw. Status `error`, with a short message from the error that never includes API keys or request headers. The partial parts are kept. +- **Error.** The stream yielded an `error` part, or anything in the turn threw. Status `error`, with a short message from the error that never includes API keys or request headers. The partial parts are kept. A provider that turns a request away with a rate limit or a server error is retried five times, about a minute in all, or after the waits it asks for through `Retry-After`. If it is still rate limited after that, the message says the model provider is limiting requests, in plain words and without naming it. A key that has run out of credit or budget, which OpenAI reports as `insufficient_quota` and LiteLLM as `budget_exceeded`, is not retried. When it is the admin's shared key, the message says the shared key has reached its usage limit and to add their own key under Settings > API keys. When it is the user's own key, it says their key has run out and to add credit to its account or use another key. These messages never name the provider, since users may not know it. - **Cancelled.** The abort signal fired. Status `cancelled`, with the partial parts kept. - **Too large.** If the final record's JSON is over 900 KB, the largest tool result outputs are uploaded as blobs and referenced from each part's `outputBlob`, and `output` keeps a short summary. If the record is still too large, the turn ends with status `error`. @@ -93,6 +93,7 @@ On startup, the server rewrites any `pending` reply in its own `turn_claim` tabl | `delta` | The part index and appended text | | `part` | A finished tool call, tool result, or source part | | `queued` | The turn is waiting for the rate limit | +| `retrying` | The provider turned a request away and it will be tried again. The web UI says the provider is busy until the next part arrives | | `status` | The final status and error, if any | The stream closes after `status`. If this server is not running that reply, the stream sends one `status` event with the reply's stored status and closes. The web UI then reads the record through the conversation API. A buffer is dropped a minute after its turn ends. @@ -149,6 +150,8 @@ The `turns` admin setting, from the admin-settings spec, holds `ratePerMinute`, - [ ] Error and pending replies are left out of the prompt. - [ ] Stream parts map to lexicon parts as in the table, with tool input and output JSON-encoded. - [ ] A provider error ends the turn with status `error`, a message without secrets, and the partial parts. +- [ ] A rate limited provider is retried with a `retrying` event each time. +- [ ] A key out of credit or over its budget is not retried. On the admin's key the message tells the user to add their own key, and on their own key it does not. - [ ] Cancelling ends the turn with status `cancelled` and the partial parts. - [ ] A turn past the timeout is cancelled. - [ ] A final record over 900 KB moves the largest tool outputs into `outputBlob` blobs.