Something went wrong. Try again.
fork of Kludex/zigai, ported to zig 0.17
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342//! Ordered fallback over provider-neutral models.
const std = @import("std");const model_types = @import("../model.zig");
pub const Event = struct { failed_index: usize, next_index: usize, failure: anyerror,};
/// Tries models in order when a request fails with a fallback-safe error.pub const Fallback = struct { models: []const model_types.Model, context: ?*anyopaque = null, shouldFallbackFn: ?*const fn (context: ?*anyopaque, failure: anyerror) bool = null, onFallbackFn: ?*const fn (context: ?*anyopaque, event: Event) anyerror!void = null,
pub fn model(self: *Fallback) model_types.Model { return .{ .context = self, .profile = commonProfile(self.models), .requestFn = request, .streamFn = stream, }; }
fn request( context: *anyopaque, allocator: std.mem.Allocator, request_value: model_types.ModelRequest, ) !model_types.ModelResponse { const self: *Fallback = @ptrCast(@alignCast(context)); if (self.models.len == 0) return error.NoFallbackModels; for (self.models, 0..) |candidate, index| { var candidate_request = request_value; candidate_request.settings = candidate.settings.overrideWith(request_value.settings); return candidate.request(allocator, candidate_request) catch |failure| { if (index + 1 == self.models.len or !self.shouldFallback(failure)) return failure; try self.notify(.{ .failed_index = index, .next_index = index + 1, .failure = failure }); continue; }; } unreachable; }
fn stream( context: *anyopaque, allocator: std.mem.Allocator, request_value: model_types.ModelRequest, sink: model_types.ModelStreamSink, ) !model_types.ModelResponse { const self: *Fallback = @ptrCast(@alignCast(context)); if (self.models.len == 0) return error.NoFallbackModels; for (self.models, 0..) |candidate, index| { var emitted = false; var forwarding = ForwardingSink{ .sink = sink, .emitted = &emitted }; var candidate_request = request_value; candidate_request.settings = candidate.settings.overrideWith(request_value.settings); return candidate.stream(allocator, candidate_request, forwarding.sinkValue()) catch |failure| { if (emitted or index + 1 == self.models.len or !self.shouldFallback(failure)) return failure; try self.notify(.{ .failed_index = index, .next_index = index + 1, .failure = failure }); continue; }; } unreachable; }
fn shouldFallback(self: Fallback, failure: anyerror) bool { if (self.shouldFallbackFn) |classify| return classify(self.context, failure); return switch (failure) { error.ProviderRateLimited, error.ProviderServerError, error.ProviderRequestFailed, error.RequestTimedOut, error.InvalidProviderResponse, => true, else => false, }; }
fn notify(self: Fallback, event: Event) !void { const callback = self.onFallbackFn orelse return; return callback(self.context, event); }};
const ForwardingSink = struct { sink: model_types.ModelStreamSink, emitted: *bool,
fn sinkValue(self: *ForwardingSink) model_types.ModelStreamSink { return .{ .context = self, .eventFn = emit }; }
fn emit(context: *anyopaque, event: model_types.ModelStreamEvent) !void { const self: *ForwardingSink = @ptrCast(@alignCast(context)); try self.sink.emit(event); self.emitted.* = true; }};
fn commonProfile(models: []const model_types.Model) model_types.ModelProfile { if (models.len == 0) return .{ .supports_tools = false, .supports_parallel_tool_calls = false, .supports_system_messages = false, .supports_max_tokens = false, }; var profile = models[0].profile; for (models[1..]) |candidate| { const other = candidate.profile; profile.supports_tools = profile.supports_tools and other.supports_tools; profile.supports_parallel_tool_calls = profile.supports_parallel_tool_calls and other.supports_parallel_tool_calls; profile.supports_json_schema_output = profile.supports_json_schema_output and other.supports_json_schema_output; profile.supports_json_object_output = profile.supports_json_object_output and other.supports_json_object_output; profile.supports_system_messages = profile.supports_system_messages and other.supports_system_messages; profile.supports_thinking = profile.supports_thinking and other.supports_thinking; profile.supports_streaming = profile.supports_streaming and other.supports_streaming; profile.supports_temperature = profile.supports_temperature and other.supports_temperature; profile.supports_max_tokens = profile.supports_max_tokens and other.supports_max_tokens; profile.supports_stop_sequences = profile.supports_stop_sequences and other.supports_stop_sequences; profile.supports_seed = profile.supports_seed and other.supports_seed; profile.supports_top_p = profile.supports_top_p and other.supports_top_p; profile.supports_top_k = profile.supports_top_k and other.supports_top_k; profile.supports_presence_penalty = profile.supports_presence_penalty and other.supports_presence_penalty; profile.supports_frequency_penalty = profile.supports_frequency_penalty and other.supports_frequency_penalty; profile.supports_logprobs = profile.supports_logprobs and other.supports_logprobs; profile.supports_tool_choice = profile.supports_tool_choice and other.supports_tool_choice; profile.supports_parallel_tool_call_setting = profile.supports_parallel_tool_call_setting and other.supports_parallel_tool_call_setting; profile.supports_thinking_budget = profile.supports_thinking_budget and other.supports_thinking_budget; profile.supports_truncation = profile.supports_truncation and other.supports_truncation; profile.supports_request_headers = profile.supports_request_headers and other.supports_request_headers; if (profile.extra_body_kind != other.extra_body_kind) profile.extra_body_kind = null; profile.reasoning_efforts.setIntersection(other.reasoning_efforts); profile.service_tiers.setIntersection(other.service_tiers); profile.builtin_tools.setIntersection(other.builtin_tools); profile.content_types.setIntersection(other.content_types); } return profile;}
test "fallback tries the next model and preserves candidate settings" { const State = struct { calls: usize = 0, fn fail(context: *anyopaque, _: std.mem.Allocator, request_value: model_types.ModelRequest) !model_types.ModelResponse { const self: *@This() = @ptrCast(@alignCast(context)); self.calls += 1; try std.testing.expectEqual(@as(?f64, 0.3), request_value.settings.temperature); return error.ProviderServerError; } fn succeed(context: *anyopaque, _: std.mem.Allocator, request_value: model_types.ModelRequest) !model_types.ModelResponse { const self: *@This() = @ptrCast(@alignCast(context)); self.calls += 1; try std.testing.expectEqual(@as(?u64, 200), request_value.settings.max_tokens); return .{ .parts = &.{.{ .text = "ok" }} }; } }; const Capture = struct { calls: usize = 0, fn fallback(context: ?*anyopaque, event: Event) !void { const self: *@This() = @ptrCast(@alignCast(context.?)); try std.testing.expectEqual(@as(usize, 0), event.failed_index); try std.testing.expectEqual(error.ProviderServerError, event.failure); self.calls += 1; } }; var first_state: State = .{}; var second_state: State = .{}; var capture: Capture = .{}; const candidates = [_]model_types.Model{ .{ .context = &first_state, .profile = .{}, .settings = .{ .temperature = 0.3 }, .requestFn = State.fail, }, .{ .context = &second_state, .profile = .{}, .settings = .{ .max_tokens = 100 }, .requestFn = State.succeed, }, }; var fallback = Fallback{ .models = &candidates, .context = &capture, .onFallbackFn = Capture.fallback, }; const response = try fallback.model().request(std.testing.allocator, .{ .messages = &.{}, .settings = .{ .max_tokens = 200 }, }); try std.testing.expectEqualStrings("ok", response.parts[0].text); try std.testing.expectEqual(@as(usize, 1), first_state.calls); try std.testing.expectEqual(@as(usize, 1), second_state.calls); try std.testing.expectEqual(@as(usize, 1), capture.calls);}
test "fallback does not retry unsafe failures" { const Failing = struct { fn request(_: *anyopaque, _: std.mem.Allocator, _: model_types.ModelRequest) !model_types.ModelResponse { return error.InvalidRequestEncoding; // kcov-ignore } }; var unused: u8 = 0; const candidates = [_]model_types.Model{ .{ .context = &unused, .profile = .{}, .requestFn = Failing.request }, .{ .context = &unused, .profile = .{}, .requestFn = Failing.request }, }; var fallback = Fallback{ .models = &candidates }; try std.testing.expectError( error.InvalidRequestEncoding, fallback.model().request(std.testing.allocator, .{ .messages = &.{} }), );}
test "fallback profile intersects builtin tools" { const Stub = struct { fn request(_: *anyopaque, _: std.mem.Allocator, _: model_types.ModelRequest) !model_types.ModelResponse { return .{ .parts = &.{.{ .text = "unused" }} }; } }; var unused: u8 = 0; const candidates = [_]model_types.Model{ .{ .context = &unused, .profile = .{ .supports_top_p = true, .supports_top_k = true, .supports_presence_penalty = true, .supports_frequency_penalty = true, .supports_logprobs = true, .supports_tool_choice = true, .supports_parallel_tool_call_setting = true, .supports_thinking_budget = true, .supports_truncation = true, .supports_request_headers = true, .extra_body_kind = .openai, .service_tiers = model_types.ModelProfile.ServiceTierSet.full, .builtin_tools = model_types.ModelProfile.BuiltinToolSet.initMany(&.{ .web_search, .web_fetch }), .content_types = model_types.ModelProfile.ContentTypeSet.initMany(&.{ .image, .audio }), }, .requestFn = Stub.request }, .{ .context = &unused, .profile = .{ .supports_top_p = true, .extra_body_kind = .anthropic, .service_tiers = model_types.ModelProfile.ServiceTierSet.initOne(.default), .builtin_tools = model_types.ModelProfile.BuiltinToolSet.initMany(&.{.web_search}), .content_types = model_types.ModelProfile.ContentTypeSet.initMany(&.{.image}), }, .requestFn = Stub.request }, }; const direct = try candidates[0].request(std.testing.allocator, .{ .messages = &.{} }); try std.testing.expectEqualStrings("unused", direct.parts[0].text); var fallback = Fallback{ .models = &candidates }; const profile = fallback.model().profile; try std.testing.expect(profile.supportsBuiltinTool(.web_search)); try std.testing.expect(!profile.supportsBuiltinTool(.web_fetch)); try std.testing.expect(profile.supportsContentType(.image)); try std.testing.expect(!profile.supportsContentType(.audio)); try std.testing.expect(profile.supports_top_p); try std.testing.expect(!profile.supports_top_k); try std.testing.expect(!profile.supports_presence_penalty); try std.testing.expect(!profile.supports_frequency_penalty); try std.testing.expect(!profile.supports_logprobs); try std.testing.expect(!profile.supports_tool_choice); try std.testing.expect(!profile.supports_parallel_tool_call_setting); try std.testing.expect(!profile.supports_thinking_budget); try std.testing.expect(!profile.supports_truncation); try std.testing.expect(!profile.supports_request_headers); try std.testing.expectEqual(@as(?model_types.ExtraBodyKind, null), profile.extra_body_kind); try std.testing.expect(profile.supportsServiceTier(.default)); try std.testing.expect(!profile.supportsServiceTier(.priority)); const response = try fallback.model().request(std.testing.allocator, .{ .messages = &.{} }); try std.testing.expectEqualStrings("unused", response.parts[0].text);}
test "stream fallback stops after exposing output" { const State = struct { calls: usize = 0, emit_before_failure: bool, succeeds: bool = false, fn request(_: *anyopaque, _: std.mem.Allocator, _: model_types.ModelRequest) !model_types.ModelResponse { return error.Unused; // kcov-ignore } fn stream( context: *anyopaque, _: std.mem.Allocator, _: model_types.ModelRequest, sink: model_types.ModelStreamSink, ) !model_types.ModelResponse { const self: *@This() = @ptrCast(@alignCast(context)); self.calls += 1; if (self.succeeds) return .{ .parts = &.{.{ .text = "recovered" }} }; if (self.emit_before_failure) try sink.emit(.{ .part_delta = .{ .index = 0, .delta = .{ .text = .{ .content_delta = "visible" } }, } }); return error.ProviderServerError; } }; const Sink = struct { events: usize = 0, fn emit(context: *anyopaque, _: model_types.ModelStreamEvent) !void { const self: *@This() = @ptrCast(@alignCast(context)); self.events += 1; } }; var first = State{ .emit_before_failure = true }; var second = State{ .emit_before_failure = false }; const candidates = [_]model_types.Model{ .{ .context = &first, .profile = .{ .supports_streaming = true }, .requestFn = State.request, .streamFn = State.stream }, .{ .context = &second, .profile = .{ .supports_streaming = true }, .requestFn = State.request, .streamFn = State.stream }, }; var fallback = Fallback{ .models = &candidates }; var sink_state: Sink = .{}; try std.testing.expectError(error.ProviderServerError, fallback.model().stream( std.testing.allocator, .{ .messages = &.{} }, .{ .context = &sink_state, .eventFn = Sink.emit }, )); try std.testing.expectEqual(@as(usize, 1), first.calls); try std.testing.expectEqual(@as(usize, 0), second.calls); try std.testing.expectEqual(@as(usize, 1), sink_state.events);
var retry_first = State{ .emit_before_failure = false }; var retry_second = State{ .emit_before_failure = false, .succeeds = true }; const retry_candidates = [_]model_types.Model{ .{ .context = &retry_first, .profile = .{ .supports_streaming = true }, .requestFn = State.request, .streamFn = State.stream }, .{ .context = &retry_second, .profile = .{ .supports_streaming = true }, .requestFn = State.request, .streamFn = State.stream }, }; var retrying = Fallback{ .models = &retry_candidates }; const recovered = try retrying.model().stream( std.testing.allocator, .{ .messages = &.{} }, .{ .context = &sink_state, .eventFn = Sink.emit }, ); try std.testing.expectEqualStrings("recovered", recovered.parts[0].text); try std.testing.expectEqual(@as(usize, 1), retry_first.calls); try std.testing.expectEqual(@as(usize, 1), retry_second.calls); try std.testing.expectError(error.Unused, retry_candidates[0].request(std.testing.allocator, .{ .messages = &.{} }));}