Something went wrong. Try again.
Provider-neutral LLM requests, transcripts and responses, I/O-free
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263(*--------------------------------------------------------------------------- Copyright (c) 2026 Invariant Systems. All rights reserved. SPDX-License-Identifier: ISC ---------------------------------------------------------------------------*)
(* The contracts of [Llm.Transcript]. *)
open Llmopen Fixture
(* Transcript grammar and [last_assistant_text] *)
let suite = group "Transcript" [ test "accepts ordinary messages" (fun () -> let t = ready_transcript () in is_true ~msg:"ready" (Transcript.is_ready t); equal int ~msg:"count" 2 (List.length (Transcript.messages t)); roundtrip "transcript" (opaque "transcript") Transcript.jsont t); test "tool loop: calls become pending, results clear them" (fun () -> let first = call ~id:"call_1" () in let second = call ~id:"call_2" () in let t = Transcript.add_exn (Message.assistant (assistant_with_calls [ first; second ])) (ready_transcript ()) in is_true ~msg:"awaiting" (not (Transcript.is_ready t)); equal (list call_t) ~msg:"pending in output order" [ first; second ] (Transcript.pending t); let t = match Transcript.add (Message.tool_result (Tool.Result.text second "second")) t with | Ok t -> t | Error e -> failf "answer second: %a" Transcript.Error.pp e in equal (list call_t) ~msg:"remaining" [ first ] (Transcript.pending t); let t = match Transcript.add (Message.tool_result (Tool.Result.empty first)) t with | Ok t -> t | Error e -> failf "answer first: %a" Transcript.Error.pp e in is_true ~msg:"ready after all results" (Transcript.is_ready t); equal int ~msg:"message count" 5 (List.length (Transcript.messages t))); test "each grammar error is producible via add/of_list" (fun () -> let first = call ~id:"call_1" () in let second = call ~id:"call_2" () in let pending = Transcript.add_exn (Message.assistant (assistant_with_calls [ first ])) (ready_transcript ()) in (* Ordinary message while pending. *) expect_error "ordinary while pending" (Transcript.add (Message.user_text "continue") pending) (fun e -> equal transcript_error_t ~msg:"pending" (Transcript.Error.Pending_tool_results [ first ]) e); (* Unknown tool result. *) expect_error "unknown result" (Transcript.add (Message.tool_result (Tool.Result.text second "x")) pending) (fun e -> equal transcript_error_t ~msg:"unknown" (Transcript.Error.Unknown_tool_result { call_id = "call_2" }) e); (* Name mismatch. *) expect_error "name mismatch" (Transcript.add (Message.tool_result (Tool.Result.empty (Tool.Call.v ~id:"call_1" ~name:"other_tool" ~input:empty_object ()))) pending) (fun e -> equal transcript_error_t ~msg:"mismatch" (Transcript.Error.Tool_result_name_mismatch { call_id = "call_1"; expected = "read_file"; actual = "other_tool"; }) e); (* Duplicate tool call id within one assistant message. *) expect_error "duplicate call id" (Transcript.add (Message.assistant (assistant_with_calls [ first; call ~id:"call_1" () ])) (ready_transcript ())) (fun e -> equal transcript_error_t ~msg:"duplicate call" (Transcript.Error.Duplicate_tool_call { call_id = "call_1" }) e); (* Tool result without a preceding call. *) expect_error "result without call" (Transcript.add (Message.tool_result (Tool.Result.empty first)) (ready_transcript ())) (fun e -> match e with | Transcript.Error.Tool_result_without_call r -> equal string ~msg:"orphan id" "call_1" (Tool.Result.call_id r) | _ -> failf "unexpected error")); test "duplicate tool result is rejected" (fun () -> let first = call ~id:"call_1" () in let second = call ~id:"call_2" () in let t = Transcript.add_exn (Message.assistant (assistant_with_calls [ first; second ])) (ready_transcript ()) in let t = match Transcript.add (Message.tool_result (Tool.Result.empty first)) t with | Ok t -> t | Error e -> failf "first: %a" Transcript.Error.pp e in expect_error "duplicate result" (Transcript.add (Message.tool_result (Tool.Result.text first "again")) t) (fun e -> equal transcript_error_t ~msg:"duplicate" (Transcript.Error.Duplicate_tool_result { call_id = "call_1" }) e)); test "add_response of a well-formed response never errors" (fun () -> let t = ready_transcript () in (match Transcript.add_response (Response.v ~model:gpt (Message.Assistant.text "Done.")) t with | Ok t' -> is_true ~msg:"still ready" (Transcript.is_ready t') | Error e -> failf "add_response: %a" Transcript.Error.pp e); (* A tool-call response leaves the transcript pending, still no error. *) match Transcript.add_response (Response.v ~model:gpt (assistant_with_calls [ call () ])) t with | Ok t' -> is_true ~msg:"pending" (not (Transcript.is_ready t')) | Error e -> failf "add_response tool: %a" Transcript.Error.pp e); test "of_list agrees with folding add" (fun () -> let first = call ~id:"call_1" () in let messages = [ Message.developer "You are Mentat."; Message.user_text "Use a tool."; Message.assistant (assistant_with_calls [ first ]); Message.tool_result (Tool.Result.text first "contents"); Message.assistant_text "Done."; ] in let from_list = match Transcript.of_list messages with | Ok t -> t | Error e -> failf "of_list: %a" Transcript.Error.pp e in let from_add = List.fold_left (fun t m -> Transcript.add_exn m t) Transcript.empty messages in equal (opaque "transcript") ~msg:"agree" from_add from_list; is_true ~msg:"reversed order rejected" (Result.is_error (Transcript.of_list (List.rev messages)))); test "last_assistant returns the raw most recent assistant" (fun () -> equal (option pass) ~msg:"none yet" None (Transcript.last_assistant (ready_transcript ())); let two_turns = Transcript.of_list_exn [ Message.user_text "First."; Message.assistant_text "First answer."; Message.user_text "Second."; Message.assistant_text "Second answer."; ] in (match Transcript.last_assistant two_turns with | Some a -> equal (list string) ~msg:"most recent" [ "Second answer." ] (Message.Assistant.texts a) | None -> failf "expected an assistant"); (* An empty assistant turn is returned unfiltered. *) let empty_turn = Transcript.of_list_exn [ Message.user_text "hi"; Message.assistant Message.Assistant.empty; ] in match Transcript.last_assistant empty_turn with | Some a -> equal (list string) ~msg:"empty returned" [] (Message.Assistant.texts a) | None -> failf "expected the empty assistant"); test "last_assistant_text skips tool-call-only and empty turns" (fun () -> (* Most recent visible text is "First answer."; the later turns are a tool-call-only assistant then an empty assistant. *) let first = call ~id:"call_9" () in let t = Transcript.of_list_exn [ Message.user_text "q1"; Message.assistant_text "First answer."; Message.user_text "q2"; Message.assistant Message.Assistant.empty; ] in equal (option string) ~msg:"skips empty" (Some "First answer.") (Transcript.last_assistant_text t); let t2 = Transcript.add_exn (Message.assistant (Message.Assistant.v [ Message.Assistant.tool_call first ])) t in equal (option string) ~msg:"skips tool-call-only" (Some "First answer.") (Transcript.last_assistant_text t2); (* [last_assistant] returns the raw latest (the tool-call turn). *) (match Transcript.last_assistant t2 with | Some a -> equal int ~msg:"latest is the tool-call turn" 1 (List.length (Message.Assistant.tool_calls a)) | None -> failf "expected the tool-call assistant"); (* Joined text is untrimmed and joined with sep. *) let joined = Transcript.of_list_exn [ Message.user_text "q"; Message.assistant (Message.Assistant.v [ Message.Assistant.text_part " a "; Message.Assistant.text_part " b "; ]); ] in equal (option string) ~msg:"untrimmed join" (Some " a | b ") (Transcript.last_assistant_text ~sep:"|" joined); equal (option pass) ~msg:"no visible text yields None" None (Transcript.last_assistant_text (ready_transcript ()))); prop "of_list (messages t) = t for valid transcripts" transcript_gen (fun t -> match Transcript.of_list (Transcript.messages t) with | Ok t' -> is_true (List.equal Message.equal (Transcript.messages t') (Transcript.messages t)) | Error _ -> Alcotest.fail "of_list refused a valid transcript"); prop "jsont round-trips (property)" transcript_gen (fun t -> roundtrip "transcript" (opaque "transcript") Transcript.jsont t); ]