"""The checklist extension, the worked example of docs/extensions.md, through every contribution it makes: the model's Python binding and its host route, the /checklist command, the stop hook, the transfer adapter that carries its rows to another daemon and posts the moved session a note, and the cleaner that deletes them with their session. Each part only works once the extension is installed, enabled for the session, prepared into its composition and reached from a live kernel, so only the real daemon shows it. The move needs a second daemon.""" import ast from contextlib import closing import json import sqlite3 import tempfile import unittest from pathlib import Path from _transfer_support import ( activate, deliver, exported, idle, import_status, ) from harness import ( Albedo, Provider, operation_id, peer_daemon, python, text, wait_until, ) REMINDER = "Before you finish, these checklist items are still open" MOVED = "Its open checklist items came with it" def answer(daemon, method, path, body=None, headers=None): with daemon.api(path, body, method=method, headers=headers) as response: return json.load(response) def outputs(history): """What each python cell of a session printed, oldest first.""" return [ json.loads(part["value"])["output"] for entry in history["items"] if entry["kind"] == "tool_result" and entry["tool"]["name"] == "python" for part in entry["content"] if part["kind"] == "json" and part["field"] == "result" ] class ChecklistTests(unittest.TestCase): def setUp(self): # Marker in the user's words -> the cell the model answers with. self.cells = {} def reply(request): last = request["input"][-1] if last.get("type") == "function_call_output": return text("done") said = json.dumps(last.get("content", "")) if REMINDER in said: return text("the CI item stays open until the run is green") for marker, cell in self.cells.items(): if marker in said: return python(cell) return text("ok") self.provider = Provider(reply) self.addCleanup(self.provider.close) self.app = Albedo(self.provider, protocol="responses") self.app.__enter__() self.addCleanup(self.app.__exit__, None, None, None) def session(self): session = self.app.session() self.app.patch_session( session, {"selection": {"extensions": {"checklist": True}}} ) return session def say(self, session, words): self.app.prompt(session, words).close() self.app.idle(session) def said(self, session, words): """What the cell the model ran for `words` printed.""" self.say(session, words) return outputs(self.app.history(session))[-1] def user_texts(self): """Every user message the provider was sent, as JSON text.""" return [ json.dumps(item.get("content", "")) for entry in self.provider.requests for item in entry["request"]["input"] if item.get("role") == "user" ] def stored(self, home, session): path = home / "albedo.sqlite" with closing(sqlite3.connect(f"file:{path}?mode=ro", uri=True)) as db: return db.execute( "SELECT text, done FROM checklist_items WHERE session=? ORDER BY id", (session,), ).fetchall() def test_the_model_keeps_its_checklist_from_python(self): self.cells = { "add two": ( "a = await checklist.add('rerun the flaky test')\n" "b = await checklist.add(' bump the version ')\n" "await checklist.check(a['id'])\n" "print([(i['text'], i['done']) for i in await checklist.items()])\n" "print(b['id'])" ), "refused": ( "for call in (lambda: checklist.add(' ')," " lambda: checklist.check(int(ID))):\n" " try:\n" " await call()\n" " except Exception as error:\n" " print(type(error).__name__, error)" ), } mine = self.session() listed, own = self.said(mine, "add two items").strip().split("\n") self.assertEqual( listed, "[('bump the version', False), ('rerun the flaky test', True)]" ) # Another session sees neither the item nor a way to check it off. self.cells["refused"] = self.cells["refused"].replace("ID", own) refusals = self.said(self.session(), "refused calls").strip().split("\n") self.assertEqual(len(refusals), 2, refusals) self.assertIn("an item needs text", refusals[0]) self.assertIn("no such checklist item in this session", refusals[1]) self.assertEqual( self.stored(self.app.home, mine), [("rerun the flaky test", 1), ("bump the version", 0)], ) def test_open_items_hold_the_turn_once(self): self.cells = {"remember": "await checklist.add('wait for CI')"} session = self.session() self.say(session, "remember to wait") reminders = {said for said in self.user_texts() if REMINDER in said} self.assertEqual(len(reminders), 1, reminders) self.assertIn("wait for CI", reminders.pop()) # The cell, the reply the hook held, and the answer to the reminder: # the item is still open, but the hook lets the turn end. self.assertEqual(len(self.provider.requests), 3) def test_the_command_asks_the_model_to_add_an_item(self): self.cells = {"then carry on:": "await checklist.add('update the changelog')"} session = self.session() with self.app.api( f"/sessions/{session}/inputs/{operation_id()}", { "kind": "command", "command_id": "/checklist", "arguments": {"item": "update the changelog"}, }, method="PUT", ) as response: json.load(response) self.app.idle(session) self.assertTrue( any( "Add this to the session checklist" in said and "update the changelog" in said for said in self.user_texts() ) ) self.assertEqual( self.stored(self.app.home, session), [("update the changelog", 0)] ) def test_full_checklist_refuses_an_add_without_losing_items(self): self.cells = { "fill the checklist": ( "for n in range(50):\n" " await checklist.add(f'item {n}')\n" "try:\n" " await checklist.add('one too many')\n" "except Exception as error:\n" " print(type(error).__name__, error)" ), "make room": ( "items = await checklist.items()\n" "await checklist.check(items[0]['id'])\n" "await checklist.add('replacement')" ), } session = self.session() refusal = self.said(session, "fill the checklist") self.assertIn("50 open items; check some off first", refusal) self.assertEqual( self.stored(self.app.home, session), [(f"item {n}", 0) for n in range(50)], ) self.say(session, "make room") rows = self.stored(self.app.home, session) self.assertEqual(sum(done == 0 for _, done in rows), 50) self.assertEqual(rows[-1], ("replacement", 0)) def test_a_session_keeps_only_its_newest_checked_items(self): # 55 add-and-check rounds, then one open item: the open item stays, # and only the 50 newest checked items do. self.cells = { "churn": ( "for n in range(55):\n" " await checklist.check((await checklist.add(f'done {n}'))['id'])\n" "await checklist.add('still open')" ) } session = self.session() self.say(session, "churn through items") rows = self.stored(self.app.home, session) self.assertEqual(len([row for row in rows if row[1] == 1]), 50) self.assertEqual(rows[0], ("done 5", 1)) self.assertEqual(rows[-1], ("still open", 0)) def test_deleting_a_session_deletes_its_items(self): self.cells = {"remember": "await checklist.add('a doomed item')"} session = self.session() kept = self.session() self.say(session, "remember this") self.say(kept, "remember that") self.assertEqual(len(self.stored(self.app.home, session)), 1) outcome = self.app.delete_tree(session) self.assertEqual(outcome["state"], "complete", outcome) self.assertEqual(self.stored(self.app.home, session), []) self.assertEqual(self.stored(self.app.home, kept), [("a doomed item", 0)]) def test_items_move_with_their_session(self): target = self.target() destination = Path(tempfile.mkdtemp(prefix="moved-", dir=target.root)) self.cells = { "travel": ( "a = await checklist.add('pack the bags')\n" "b = await checklist.add('book the train')\n" "await checklist.check(b['id'])\n" "print(b['id'])" ), "after the move": ( "print(sorted((i['id'], i['text'], i['done'])" " for i in await checklist.items()))" ), } session = self.session() top = int(self.said(session, "travel plans")) # The target's own session numbers its items past the moved ones, so # an import that kept the source's ids would collide with them. self.cells["neighbour"] = ( f"for _ in range({top}):\n" " await checklist.check((await checklist.add('filler'))['id'])\n" "print((await checklist.add('the target keeps this'))['id'])" ) neighbour = self.target_session(target, destination) self.say_on(target, neighbour, "neighbour items") kept = int(outputs(self.history_on(target, neighbour))[-1]) self.assertGreater(kept, top) self.move(target, session, destination) self.say_on(target, session, "list after the move") moved = ast.literal_eval(outputs(self.history_on(target, session))[-1]) self.assertEqual( [(text, done) for _, text, done in moved], [("pack the bags", False), ("book the train", True)], ) self.assertTrue(all(row[0] > kept for row in moved), moved) # The note activation posted rode with the moved session's next turn, # naming only the open item. notes = {said for said in self.user_texts() if MOVED in said} self.assertEqual(len(notes), 1, notes) note = notes.pop() self.assertIn("pack the bags", note) self.assertNotIn("book the train", note) # ---- the move --------------------------------------------------------- def target(self): profile = self.app.profile url = self.provider.url def configure(daemon): config = { "providers": { profile: { "extension": "openai", "baseUrl": url, "apiKey": "fixture-key", "model": "fixture-model", "protocol": "responses", } }, "active": profile, } (daemon.home / "config.json").write_text(json.dumps(config)) return peer_daemon("checklist-target", prepare=configure) def target_session(self, target, workspace): session = operation_id() answer( target, "PUT", f"/sessions/{session}", { "kind": "new", "workspace": str(workspace), "provider_profile": self.app.profile, }, {"If-None-Match": "*"}, ) route = f"/sessions/{session}?view=configuration" with target.api(route) as response: json.load(response) etag = response.headers["ETag"] answer( target, "PATCH", route, {"selection": {"extensions": {"checklist": True}}}, {"If-Match": etag}, ) return session def history_on(self, target, session): return answer(target, "GET", f"/sessions/{session}/history") def say_on(self, target, session, words): answer( target, "PUT", f"/sessions/{session}/inputs/{operation_id()}", {"kind": "message", "text": words}, ) idle(target, session, 30, f"session {session} did not settle on the target") def move(self, target, session, destination): """Exports `session` here, stages it into `destination` on the target, and activates it there, as the /cd move's transport does.""" transfer, manifest, segments = exported(self.app.daemon, session) ready = deliver( target, transfer, json.dumps(manifest).encode(), segments, destination ) self.assertEqual(ready["state"], "ready", ready) activate(target, transfer) seen = {} def settled(): seen.update(import_status(target, transfer)) # A refused activation goes back to ready, with its error. return seen["state"] == "active" or seen["error"] is not None wait_until(settled, 120, f"transfer {transfer} did not settle") self.assertEqual((seen["state"], seen["error"]), ("active", None)) if __name__ == "__main__": unittest.main()