Something went wrong. Try again.
small gleam coding and (not yet) persistent agent daemon with a detachable cli forked from okami.mom/albedo
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312"""A kernel that cannot run Python is diagnosed, held to its deadline, and replaced.
Native code can hold the interpreter so that nothing the daemon sends getsthrough: no interrupt, no deadline timer, no background notice. The port ownerprobes the kernel's link while it has work and reads the process when the probegoes unanswered. A verdict informs, since a C call that blocks for a whilelooks like one that never returns: a waiting turn is answered as backgroundedwith the diagnosis so the model can tell the user, and the owner enforces thedeadline the kernel's own timer cannot, ending the kernel with a result and awake that say why; the next cell restores the variables last saved. A kernelthat answers its probes, however busy, is left alone. A kernel found busy aftera daemon restart because it is asleep in native code, which nothing bounds, isended after the idle limit instead of being waited for forever."""
import astfrom contextlib import suppressimport jsonimport osimport signalimport unittest
import harnessfrom harness import ( Albedo, Provider, Reply, alive, exclusive, exclusive_class, latest_user, text,)
# Records this kernel's pid and that of a job it holds, then blocks in libc# with the GIL held: no Python thread can run, so neither the deadline timer# nor an interrupt reaches the cell, and the probe goes unanswered.ASLEEP = ( "import asyncio, ctypes, os, pathlib\n" "hold = run('sleep', '120')\n" "while hold.process is None:\n" " await asyncio.sleep(0.01)\n" "pathlib.Path({marker!r}).write_text(f'{{os.getpid()}} {{hold.process.pid}}')\n" "ctypes.PyDLL(None).sleep(600)")
# Records the pid, then backtracks forever in the regex engine, which holds# the GIL and never checks for signals: the kernel computes at full CPU and# cannot say anything.SPINNING = ( "import os, pathlib, re\n" "pathlib.Path({marker!r}).write_text(str(os.getpid()))\n" "re.match(r'(a+)+$', 'a' * 40 + '!')")
PROBE = "import os\n(globals().get('survivor'), os.getpid())"
def end(*pids): for pid in pids: with suppress(OSError): os.kill(pid, signal.SIGKILL)
def pids(marker): harness.wait_until(marker.exists, 30, "the wedging cell never started") return [int(word) for word in marker.read_text().split()]
class Wedges(unittest.TestCase): def boot(self, prepare=None): self.cells = [] self.turn = []
def script(request): last = request["input"][-1] if last.get("role") == "user" and "cells finished" in latest_user(request): return text("completion received") if last.get("role") == "user" and self.cells: code, timeout_ms = self.cells.pop(0) return Reply( "python", tool_arguments={"code": code, "timeout_ms": timeout_ms} ) return text("ok")
def environment(app): # Probes every 250 ms, so a verdict takes about a second; the # background threshold is the kernel's own, in wall time. app.env.update( ALBEDO_TEST_TIME_SCALE="20", ALBEDO_CELL_BACKGROUND_SECONDS="0.5", ALBEDO_IDLE_SECONDS="600", ) if prepare is not None: prepare(app)
self.provider = Provider(script) self.addCleanup(self.provider.close) self.app = Albedo(self.provider, protocol="responses", prepare=environment) self.app.__enter__() self.addCleanup(self.app.__exit__, None, None, None) self.session = self.app.session() self.log_offset = self.log_size()
def log_size(self): log = self.app.home / "daemon.log" return log.stat().st_size if log.exists() else 0
def log(self): """The daemon log written since this test began.""" log = self.app.home / "daemon.log" return log.read_bytes()[self.log_offset :].decode(errors="replace")
def cell(self, code, timeout_ms=60_000, settle=30): """Run one cell through a turn: its result, and every daemon note the model saw during the turn, whether it rode on the prompt or came as a message of its own.""" self.cells.append((code, timeout_ms)) start = len(self.provider.requests) self.app.prompt(self.session, "run it").close() self.app.idle(self.session, timeout=settle) results = [ json.loads(part["value"]) for entry in self.app.history(self.session)["items"] if entry["kind"] == "tool_result" and entry["tool"]["name"] == "python" for part in entry["content"] if part["kind"] == "json" and part["field"] == "result" ] self.turn = self.provider.requests[start:] return results[-1], self.notes()
def notes(self): return "".join( str(item.get("content", "")) for record in self.turn for item in record["request"]["input"] if item.get("role") == "user" and "<system-note>" in str(item.get("content", "")) )
def wakes(self): return [ latest_user(record["request"]) for record in self.provider.requests if "cells finished" in latest_user(record["request"]) ]
def assert_ended(self, *pids): harness.wait_until( lambda: not any(alive(pid) for pid in pids), 15, "the wedged kernel or its job outlived its replacement", )
def saved(self): state = self.app.home / "kernels" / f"{self.session}.state" harness.wait_until(state.exists, 30, "the resting kernel was never saved") return state
@exclusive_classclass KernelWedgeTests(Wedges): def setUp(self): self.boot()
def test_a_kernel_asleep_in_native_code_is_reported_then_replaced_at_its_deadline_with_its_saved_variables( self, ): first, _ = self.cell("import os\nsurvivor = 41\nos.getpid()") self.assertEqual(first["status"], "ok", first) self.saved() # the resting release wrote the namespace to disk marker = self.app.workspace / "asleep"
# The deadline outlasts the verdict; on the test clock it is 3 s away. result, _ = self.cell(ASLEEP.format(marker=str(marker)), timeout_ms=60_000)
kernel, job = pids(marker) self.addCleanup(end, kernel, job) # Asleep in libc with the GIL: told to the model, not ended on that. self.assertEqual(result["status"], "backgrounded", result) self.assertIn("using no CPU", result["output"]) self.assertIn("deadline (60 s)", result["output"]) self.assertIn("kernel_unresponsive", self.log()) self.assertIn("asleep", self.log()) self.assertNotIn("kernel_wedged", self.log()) # At the deadline the owner interrupts, and ends what will not stop. harness.wait_until(self.wakes, 20, "the deadline never woke the session") wake = self.wakes()[-1] self.assertIn("status=interrupted", wake) self.assertIn("ran past its deadline of 60 s", wake) self.assertIn("using no CPU", wake) self.assert_ended(kernel, job) self.assertIn("{deadline,", self.log()) self.app.idle(self.session)
# The replacement restores the save, whole, so the restore itself is # silent: what the model was told is in the result and the wake. The # job handle made after the save is not in it. after, _ = self.cell( "import os\n(globals().get('survivor'), 'hold' in globals(), os.getpid())" ) survivor, held, pid = ast.literal_eval(after["value"]) self.assertEqual(survivor, 41, "the replacement did not restore the save") self.assertFalse(held) self.assertNotEqual(pid, kernel)
def test_a_kernel_spinning_in_native_code_is_backgrounded_for_the_model_then_held_to_its_deadline( self, ): marker = self.app.workspace / "spinning"
# The deadline outlasts the verdict; on the test clock it is 3 s away. result, _ = self.cell(SPINNING.format(marker=str(marker)), timeout_ms=60_000)
(kernel,) = pids(marker) self.addCleanup(end, kernel) # The kernel could not send its own background notice; the owner did, # so the turn ended instead of waiting out the deadline. self.assertEqual(result["status"], "backgrounded", result) self.assertIn("has not answered", result["output"]) self.assertIn("deadline (60 s)", result["output"]) self.assertIn("kernel_unresponsive", self.log()) self.assertIn("spinning", self.log()) self.assertNotIn("kernel_wedged", self.log()) # At the deadline the owner interrupts, and ends what will not stop. harness.wait_until(self.wakes, 20, "the deadline never woke the session") wake = self.wakes()[-1] self.assertIn("status=interrupted", wake) self.assertIn("ran past its deadline of 60 s", wake) self.assert_ended(kernel) self.assertIn("kernel_wedged", self.log()) self.app.idle(self.session)
cell_id = result["cell_id"] after, _ = self.cell(f"print((await cells.info({cell_id!r}))['status'])") self.assertEqual(after["output"].strip(), "interrupted")
def test_a_cell_that_spins_in_python_answers_probes_and_is_left_alone(self): result, _ = self.cell( "import os, time\n" "started = time.monotonic()\n" "while time.monotonic() - started < 2.5:\n" " pass\n" "os.getpid()", timeout_ms=30_000, )
# Backgrounded by the kernel itself at the threshold, at full CPU. self.assertEqual(result["status"], "backgrounded", result) self.assertIn("wake automatically", result["output"]) harness.wait_until(self.wakes, 20, "the cell never finished") self.assertIn("status=ok", self.wakes()[-1]) kernel = int( harness.cell_result( next( i["output"] for r in reversed(self.provider.requests) for i in r["request"]["input"] if i.get("type") == "function_call_output" ) ).get("value", "0") or 0 ) log = self.log() self.assertNotIn("kernel_unresponsive", log) self.assertNotIn("kernel_wedged", log) self.app.idle(self.session) after, _ = self.cell(PROBE) _, pid = ast.literal_eval(after["value"]) self.assertTrue(alive(pid), "the kernel did not survive its busy cell") if kernel: self.assertEqual(pid, kernel)
# exclusive: restarts the daemon@exclusiveclass KernelWedgeAfterDaemonRestartTests(Wedges): def setUp(self): # Nothing bounds a kernel busy since a restart: it gets the idle limit, # 1 s on this test clock. self.boot(prepare=lambda app: app.env.update(ALBEDO_IDLE_SECONDS="20"))
def test_a_kernel_busy_after_a_restart_because_it_is_asleep_is_ended_after_the_idle_limit( self, ): gate = self.app.workspace / "busy-gate" first, _ = self.cell( "import ctypes, os, threading, time\n" "def hold():\n" f" while not os.path.exists({str(gate)!r}):\n" " time.sleep(0.05)\n" " ctypes.PyDLL(None).sleep(600) # libc sleep, GIL held\n" "threading.Thread(target=hold, daemon=True).start()\n" "os.getpid()" ) kernel = int(first["value"]) self.addCleanup(end, kernel)
self.app.restart(prepare=lambda _: gate.touch()) self.log_offset = 0
# Reattached, no hello: busy. The probe finds it asleep; once silent # for the idle limit it is ended, where the daemon used to wait for # it forever, and the next cell boots a working kernel. self.assert_ended(kernel) log = self.log() self.assertIn("kernel_busy_at_attach", log) self.assertIn("{idle,asleep}", log) after, _ = self.cell(PROBE) _, pid = ast.literal_eval(after["value"]) self.assertNotEqual(pid, kernel)