Something went wrong. Try again.
llm chat done the right way™
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630import { expect, test } from "bun:test";import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs";import { tmpdir } from "node:os";import { join } from "node:path";import { createExecutor, formatExecResult, isDaemonDown, isMissingContainer, LocalDockerExecutor, overDormantCap,} from "../src/executor";
// docker isn't guaranteed in CI; probe once and skip the live cases without it.async function dockerAvailable(): Promise<boolean> { try { return ( (await Bun.spawn(["docker", "version"], { stdout: "ignore", stderr: "ignore" }).exited) === 0 ); } catch { return false; }}const HAS_DOCKER = await dockerAvailable();
const SANDBOX = { enabled: true, backend: "docker", image: "alpine:3.20", timeoutMs: 30_000, maxTimeoutMs: 300_000, idleMs: 600_000, network: false,} as const;const liveTest = HAS_DOCKER ? test : test.skip;
// A bind-mounted workspace has to live somewhere the DAEMON can see, which on// macOS rules out mkdtemp's default: /var/folders is not shared into the VM// docker runs in, while /private/tmp is. On Linux the daemon is right here.const TMP_ROOT = existsSync("/private/tmp") ? "/private/tmp" : tmpdir();
/** * Wait for a fire-and-forget docker call to land, instead of guessing at how * long it takes. Reaping spawns a container, and a fixed sleep that is fine on * an idle daemon is a coin flip when the rest of the suite is using it. */async function eventually(done: () => boolean, ms = 30_000): Promise<void> { const deadline = Date.now() + ms; while (Date.now() < deadline && !done()) await Bun.sleep(250);}
/** * Empty a workspace root the way its files were made: from inside a container. * * The sandbox runs as root, so on Linux everything it wrote into the bind mount * is root-owned and the test process cannot remove it. (On macOS the daemon's * VM maps it back to the calling user, which is why a plain rm looks fine * there.) Clearing the contents as root leaves an empty directory that belongs * to whoever made it. */async function inWorkspaceRoot(root: string, script: string): Promise<void> { await Bun.spawn( ["docker", "run", "--rm", "-v", `${root}:/x`, "alpine:3.20", "sh", "-c", script], { stdout: "ignore", stderr: "ignore" }, ).exited;}
/** Age a workspace past any TTL. Root's files, so root's `touch`. */function backdate(root: string, names: string[]): Promise<void> { return inWorkspaceRoot(root, `touch -t 200001010000 ${names.map((n) => `/x/${n}`).join(" ")}`);}
async function clearAsRoot(root: string): Promise<void> { await inWorkspaceRoot(root, "rm -rf /x/* /x/.??*"); rmSync(root, { recursive: true, force: true });}
test("createExecutor returns null when the sandbox is disabled", () => { expect( createExecutor({ enabled: false, backend: "docker", image: "alpine:3.20", timeoutMs: 30_000, idleMs: 600_000, network: false, }), ).toBeNull();});
test("createExecutor builds a docker executor when enabled", () => { const e = createExecutor({ enabled: true, backend: "docker", image: "alpine:3.20", timeoutMs: 30_000, maxTimeoutMs: 300_000, idleMs: 600_000, network: false, }); expect(e?.kind).toBe("docker");});
test("createExecutor reflects the configured runtime in kind (kata → microVM)", () => { const e = createExecutor({ enabled: true, backend: "docker", image: "alpine:3.20", runtime: "kata", dockerHost: "ssh://kloe@prattle", timeoutMs: 30_000, maxTimeoutMs: 300_000, idleMs: 600_000, network: false, }); expect(e?.kind).toBe("kata");});
test("formatExecResult surfaces exit code, stdout, and stderr", () => { const out = formatExecResult({ stdout: "hello", stderr: "oops", exitCode: 3, timedOut: false }); expect(out).toContain("exit code: 3"); expect(out).toContain("hello"); expect(out).toContain("oops");});
liveTest( "docker executor runs a command and captures stdout + exit 0", async () => { const e = new LocalDockerExecutor(SANDBOX); const r = await e.run({ command: "echo hi from sandbox" }); expect(r.exitCode).toBe(0); expect(r.stdout).toContain("hi from sandbox"); expect(r.timedOut).toBe(false); }, 60_000,);
liveTest( "a runaway command is killed inside the container, on time", async () => { const e = new LocalDockerExecutor(SANDBOX); const session = "test-" + Math.random().toString(36).slice(2); try { const started = Date.now(); const r = await e.run({ command: "sleep 60", session, timeoutMs: 2_000 }); expect(r.timedOut).toBe(true); // The in-container `timeout` fires before the client-side backstop, so the // process is actually dead rather than merely abandoned by the CLI. expect(Date.now() - started).toBeLessThan(4_000); // `[s]leep` so this probe's own command line doesn't match itself. const still = await e.run({ command: "ps -o args= | grep -c '[s]leep 60'", session }); expect(still.stdout.trim()).toBe("0"); } finally { e.disposeSession(session); } }, 90_000,);
liveTest( "a session's timeout may be raised, never past the configured max", async () => { const e = new LocalDockerExecutor({ ...SANDBOX, timeoutMs: 1_000, maxTimeoutMs: 3_000 }); const session = "test-" + Math.random().toString(36).slice(2); try { // Longer than the default and under the max: honored. expect((await e.run({ command: "sleep 2", session, timeoutMs: 5_000 })).timedOut).toBe(false); // Asking past the max is clamped to it, so this still dies. expect((await e.run({ command: "sleep 10", session, timeoutMs: 60_000 })).timedOut).toBe( true, ); } finally { e.disposeSession(session); } }, 90_000,);
liveTest( "a session starts with the inbox and outbox already there", async () => { const e = new LocalDockerExecutor(SANDBOX); const session = "test-" + Math.random().toString(36).slice(2); try { // No `mkdir -p` first: writing a file to keep is a one-liner, as the tool // description promises. const r = await e.run({ command: "echo kept > outputs/note.txt && ls inputs", session }); expect(r.exitCode).toBe(0); const got = await e.harvest(session, "/workspace/outputs", { maxFiles: 5, maxBytes: 1e6 }); expect(got.map((f) => f.path)).toEqual(["note.txt"]); } finally { e.disposeSession(session); } }, 90_000,);
liveTest( "a restart adopts the conversation's sandbox, but a policy change replaces it", async () => { const session = "test-" + Math.random().toString(36).slice(2); const first = new LocalDockerExecutor(SANDBOX); try { await first.run({ command: "echo kept > /workspace/notes.txt", session }); // A new executor over the same conversation is exactly what a server // restart looks like: the workspace should still be there. const restarted = new LocalDockerExecutor(SANDBOX); expect( (await restarted.run({ command: "cat /workspace/notes.txt", session })).stdout, ).toContain("kept"); // Change what the sandbox is allowed to do and the old container is no // longer fit to adopt, however convenient its contents. const tightened = new LocalDockerExecutor({ ...SANDBOX, network: true }); const r = await tightened.run({ command: "cat /workspace/notes.txt 2>/dev/null || echo RECREATED", session, }); expect(r.stdout).toContain("RECREATED"); } finally { first.disposeSession(session); } }, 120_000,);
liveTest( "a networked sandbox cannot reach the daemon host by its docker aliases", async () => { // The realistic path to the host is a model talked into fetching // `host.docker.internal:5432`, so the names go nowhere. This is name-level // only and deliberately so: the raw gateway IP and the open internet are // still reachable, and constraining those is firewall state on the daemon's // host (see `dockerNetwork`), not something this process can enforce. const e = new LocalDockerExecutor({ ...SANDBOX, network: true }); const session = "test-" + Math.random().toString(36).slice(2); try { const r = await e.run({ command: "for h in host.docker.internal host.internal gateway.docker.internal; do " + 'echo "$h=$(getent hosts $h | cut -d" " -f1)"; done', session, }); for (const line of r.stdout.trim().split("\n")) expect(line).toContain("=127.0.0.1"); } finally { e.disposeSession(session); } }, 90_000,);
liveTest( "the sandbox drops capabilities without breaking package installs", async () => { const e = new LocalDockerExecutor(SANDBOX); const session = "test-" + Math.random().toString(36).slice(2); try { // The effective capability set, read straight from the kernel, so this // asserts what the drop actually changed. Docker's default set is // 0xa80425fb; ours is 0xdb — chown, dac_override, fowner, fsetid, setgid, // setuid and nothing else. (An earlier version of this test asserted that // `mount` fails, which proved nothing: CAP_SYS_ADMIN was never in the // default set, so it failed before the hardening too.) const caps = await e.run({ command: "grep -E 'CapEff|NoNewPrivs' /proc/self/status", session, }); expect(caps.stdout).toContain("00000000000000db"); expect(caps.stdout).toMatch(/NoNewPrivs:\s*1/); // ...while the file-ownership caps a package manager needs survive, which // is the whole reason the drop isn't a blanket one. const unpacked = await e.run({ command: "mkdir -p /tmp/c && echo x > /tmp/c/f && chown 1000:1000 /tmp/c/f && " + "chmod u+s /tmp/c/f && echo CAPS-OK", session, }); expect(unpacked.stdout).toContain("CAPS-OK"); } finally { e.disposeSession(session); } }, 90_000,);
liveTest( "docker executor reports a nonzero exit code", async () => { const e = new LocalDockerExecutor(SANDBOX); const r = await e.run({ command: "exit 7" }); expect(r.exitCode).toBe(7); }, 60_000,);
liveTest( "a session's /workspace persists across calls; disposeSession clears it", async () => { const e = new LocalDockerExecutor(SANDBOX); const session = "test-" + Math.random().toString(36).slice(2); try { const w = await e.run({ command: "echo persisted > /workspace/note.txt", session }); expect(w.exitCode).toBe(0); const r = await e.run({ command: "cat /workspace/note.txt", session }); expect(r.stdout).toContain("persisted"); // same container → the file survived } finally { e.disposeSession(session); } // A session-less (one-off) call shares nothing with the disposed session. const fresh = await e.run({ command: "cat /workspace/note.txt 2>&1 || echo GONE" }); expect(fresh.stdout).toContain("GONE"); }, 90_000,);
liveTest( "a blob written in comes back out byte-identical, including binary", async () => { const e = new LocalDockerExecutor(SANDBOX); const session = "test-" + Math.random().toString(36).slice(2); // Bytes that would not survive a text round-trip: NUL, high bytes, a lone // 0xff. `docker exec -i` pipes stdin through untouched, which is the whole // reason this path uses `cat >` rather than a tar or a base64 hop. const bytes = new Uint8Array([0, 1, 2, 255, 254, 10, 13, 0, 65, 66]); try { await e.putFile(session, "/workspace/inputs/blob.bin", bytes); const r = await e.run({ command: "wc -c < /workspace/inputs/blob.bin", session }); expect(r.stdout.trim()).toBe(String(bytes.length));
// Promotion: the outbox is drained, not merely read. await e.run({ command: "mkdir -p /workspace/outputs && cp /workspace/inputs/blob.bin /workspace/outputs/out.bin", session, }); const got = await e.harvest(session, "/workspace/outputs", { maxFiles: 10, maxBytes: 1_000_000, }); expect(got).toHaveLength(1); expect(got[0]!.path).toBe("out.bin"); expect(Array.from(got[0]!.bytes)).toEqual(Array.from(bytes)); // Drained: a second harvest finds nothing, so a later step can't re-promote it. expect( await e.harvest(session, "/workspace/outputs", { maxFiles: 10, maxBytes: 1e6 }), ).toEqual([]);
// Nested files keep their path relative to the outbox. Promotion names a // document by that whole path, so two `notes.md` in different folders stay // two documents instead of collapsing into versions of each other. await e.run({ command: "mkdir -p /workspace/outputs/a /workspace/outputs/b && " + "echo one > /workspace/outputs/a/notes.md && echo two > /workspace/outputs/b/notes.md", session, }); const nested = await e.harvest(session, "/workspace/outputs", { maxFiles: 10, maxBytes: 1e6, }); expect(nested.map((f) => f.path).sort()).toEqual(["a/notes.md", "b/notes.md"]); } finally { e.disposeSession(session); } }, 120_000,);
liveTest( "harvesting a workspace with no outbox is empty, not an error", async () => { const e = new LocalDockerExecutor(SANDBOX); const session = "test-" + Math.random().toString(36).slice(2); try { await e.run({ command: "true", session }); expect( await e.harvest(session, "/workspace/outputs", { maxFiles: 5, maxBytes: 1e6 }), ).toEqual([]); } finally { e.disposeSession(session); } }, 90_000,);
liveTest( "readFile classifies what it found, not just whether it worked", async () => { const e = new LocalDockerExecutor(SANDBOX); const session = "test-" + Math.random().toString(36).slice(2); try { await e.putFile(session, "/workspace/notes.md", new TextEncoder().encode("one\ntwo\n")); expect(await e.readFile(session, "/workspace/notes.md")).toMatchObject({ kind: "file", text: "one\ntwo\n", });
// The three ways a read fails are three different mistakes, and the tool // above phrases a different fix for each. expect((await e.readFile(session, "/workspace/nope.md")).kind).toBe("missing"); expect((await e.readFile(session, "/workspace")).kind).toBe("directory");
// Bytes that were never text: refused rather than mangled into a string. await e.putFile(session, "/workspace/blob.bin", new Uint8Array([0, 1, 2, 0, 255])); expect((await e.readFile(session, "/workspace/blob.bin")).kind).toBe("binary");
// A file too big to inline is a different answer from a file that failed. await e.run({ command: "head -c 300000 /dev/zero | tr '\\0' 'x' > /workspace/big.txt", session, }); expect((await e.readFile(session, "/workspace/big.txt")).kind).toBe("too-large"); } finally { e.disposeSession(session); } }, 120_000,);
test("a stopped daemon is told apart from a container that wouldn't start", () => { // The real message from a Mac with OrbStack shut down. expect( isDaemonDown( "failed to connect to the docker API at unix:///var/run/docker.sock; check if the path is correct and if the daemon is running: dial unix /var/run/docker.sock: connect: no such file or directory", ), ).toBe(true); expect(isDaemonDown("Cannot connect to the Docker daemon at unix:///var/run/docker.sock.")).toBe( true, ); // A container that genuinely failed is a different problem with a different // answer, and must keep its own message. expect(isDaemonDown("Error response from daemon: No such image: alpine:9.99")).toBe(false); expect(isDaemonDown("invalid CapDrop: unknown capability")).toBe(false);});
liveTest( "the daemon-down answer tells the model to stop trying", async () => { // Pointed at a socket that cannot exist: what a stopped runtime looks like. const e = new LocalDockerExecutor({ ...SANDBOX, dockerHost: "unix:///nope/docker.sock" }); const r = await e.run({ command: "echo hi", session: "test-down" }).catch((err: Error) => err); expect(r).toBeInstanceOf(Error); const msg = (r as Error).message; expect(msg).toContain("sandbox is unavailable"); expect(msg).toContain("do not retry"); }, 60_000,);
test("a removed container is recognised, and the daemon being down is not it", () => { expect(isMissingContainer("Error response from daemon: No such container: kloe-sbx-abc")).toBe( true, ); expect(isMissingContainer("exec failed: no such file or directory")).toBe(false);});
liveTest( "a container removed behind the executor's back is rebuilt, not reported to the model", async () => { // What a restart used to do: the startup sweep culled a container the // conversation had just adopted, and every later call in that turn handed // the model "No such container" as if the command had said it. const e = new LocalDockerExecutor(SANDBOX); const session = "test-" + Math.random().toString(36).slice(2); try { expect((await e.run({ command: "echo hi > /workspace/note.txt", session })).exitCode).toBe(0); const name = (e as unknown as { containerName(s: string): string }).containerName(session); await Bun.spawn(["docker", "rm", "-f", name], { stdout: "ignore", stderr: "ignore" }).exited;
const r = await e.run({ command: "echo alive", session }); expect(r.stderr).not.toContain("No such container"); expect(r.exitCode).toBe(0); expect(r.stdout).toContain("alive"); // The workspace was in the container, so it is genuinely gone. That is // the honest outcome; a dead session for the rest of the turn is not. expect((await e.readFile(session, "/workspace/note.txt")).kind).toBe("missing"); } finally { e.disposeSession(session); } }, 120_000,);
liveTest( "a restart adopts the containers it finds instead of sweeping them at boot", async () => { // The container outlived the process, so by uptime it is always older than // the idle TTL. Culling on that at boot cost every open conversation its // workspace, which is the opposite of what surviving a restart is for. const name = "kloe-sbx-test-" + Math.random().toString(36).slice(2); await Bun.spawn( [ "docker", "run", "-d", "--name", name, "--label", "kloe-sandbox=1", SANDBOX.image, "sh", "-c", "exec tail -f /dev/null", ], { stdout: "ignore", stderr: "ignore" }, ).exited; try { // Old by uptime, which is the only thing a fresh process can measure, and // the thing the boot sweep used to cull on. const idleMs = 3_000; await Bun.sleep(idleMs + 1_000); new LocalDockerExecutor({ ...SANDBOX, idleMs }); await Bun.sleep(1_000); // let the constructor's sweep finish const alive = await Bun.spawn(["docker", "inspect", "-f", "{{.State.Running}}", name], { stdout: "pipe", stderr: "ignore", }); expect((await new Response(alive.stdout).text()).trim()).toBe("true"); } finally { await Bun.spawn(["docker", "rm", "-f", name], { stdout: "ignore", stderr: "ignore" }).exited; } }, 120_000,);
liveTest( "an idle sandbox is stopped, not destroyed: coming back restores what it had", async () => { const e = new LocalDockerExecutor(SANDBOX); const session = "test-" + Math.random().toString(36).slice(2); try { await e.run({ command: "echo kept > /workspace/notes.txt && touch /usr/local/marker", session, }); // What eviction now does. Stopping keeps the writable layer, so both the // chat's files and whatever it installed are still there. await Bun.spawn(["docker", "stop", "-t", "1", "kloe-sbx-" + session], { stdout: "ignore", stderr: "ignore", }).exited; const back = await e.run({ command: "cat /workspace/notes.txt && ls /usr/local/marker", session, }); expect(back.exitCode).toBe(0); expect(back.stdout).toContain("kept"); expect(back.stdout).toContain("/usr/local/marker"); } finally { e.disposeSession(session); } }, 120_000,);
liveTest( "with a workspace root, the chat's files outlive the container entirely", async () => { const root = mkdtempSync(join(TMP_ROOT, "kloe-ws-")); const cfg = { ...SANDBOX, workspaceRoot: root }; const session = "test-" + Math.random().toString(36).slice(2); const e = new LocalDockerExecutor(cfg); try { await e.run({ command: "echo durable > /workspace/notes.txt", session }); // The files are on the host, not in the container — which is the whole // point: they're on a disk you can snapshot, and they don't answer to the // container's lifetime. expect(readFileSync(join(root, session, "notes.txt"), "utf8")).toContain("durable");
// A policy change replaces the container, as it must. Under the old // arrangement that cost the chat its work; now it costs only the packages. const tightened = new LocalDockerExecutor({ ...cfg, network: true }); expect( (await tightened.run({ command: "cat /workspace/notes.txt", session })).stdout, ).toContain("durable");
// Deleting the conversation is the one thing that does take the files. tightened.disposeSession(session); await eventually(() => !existsSync(join(root, session))); expect(existsSync(join(root, session))).toBe(false); } finally { e.disposeSession(session); await clearAsRoot(root); } }, 180_000,);
liveTest( "the reaper takes orphaned workspaces, and leaves claimed ones alone", async () => { const root = mkdtempSync(join(TMP_ROOT, "kloe-ws-")); // A one-day TTL and a backdated directory stand in for the 30-day default; // `find -mtime` reads the same mtime either way. const e = new LocalDockerExecutor({ ...SANDBOX, workspaceRoot: root, ttlMs: 86_400_000 }); const live = "test-" + Math.random().toString(36).slice(2); try { await e.run({ command: "echo working > /workspace/notes.txt", session: live }); mkdirSync(join(root, "abandoned")); // Both are equally old; only one still has a container behind it, and that // is what decides. await backdate(root, ["abandoned", live]);
await e.sweepWorkspaces(); await eventually(() => !existsSync(join(root, "abandoned"))); expect(existsSync(join(root, "abandoned"))).toBe(false); expect(existsSync(join(root, live))).toBe(true); } finally { e.disposeSession(live); await clearAsRoot(root); } }, 180_000,);
test("the dormant cap retires the longest-cold sandboxes, and nothing under it", () => { const at = (name: string, days: number) => ({ name, finished: 1_000_000_000_000 - days * 86_400_000, }); const fleet = [at("c", 1), at("a", 30), at("b", 7)];
// Under the cap, everyone stays; over it, the coldest go first and only as // many as the cap demands. expect(overDormantCap(fleet, 5)).toEqual([]); expect(overDormantCap(fleet, 2)).toEqual(["a"]); expect(overDormantCap(fleet, 1)).toEqual(["a", "b"]); // 0 lifts the cap rather than retiring everything — the difference matters. expect(overDormantCap(fleet, 0)).toEqual([]);});