Something went wrong. Try again.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545/** * Lifecycle supervisor — ARCHITECTURE §8 responsibility 1 (saga tb4593.1). * * The process-supervisor core of `shardd`: spawn `shard agent` child processes, track a * PID table, reap on exit, and apply a configurable restart policy. Carries **no** agent * state beyond the live child table — the registry (tb4593.2) consumes exits via * {@link Supervisor.onExit}. This is deliberately the thinnest possible supervisor: no * socket, no proxy, no registry; just `child_process` lifecycle. * * ## Substrate * ARCHITECTURE §8 diagrams the daemon spawning `shard agent --id --socket`; today the * proven, working substrate is the **stdio** `shard agent` server (src/transports/stdio, * saga uoxl20) — `--socket` is reserved (see src/cli/index.ts). The supervisor therefore * spawns `shard agent --id <id> [--cwd] [--model] [...]` with piped stdio; the proxy * (tb4593.4) bridges that direct-connection stdio to daemon-connection clients by * stripping/adding `agent_id` (§5.2). The spawn command is fully configurable so the * daemon wiring (tb4593.6) and tests can substitute a stub child. * * ## Restart semantics * Runtime generations are ephemeral; durable agent identity and JSONL session data persist * across exits and can be restored on a later attach. * So: * - `'never'` (default): a clean OR crashing exit is terminal. No respawn. * - `'on-crash'`: respawns only on a non-zero / signal exit, up to `maxRestarts`. * - `'always'`: respawns on any exit (including clean 0), up to `maxRestarts`. * Restart exhaustion emits a terminal exit (`reason: 'restart-failed'`) and the child * stays dead — honoring the §8 invariant even under the crash-loop policy. Backoff is a * fixed delay (no jitter in v0.1); the registry learns the final fate via `onExit`. * * ## Promise identity across restarts * A caller awaiting `handle.exited` awaits ONE terminal resolution per agent lifecycle, * not per process incarnation. The `exited` promise + its resolver live on a persistent * {@link AgentLifecycle} created at first spawn and carried across respawns; each * respawn swaps the underlying `ChildProcess` + stdio streams but not the promise. */import { spawn, type ChildProcess } from 'node:child_process';import { existsSync } from 'node:fs';import { resolve } from 'node:path';import { Readable } from 'node:stream';
/** When to respawn a child that has exited. §8 default = `'never'`. */export type RestartPolicy = 'never' | 'always' | 'on-crash';
/** Exit descriptor pushed to {@link ExitListener}s and carried by `ChildHandle.exited`. */export interface ExitInfo { /** Which agent exited. */ agentId: string; /** OS PID of the exited process. */ pid: number; /** Numeric exit code, or `null` if terminated by signal. */ exitCode: number | null; /** Signal that killed the process, or `null` for a clean code exit. */ signal: NodeJS.Signals | null; /** Did the exit count as a crash (for `on-crash`)? Non-zero code OR any signal. */ crashed: boolean; /** Why the supervisor considers this terminal. */ reason: 'exited' | 'killed' | 'restart-failed'; /** Restart attempt index (0 = first spawn, 1+ = respawns). */ attempt: number;}
/** Live handle to a spawned agent child. stdin/stdout/stderr + PID + exit promise. */export interface ChildHandle { /** The agent id this child runs (set at spawn, stable across restarts). */ agentId: string; /** Current PID (changes across restarts; read fresh each time). */ pid: number; /** Stdin — the daemon writes per-agent COMMANDS here (direct §5.2). */ stdin: NodeJS.WritableStream; /** Stdout — the daemon reads agent EVENTS here (direct §5.2). */ stdout: NodeJS.ReadableStream; /** Stderr — diagnostics only, never on the wire. */ stderr: NodeJS.ReadableStream; /** Resolves with terminal {@link ExitInfo} when the agent is gone for good. */ exited: Promise<ExitInfo>;}
/** Listener invoked once per *terminal* exit (no invocation on intermediate restarts). */export type ExitListener = (info: ExitInfo) => void;
/** * Supervisor-wide config. All fields optional — sane defaults match §8 / dev usage. */export interface SupervisorConfig { /** * Executable to spawn. Default: the current runtime (`process.execPath`, i.e. `bun` or * `node`) so a dev `shardd` can launch `bun ... agent` without a packaged binary. */ command?: string; /** * Base argv prepended to every spawn (before agent-specific flags). Default: * if `process.argv[1]` names a REAL on-disk CLI entry script (`bun src/cli/index.ts` in * dev, or the extensionless bundled `dist/shard` shebang binary), the resolved ABSOLUTE * path + `'agent'`; otherwise `['agent']` so a `bun --compile` single-file binary (whose * `argv[1]` is its `/$bunfs/root/<bin>` virtual entry, not a re-invokable script) runs its * own entry via `execPath agent ...`. The path is resolved against the daemon's cwd * because children are spawned with their own agent cwd. See {@link resolveBaseArgs}. */ baseArgs?: string[]; /** Restart policy. Default `'never'`; callers explicitly request new generations. */ restart?: RestartPolicy; /** Delay between a crash and the respawn attempt, ms. Default `1000`. */ backoffMs?: number; /** Max respawn attempts before giving up (policies `always`/`on-crash`). Default `5`. */ maxRestarts?: number; /** Escalate shutdown SIGTERM to SIGKILL after this delay. Set to 0 to disable. */ shutdownGraceMs?: number; /** Env for child processes. Default: `process.env` (inherit). */ env?: NodeJS.ProcessEnv;}
/** Per-spawn agent config. Maps 1:1 to `shard agent` flags; `extraArgs` is the escape hatch. */export interface AgentSpawnConfig { /** Unique agent id (becomes `--id`). Required. */ agentId: string; /** Working directory (becomes `--cwd`). Required. */ cwd: string; /** Resolved model name (becomes `--model`). */ model?: string; /** Provider key (becomes `--provider`). */ provider?: string; /** * Additional argv appended AFTER the standard flags — for mode, tools, skills, session * path, `--api-key`/`--base-url` test hatches, etc. The supervisor stays agnostic to * their meaning; the daemon wiring (tb4593.6) builds these from `agent.create` config. */ extraArgs?: readonly string[]; /** Fencing identities propagated to the child session writer lease. */ runtimeGeneration?: string; daemonLease?: string; /** * Managed-manifest path (saga u4uo9m.1). When set, the child is a daemon-managed * subagent: the CLI loads its pinned system_prompt/launch_policy/depth/timeout/alias/ * definition config from this manifest instead of argv, and connects back to the daemon * over `daemonSocket`. Prompt/task/config stay OFF argv and out of AgentInfo/logs/errors. */ managedManifest?: string; /** Daemon unix-socket path for a managed child to connect back to (saga u4uo9m.1). */ daemonSocket?: string;}
const DEFAULT_BACKOFF_MS = 1000;const DEFAULT_MAX_RESTARTS = 5;const DEFAULT_SHUTDOWN_GRACE_MS = 5000;
/** * Persistent, per-agent lifecycle state. Lives across restarts: the `exited` promise + * resolver are created once (at first spawn) and resolved only on a terminal exit. Each * restart swaps `process`/stdlib streams + bumps `attempt`, but the promise identity is * preserved so a caller awaiting `handle.exited` sees exactly one resolution. */interface AgentLifecycle { agentId: string; /** The current child process (replaced on restart). `undefined` during backoff gap. */ process: ChildProcess | undefined; /** Public handle. `stdin/stdout/stderr/pid` are live getters reading off `process`. */ handle: ChildHandle; /** Resolves `handle.exited` on terminal exit. */ exitResolve: (info: ExitInfo) => void; /** Restart attempt index the CURRENT process incarnation represents. */ attempt: number; /** True once a terminal exit has resolved (guards against double-resolve). */ terminal: boolean;}
/** * Process supervisor. Owns the live child table; the registry (tb4593.2) and proxy * (tb4593.4) read handles via {@link Supervisor.get}; the orchestrator (tb4593.5) drives * {@link Supervisor.kill} + {@link Supervisor.spawn}. * * Thread-safety: single-threaded Node event loop; the exit handler is non-reentrant. A * restart schedules a `setTimeout` rather than recursing inline, so a synchronously * crashing child can't stack-overflow the supervisor. */export class Supervisor { private readonly command: string; private readonly baseArgs: readonly string[]; private readonly restart: RestartPolicy; private readonly backoffMs: number; private readonly maxRestarts: number; private readonly shutdownGraceMs: number; private readonly env: NodeJS.ProcessEnv;
/** Live + backoff-pending lifecycles keyed by agent id. */ private readonly lifecycles = new Map<string, AgentLifecycle>(); /** Terminal-exit listeners (sibling sagas: registry, orchestrator). */ private readonly exitListeners = new Set<ExitListener>(); /** Pending restart timers (cleared on shutdown / kill). */ private readonly restartTimers = new Map<string, NodeJS.Timeout>(); /** Agent ids signalled intentionally (via kill/shutdown) — suppresses restart. */ private readonly intentional = new Set<string>();
constructor(config: SupervisorConfig = {}) { this.command = config.command ?? process.execPath; this.baseArgs = config.baseArgs ?? defaultBaseArgs(); this.restart = config.restart ?? 'never'; this.backoffMs = config.backoffMs ?? DEFAULT_BACKOFF_MS; this.maxRestarts = config.maxRestarts ?? DEFAULT_MAX_RESTARTS; this.shutdownGraceMs = config.shutdownGraceMs ?? DEFAULT_SHUTDOWN_GRACE_MS; if (!Number.isFinite(this.shutdownGraceMs) || this.shutdownGraceMs < 0) { throw new RangeError('shutdownGraceMs must be a finite non-negative number'); } this.env = config.env ?? process.env; }
/** * Spawn an agent child. If a live child for `agentId` already exists, returns it * unchanged (idempotent — the orchestrator may retry `agent.create` under load). * Throws if a previous child for `agentId` is mid-restart-backoff (caller should await * the prior `exited` first). */ spawn(cfg: AgentSpawnConfig): ChildHandle { const existing = this.lifecycles.get(cfg.agentId); if (existing !== undefined) { // Already-live (process present) => return the handle. if (existing.process !== undefined) { return existing.handle; } // Pending a backoff restart — refuse; the caller raced ahead of `exited`. throw new Error( `agent "${cfg.agentId}" is pending a restart; await its prior exit first`, ); } // First spawn: create the persistent lifecycle (promise + resolver) once, then spawn. const lifecycle = this.createLifecycle(cfg.agentId); this.lifecycles.set(cfg.agentId, lifecycle); this.doSpawn(cfg, lifecycle, 0); return lifecycle.handle; }
/** The live child for `agentId`, or `undefined` if not running (or mid-backoff). */ get(agentId: string): ChildHandle | undefined { const lc = this.lifecycles.get(agentId); if (lc === undefined || lc.process === undefined) { return undefined; } return lc.handle; }
/** Snapshot of all live children (those with a running process). */ list(): ChildHandle[] { return [...this.lifecycles.values()] .filter((lc) => lc.process !== undefined) .map((lc) => lc.handle); }
/** * Signal an agent's process. `force=false` (default) sends SIGTERM (graceful — * §8 "signal agents to finish turns"); `force=true` sends SIGKILL. Returns `false` if * no live child exists. The terminal {@link ExitInfo} flows through `onExit` once the * OS reaps the process; `kill` itself does not wait. */ kill(agentId: string, force = false): boolean { // Cancel any pending respawn so a late timer can't revive a deliberately-killed agent. const timer = this.restartTimers.get(agentId); if (timer !== undefined) { clearTimeout(timer); this.restartTimers.delete(agentId); } const lc = this.lifecycles.get(agentId); if (lc === undefined || lc.process === undefined) { return false; } this.intentional.add(agentId); try { lc.process.kill(force ? 'SIGKILL' : 'SIGTERM'); } catch { // ESRCH: already gone. The exit handler will fire (or already has); no-op here. } return true; }
/** * Subscribe to *terminal* exits across all agents. Invoked once per agent death — not * for intermediate restarts (those are internal). Returns an unsubscribe function. */ onExit(listener: ExitListener): () => void { this.exitListeners.add(listener); return () => { this.exitListeners.delete(listener); }; }
/** * Graceful shutdown: SIGTERM every live child, clear pending restarts. Resolves when * every child has been reaped. Does not force-kill (call {@link Supervisor.kill} with * `force=true` per agent for that). */ async shutdown(): Promise<void> { // Cancel all pending restarts first — no revivals during shutdown. for (const timer of this.restartTimers.values()) { clearTimeout(timer); } this.restartTimers.clear();
const exits: Promise<ExitInfo>[] = []; for (const [agentId, lc] of this.lifecycles) { if (lc.process !== undefined) { this.intentional.add(agentId); exits.push(lc.handle.exited); try { lc.process.kill('SIGTERM'); } catch { // ESRCH: already exiting; the exit handler is (about to be) in flight. } } } let forceTimer: ReturnType<typeof setTimeout> | undefined; if (exits.length > 0 && this.shutdownGraceMs > 0) { forceTimer = setTimeout(() => { for (const [agentId, lc] of this.lifecycles) { if (lc.process !== undefined) this.kill(agentId, true); } }, this.shutdownGraceMs); forceTimer.unref?.(); } try { await Promise.all(exits); } finally { if (forceTimer !== undefined) clearTimeout(forceTimer); } }
// ─── Internal ──────────────────────────────────────────────────────────────────
/** Create the persistent lifecycle for an agent: the `exited` promise + resolver. */ private createLifecycle(agentId: string): AgentLifecycle { let exitResolve!: (info: ExitInfo) => void; const exited = new Promise<ExitInfo>((resolve) => { exitResolve = resolve; }); // `process` is set by doSpawn immediately after. The handle's stream/pid getters // read off `lc.process` so they stay valid across restarts (caller always sees the // CURRENT incarnation's streams via the same handle reference). const lc: AgentLifecycle = { agentId, process: undefined, handle: { agentId, get pid(): number { return lc.process?.pid ?? -1; }, get stdin(): NodeJS.WritableStream { return lc.process?.stdin ?? closedWritable; }, get stdout(): NodeJS.ReadableStream { return lc.process?.stdout ?? closedReadable; }, get stderr(): NodeJS.ReadableStream { return lc.process?.stderr ?? closedReadable; }, exited, }, exitResolve, attempt: 0, terminal: false, }; return lc; }
/** * The actual spawn. `attempt` is the restart index (0 = first). Builds argv, spawns * with piped stdio, wires the exit handler onto the lifecycle. Stamps `process` + * `attempt` so the public handle getters reflect the new incarnation. */ private doSpawn(cfg: AgentSpawnConfig, lc: AgentLifecycle, attempt: number): void { const args = this.buildArgs(cfg); const child = spawn(this.command, args, { cwd: cfg.cwd, stdio: ['pipe', 'pipe', 'pipe'], env: this.env, }); lc.process = child; lc.attempt = attempt;
child.on('exit', (code, signal) => { // `close` is NOT used — we want the exit code ASAP and stdio streams stay readable // until drained by the proxy. The proxy (tb4593.4) owns stream consumption. this.onChildExit(cfg, lc, code, signal ?? null, false); }); child.on('error', (err) => { // spawn failure (ENOENT etc.) — no exit event fires. Synthesize a crash exit so // the caller's `exited` always resolves and the restart policy gets to weigh in. this.onChildExit(cfg, lc, null, null, true, err.message); }); }
/** * Exit handler — reaps the child, decides restart vs terminal. `code`/`signal` are the * Node `exit` event args; `synthetic` marks a spawn-error path (treated as a crash). */ private onChildExit( cfg: AgentSpawnConfig, lc: AgentLifecycle, code: number | null, signal: NodeJS.Signals | null, synthetic: boolean, syntheticMessage?: string, ): void { if (lc.terminal) { // Already terminal (e.g. synthetic error after a real exit raced). Idempotent no-op. return; } // Capture the incarnation PID before clearing it. RuntimeManager uses this PID to fence // late exit callbacks from an older generation; reporting -1 here caused real exits to // be ignored and left runtimes permanently marked running. const pid = lc.process?.pid ?? -1; // Clear the (now-dead) process so handle getters return the closed fallbacks and // kill()/get() see "not running". lc.process = undefined; const crashed = synthetic || code === null ? true : code !== 0; const intentional = this.intentional.has(cfg.agentId);
// Restart decision. Killed-intentionally => never restart (kill is a deliberate act). const shouldRestart = !intentional && this.shouldRestart(this.restart, crashed);
if (shouldRestart && lc.attempt < this.maxRestarts) { const timer = setTimeout(() => { this.restartTimers.delete(cfg.agentId); this.intentional.delete(cfg.agentId); // Preserve lc (its `exited` promise + resolver) across the respawn; bump attempt. this.doSpawn(cfg, lc, lc.attempt + 1); }, this.backoffMs); this.restartTimers.set(cfg.agentId, timer); return; }
// Terminal. Resolve the caller's `exited` + fan out to listeners. lc.terminal = true; this.intentional.delete(cfg.agentId); this.lifecycles.delete(cfg.agentId); const reason: ExitInfo['reason'] = intentional ? 'killed' : shouldRestart ? 'restart-failed' : 'exited'; if (synthetic && syntheticMessage !== undefined) { // Spawn-error: log to the supervisor's own stderr (no child stderr available). process.stderr.write( `shardd: agent "${cfg.agentId}" spawn failed: ${syntheticMessage}\n`, ); } const info: ExitInfo = { agentId: cfg.agentId, pid, exitCode: code, signal, crashed, reason, attempt: lc.attempt, }; lc.exitResolve(info); for (const listener of this.exitListeners) { try { listener(info); } catch (err) { // A listener throwing must not break sibling listeners or the supervisor. process.stderr.write( `shardd: onExit listener threw: ${(err as Error).message}\n`, ); } } }
/** Restart-policy predicate. Pure — no side effects. */ private shouldRestart(policy: RestartPolicy, crashed: boolean): boolean { switch (policy) { case 'never': return false; case 'always': return true; case 'on-crash': return crashed; } }
/** Build the agent argv from supervisor config + spawn config. */ private buildArgs(cfg: AgentSpawnConfig): string[] { const args = [...this.baseArgs, '--id', cfg.agentId, '--cwd', cfg.cwd]; if (cfg.model !== undefined) args.push('--model', cfg.model); if (cfg.provider !== undefined) args.push('--provider', cfg.provider); if (cfg.runtimeGeneration !== undefined) args.push('--runtime-generation', cfg.runtimeGeneration); if (cfg.daemonLease !== undefined) args.push('--daemon-lease', cfg.daemonLease); // saga u4uo9m.1: managed spawn context. The manifest + socket pin the child's durable // config; prompt/task/config never appear on argv (the manifest is the authority). if (cfg.managedManifest !== undefined) args.push('--managed-manifest', cfg.managedManifest); if (cfg.daemonSocket !== undefined) args.push('--daemon-socket', cfg.daemonSocket); if (cfg.extraArgs !== undefined) args.push(...cfg.extraArgs); return args; }}
// ─── Closed-stream fallbacks (used when no live process is attached to a lifecycle) ──
/** A writable that discards writes (used when the child is mid-backoff / dead). */const closedWritable: NodeJS.WritableStream = { write: () => false, // EAGAIN-ish: caller should backpressure / drop end: () => {}, // Minimal stub — the proxy (tb4593.4) guards against dead children before writing.} as unknown as NodeJS.WritableStream;
/** A readable that yields nothing and is already ended (used when the child is gone). */const closedReadable: NodeJS.ReadableStream = Readable.from([]);
/** * Resolve the base argv for agent children from the daemon's own `process.argv[1]`. * `entry` names a re-invokable CLI entry script whenever it resolves to a REAL on-disk file * — in dev (`bun src/cli/index.ts`) AND for the bundled shebang binary (`dist/shard`, which * is extensionless, so an extension test misclassifies it). Otherwise fall back to the bare * `'agent'` subcommand so `execPath agent ...` works. The entry is resolved ABSOLUTELY * because children are spawned with their own agent cwd, not the daemon's cwd. * * The fall-back also covers a `bun --compile` single-file binary (`dist/shard` built with * `--compile`). There `process.execPath` is the runnable binary itself and `argv[1]` is its * EMBEDDED entry exposed via Bun's virtual filesystem at `/$bunfs/root/<bin>` — NOT a * script to re-invoke. `existsSync` returns `true` for that virtual path (Bun intercepts * the call), so a real-file test alone misclassifies it: re-invoking it prepends a bogus * argv element the child's CLI parses as an unknown subcommand and every agent dies with * `shard: unknown subcommand: /$bunfs/root/shard`. Detect the virtual mount explicitly and * let the compiled binary run its own entry via `execPath agent ...`. */export function resolveBaseArgs(entry: string | undefined, cwd: string): string[] { if (entry === undefined) return ['agent']; const resolved = resolve(cwd, entry); if (isCompiledBinaryVirtualEntry(resolved)) return ['agent']; if (existsSync(resolved)) return [resolved, 'agent']; return ['agent'];}
/** * Whether `path` is the embedded entry of a `bun --compile` single-file binary, exposed * through Bun's virtual filesystem mounted at `/$bunfs/` (e.g. `/$bunfs/root/shard`). The * `/$bunfs/` root uses forward slashes in the virtual namespace on every platform — it is * a URL-style mount, not a real OS path — but we also tolerate the backslash form so a * platform that joins with `\` is still recognized. */function isCompiledBinaryVirtualEntry(path: string): boolean { return path.includes('/$bunfs/') || path.includes('\\$bunfs\\');}
/** Default base argv from this daemon process's argv. */function defaultBaseArgs(): string[] { return resolveBaseArgs(process.argv[1], process.cwd());}