import { execFile } from "node:child_process"; import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { dirname, join, resolve } from "node:path"; import { promisify } from "node:util"; import { getOk, pdsCall } from "./api"; import { config, type Account } from "./config"; import type { PublicKey } from "./lexicons"; import { sleep, xrpc } from "./net"; import type { Ctx } from "./vocabulary"; const exec = promisify(execFile); const KEYS = resolve(".auth/keys"); const keygen = async (who: Account) => { const path = join(KEYS, who.name); await mkdir(KEYS, { recursive: true }); await exec("ssh-keygen", [ "-q", "-t", "ed25519", "-N", "", "-C", `e2e-${who.name}`, "-f", path ]); return path; }; // keyed by the key too, makeKey rewrites the same path with a new one const registered = new Set(); // the knot picks up org.tangled.key.sshKey from the pds when it connects const keyFileOf = async (who: Account) => { const path = join(KEYS, who.name); await readFile(path).catch(() => keygen(who)); const line = (await readFile(`${path}.pub`, "utf8")).trim(); if (registered.has(`${who.did} ${line}`)) return path; const listed = await getOk( xrpc(config.pdsUrl, "com.atproto.repo.listRecords", { repo: who.did, collection: "org.tangled.key.sshKey" }) ); if (!listed.records.some((r: any) => r.value.publicKey === line)) { await pdsCall(who, "com.atproto.repo.createRecord", { collection: "org.tangled.key.sshKey", record: { $type: "org.tangled.key.sshKey", publicKey: line, name: `e2e-${who.name}`, createdAt: new Date().toISOString() } satisfies PublicKey }); } registered.add(`${who.did} ${line}`); return path; }; // a key is only worth keeping while its account exists export const forgetKey = async (who: Account) => { const path = join(KEYS, who.name); await rm(path, { force: true }); await rm(`${path}.pub`, { force: true }); }; // BatchMode and ConnectTimeout make a dead host fail fast instead of prompting // -F /dev/null so that a personal ssh config isn't used const sshFor = (key: string) => `ssh -F /dev/null -i ${key} -o IdentitiesOnly=yes -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR`; // as the ssh user the knot reads keys on the spot, plain "git" only sees the ones gathered so far const remoteOf = (who: Account, repoDid: string) => { const [host, port] = config.knotSsh.split(":"); return `ssh://${who.handle}@${host}:${port}/${repoDid}`; }; export const sshAs = async (who: Account) => sshFor(await keyFileOf(who)); const gitTo = async (who: Account, repoDid: string) => ({ ssh: await sshAs(who), remote: remoteOf(who, repoDid) }); // the knot allows four git ops at once per address, and every worker shares one const retryWhenBusy = async (run: () => Promise): Promise => { for (let attempt = 1; ; attempt++) { try { return await run(); } catch (cause: any) { const busy = /too many concurrent operations/.test(String(cause.stderr ?? cause)); if (!busy || attempt >= 30) throw cause; await sleep(200 + Math.random() * 400); } } }; const GIT_TIMEOUT = 120_000; const git = async (cwd: string, ssh: string, ...args: string[]) => retryWhenBusy(() => exec("git", ["-c", "user.name=e2e", "-c", "user.email=e2e@example.com", ...args], { cwd, env: { ...process.env, GIT_SSH_COMMAND: ssh }, timeout: GIT_TIMEOUT }) ); const inScratch = async (run: (dir: string) => Promise) => { const dir = await mkdtemp(join(tmpdir(), "e2e-git-")); try { return await run(dir); } finally { await rm(dir, { recursive: true, force: true }); } }; export const push = async ( who: Account, repoDid: string, files: Record, { branch = "main", base, options = [] }: { branch?: string; base?: string; options?: string[] } = {} ) => inScratch(async (dir) => { const { ssh, remote } = await gitTo(who, repoDid); const heads = Object.keys(await lsRemote(who, repoDid, "refs/heads/")); if (heads.length > 0) { const known = heads.includes(`refs/heads/${branch}`); const start = known ? branch : base; await git( dir, ssh, "clone", "-q", "--depth=1", ...(start ? ["--single-branch", "--branch", start] : []), remote, "." ); if (!known) await git(dir, ssh, "checkout", "-q", "-b", branch); } else { // knot repos are sha256, and git refuses a sha1 history pushed into one await git(dir, ssh, "init", "-q", "--object-format=sha256", "-b", branch); } for (const [file, text] of Object.entries(files)) { await mkdir(dirname(join(dir, file)), { recursive: true }); await writeFile(join(dir, file), text); } await git(dir, ssh, "add", "-A"); await git(dir, ssh, "commit", "-q", "-m", "e2e"); const sha = (await git(dir, ssh, "rev-parse", "HEAD")).stdout.trim(); const { stderr } = await git( dir, ssh, "push", ...options.flatMap((option) => ["-o", option]), remote, `${branch}:${branch}` ); return { sha, output: stderr }; }); export const lsRemote = async (who: Account, repoDid: string, prefix = "") => { const { ssh, remote } = await gitTo(who, repoDid); const { stdout } = await git(tmpdir(), ssh, "ls-remote", remote); return Object.fromEntries( stdout .trim() .split("\n") .filter(Boolean) .map((line) => line.split("\t").reverse()) .filter(([ref]) => ref.startsWith(prefix)) ) as Record; }; // git can't resolve the stack's names or trust its dev ca either const overHttps = (cwd: string, ...args: string[]) => { const resolve = [config.webUrl, config.knotUrl].flatMap((url) => { const { hostname, port } = new URL(url); return ["-c", `http.curloptResolve=${hostname}:${port || 443}:127.0.0.1`]; }); return retryWhenBusy(() => exec("git", ["-c", "http.sslVerify=false", ...resolve, ...args], { cwd, timeout: GIT_TIMEOUT }) ); }; export const lsRemoteOverHttps = async (url: string) => (await overHttps(tmpdir(), "ls-remote", url)).stdout; export const cloneOverHttps = (url: string) => inScratch(async (dir) => { await overHttps(dir, "clone", "-q", url, "."); return (await overHttps(dir, "rev-parse", "HEAD")).stdout.trim(); }); export const headOf = async (who: Account, repoDid: string) => { const { ssh, remote } = await gitTo(who, repoDid); const { stdout } = await git(tmpdir(), ssh, "ls-remote", "--symref", remote, "HEAD"); return /^ref: refs\/heads\/(\S+)\tHEAD/m.exec(stdout)?.[1]; }; export const deleteBranch = async (who: Account, repoDid: string, branch: string) => inScratch(async (dir) => { const { ssh, remote } = await gitTo(who, repoDid); await git(dir, ssh, "init", "-q", "--object-format=sha256"); await git(dir, ssh, "push", remote, "--delete", branch); }); export const pushAnnotatedTag = async ( who: Account, repoDid: string, tag: string, message: string ) => inScratch(async (dir) => { const { ssh, remote } = await gitTo(who, repoDid); await git(dir, ssh, "clone", "-q", "--depth=1", remote, "."); await git(dir, ssh, "tag", "-a", tag, "-m", message); await git(dir, ssh, "push", "-q", remote, tag); }); interface Key { path: string; pub: string; fingerprint: string; } const made = new WeakMap(); // writes to the path pushes read, so a push uses this key export const makeKey = async (ctx: Ctx) => { const path = await keygen(ctx.actor); const { stdout } = await exec("ssh-keygen", ["-lf", `${path}.pub`]); const pub = (await readFile(`${path}.pub`, "utf8")).trim(); made.set(ctx, { path, pub, fingerprint: stdout.split(" ")[1] }); }; export const keyOf = (ctx: Ctx) => made.get(ctx)!; // retryWhenBusy keeps a busy knot from reading as a bad key export const lsRemoteWith = async (ctx: Ctx) => { try { await retryWhenBusy(() => exec("git", ["ls-remote", remoteOf(ctx.actor, ctx.repo.did)], { env: { ...process.env, GIT_SSH_COMMAND: sshFor(keyOf(ctx).path) }, timeout: 60_000 }) ); return "listed"; } catch (cause: any) { return String(cause.stderr ?? cause); } };