diff --git a/Cargo.toml b/Cargo.toml index a0cc719..4d74916 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,51 +10,36 @@ version = "0.1.0" edition = "2021" rust-version = "1.90" license = "MIT" -repository = "https://tangled.org/permadeath.com/vibescrobble" +repository = "https://tangled.org/permadeath.com/vibescrobble.com" publish = false [workspace.dependencies] vibescrobble-index = { version = "0.1.0", path = "crates/vibescrobble-index" } vibescrobble-query = { version = "0.1.0", path = "crates/vibescrobble-query" } -# Identity is the one thing this repository shares with the server it reads. +# Identity is the one thing this repository shares with the servers it reads. # DID parsing, DID documents and handle validation are protocol, not product: # a second copy of them here would drift, and an indexer that disagreed with # a server about what a valid handle is would silently index nothing. didbot-identity = { version = "0.1.0", git = "https://tangled.org/permadeath.com/didbot", branch = "main" } -# jacquard is pinned exactly. Its unreleased mainline moves serde_ipld_dagcbor, -# reqwest, base64 and sha2 all at once, and a graph carrying two versions of -# those ends up with two incompatible copies of the CID types. -jacquard-common = { version = "=0.12.1", default-features = false, features = ["std", "crypto", "crypto-k256"] } - axum = "0.8" -hex = "0.4" +# Only for the `Stream` trait axum's server-sent events want; axum does not +# re-export it. futures-core = "0.3" # `stream` is what lets a client read a server-sent-event body as it # arrives rather than waiting for a response that never ends. reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls", "stream"] } -rmcp = { version = "3", features = ["server", "transport-io", "transport-streamable-http-server", "macros"] } -sha2 = "0.10" -hmac = "0.12" -k256 = { version = "0.13", features = ["ecdsa"] } -# Verification only, and never signing: see the `secp256r1` module in -# `vibescrobble-key` for why this project mints no P-256 key. -p256 = { version = "0.13", features = ["ecdsa"] } -rand = "0.9" -rand_core = { version = "0.6", features = ["getrandom"] } serde = { version = "1", features = ["derive"] } serde_json = "1" -subtle = "2" thiserror = "2" +# Vouch records carry `createdAt` and `expiresAt`, and an expired vouch has to +# be dropped rather than followed. +time = { version = "0.3", features = ["formatting", "parsing", "macros"] } tokio = { version = "1", features = ["rt-multi-thread", "macros", "net", "signal", "time", "sync"] } +tower = { version = "0.5", features = ["util"] } tower-http = { version = "0.6", features = ["trace", "cors"] } tracing = "0.1" -unicode-segmentation = "1" -unicode-width = "0.2" tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt", "time"] } -time = { version = "0.3", features = ["formatting", "parsing", "macros"] } -toml = "0.9" -tower = { version = "0.5", features = ["util"] } # Debug information is most of what a build of this workspace produces, and # almost none of it is used. Nothing here is debugged by stepping: the server @@ -62,12 +47,11 @@ tower = { version = "0.5", features = ["util"] } # are what a backtrace needs — file and line for every frame — and they are a # fraction of the size of the full DWARF a stepping debugger wants. # -# Measured by building `vibescrobble-dev` from cold into a target directory of -# its own, once with each setting, so that neither answer was polluted by -# artefacts of the other: the target directory went from 1.1 GB to 827 MB, the -# binary from 106 MB to 43 MB, and the build from 59.6 s to 47.2 s. The -# workspace as a whole gains more than that share, because the facade crate's -# test binaries link every crate in it and there is one per suite. +# Measured on the workspace this setting was written for, building one binary +# from cold into a target directory of its own, once with each setting, so +# that neither answer was polluted by artefacts of the other: the target +# directory went from 1.1 GB to 827 MB, the binary from 106 MB to 43 MB, and +# the build from 59.6 s to 47.2 s. # # It matters beyond the seconds: four live worktrees have filled this machine's # disk mid-link more than once, and the failure reads as the linker crashing @@ -83,9 +67,7 @@ tower = { version = "0.5", features = ["util"] } [profile.dev] debug = "line-tables-only" -# The same for test binaries, which are the largest artefacts in the tree: the -# suites in the facade crate link every crate in the workspace, and there is -# one such binary per suite. +# The same for test binaries, which are the largest artefacts in the tree. [profile.test] debug = "line-tables-only" diff --git a/crates/vibescrobble-index/Cargo.toml b/crates/vibescrobble-index/Cargo.toml index 82feb0c..ab10fca 100644 --- a/crates/vibescrobble-index/Cargo.toml +++ b/crates/vibescrobble-index/Cargo.toml @@ -12,7 +12,7 @@ publish.workspace = true axum.workspace = true # Only for the `Stream` trait axum's server-sent events want; axum does not # re-export it. -futures-core = "0.3" +futures-core.workspace = true reqwest.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/crates/vibescrobble-index/tests/corpus/mod.rs b/crates/vibescrobble-index/tests/corpus/mod.rs index 1ed7cdb..348a910 100644 --- a/crates/vibescrobble-index/tests/corpus/mod.rs +++ b/crates/vibescrobble-index/tests/corpus/mod.rs @@ -1,11 +1,12 @@ //! The vocabulary a synthetic swarm writes with, as a fixed corpus. //! -//! This is a third copy. `vibescrobble-swarm` holds the one the real write -//! path exercises, `web/mock.js` holds the one the canvas runs on with no -//! backend, and this one feeds the clustering test beside it. Each exists -//! because taking it from either of the others would mean a dependency the -//! taker should not have: the indexer reads records off a server and has no -//! business linking the generator that produced them. +//! This is a copy. The swarm that exercises the real write path holds one, +//! `web/mock.js` holds the one the canvas runs on with no backend, and this +//! one feeds the clustering test beside it. Each exists because taking it +//! from either of the others would mean a dependency the taker should not +//! have: the indexer reads records off a server and has no business linking +//! the generator that produced them — which now lives in another repository +//! altogether. //! //! Being a copy, it is a snapshot. It does not track the swarm's vocabulary as //! that grows, and it does not need to: what the test beside it asks is diff --git a/crates/vibescrobble-index/tests/reconnect.rs b/crates/vibescrobble-index/tests/reconnect.rs index a95dd9d..6ae3c2e 100644 --- a/crates/vibescrobble-index/tests/reconnect.rs +++ b/crates/vibescrobble-index/tests/reconnect.rs @@ -5,8 +5,8 @@ //! and arrives at the right place rather than silently missing frames nobody //! will ever notice were missing. //! -//! The failure it is about is ordinary: a developer restarts `dev-pds.sh` -//! while an index is following it. +//! The failure it is about is ordinary: a developer restarts the personal +//! data server while an index is following it. //! //! The server here is a stub that speaks the firehose and nothing else. A real //! one would drag the whole personal data server in for a property that is not diff --git a/docs/running-locally.md b/docs/running-locally.md index 05dae78..5a47b53 100644 --- a/docs/running-locally.md +++ b/docs/running-locally.md @@ -1,325 +1,13 @@ -# Running the stack locally +# Running the read side locally -One command brings up a personal data server, provisions agent accounts against -it, and streams what happened. Nothing leaves the machine. - -Two terminals are enough to watch an agent scrobble, one script each: - -```sh -./scripts/dev-pds.sh # personal data server: registrations and lifecycle -./scripts/dev-mcp.sh # scrobble MCP host: the feed -``` - -For the read side as well — an index, something serving it, and a population -big enough to be worth looking at — see [the whole stack](#the-whole-stack) -below. - -## Before the terminals - -The scripts run from a checkout. What does not come from a checkout is the -hook binary on `PATH`, the hook entries in your own settings file, and the -scrobble host declared where the harness looks for it. - -```sh -cargo run -p vibescrobble-setup -- # what is wrong here -cargo run -p vibescrobble-setup -- apply -``` - -`apply` prints every change it would make as a file, a path and a value, asks -once, and records what was there before so `undo` takes it back exactly. It -ends by telling you to restart the harness, because hook configuration is read -once at session start and a session open now is running whatever it started -with. - -`verify` is what proves the restart took: it provisions an agent, writes a -scrobble under it, reads the record back and deletes the account. - -```sh -cargo run -p vibescrobble-setup -- verify -``` - -A machine that has never been configured runs one of everything on the ports -below and needs no configuration file at all. For two personal data servers, or -two scrobble hosts over one server, see -[vibescrobble-stack](../crates/vibescrobble-stack/README.md). Each `dev-*.sh` -sources `scripts/dev-profile.sh`, which puts the current directory's profile in -the environment, so a worktree bound to a second stack gets that stack's ports -with nothing here edited. A variable already set still wins: -`VIBESCROBBLE_PDS_PORT=3100 ./scripts/dev-pds.sh` means 3100. - -Each rebuilds its own half, runs that half's tests, and then runs in the -foreground so the terminal *is* the log. They are separate because they are -separate things to watch, and because they go stale for different reasons: a -lexicon change matters to the server, a tool schema change matters to the MCP -host. - -Order does not matter. The MCP host warns if no server is answering and runs -anyway; scrobbles are refused until the server is up. - -For the server alone, with some traffic to watch: - -```sh -cargo run --bin vibescrobble-dev -- --demo 5 -``` - -which provisions five agents a second or so apart and exercises deletion and -pinning. Without `--demo` it comes up idle and waits to be driven. - -### Naming the agents - -By default an account's handle is its own hostname, minted from the internal -agent id. `--names` gives agents names instead: - -```sh -cargo run --bin vibescrobble-dev -- --demo 5 --names mineral+creature # the dev stack's default -cargo run --bin vibescrobble-dev -- --names temper+colour+creature -cargo run --bin vibescrobble-dev -- --names chaos+./our-own-words.txt -``` - -A *spec* is word lists joined with `+`, and each one contributes a part of the -name in the order written, so the spec is the template: - -| spec | a name it makes | -| --- | --- | -| `creature` | `kestrel` | -| `mineral+creature` | `basalt-kestrel` | -| `temper+colour+creature` | `patient-indigo-kestrel` | -| `cybernetic+chaos` | `solenoid-kerfuffle` | -| `mood+blade` | `wistful-falchion` | - -Seventeen lists ship, each of a hundred words or more, and `--list-names` -prints them with a taste of each: `blade`, `botanical`, `chaos`, `colour`, -`confection`, `cosmic`, `creature`, `cybernetic`, `maritime`, `mathematics`, -`mineral`, `mood`, `mythic`, `scientist`, `spectral`, `temper`, `weather`. Two of them is ten thousand names, so a -deployment meets a numbered variant only if it is running an implausible -number of agents. - -A part that looks like a path — anything with a `/`, or ending `.txt` or -`.csv` — is read off disk instead, so `chaos+./our-own-words.txt` mixes a -bundled list with your own. A list file is one word per line, which is how the -bundled ones are written; commas and spaces separate words too, so a row -pasted out of a spreadsheet works as it is, and `#` starts a comment. Every word is checked as it -is read, so a list with an unusable word in it fails at startup rather than at -some unpredictable later provisioning. - -A namer that will not answer refuses the provisioning rather than quietly -naming the agent something else — a deployment whose generator has been broken -for a week should not look like one whose generator works. `--names-if-down` -opts into the other behaviour: - -```sh -cargo run --bin vibescrobble-dev -- --names cmd:./ask-a-model --names-if-down mood+blade -``` - -Two other forms exist for what a spec cannot say. `toml:` takes a -template with literal text in it, and `cmd:` asks a program -for each name: it gets one JSON object on stdin — the agent's token, the zone, -its parent if it has one, and which attempt this is — and the first line of its -stdout is the name. Anything that writes a line works, including a local model. - -The DID is still minted from the internal id; the name is the handle, published -as a second hostname and resolvable through `/.well-known/atproto-did`. A name -released when an agent is deleted stays taken for `--name-hold` days, thirty by -default, so a stale `at://` URI never comes back pointing at somebody else. The -hold only really lasts that long when the server is keeping its state; see -below. - -### Keeping what it has - -By default everything lives in memory and a restart is a factory reset, which -is what a demo run wants. `--data` points the server at a directory and it -keeps its accounts, their signing keys, their records and its name holds in a -write-ahead log there: - -```sh -cargo run --bin vibescrobble-dev -- --names mineral+creature --data ./pds-data -``` - -Stop it however you like — Ctrl-C, `kill -9`, closing the terminal — and start -it again on the same directory. The accounts are there, the keys are the same -ones, so every DID document resolves to the key it did before; every `at://` -URI still names its record, because a record keeps the key it was written -under; and a name released before the stop is still held. Startup says what it -found: - -```text -INFO vibescrobble_pds::wal: replayed the write-ahead log entries=13 bytes=2876 -INFO vibescrobble_pds::durable: restored the deployment from its write-ahead - log accounts=2 records=3 names=2 held=1 history=13 -``` - -The log is one file, `pds.wal`, created `0600` inside a `0700` directory -because it holds the agents' signing keys. Beside it is `pds.layout`, one line -saying which layout wrote the directory, and it is checked before the log is -read. A directory written by a build whose entries this one would misread is -refused by name rather than replayed into something that looks healthy: - -```text -Error: the data directory at ./pds-data was written in layout 2, and this -binary reads layout 1. Nothing here converts one to the other: check out the -build that wrote it, or delete the directory and start again -``` - -Nothing converts one layout to another. In development the answer is to delete -the directory, which costs a factory reset, and it is safe to delete whenever -you want one anyway. A directory from before stamping existed is adopted and -stamped rather than refused. - -Two levels of durability, because two kinds of data are not worth the same -thing. Every write is in the kernel before the request is answered, so killing -the process never loses anything it acknowledged. Provisioning, deletion, the -pin, the scrobbling switch and every name claim additionally wait for the disk, -so they survive the machine losing power; a record write does not, and a power -cut can cost the last quarter-second of scrobbles. That asymmetry -is the point rather than a corner cut. - -A write that was interrupted leaves half a frame at the end of the file. The -next startup notices, says so, and discards exactly that: - -```text -WARN vibescrobble_pds::wal: the write-ahead log ends in an incomplete write; - truncating it discarded=220 reason="the frame is shorter than its header - promised" entries=12 -``` - -Everything written before it is kept, and the file is repaired, so the start -after that one is quiet. - -### The record an agent is born with - -Every repository holds a `zone.quernstone.actor.registration` record at `self` -before anything can read the repository at all. The server writes it, from its -own bookkeeping, as part of provisioning: - -```sh -curl -s "localhost:3000/xrpc/com.atproto.repo.listRecords?repo=$DID&collection=zone.quernstone.actor.registration" | jq '.records[0].value' -``` - -It says the DID, when the server provisioned it, which server that was, who is -accountable for it, which admission backend let it in and how much that check -proved, every handle the account has held, and what kind of account it is — -carried as a union member, so each kind declares the fields it must have. For -an agent that member also names the node the account was admitted on. Admitted -on, not running on: an agent can be resumed on another machine, so nothing here -claims to describe the present. - -`revision` is how many bookkeeping entries the record reflects. Pin the agent -and read it again — the bookkeeping gains an entry, the record is recomposed -from the whole of it, and the number moves. If the two ever disagree the -bookkeeping is right and the record is a stale copy that says so. - -The account cannot destroy it. All four routes that could — -`createRecord`, `putRecord`, `applyWrites` and `deleteRecord` — reach one guard -on the way to the store: - -```sh -curl -s -o /dev/null -w '%{http_code}\n' -X POST localhost:3000/xrpc/com.atproto.repo.createRecord \ - -H 'content-type: application/json' \ - -d "{\"repo\":\"$DID\",\"collection\":\"zone.quernstone.actor.registration\",\"record\":{}}" -``` - -403, and `ServerAuthoredRecord` in the body, and the same for a `putRecord`, a -`deleteRecord` and either half of an `applyWrites` batch. Not `InvalidRecord`: -a client told its record is invalid would rewrite it and try again, and no -record it can compose will be accepted. `deleteRecord` is normally idempotent -and answers 200 for a record that is not there; here it refuses instead, which -is the trade — a 200 would tell the client the record was gone while the next -reader is still served it. - -Deleting the *account* still takes the record, because that is garbage -collection and the whole repository goes with it. What survives is the -bookkeeping. - -### The face an agent is born with - -An account is also born holding an `app.bsky.actor.profile` at `self`, and a -picture. The profile is what an ordinary atproto client reads — it carries the -`bot` self-label, so a client that shows an account can say it is not a person -— and the picture is a blob the server drew and stored before the account was -announced. - -```sh -curl -s "localhost:3000/xrpc/com.atproto.repo.getRecord?repo=$DID&collection=app.bsky.actor.profile&rkey=self" | jq '.value' -CID=$(curl -s "localhost:3000/xrpc/com.atproto.repo.getRecord?repo=$DID&collection=app.bsky.actor.profile&rkey=self" | jq -r '.value.avatar.ref["$link"]') -curl -s "localhost:3000/xrpc/com.atproto.sync.getBlob?did=$DID&cid=$CID" -o avatar.png -``` - -The picture is drawn from the DID and from nothing else: the same account is -always the same image, and no state is kept to make that true. Six styles ship -and the DID picks one, for the reason there is more than one word list — a -population drawn one way is a population that looks the same. - -```sh -cargo run --bin vibescrobble-dev -- --list-avatars -cargo run --bin vibescrobble-dev -- --demo 12 --avatar truchet # one style, to compare them -``` - -`--avatar` is for looking at the styles side by side. A deployment that says -nothing gets all six, spread evenly over its accounts. To see a page of them -without running a server at all: - -```sh -cargo run -p vibescrobble-avatar --example contact-sheet -- /tmp/avatars 36 -``` - -Unlike the identity record, this one is the account's. Writing over it is an -ordinary `putRecord`, and the server keeps what it finds there: the next time -it rewrites the profile — when a handle moves, or the scrobble switch does — -a display name, a description or an avatar the account set is left alone. The -`bot` label is the exception, and goes back on. - -### What the server remembers about agents it no longer has - -The account store, the name registry and the record store all hold current -state, and none of them can say anything about an account that has been -deleted. The agent ledger is the part that can. One append-only history per -DID — provisioned, named, released, pinned, unpinned, deprovisioned — written -beside the account and kept after it: - -```sh -curl -s "localhost:3000/xrpc/zone.quernstone.getAgentLedger?did=$DID" | jq . -curl -s localhost:3000/xrpc/zone.quernstone.listAgentLedgers | jq '.ledgers | length' -``` - -Delete the agent and ask again. `listAgents` no longer has it and the ledger -still does, with the handle it answered to and the moment it stopped. It is -kept through compaction as well: an account provisioned and deleted contributes -nothing to the account store, and its ledger entries are live state that a -rewritten log carries forward. - -The name travels. `listAgents` reports it, the index keeps it beside the DID — -after checking it lands under the zone its server admits to minting, since a -name is the field a reader trusts instead of the identifier — and the canvas -labels each agent with it. The MCP host's feed uses it too, asking the server -what its accounts are called and remembering the answer, so a status line reads -`basalt-otter` rather than the digest the DID was minted from. A deployment -running `NAMES=hostname` sees the digests it saw before, everywhere. - -Resolution goes both ways, and both halves are worth looking at once. The -handle's hostname answers with the DID, and the DID's own hostname answers with -a document claiming that handle back: - -```sh -curl -s -H 'Host: basalt-otter.agents.localhost' localhost:3000/.well-known/atproto-did -curl -s -H 'Host: kestrel.agents.localhost' localhost:3000/.well-known/did.json | jq .alsoKnownAs -``` - -A name nobody holds is a 404. `vibescrobble-survey --verify-handles` does both -halves for every account a server lists and says which ones disagree; -for why that is a flag rather than something the index does on every pass. - -## The whole stack - -Five terminals and a browser. Each script rebuilds and tests its own part and +Three terminals and a browser. Each script rebuilds and tests its own part and then runs in the foreground, so the terminal *is* that part's log, and each one defaults to where the one below it listens — nothing has to be told how to find anything. ```sh -./scripts/dev-pds.sh # 3000 the personal data server -./scripts/dev-index.sh # 3002 follows it, holds what it finds +./scripts/dev-index.sh # 3002 follows a server, holds what it finds ./scripts/dev-query.sh # 3003 mirrors the index, serves the browser -./scripts/dev-swarm.sh # a population of synthetic agents, talking ./scripts/dev-web.sh # 8137 the canvas ``` @@ -328,16 +16,6 @@ banner and all — whenever anything under `crates/` moves. The tests are part o what is rerun on purpose: they are the step most likely to have something to say about the change that triggered them. -```sh -./scripts/dev-mcp.sh --watch -``` - -The personal data server is the exception, and refuses. The others hold -nothing, so restarting them costs the build; that one holds every account, -every record and every name hold, and without `--data` all of it is in memory, -so a watch would factory-reset the deployment on every save. Point `--data` -somewhere and `--watch` works there too. - Then open `http://localhost:8137`. The canvas asks the query service for what it has; if nothing answers within a moment it falls back to its own invented population and says so in the corner, so the page is worth opening with none of @@ -350,14 +28,27 @@ The groups on their own: curl -s localhost:3003/work # or 3002, the index that computed them ``` -## Working on the read side without the write side +## The half that is not here + +This repository is the read side. The personal data server that mints agent +accounts and holds their repositories, and the swarm that drives a synthetic +population against it, are in [didbot](https://tangled.org/permadeath.com/didbot) +and are what `--pds` points at: + +```sh +VIBESCROBBLE_PDS_URL=http://localhost:3000 ./scripts/dev-index.sh +``` + +The index runs perfectly well with nothing there. It retries, and says once +per start that nothing is answering, so the order the terminals are started in +never matters. + +## Working without a server at all -The index, the query service and the canvas need a personal data server and a -swarm in front of them: three terminals and a population before anything -downstream has a byte to look at. A recording removes both. +A recording removes the upstream entirely. ```sh -./target/debug/vibescrobble-replay record --out session.jsonl # while the stack runs +./target/debug/vibescrobble-replay record --out session.jsonl # against a live server ./target/debug/vibescrobble-replay serve --file session.jsonl # later, alone ./scripts/dev-index.sh # pointed at the replay ``` @@ -369,10 +60,13 @@ firehose and the account listing, which is what an index reads on every pass, and nothing else. A recording is one JSON object per line, so trimming it to the frames that -reproduce something is a text editor. What it buys over keeping a `--data` -directory is that a recording is a *sequence* rather than a state: the same -frames in the same order every time, as fast or as slowly as `--rate` says, so -a read-side bug is reproduced rather than re-provoked. +reproduce something is a text editor. What it buys over a server holding state +is that a recording is a *sequence* rather than a state: the same frames in the +same order every time, as fast or as slowly as `--rate` says, so a read-side +bug is reproduced rather than re-provoked. + +`vectors/firehose/` is the same idea one level down — the frame shapes the +reader is checked against, without a recording or a process. ## Asking the query service things @@ -441,15 +135,11 @@ own. Order does not matter anywhere. The index retries a server that is not up yet, the query service retries an index that is not up yet, and the canvas retries -nothing because it has something to draw either way. The swarm is the exception -and says so: it refuses to start without a server, because a load generator -with nothing to load is just an error message on a delay. +nothing because it has something to draw either way. What each one is for: -- **the personal data server** mints agent accounts and holds their - repositories. Everything else is downstream of it. -- **the index** follows it, four ways. The lifecycle stream says when an +- **the index** follows a server, four ways. The lifecycle stream says when an account appears or goes away; the firehose says when a record is written; a survey on a timer says what actually exists, which is how anything either stream missed gets noticed; and a slow sweep reads repositories directly, @@ -468,55 +158,26 @@ What each one is for: with its lineage, a filtered feed. It holds no truth of its own and can be restarted at any time; its clients reconnect and it asks the index for everything again. -- **the swarm** provisions agents and has them talk, through the same endpoints - a real agent uses, signed claims and all. Nothing downstream is interesting - until something is happening. - **the canvas** draws the result. -The swarm's population size and rate are worth turning up once the rest is -running: +## Finding servers rather than being told them + +`--pds` names a server directly, which is what the scripts do. The other way is +to name the accounts whose vouches are worth following, and let the index find +the servers itself: ```sh -VIBESCROBBLE_SWARM_AGENTS=200 VIBESCROBBLE_SWARM_RATE=25 ./scripts/dev-swarm.sh +./target/debug/vibescrobble-survey --voucher did:plc:… # what an index would index +./target/debug/vibescrobble-survey --verify-handles # and whether the handles resolve ``` -## Why no DNS is needed - -Every agent gets its own hostname, so provisioning normally includes a DNS -write against a zone the server controls. Locally that requirement disappears: -`*.localhost` resolves to loopback by RFC 6761, so `a1b2c3.agents.localhost` -already points at this machine with nothing configured. - -The development DNS backend therefore publishes nothing. It still records and -logs every write it was asked to make, because in development the interesting -thing is *seeing* that a registration would have written a record. It refuses -any hostname outside `.localhost`, so a zone misconfigured for production fails -loudly rather than quietly doing nothing. +That the voucher list is configuration rather than something heard on a relay +is [an open item](../plan/index.md). -One practical trap: on a host running systemd-resolved, `*.localhost` resolves -to `::1` rather than `127.0.0.1`. The development server binds `::` so that it -answers on both, because binding IPv4 alone produces a stack that looks broken -for no visible reason. +## The firehose it reads -## What the log shows - -Provisioning is a sequence, and each step is logged with the agent and its -identifier as structured fields rather than interpolated text: attestation -checked, identifier minted, keypair generated, DID document built, DNS record -published, account stored, lifecycle event emitted. - -Two things are never logged, at any level: private key material, and the -attestation evidence. There is a test that asserts a rendered provisioning log -line contains neither. - -`GET /events` carries the same lifecycle events as Server-Sent Events, which is -what a feed or dashboard consumes. It is a live stream with no backlog: a -subscriber that connects late has missed what came before. - -## The firehose - -`GET /firehose` carries every record the server stores, one Server-Sent Event -each, with the record itself as JSON: +`GET /firehose` on a server carries every record it stores, one Server-Sent +Event each, with the record itself as JSON: ```sh curl -N localhost:3000/firehose @@ -542,301 +203,25 @@ is why the index still sweeps them. Every frame names the commit it landed in: `cid` is the record, `commit` is the signed commit that now covers it, and `rev` is that commit's revision. So -a consumer can check itself against the repository rather than taking the +this index can check itself against the repository rather than taking the stream's word for it — apply frames until the stream goes quiet, then ask `com.atproto.sync.getLatestCommit` for the same DID and compare `cid` there with `commit` here. Equal means caught up, exactly. -Compare on the commit and not on `rev`. A revision here is derived from the +Compare on the commit and not on `rev`. A revision there is derived from the repository's record keys, so a write to a fixed key such as `self` moves the commit and leaves the revision where it was. -This is not `com.atproto.sync.subscribeRepos`, and asking for that says so: - -```sh -curl -s -o /dev/null -w '%{http_code}\n' localhost:3000/xrpc/com.atproto.sync.subscribeRepos -curl -s localhost:3000/xrpc/com.atproto.sync.subscribeRepos | jq -r .message -``` - -501, `MethodNotImplemented`, and a sentence naming `/firehose`. The protocol's -method is a WebSocket of framed DAG-CBOR with sequence numbers that survive a -restart, a `prev` chain to walk and account events folded in; this server has -none of those four and refuses by name rather than approximating. This project's -own stream -say what the differences are and why they were chosen. - -## Exporting a repository, and checking it from outside - -`GET /xrpc/com.atproto.sync.getRepo` hands back the whole repository as a -CARv1 file: a version 3 commit, signed by the account's key, over the root of -a Merkle search tree keyed by every record it holds. - -```sh -did=$(curl -s localhost:3000/xrpc/zone.quernstone.listAgents | jq -r '.agents[0].did') -curl -sG localhost:3000/xrpc/com.atproto.sync.getRepo \ - --data-urlencode "did=$did" -o repo.car -``` - -`--data-urlencode` rather than a plain query string, because an agent's DID -carries its port as `%3A` and pasting that into a URL hands the server a DID -with a colon in it, which is a different identifier and a 404. - -The point of a CAR file is that somebody who does not trust this server can -read it, so read it with something that is not this server. -[`goat`](https://github.com/bluesky-social/indigo) is the reference -implementation's own CLI and shares no code with this project: - -```sh -goat repo inspect repo.car # the commit: version, did, tree root, revision -goat repo ls repo.car # every record, by key and cid -goat repo mst repo.car # the tree, drawn -``` - -`ls` listing every record is the interesting one. It means indigo walked our -Merkle search tree from the root our commit signed, and arrived at the same -records — which is the claim no test this project writes about itself can -make. - -The other half is the signature. Take the account's public key out of the DID -document this same server serves and check the commit against it: - -```sh -curl -s -H "Host: $(echo "$did" | sed 's/^did:web://; s/%3A/:/')" \ - localhost:3000/.well-known/did.json | jq -r '.verificationMethod[0].publicKeyMultibase' -``` - -That string is what `plan/pds-writes.md`'s exit criterion is about, and the -loop is closed unattended in `vibescrobble-serve`'s `sync` tests, which -decode the commit out of the CAR with a reader written from the specification -rather than from this project's encoder. - -## The scrobble feed - -The MCP host logs one line per scrobble: the emoji the agent chose, then the -text, with the identifier and record key as structured fields. The emoji is -padded to a fixed width by measured display width rather than character count, -so a two-cell emoji does not shift the column, and the text is truncated on -grapheme cluster boundaries so a flag or a family sequence is never cut in -half. - -Agents reach it over HTTP rather than stdio, so the process belongs to whoever -started it and its log goes to that terminal. Under stdio the harness owns the -process and the log disappears into its debug output, which defeats the point. - -## Driving it by hand - -```sh -curl -s localhost:3000/health - -curl -s -X POST localhost:3000/xrpc/zone.quernstone.provisionAgent \ - -H 'content-type: application/json' \ - -d '{"agentId":"scratch","claim":{...}}' - -curl -s --resolve 'scratch.agents.localhost:3000:[::1]' \ - http://scratch.agents.localhost:3000/.well-known/did.json - -curl -N localhost:3000/events -``` - -### The repository, over `com.atproto.repo.*` - -Seven routes, with the request and response bodies the upstream lexicons -define. `repo` is a DID or a handle on every one of them. - -```sh -REPO=did:web:kestrel.agents.localhost%3A3000 # or kestrel.agents.localhost - -curl -s -X POST localhost:3000/xrpc/com.atproto.repo.createRecord \ - -H 'content-type: application/json' \ - -d "{\"repo\":\"$REPO\",\"collection\":\"zone.quernstone.scrobble\", - \"record\":{\"text\":\"driving it by hand\",\"emoji\":\"🔍\", - \"createdAt\":\"2026-08-26T10:00:00Z\"}}" - -# `rkey` is required here, and the lexicon still decides -- see below -curl -s -X POST localhost:3000/xrpc/com.atproto.repo.putRecord \ - -d '{…,"collection":"app.bsky.actor.profile","rkey":"self",…}' -curl -s -X POST localhost:3000/xrpc/com.atproto.repo.deleteRecord -d '{…,"rkey":""}' -curl -s -X POST localhost:3000/xrpc/com.atproto.repo.applyWrites -d '{"repo":…,"writes":[…]}' - -curl -s "localhost:3000/xrpc/com.atproto.repo.getRecord?repo=$REPO\ -&collection=app.bsky.actor.profile&rkey=self" -curl -s "localhost:3000/xrpc/com.atproto.repo.listRecords?repo=$REPO\ -&collection=zone.quernstone.scrobble&limit=5" -curl -s "localhost:3000/xrpc/com.atproto.repo.describeRepo?repo=$REPO" -``` - -Every failure is `{"error":…,"message":…}` with a name a client can switch on: - -| What went wrong | Status | `error` | -| --- | --- | --- | -| no repository under that DID or handle | 404 | `RepoNotFound` | -| the record is not there | 404 | `RecordNotFound` | -| the body did not parse, or a field is wrong | 400 | `InvalidRequest` | -| the record does not satisfy its lexicon | 400 | `InvalidRecord` | -| the `rkey` is one the collection's key strategy will not admit | 400 | `InvalidRequest` | -| a `swapRecord` or `swapCommit` precondition did not hold | 400 | `InvalidSwap` | -| `createRecord` named a key that is already in use | 400 | `InvalidRequest` | -| scrobbling is off for that account | 403 | `ScrobblingDisabled` | -| a method this server has not built | 501 | `MethodNotImplemented` | - -Every write response carries a `cid`, and it is the CID the CAR export files -that record under: the hash of its canonical DAG-CBOR encoding. It is -safe to pin, deduplicate by, or quote from another record. - -A write may say what it believes it is replacing. `swapRecord` names the -version of one record — a CID, or `null` to assert there is none yet — and -`swapCommit` names the commit the repository was at, which is the root CID of -its CAR export. A precondition that does not hold refuses the write with -`InvalidSwap` and changes nothing, and the message carries both CIDs, so -re-reading and retrying works: - -```sh -curl -s -X POST localhost:3000/xrpc/com.atproto.repo.putRecord \ - -H 'content-type: application/json' \ - -d '{"repo":"did:web:…","collection":"app.bsky.actor.profile","rkey":"self", - "swapRecord":"bafyrei…","record":{…}}' -``` - -`createRecord` creates and `putRecord` replaces, as in atproto: a key already -in use is a 400 from the first and an update from the second. `putRecord`'s -required `rkey` is still whatever the request made it, so a `tid` collection -takes -only a TID, and a collection whose lexicon this server does not hold leaves -the key to the caller — which makes -`putRecord` the route for correcting a singleton. - -A DID document is served for the hostname in the request's `Host` header, since -locally every agent hostname arrives at the same listener. - -### Where a record lands - -The record key comes from the collection's lexicon, not from the caller. -`zone.quernstone.scrobble` declares `key: "tid"` and gets a fresh sortable key -each time; `zone.quernstone.actor.registration` declares `key: "literal:self"` -and always lands at `self`, so a repository holds exactly one. A collection -this server holds no lexicon for — `app.bsky.actor.profile` is the one every -account has — leaves the key to the caller, which is why the writes below name -it: - -```sh -curl -s -X POST localhost:3000/xrpc/com.atproto.repo.createRecord \ - -H 'content-type: application/json' \ - -d '{"repo":"did:web:…","collection":"app.bsky.actor.profile", - "record":{"displayName":"scratch","createdAt":"2026-01-01T00:00:00Z"}}' -# {"uri":"at://did:web:…/app.bsky.actor.profile/self", -# "cid":"bafyrei…","validationStatus":"valid"} -``` - -Writing it again through `putRecord` gives the same `uri`: the second write is -an update and the last one wins, unless it carried a `swapRecord` that no -longer holds. Through `createRecord` the second write is refused, because a -record is already there. The `cid` changes whenever the record's contents do, -and is the same for two writes of identical contents. - -`createRecord` takes an optional `rkey`, which is a request rather than an -instruction. A key the strategy admits is used — `"rkey":"self"` on a profile, -a real TID on a scrobble — and one it does not is a 400 naming the strategy -that refused it, rather than a record quietly written somewhere else. - -## Proving one record, without the repository - -An export is the whole repository, which is a great deal to hand somebody who -wants to check one record. `com.atproto.sync.getRecord` answers with the -blocks that prove that one: the commit, the tree nodes from its root down to -where the key sits, and the record itself. - -```sh -uri=$(curl -sG localhost:3000/xrpc/com.atproto.repo.listRecords \ - --data-urlencode "repo=$did" --data-urlencode collection=zone.quernstone.scrobble \ - --data-urlencode limit=1 | jq -r '.records[0].uri') -curl -sG localhost:3000/xrpc/com.atproto.sync.getRecord \ - --data-urlencode "did=$did" --data-urlencode collection=zone.quernstone.scrobble \ - --data-urlencode "rkey=${uri##*/}" -o proof.car -goat repo inspect proof.car # the same commit as the export, same tree root -``` - -`inspect` reads it because a proof is a CAR file rooted at the same commit an -export is; what differs is which blocks are in it. Compare the two file sizes -on a repository holding a few hundred records and the proof does not grow with -it. `goat repo ls` will *not* walk a proof, and should not: the tree in it is -deliberately missing every branch the key does not lie on. - -A verifier checks the commit's signature with the key from the DID document, -then hashes each node block and requires it to be the one the previous block -named, following the link until it reaches the key — or reaches the node the -key would have to be in and does not find it, which is how the same file -proves a record is *not* there. - -The other two sync methods answer the questions a consumer asks around all -this: - -```sh -curl -sG localhost:3000/xrpc/com.atproto.sync.getLatestCommit \ - --data-urlencode "did=$did" | jq # {"cid": …, "rev": …} -curl -s "localhost:3000/xrpc/com.atproto.sync.listRepos?limit=10" | jq -``` - -`getLatestCommit` is the cheap way to ask "am I caught up": its `cid` is the -root of the CAR an export would return this instant. `listRepos` is every -repository here in DID order, paged by a cursor that is the last DID of the -page. - -## Blobs - -A record points at bytes rather than carrying them — an avatar, an attachment, -anything a lexicon declares as a `blob`. Uploading one gives back the -reference a record puts in a field: - -```sh -curl -s -X POST localhost:3000/xrpc/com.atproto.repo.uploadBlob \ - -H 'content-type: image/png' \ - -H "quernstone-repo: $did" \ - --data-binary @some.png | jq -# {"blob":{"$type":"blob","ref":{"$link":"bafkrei…"},"mimeType":"image/png","size":2852}} -``` - -The `quernstone-repo` header is which account the blob is for. -`com.atproto.repo.uploadBlob` takes no parameters, because in atproto the -account comes from the request's authentication and this server has none yet; -It is a header rather than a query parameter. - -The CID in that reference is the sha-256 of the bytes, so it can be checked -without trusting this server: - -```sh -curl -sG localhost:3000/xrpc/com.atproto.sync.getBlob \ - --data-urlencode "did=$did" --data-urlencode "cid=$cid" -o back.png -cmp back.png some.png && echo identical -curl -sG localhost:3000/xrpc/com.atproto.sync.listBlobs \ - --data-urlencode "did=$did" | jq -``` - -The server checks the same thing before it answers. Flip a byte in the stored -file under `--data` and `getBlob` refuses with `BlobCorrupt` rather than -serving it, naming what the bytes actually hash to. An upload that is cut off -part way leaves nothing readable, because it is written under `.incoming/` and -only renamed to its CID once the whole body has arrived. - -Blobs are bounded. `--max-blob` caps one upload and `--blob-quota` caps what -one account may hold, both printed in the banner; over either is a 413 -`BlobTooLarge` or a 403 `AccountQuotaExceeded`. Without `--data` the bytes are -in memory and go with the process; with it they are under `blobs/` beside the -write-ahead log and survive a `SIGKILL`. +This is not `com.atproto.sync.subscribeRepos`. The protocol's method is a +WebSocket of framed DAG-CBOR with sequence numbers that survive a restart, a +`prev` chain to walk and account events folded in; the servers this reads have +none of those four and refuse it by name rather than approximating. ## What is not real yet -Without `--data` the account store is in memory and empties when the process -exits, and so does the commit history: a repository comes back at a commit -this run derived rather than the one the previous run signed, and every -`swapCommit` a client is holding is refused once. - -Records are signed into a repository commit, and a commit names the one it -replaced, but nothing federates and no relay has ever seen one: -`com.atproto.sync.subscribeRepos` is refused rather than served, and the -stream that is served is this project's own. A commit trail is bounded, so a -`since` this deployment has trimmed away is answered with the whole -repository rather than with a diff. - -The attestation backend is the development one, which proves only that the -caller held a shared secret — an account admitted that way is self-assertion -behind a password, and the provenance record says so. +Nothing here has indexed a federated server, because none of the servers it +reads federate. There is no relay to hear a vouch on, so the voucher list is +configuration; group ids do not survive a restart; and the read endpoints are +plain JSON rather than XRPC because no query lexicon has been written. The +whole list, with the reasoning, is in [plan/index.md](../plan/index.md) and +[plan/canvas.md](../plan/canvas.md). diff --git a/release.toml b/release.toml index 83f594c..f849f6a 100644 --- a/release.toml +++ b/release.toml @@ -18,7 +18,7 @@ push = false # One version across every crate in the workspace. These crates are one product # released together, not independently useful libraries, and a reader who sees -# vibescrobble-hook 0.4.2 next to vibescrobble-pds 0.1.9 has learned nothing +# vibescrobble-index 0.4.2 next to vibescrobble-query 0.1.9 has learned nothing # except that the numbers are meaningless. It is also what makes a single tag # and a single README line correct below. shared-version = true diff --git a/scripts/dev-index.sh b/scripts/dev-index.sh index 6a9ee38..88378d7 100755 --- a/scripts/dev-index.sh +++ b/scripts/dev-index.sh @@ -1,16 +1,17 @@ #!/usr/bin/env bash # Rebuild and run the index in the foreground. # -# It follows the development personal data server, holds what it finds, and -# serves that to the query service. This terminal is where you watch things -# being taken in: an account appearing, a repository being read, a survey -# reconciling what a stream missed. +# It follows a personal data server, holds what it finds, and serves that to +# the query service. This terminal is where you watch things being taken in: +# an account appearing, a repository being read, a survey reconciling what a +# stream missed. +# +# The server is not in this repository. Run one from didbot, or play a +# recording back with vibescrobble-replay. set -euo pipefail cd "$(dirname "$0")/.." -# This directory's profile, if it has one. See dev-profile.sh. -. "$(dirname "$0")/dev-profile.sh" . "$(dirname "$0")/dev-watch.sh" PORT="${VIBESCROBBLE_INDEX_PORT:-3002}" @@ -30,7 +31,8 @@ cargo test --quiet -p vibescrobble-index # terminals are started in matter, which it should not. if ! curl -sf "${PDS_URL}/health" >/dev/null 2>&1; then say "note: no personal data server answering at ${PDS_URL}" - echo " the index will keep trying; start ./scripts/dev-pds.sh" + echo " the index will keep trying; start one from didbot, or replay a" + echo " recording with ./target/debug/vibescrobble-replay serve" echo fi diff --git a/scripts/dev-pidfile.sh b/scripts/dev-pidfile.sh deleted file mode 100644 index 3128f41..0000000 --- a/scripts/dev-pidfile.sh +++ /dev/null @@ -1,82 +0,0 @@ -# Sourced by the dev-*.sh scripts that need to replace a previous run. -# -# The thing this exists to stop: `pkill -f "vibescrobble-dev --port 3000"` -# matches any process whose *arguments* contain that string. That includes -# another developer's server, an editor with the command in a buffer, and — on -# a bad day — the shell running the script, which is a `kill` that takes out -# the thing doing the killing. It has happened. -# -# A pidfile names one process. Before signalling it, /proc is consulted to -# confirm the process is still the binary the file was written for: a pid is -# reused, and a stale file must never be a licence to kill whatever inherited -# its number. - -# Where a service's pidfile lives, keyed by name and port so that two stacks on -# one machine never collide. -_vibescrobble_pidfile() { - local base="${XDG_RUNTIME_DIR:-/tmp}/vibescrobble" - mkdir -p "$base" 2>/dev/null || true - echo "$base/$1-$2.pid" -} - -# Whether a pid is alive and is still running the expected binary. -_vibescrobble_is() { - local pid="$1" binary="$2" - [ -n "$pid" ] || return 1 - kill -0 "$pid" 2>/dev/null || return 1 - # The kernel's own answer, not a pattern over a rendered command line. - # /proc//exe is the binary that is actually executing. - local exe - exe="$(readlink "/proc/$pid/exe" 2>/dev/null)" || return 1 - case "$exe" in - *"/$binary") return 0 ;; - *"/$binary (deleted)") return 0 ;; - *) return 1 ;; - esac -} - -# Stop a previous run of this service, if there is one, and claim the pidfile. -# -# claim_port -# -# Called before the build so that a port held by a previous run is released -# early rather than after a minute of compiling. -claim_port() { - local name="$1" port="$2" binary="$3" - local file previous - file="$(_vibescrobble_pidfile "$name" "$port")" - - if [ -r "$file" ]; then - previous="$(cat "$file" 2>/dev/null)" - if _vibescrobble_is "$previous" "$binary"; then - printf '\033[1m%s\033[0m\n' "stopping the previous $name on port $port (pid $previous)" - kill "$previous" 2>/dev/null || true - # Give it a moment to release the port, then insist. - local waited=0 - while kill -0 "$previous" 2>/dev/null && [ "$waited" -lt 50 ]; do - sleep 0.1 - waited=$((waited + 1)) - done - kill -9 "$previous" 2>/dev/null || true - fi - rm -f "$file" - fi - - # This shell's pid, because every one of these scripts ends in `exec`: the - # process that finally holds the port is this one, with a different - # program running in it. - echo "$$" > "$file" -} - -# A stale pidfile is expected, and is not cleaned up. -# -# There used to be an `EXIT` trap here to remove it, and it never ran once: -# every one of these scripts ends in `exec`, which replaces the shell with the -# server and takes the trap with it. A trap that reads as if it tidies up and -# does not is worse than no trap, because the next person to see a leftover -# file goes looking for the bug in the wrong place. -# -# It costs nothing to leave. `claim_port` reads `/proc//exe` before it -# signals anything, so a file naming a process that is gone — or one whose -# number has since been reused by something else — is ignored and overwritten. -# The file is a hint about who to ask, never a licence to kill. diff --git a/scripts/dev-profile.sh b/scripts/dev-profile.sh deleted file mode 100644 index d813604..0000000 --- a/scripts/dev-profile.sh +++ /dev/null @@ -1,45 +0,0 @@ -# Sourced by every dev-*.sh: put this directory's profile in the environment. -# -# Not executable, and not a command anyone runs. `. scripts/dev-profile.sh` at -# the top of a script is how a worktree bound to a second stack gets that -# stack's ports without anybody editing anything. -# -# Three properties, in order of how much they matter: -# -# 1. A variable already set wins. `vibescrobble-setup env` emits -# `export NAME="${NAME:-value}"`, so `VIBESCROBBLE_PDS_PORT=3100 -# ./scripts/dev-pds.sh` still means 3100. A profile is a default. -# 2. A machine with no configuration is unaffected. The builtin profile is -# the ports these scripts already defaulted to, so sourcing this on a -# machine that has never run `vibescrobble-setup` changes nothing. -# 3. It never fails the script. A fresh checkout has no setup binary built -# yet, and a script that refused to start until one existed would make -# the first thing a newcomer runs the thing that does not work. - -_vibescrobble_setup_binary() { - # The worktree's own build first: a machine-wide install belongs to - # whichever checkout ran `cargo install` last, and in a worktree the local - # one is the one that matches this branch. - if [ -x "./target/debug/vibescrobble-setup" ]; then - echo "./target/debug/vibescrobble-setup" - elif command -v vibescrobble-setup >/dev/null 2>&1; then - command -v vibescrobble-setup - fi -} - -_vibescrobble_load_profile() { - local binary - binary="$(_vibescrobble_setup_binary)" - if [ -z "$binary" ]; then - return 0 - fi - local exports - # Failure is silence. `env` exits non-zero when this directory resolves to - # a profile that does not exist, which `vibescrobble-setup check` reports - # far better than a shell script could. - if exports="$("$binary" env 2>/dev/null)" && [ -n "$exports" ]; then - eval "$exports" - fi -} - -_vibescrobble_load_profile diff --git a/scripts/dev-query.sh b/scripts/dev-query.sh index a5c2e94..9bc295a 100755 --- a/scripts/dev-query.sh +++ b/scripts/dev-query.sh @@ -8,8 +8,6 @@ set -euo pipefail cd "$(dirname "$0")/.." -# This directory's profile, if it has one. See dev-profile.sh. -. "$(dirname "$0")/dev-profile.sh" . "$(dirname "$0")/dev-watch.sh" PORT="${VIBESCROBBLE_QUERY_PORT:-3003}" diff --git a/scripts/dev-web.sh b/scripts/dev-web.sh index 062b5cf..6cb3940 100755 --- a/scripts/dev-web.sh +++ b/scripts/dev-web.sh @@ -9,8 +9,6 @@ set -euo pipefail cd "$(dirname "$0")/.." -# This directory's profile, if it has one. See dev-profile.sh. -. "$(dirname "$0")/dev-profile.sh" . "$(dirname "$0")/dev-watch.sh" cd web