diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ab4647d..9dbe018 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,9 +25,9 @@ jobs: - name: Check formatting run: cargo fmt --all -- --check - name: Run Clippy - run: cargo clippy --all-targets --locked -- -D warnings + run: cargo clippy --workspace --all-targets --locked -- -D warnings - name: Run tests - run: cargo test --all-targets --locked + run: cargo test --workspace --all-targets --locked supply-chain: runs-on: ubuntu-24.04 diff --git a/Cargo.lock b/Cargo.lock index b3cd77d..e5ff98e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -490,6 +490,7 @@ dependencies = [ "base64", "blake3", "cid", + "cpds-protocol", "futures", "governor", "hickory-resolver", @@ -517,9 +518,25 @@ dependencies = [ "tower_governor", "tracing", "tracing-subscriber", + "unsigned-varint", "url", ] +[[package]] +name = "cpds-protocol" +version = "0.1.0" +dependencies = [ + "atrium-api", + "base64", + "cid", + "serde", + "serde_bytes", + "serde_ipld_dagcbor", + "serde_json", + "sha2", + "thiserror 2.0.19", +] + [[package]] name = "cpufeatures" version = "0.2.17" diff --git a/Cargo.toml b/Cargo.toml index 80983d9..fca7ac9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,18 @@ +[workspace] +members = ["crates/cpds-protocol"] +resolver = "3" + +[workspace.dependencies] +atrium-api = { version = "=0.25.8", default-features = false } +base64 = "0.22.1" +cid = "0.11.1" +serde = { version = "1.0.219", features = ["derive"] } +serde_bytes = "0.11.19" +serde_ipld_dagcbor = "0.6.4" +serde_json = "1.0.143" +sha2 = "0.10.9" +thiserror = "2.0.16" + [package] name = "cpds" version = "0.1.0" @@ -7,9 +22,10 @@ license = "MIT" description = "A small, readable Personal Data Server for the AT Protocol" [dependencies] +cpds-protocol = { version = "0.1.0", path = "crates/cpds-protocol" } argon2 = "0.5.3" async-trait = "0.1.89" -atrium-api = { version = "=0.25.8", default-features = false } +atrium-api.workspace = true atrium-common = "=0.1.4" atrium-crypto = "=0.1.3" atrium-identity = { version = "=0.1.9", default-features = false } @@ -17,9 +33,9 @@ atrium-repo = "=0.1.8" atrium-xrpc = "=0.12.4" axum = { version = "0.8.4", features = ["form", "ws"] } axum-server = { version = "0.8.0", default-features = false } -base64 = "0.22.1" +base64.workspace = true blake3 = "1.8.2" -cid = "0.11.1" +cid.workspace = true futures = { version = "0.3.33", default-features = false, features = ["alloc", "std"] } governor = "0.10.4" hickory-resolver = { version = "=0.26.1", default-features = false, features = ["system-config", "tokio"] } @@ -31,13 +47,13 @@ percent-encoding = "2.3.2" rand = "0.9.2" reqwest = { version = "0.12.24", default-features = false, features = ["json", "rustls-tls"] } rusqlite = { version = "0.40.2", default-features = false, features = ["backup", "bundled"] } -serde = { version = "1.0.219", features = ["derive"] } +serde.workspace = true serde_html_form = "=0.2.8" -serde_ipld_dagcbor = "0.6.4" -serde_json = "1.0.143" -serde_bytes = "0.11.19" -sha2 = "0.10.9" -thiserror = "2.0.16" +serde_ipld_dagcbor.workspace = true +serde_json.workspace = true +serde_bytes.workspace = true +sha2.workspace = true +thiserror.workspace = true time = "0.3.43" tokio = { version = "1.47.1", features = ["fs", "io-util", "macros", "net", "rt-multi-thread", "signal", "sync", "time"] } tower = { version = "0.5.2", features = ["limit", "load-shed", "timeout"] } @@ -45,6 +61,7 @@ tower_governor = { version = "0.8.0", default-features = false, features = ["axu tower-http = { version = "0.6.6", features = ["cors", "request-id", "trace"] } tracing = "0.1.41" tracing-subscriber = { version = "0.3.20", features = ["env-filter"] } +unsigned-varint = "=0.8.0" url = { version = "2.5.7", features = ["serde"] } [dev-dependencies] @@ -52,9 +69,12 @@ k256 = { version = "0.13.4", features = ["ecdsa"] } tempfile = "3.21.0" tower = { version = "0.5.2", features = ["util"] } -[lints.rust] +[lints] +workspace = true + +[workspace.lints.rust] unsafe_code = "forbid" -[lints.clippy] +[workspace.lints.clippy] all = "warn" pedantic = "warn" diff --git a/README.md b/README.md index 0dc4b38..7ed7491 100644 --- a/README.md +++ b/README.md @@ -4,14 +4,17 @@ It hosts one AT Protocol identity, grants applications access through OAuth, stores content-addressed records and blobs, and exposes a focused XRPC surface. -The project favors visible seams over frameworks. Each file owns one idea; the -repository service contains policy; HTTP only translates; SQLite only persists. +Start with the [repository guide](docs/repository-guide.md) for a plain-language +tour, the source map, and small exercises. HTTP translates requests, services +enforce policy, and SQLite commits durable state. The shared `cpds-protocol` +crate handles record validation and content addressing without running a server. ## What works - OAuth 2.0 authorization code flow with exact redirect matching and S256 PKCE - short-lived, one-time authorization codes and hashed opaque access tokens -- `createRecord`, `putRecord`, `getRecord`, `listRecords`, and `deleteRecord` +- `createRecord`, `putRecord`, `getRecord`, `listRecords`, `deleteRecord`, and + ordered, atomic `applyWrites` producing one signed repository commit - official-fixture-matched AT Protocol data-model validation and encoding, including native DAG-CBOR CID links and bytes, deterministic record CIDs, signed repository-v3 MSTs, and AT URIs @@ -99,6 +102,20 @@ crates.io on the first build. cargo run --locked ``` +No separate database server or manual SQL setup is needed. On a normal start, +CPDS opens or creates `cpds.sqlite3` in the current directory and applies its +schema migrations. Blob bytes live separately in `blobs/`. Set +`CPDS_DATABASE` and `CPDS_BLOB_DIRECTORY` to change those locations; the database +file's parent directory must already exist and be writable. Preserve both +locations, plus the separately configured signing key, between runs. + +Successful startup logs `cpds is listening` after the listener has bound. INFO +logging is the default; `RUST_LOG=cpds=info cargo run --locked` explicitly enables +CPDS's messages if an existing `RUST_LOG` filter hides them. In another terminal, +`curl -i http://127.0.0.1:3000/ready` checks storage readiness; `/health` only +checks that HTTP is responding. A failed startup exits with an error before +that listening message. + Before production binds its listener, CPDS resolves the configured handle using DNS `_atproto` TXT records (preferred) or HTTPS `/.well-known/atproto-did`, resolves the resulting `did:plc` or `did:web` document, and requires the exact @@ -208,9 +225,11 @@ addition to the SQLite policy and adapter checks; corruption or loss of the federation stream table therefore fails readiness without repeating the full startup graph walk. Blob downloads carry the AT Protocol CSP and `nosniff` isolation headers. -OAuth/browser requests support CORS; mutable and sensitive responses are -`no-store`, while well-known metadata and content-addressed blobs cache for five -minutes. Forwarding headers never override the configured public origin. +OAuth/browser requests support CORS. Successful well-known metadata and public +`com.atproto.sync.getBlob` responses cache for five minutes; Space blob responses, +errors, and other responses are `no-store`. A public blob download requires a +current public-record reference, not merely a successful upload or a Space +reference. Forwarding headers never override the configured public origin. All configuration variables are parsed in [`src/config.rs`](src/config.rs). `CPDS_REPO_EVENT_RETENTION` bounds the durable firehose window between 1 and @@ -379,7 +398,7 @@ whose `ath` binds that request to the credential. ## Extend it -To add an object-storage backend, implement the three methods of +To add an object-storage backend, implement `check`, `put`, `get`, and `delete` on [`BlobStore`](src/blob_store/mod.rs). Run [`tests/blob_store_contract.rs`](tests/blob_store_contract.rs) against it; the same contract currently proves both bundled implementations. @@ -390,12 +409,13 @@ create/update/delete and blob store produces a typed `RepositoryEvent`. This is an optional best-effort application hook; public federation uses the separate transactional repository outbox and `subscribeRepos` WebSocket. -See [`docs/architecture.md`](docs/architecture.md) for the dependency direction -and a guided tour. The evidence-driven roadmap in +See the [repository guide](docs/repository-guide.md) for the teaching sequence +and [`docs/architecture.md`](docs/architecture.md) for design boundaries. The +evidence-driven roadmap in [`docs/production-plan.md`](docs/production-plan.md) tracks every remaining federation, reliability, security, operations, and local-test gate. The -[`ATrium evaluation`](docs/atrium-evaluation.md) records which ecosystem crates -will replace custom protocol machinery and the evidence required before use. +[`ATrium evaluation`](docs/atrium-evaluation.md) records ecosystem adoption +decisions and their dated evidence; it is not a current vulnerability report. ## Verify @@ -403,11 +423,15 @@ Run the formatter, linter, complete test suite, and locked dependency policy: ```sh cargo fmt --all -- --check -cargo clippy --all-targets -- -D warnings -cargo test --all-targets --locked +cargo clippy --workspace --all-targets --locked -- -D warnings +cargo test --workspace --all-targets --locked cargo deny --locked check ``` -CI runs these gates on Rust 1.88. The dependency policy rejects advisories, +Use `--workspace` to include `cpds-protocol`; plain `cargo test` defaults to the +root package. Some tests open loopback sockets or start child processes, so a +runner that prohibits those operations cannot execute the complete suite. + +CI is configured to run these gates on Rust 1.88. The dependency policy rejects advisories, unapproved licenses, wildcard requirements, and unapproved registries or Git sources; duplicate versions remain visible for deliberate review. diff --git a/crates/cpds-protocol/Cargo.toml b/crates/cpds-protocol/Cargo.toml new file mode 100644 index 0000000..0603649 --- /dev/null +++ b/crates/cpds-protocol/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "cpds-protocol" +version = "0.1.0" +edition = "2024" +rust-version = "1.88" +license = "MIT" +description = "Shared AT Protocol record validation and content addressing for CPDS" + +[dependencies] +atrium-api.workspace = true +base64.workspace = true +cid.workspace = true +serde.workspace = true +serde_bytes.workspace = true +serde_ipld_dagcbor.workspace = true +serde_json.workspace = true +sha2.workspace = true +thiserror.workspace = true + +[lints] +workspace = true diff --git a/crates/cpds-protocol/src/lib.rs b/crates/cpds-protocol/src/lib.rs new file mode 100644 index 0000000..16c8928 --- /dev/null +++ b/crates/cpds-protocol/src/lib.rs @@ -0,0 +1,250 @@ +//! Pure AT Protocol record validation, canonical encoding, and content addressing. +//! +//! Shared by public repositories and permissioned Spaces. This crate has no +//! database, HTTP, account, or runtime dependency of its own. + +use std::collections::BTreeMap; + +use atrium_api::types::{BlobRef as AtriumBlobRef, CidLink, TypedBlobRef}; +use base64::{ + Engine as _, + engine::general_purpose::{STANDARD, STANDARD_NO_PAD}, +}; +use cid::Cid; +use serde::{Deserialize as _, Serialize}; +use serde_json::Value; +use sha2::{Digest, Sha256}; + +const SHA2_256: u64 = 0x12; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("{0}")] + InvalidRecord(String), +} + +pub type Result = std::result::Result; + +const DAG_CBOR_CODEC: u64 = 0x71; +const RAW_CODEC: u64 = 0x55; +const MAX_RECORD_BYTES: usize = 1_000_000; +const MAX_RECORD_DEPTH: usize = 32; +const MAX_RECORD_KEY_BYTES: usize = 8 * 1024; +const MAX_SAFE_JSON_INTEGER: f64 = 9_007_199_254_740_991.0; + +/// Require the record's `$type` to match the target collection. +/// +/// # Errors +/// Returns an error for a missing, non-string, or mismatched type. +pub fn validate_record_type(collection: &str, value: &Value) -> Result<()> { + let record_type = value + .as_object() + .and_then(|object| object.get("$type")) + .and_then(Value::as_str) + .ok_or_else(|| Error::InvalidRecord("record requires a string `$type`".to_owned()))?; + if record_type != collection { + return Err(Error::InvalidRecord(format!( + "record `$type` must equal collection `{collection}`" + ))); + } + Ok(()) +} + +/// Validate and address a record using its canonical DAG-CBOR bytes. +/// +/// # Errors +/// Rejects invalid AT Protocol data or a record larger than one million bytes. +pub fn record_cid(value: &Value) -> Result { + let bytes = serde_ipld_dagcbor::to_vec(&atproto_record(value)?) + .map_err(|error| Error::InvalidRecord(format!("record is not valid DAG-CBOR: {error}")))?; + if bytes.len() > MAX_RECORD_BYTES { + return Err(Error::InvalidRecord(format!( + "record exceeds the {MAX_RECORD_BYTES}-byte DAG-CBOR limit" + ))); + } + Ok(dag_cbor_cid(&bytes)) +} + +#[derive(Serialize)] +#[serde(untagged)] +enum AtprotoData { + Null, + Boolean(bool), + Integer(i64), + Bytes(serde_bytes::ByteBuf), + String(String), + Link(CidLink), + List(Vec), + Map(BTreeMap), +} + +/// Convert JSON special forms into native DAG-CBOR links and bytes. +/// +/// The opaque result can be passed to Serde or `ATrium` without exposing a second +/// public data-model enum. Use `record_cid` to enforce the encoded size limit. +/// +/// # Errors +/// Rejects non-object records, malformed special values, and excessive nesting. +pub fn atproto_record(value: &Value) -> Result { + if !value.is_object() { + return Err(Error::InvalidRecord( + "record must be an AT Protocol object".to_owned(), + )); + } + atproto_data(value, 0) +} + +fn atproto_data(value: &Value, depth: usize) -> Result { + if depth > MAX_RECORD_DEPTH { + return Err(Error::InvalidRecord(format!( + "record exceeds the {MAX_RECORD_DEPTH}-level nesting limit" + ))); + } + match value { + Value::Null => Ok(AtprotoData::Null), + Value::Bool(value) => Ok(AtprotoData::Boolean(*value)), + Value::Number(value) => { + let integer = value.as_i64().or_else(|| { + value.as_f64().and_then(|value| { + (value.fract() == 0.0 && value.abs() <= MAX_SAFE_JSON_INTEGER) + .then(|| format!("{value:.0}").parse().ok()) + .flatten() + }) + }); + integer.map(AtprotoData::Integer).ok_or_else(|| { + Error::InvalidRecord("record numbers must be signed 64-bit integers".to_owned()) + }) + } + Value::String(value) => Ok(AtprotoData::String(value.clone())), + Value::Array(values) => values + .iter() + .map(|value| atproto_data(value, depth + 1)) + .collect::>>() + .map(AtprotoData::List), + Value::Object(values) => atproto_object(values, depth), + } +} + +fn atproto_object(values: &serde_json::Map, depth: usize) -> Result { + if values + .get("$type") + .is_some_and(|value| value.as_str().is_none_or(str::is_empty)) + { + return Err(Error::InvalidRecord( + "record `$type` fields must be non-empty strings".to_owned(), + )); + } + validate_blob_ref(values)?; + if let Some(value) = values.get("$link") { + return atproto_link(value, values.len()); + } + if let Some(value) = values.get("$bytes") { + return atproto_bytes(value, values.len()); + } + values + .iter() + .map(|(key, value)| { + if key.is_empty() || key.len() > MAX_RECORD_KEY_BYTES { + return Err(Error::InvalidRecord(format!( + "record object keys must contain 1 to {MAX_RECORD_KEY_BYTES} UTF-8 bytes" + ))); + } + Ok((key.clone(), atproto_data(value, depth + 1)?)) + }) + .collect::>>() + .map(AtprotoData::Map) +} + +fn validate_blob_ref(values: &serde_json::Map) -> Result<()> { + if values.get("$type").and_then(Value::as_str) != Some("blob") { + return Ok(()); + } + let blob = AtriumBlobRef::deserialize(values).map_err(|error| { + Error::InvalidRecord(format!( + "record contains an invalid blob reference: {error}" + )) + })?; + let AtriumBlobRef::Typed(TypedBlobRef::Blob(blob)) = blob else { + return Err(Error::InvalidRecord( + "record blob references must use the current typed format".to_owned(), + )); + }; + let hash = blob.r#ref.0.hash(); + if values.len() != 4 + || blob.mime_type.is_empty() + || blob.size == 0 + || blob.r#ref.0.version() != cid::Version::V1 + || blob.r#ref.0.codec() != RAW_CODEC + || hash.code() != SHA2_256 + || hash.digest().len() != 32 + { + return Err(Error::InvalidRecord( + "record contains a noncanonical blob reference".to_owned(), + )); + } + Ok(()) +} + +fn atproto_link(value: &Value, fields: usize) -> Result { + if fields != 1 { + return Err(Error::InvalidRecord( + "record CID links cannot contain extra fields".to_owned(), + )); + } + let value = value + .as_str() + .ok_or_else(|| Error::InvalidRecord("record CID link must be a string".to_owned()))?; + let link = CidLink::try_from(value) + .map_err(|error| Error::InvalidRecord(format!("record CID link is invalid: {error}")))?; + let hash = link.0.hash(); + if value != link.0.to_string() + || link.0.version() != cid::Version::V1 + || !matches!(link.0.codec(), DAG_CBOR_CODEC | RAW_CODEC) + || hash.code() != SHA2_256 + || hash.digest().len() != 32 + { + return Err(Error::InvalidRecord( + "record CID links must use canonical base32 CIDv1 with dag-cbor or raw codec and SHA-256".to_owned(), + )); + } + Ok(AtprotoData::Link(link)) +} + +fn atproto_bytes(value: &Value, fields: usize) -> Result { + if fields != 1 { + return Err(Error::InvalidRecord( + "record bytes cannot contain extra fields".to_owned(), + )); + } + let value = value + .as_str() + .ok_or_else(|| Error::InvalidRecord("record bytes must be a base64 string".to_owned()))?; + let bytes = STANDARD + .decode(value) + .or_else(|_| STANDARD_NO_PAD.decode(value)) + .map_err(|error| { + Error::InvalidRecord(format!("record bytes are invalid base64: {error}")) + })?; + Ok(AtprotoData::Bytes(bytes.into())) +} + +/// Address an already encoded DAG-CBOR block using SHA-256. +#[must_use] +pub fn dag_cbor_cid(bytes: &[u8]) -> Cid { + sha256_cid(DAG_CBOR_CODEC, bytes) +} + +/// Address bytes using SHA-256 and the supplied IPLD codec. +/// +/// # Panics +/// Panics only if the fixed SHA-256 digest exceeds the multihash capacity. +#[must_use] +pub fn sha256_cid(codec: u64, bytes: &[u8]) -> Cid { + let digest = Sha256::digest(bytes); + let multihash = cid::multihash::Multihash::<64>::wrap(0x12, &digest) + .expect("a 32-byte SHA-256 digest fits the 64-byte multihash capacity"); + Cid::new_v1(codec, multihash) +} + +#[cfg(test)] +mod tests; diff --git a/crates/cpds-protocol/src/tests.rs b/crates/cpds-protocol/src/tests.rs new file mode 100644 index 0000000..bab8f5d --- /dev/null +++ b/crates/cpds-protocol/src/tests.rs @@ -0,0 +1,251 @@ +use super::*; +use serde::Deserialize; +use serde_json::json; + +#[derive(Deserialize)] +struct DataModelFixture { + json: Value, + cbor_base64: String, + cid: String, +} + +#[derive(Deserialize)] +struct DataModelValidationFixture { + note: String, + json: Value, +} + +#[test] +fn fixed_width_sha256_cids_match_the_previous_fallible_construction() { + fn previous(codec: u64, bytes: &[u8]) -> std::result::Result { + let digest = Sha256::digest(bytes); + let multihash = cid::multihash::Multihash::<64>::wrap(0x12, &digest)?; + Ok(Cid::new_v1(codec, multihash)) + } + + let mut inputs = vec![Vec::new(), b"abc".to_vec(), (0..=255).collect()]; + for length in [55, 56, 63, 64, 65, 127, 128, 129, 1_024, 65_537] { + inputs.push((0..=255).cycle().take(length).collect()); + } + for bytes in inputs { + for codec in [0, 0x55, 0x70, DAG_CBOR_CODEC, u64::MAX] { + let expected = previous(codec, &bytes).unwrap(); + let actual = sha256_cid(codec, &bytes); + assert_eq!(actual, expected); + assert_eq!(actual.to_bytes(), expected.to_bytes()); + assert_eq!(actual.to_string(), expected.to_string()); + assert_eq!(actual.version(), cid::Version::V1); + assert_eq!(actual.codec(), codec); + assert_eq!(actual.hash().code(), 0x12); + assert_eq!(actual.hash().digest().len(), 32); + } + assert_eq!(dag_cbor_cid(&bytes), sha256_cid(DAG_CBOR_CODEC, &bytes)); + } + + for (bytes, digest) in [ + ( + b"".as_slice(), + [ + 0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, + 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, + 0x78, 0x52, 0xb8, 0x55, + ], + ), + ( + b"abc".as_slice(), + [ + 0xba, 0x78, 0x16, 0xbf, 0x8f, 0x01, 0xcf, 0xea, 0x41, 0x41, 0x40, 0xde, 0x5d, 0xae, + 0x22, 0x23, 0xb0, 0x03, 0x61, 0xa3, 0x96, 0x17, 0x7a, 0x9c, 0xb4, 0x10, 0xff, 0x61, + 0xf2, 0x00, 0x15, 0xad, + ], + ), + ] { + assert_eq!(sha256_cid(0x55, bytes).hash().digest(), digest); + } +} + +#[test] +fn borrowed_blob_map_matches_owned_decoding_values_and_errors() { + let typed = json!({ + "$type": "blob", + "ref": {"$link": "bafkreibme22gw2h7y2h7tg2fhqotaqjucnbc24deqo72b6mkl2egezxhvy"}, + "mimeType": "image/png", + "size": 3, + }); + let mut values = vec![ + typed.clone(), + json!({}), + json!({"cid": "not-a-cid", "mimeType": "image/png"}), + json!({"$type": "blob", "cid": "legacy", "mimeType": "image/png"}), + json!({"$type": "other", "cid": "legacy", "mimeType": "image/png"}), + ]; + for field in ["$type", "ref", "mimeType", "size"] { + let mut missing = typed.clone(); + missing.as_object_mut().unwrap().remove(field); + values.push(missing); + for replacement in [ + Value::Null, + json!(false), + json!(-1), + json!(1.5), + json!(u64::MAX), + json!(""), + json!([]), + json!({}), + ] { + let mut value = typed.clone(); + value[field] = replacement; + values.push(value); + } + } + let mut extended = typed; + extended["unknown"] = json!({"nested": [null, true, 1.5, "value"]}); + values.push(extended); + + for value in values { + let previous = serde_json::from_value::(value.clone()) + .map(|blob| serde_json::to_value(blob).unwrap()) + .map_err(|error| format!("{error:?}")); + let current = AtriumBlobRef::deserialize(value.as_object().unwrap()) + .map(|blob| serde_json::to_value(blob).unwrap()) + .map_err(|error| format!("{error:?}")); + assert_eq!(current, previous, "{value}"); + } +} + +#[test] +fn special_object_dispatch_preserves_error_priority_and_native_encoding() { + for (value, expected) in [ + ( + json!({"$type": false, "$link": false, "$bytes": false}), + "record `$type` fields must be non-empty strings", + ), + ( + json!({"$type": "blob", "cid": "legacy", "mimeType": "image/png", "$link": false}), + "record blob references must use the current typed format", + ), + ( + json!({"$link": false, "$bytes": false}), + "record CID links cannot contain extra fields", + ), + ( + json!({"$link": false, "extra": true}), + "record CID links cannot contain extra fields", + ), + ( + json!({"$bytes": false, "extra": true}), + "record bytes cannot contain extra fields", + ), + (json!({"$link": false}), "record CID link must be a string"), + ( + json!({"$bytes": false}), + "record bytes must be a base64 string", + ), + ] { + assert!( + matches!(atproto_record(&value), Err(Error::InvalidRecord(message)) if message == expected) + ); + } + + for (value, expected) in [ + (json!({"$bytes": ""}), vec![0x40]), + (json!({"$bytes": "AQ=="}), vec![0x41, 1]), + (json!({"$bytes": "AQ"}), vec![0x41, 1]), + (json!({"x": null}), vec![0xa1, 0x61, b'x', 0xf6]), + ] { + assert_eq!( + serde_ipld_dagcbor::to_vec(&atproto_record(&value).unwrap()).unwrap(), + expected, + ); + } +} + +#[test] +fn record_addressing_rejects_oversized_dag_cbor_before_repository_work() { + let oversized = json!({ + "$type": "app.bsky.feed.post", + "payload": "x".repeat(MAX_RECORD_BYTES) + }); + + let error = record_cid(&oversized).unwrap_err().to_string(); + + assert!(error.contains("1000000-byte DAG-CBOR limit")); +} + +#[test] +fn record_addressing_matches_official_atproto_data_model_fixtures() { + let fixtures: Vec = serde_json::from_str(include_str!( + "../../../tests/fixtures/atproto-data-model-fixtures.json" + )) + .unwrap(); + + for fixture in fixtures { + let record = atproto_record(&fixture.json).unwrap(); + let bytes = serde_ipld_dagcbor::to_vec(&record).unwrap(); + assert_eq!(bytes, STANDARD_NO_PAD.decode(fixture.cbor_base64).unwrap()); + assert_eq!(record_cid(&fixture.json).unwrap().to_string(), fixture.cid); + } +} + +#[test] +fn generic_record_validation_matches_official_atproto_data_model_fixtures() { + let valid: Vec = serde_json::from_str(include_str!( + "../../../tests/fixtures/atproto-data-model-valid.json" + )) + .unwrap(); + for fixture in valid { + assert!( + atproto_record(&fixture.json).is_ok(), + "rejected valid fixture: {}", + fixture.note + ); + } + + let invalid: Vec = serde_json::from_str(include_str!( + "../../../tests/fixtures/atproto-data-model-invalid.json" + )) + .unwrap(); + for fixture in invalid { + assert!( + atproto_record(&fixture.json).is_err(), + "accepted invalid fixture: {}", + fixture.note + ); + } +} + +#[test] +fn record_addressing_rejects_values_outside_the_atproto_data_model() { + for value in [ + json!({"value": 1.5}), + json!({"value": u64::MAX}), + json!({"value": {"$bytes": true}}), + json!({"value": {"$bytes": "", "extra": true}}), + json!({"value": {"$link": "not-a-cid"}}), + json!({ + "value": { + "$link": "bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi" + } + }), + json!({ + "value": { + "$link": "bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a", + "extra": true + } + }), + ] { + assert!(record_cid(&value).is_err(), "accepted {value}"); + } + + let integer_like: Value = serde_json::from_str(r#"{"value": 123.0}"#).unwrap(); + assert_eq!( + record_cid(&integer_like).unwrap(), + record_cid(&json!({"value": 123})).unwrap() + ); + + let mut nested = Value::Null; + for _ in 0..=MAX_RECORD_DEPTH { + nested = json!({"nested": nested}); + } + assert!(record_cid(&nested).is_err()); +} diff --git a/docs/architecture.md b/docs/architecture.md index 1bf03a1..bdee55b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,221 +1,146 @@ # Architecture -The shortest useful description of `cpds` is: - -```text -HTTP request - ↓ parse -HTTP adapter (`src/http`) - ↓ domain values -Identity, OAuth, or repository service - ↓ explicit ports -SQLite · BlobStore · EventSink -``` - -Dependencies only point downward. Persistence never calls HTTP. Blob stores do -not know about OAuth. Route handlers do not contain SQL. This makes a file's -reason to change easy to name. - -## Assembly - -[`AppState::assemble`](../src/lib.rs) is the composition root. Production and -tests call the same function. Tests replace the filesystem with -`MemoryBlobStore`, use an in-memory SQLite connection, and record events; no -mock web server or internet connection is involved. - -The `verify` process mode uses the same composition root and exhaustive startup -verification, but requires existing initialized storage, checks restored account -identity against configuration, validates a colocated backup manifest when -present, skips lifecycle collection, and exits without opening a listener. This -keeps restore proof on the production code path. - -The `backup` mode performs that proof before capture. A SQLite immediate -transaction blocks mutations and lifecycle claims while a separate read-only -connection drives SQLite's native online backup and the filesystem adapter -copies CID-verified objects. The target is private, durable, create-only, and -contains a non-secret restore manifest; credentials and the repository signing -key remain a separately protected operator concern. - -The `set-account-status` mode uses the same existing-storage composition and -verification path. Filesystem access by the configured service account is its -operator authorization boundary; no network administration route exists. It -atomically changes the constrained status and appends the corresponding retained -`#account` event, then repeats verification without opening a listener or -running collection. A production transition to `active` first repeats the -bidirectional network identity gate. The reversible `deleted` status is access -suppression, not physical erasure. - -The `request-crawl` mode deliberately does not assemble application state or -open storage. It validates the configured relay origins, then `src/relay.rs` -posts ATrium's generated `com.atproto.sync.requestCrawl` input to all relays in -parallel with redirect refusal, connection/request deadlines, and bounded -transient retries. Per-relay logs plus process exit status make the idempotent -operator action observable without turning it into an untracked startup side -effect. - -## Identity - -[`IdentityService`](../src/identity.rs) is the shared identity boundary for the -public `resolveHandle` endpoint, production startup, and remote Space token and -service resolution. It composes ATrium's generated DID/handle types, -`CommonDidResolver` for `did:plc`, `WellKnownHandleResolver` for HTTPS handle -fallback, and XRPC HTTP trait with CPDS-owned policy. Hickory supplies direct -system DNS so DNS TXT remains the preferred handle method and the HTTP client -can reject private, loopback, link-local, reserved, and documentation answers. -The adapter also bounds DNS caching, connection/request time, redirects, and -document bytes. - -The service depends internally on the three-operation `IdentityNetwork` port: -DNS handle lookup, HTTPS handle lookup, and DID resolution. Production supplies -the ATrium/Hickory adapter; deterministic tests inject the same boundary to -prove precedence, fallback, terminal conflicts, document-ID matching, and -backlinks without public network access. - -ATrium 0.1.9's web resolver maps the complete `did:web` method identifier as a -host, so CPDS retains only the standards mapping needed for path and encoded-port -forms and sends it through the same hardened transport. CPDS also rejects -conflicting valid DNS DID records and requires an explicit DID-document handle -backlink, because the current ATrium helpers do not enforce those trust rules. -Local startup remains network-free; production initializes the resolver eagerly -and verifies both directions before binding, while local federation initializes -it on first remote lookup. - -While production is serving, an independent bounded timer repeats that check. -ATrium supplies the validated DID/handle/document shapes and resolution -building blocks; CPDS classifies only confirmed `HandleNotFound` or -bidirectional mismatches as invalid and treats transport failures as -inconclusive. Confirmed invalidation atomically stores `handle.invalid` with a -retained `#identity` event. Recovery restores the configured handle the same -way. The DID and account hosting status do not change, the public DID document -continues to describe the configured identity, and local mode creates no timer. - -## OAuth - -[`OAuthService`](../src/oauth/mod.rs) owns authorization policy: - -1. resolve a client through `ClientRegistry`; -2. compare the redirect URI exactly; -3. require the `atproto` scope and S256 challenge; -4. authenticate the hosted account with Argon2; -5. store only a hash of the short-lived code; -6. redeem it once, atomically, while checking client, redirect, and PKCE; -7. store only a hash of the opaque access token. - -`StaticClientRegistry` makes local and offline operation deterministic. A future -AT Protocol metadata resolver belongs in a new `ClientRegistry` implementation; -the OAuth state machine does not need to change. - -## Repository - -[`RepositoryService`](../src/repository.rs) is the policy boundary for data: - -- an access token can mutate only its own DID; -- collections and record keys are validated before storage; -- `$type` must match the collection; -- records pass a small schema-independent AT Protocol data-model adapter that - rejects floats, malformed special objects, excessive nesting/key size, and - non-blessed CID forms; it composes ATrium's `CidLink`, typed `BlobRef`, and - repository writer so CID links and bytes become native DAG-CBOR values, then - rejects canonical blocks above the protocol's one-megabyte encoded-record - bound; reads use ATrium's native `DataModel` instead of a parallel decoder; -- generated keys are monotonically increasing AT Protocol TIDs; -- blobs are size-limited, MIME-checked, content-addressed, and indexed by record - reference in the same transaction; startup recomputes the derived index and - fails closed on drift; -- aged unreferenced blob metadata is claimed transactionally before object - deletion, preventing concurrent public or Space writes from acquiring it and - making interrupted deletion resumable; -- events are published only after the mutation commits. - -SQLite is intentionally serialized through one mutex. A personal server values -clear transactional behavior more than write concurrency, and this design can -be audited without an async database framework. - -## Adding a blob store - -`BlobStore` accepts an owner namespace, CID, and bytes. Metadata remains in -SQLite, so an adapter only answers three questions: - -```rust -#[async_trait] -pub trait BlobStore: Send + Sync { - async fn put(&self, owner: &str, cid: &Cid, bytes: &[u8]) -> Result<()>; - async fn get(&self, owner: &str, cid: &Cid) -> Result>>; - async fn delete(&self, owner: &str, cid: &Cid) -> Result; -} -``` - -An S3 adapter can derive an object key from `owner/cid`, make `put` idempotent, -and map a missing object to `Ok(None)`. The shared contract test then applies -without any service changes. - -The lifecycle collector considers both public and Space reference indexes. It -runs a bounded pass before startup verification and periodically while serving. -Uploads reserve hidden metadata before publishing bytes and mark it available -only after the object is durable. A stop at any point therefore leaves either no -object or a hidden row that the same collector can reclaim after its grace -period. - -## Adding listeners and sync - -`EventSink` is deliberately smaller than a message bus. The repository emits -facts, not callbacks into its own transaction. Durable public sync is separate: - -Account hosting status is a constrained SQLite value and a closed Rust enum. -The user-facing activation/deactivation procedures reuse ATrium's generated -Lexicon models, but CPDS owns the PDS-specific transition policy. A -compare-and-set transaction updates status, appends the matching `#account` -outbox event, and applies retention atomically. Public repo reads map inactive -states to the Lexicon's `RepoDeactivated`, `RepoSuspended`, `RepoTakendown`, or -`RepoNotFound` errors. OAuth-backed mutations fail with the corresponding -account error. Repository commits and blob reservation/publication also check -active state inside SQLite, preventing a concurrent transition from racing the -HTTP authorization check. Account credentials can reverse only their own -deactivation. The offline, storage-verified operator command can apply or -reverse any host state without exposing a network administration endpoint. - -1. the first record transaction atomically stores ATrium blocks, signed head, - index, operation, initial identity/account events, versioned commit metadata, - and ordered commit-block membership; -2. the repository service rebuilds generated ATrium identity/account models and - signature-checked commit or sync messages with CAR slices from one resumable - sequence cursor; -3. the HTTP adapter applies connection/backpressure bounds and the specified - two-object DAG-CBOR WebSocket framing for `com.atproto.sync.subscribeRepos`. - Each upgraded stream runs in a metadata-only tracing span carrying the - server-generated request ID and initial cursor; lifecycle, disconnect, and - backpressure logs add only the last durable cursor, never event payloads or - request credentials. - On shutdown it signals every stream before HTTP draining; each subscriber - receives a bounded going-away close containing its last durable cursor. - -Before binding the listener, the repository service enumerates every initialized -ATrium MST, verifies its signed head and all reachable content-addressed blocks, -and compares its ordered entries exactly with the SQL record index. It also -rebuilds every retained event through the generated ATrium wire model and CAR -writer, checks global sequence continuity and per-repository inductive links, -and requires each latest event to match its persisted head. The ordinary -readiness probe checks adapter availability without repeating that unbounded -startup walk. Its bounded database check also queries the durable outbox's -latest cursor and oldest timestamp, independently of the best-effort -`EventSink`, so loss of the federation stream store fails readiness. - -After verification, bounded repository-block collection composes ATrium's -reachable graph with CPDS's durable sync retention policy. A transaction deletes -only blocks outside every current graph, head, retained commit membership, and -legacy sync root. The periodic maintenance loop repeats the same pass, so old -blocks become reclaimable only after their replay event leaves retention. - -The best-effort sink still logs delivery errors after a successful mutation and -never reports the already-committed write as failed, which avoids unsafe client -retries. It is an application extension seam, not the durable federation path. - -## Why limitations are explicit - -`cpds` does not use placeholder implementations for MSTs, signatures, DPoP, or -firehose sequencing. Those features have cryptographic and transactional -invariants. Current code emits the initial identity/account/commit sequence and -verifies the hosted handle bidirectionally before production serves. New initial -identity events carry that canonical handle in the same repository transaction; -retained historical events are not rewritten. Later account transitions, relay -observability, and remote repository ingestion remain visibly incomplete. +For a first walkthrough, read the [repository guide](repository-guide.md). This +document records the design boundaries; [README](../README.md) owns operating +instructions, and [production-plan.md](production-plan.md) owns unfinished +production and federation work. + +## Two packages, one server + +The Cargo workspace contains the `cpds` server package and the small +[`cpds-protocol`](../crates/cpds-protocol/src/lib.rs) library package. The server +depends on the protocol package, never the reverse. Its API validates record +shape, turns JSON `$link`/`$bytes` forms into native DAG-CBOR values, and computes +content identifiers. It owns no HTTP, account, SQLite, filesystem, or network +policy and can be tested independently. + +The rest is organized into modules, not a crate per layer. Record writes, +repository blocks, blob-reference indexes, account state, and durable events +must share transaction rules. Artificially independent storage or generic CRUD +packages would obscure that coupling rather than remove it. Small modules keep +responsibilities readable without making every helper a public package API. + +## Assembly and adapters + +[`AppState::assemble`](../src/lib.rs) wires services to a concrete `Database` and +the narrow `BlobStore`, `EventSink`, and `ClientRegistry` interfaces. Services own +authorization and orchestration; HTTP handlers translate requests and errors; +persistence modules own SQL and transaction boundaries. Database code does not +call HTTP handlers, and blob adapters do not decide OAuth or record visibility. + +The ordinary HTTP test harness calls this same assembly function with real +in-memory SQLite, `MemoryBlobStore`, a static OAuth client, and +`RecordingEventSink`. Separate filesystem, process, and local fake-server tests +exercise behavior that an in-memory router cannot prove. + +`BlobStore` has four operations: `check`, `put`, `get`, and `delete`. Bytes are +namespaced by owner and CID; MIME type, size, publication state, and references +remain in SQLite. The shared [contract test](../tests/blob_store_contract.rs) +covers both bundled stores. `EventSink` has `check` and `publish`; it is a +best-effort application hook, not the federation log. + +## Public repository transactions + +[`repository/records.rs`](../src/repository/records.rs) validates identifiers, +ownership, swaps, record data, and uploaded blob references. Typed mutations +retain create/update/delete intent throughout ordered batch planning. +[`repository/commit.rs`](../src/repository/commit.rs) applies them through +ATrium's MST and stages content-addressed blocks before asking +[`database/repo.rs`](../src/database/repo.rs) to publish them atomically. +The transaction-local block overlay and CID-verified reads live separately in +[`database/blockstore.rs`](../src/database/blockstore.rs); that adapter stages +bytes but does not decide which record changes may commit. + +One successful public mutation transaction contains the final signed head, +record index, blob-reference index, repository blocks, ordered operations, and +durable sync-event metadata. An `applyWrites` batch produces one signed commit, +not a series of hidden intermediate commits. The event proof contains newly +written blocks still reachable from the final root. Compare-and-swap checks +prevent a concurrent head or record change from being silently overwritten. + +ATrium owns standard MST and repository machinery. CPDS owns staging, signing +orchestration, validation, resource limits, and the SQLite commit. Opening an +ATrium repository alone is not verification: CPDS also checks signatures, +content hashes, revision metadata, and agreement with its SQL record index. + +[`repository/export.rs`](../src/repository/export.rs) exports verified local +repositories as bounded CAR archives. Full exports, incremental exports, +record proofs, and requested-block exports have different selection rules; a +shared block table is not permission to return arbitrary known blocks. Remote +CAR ingestion remains outside the implemented public surface. + +## Public data and Spaces are different stores + +Public records and alpha Space records have separate tables, authorization, +revision histories, and commit formats. They share only appropriate mechanisms: +canonical record encoding, blob bytes/metadata, identifier checks, identity +resolution, and cryptographic dependencies. A Space is not a public repository +with an extra access flag. + +[`space/commit.rs`](../src/space/commit.rs) implements the alpha LtHash-based +deniable commit, while public repositories use repository-v3 signed MST commits. +Space CAR recovery carries its signed view and index as two roots. These formats +are deliberately not forced through one generic repository abstraction. + +Blob visibility follows references. A public download requires a current public +record reference. A Space download first requires authorization for the selected +Space repository and a matching `space_record_blobs` index entry. Merely knowing +a CID or uploading bytes grants neither public visibility nor Space access. +Space responses and errors are `no-store`; only successful public blob and +well-known responses receive the public cache policy. + +## Identity and authentication + +[`IdentityService`](../src/identity.rs) centralizes handle and DID resolution. +ATrium provides protocol models and resolver building blocks; CPDS retains DNS +conflict handling, `did:web` path/port mapping, transport policy, document-ID +checks, and the required handle backlink. Local startup uses the configured +development identity. Production requires bidirectional network verification +before listening and periodically revalidates it. A confirmed mismatch stores +`handle.invalid` with an identity event; inconclusive transport failures preserve +the last confirmed state. + +[`OAuthService`](../src/oauth/mod.rs) implements authorization codes with exact +registered redirects, S256 PKCE, Argon2 account authentication, and hashed opaque +credentials. Redeeming the code and inserting the access token is one SQLite +transaction: failed token storage must not consume the code. The static registry +is intentional; this is not the complete AT Protocol OAuth server profile. + +Space delegation and DPoP-bound credentials belong to +[`space/auth.rs`](../src/space/auth.rs), service-token/notification transport to +[`auth/service.rs`](../src/space/auth/service.rs), and client attestations to +[`auth/attestation.rs`](../src/space/auth/attestation.rs). Shared outbound checks +live in [`identity/http.rs`](../src/identity/http.rs). Space DPoP support must not be +confused with unimplemented OAuth DPoP support. The configured public origin, +not forwarding headers, determines request-URL bindings. + +## Durability, replay, and recovery + +[`repository/stream.rs`](../src/repository/stream.rs) rebuilds retained SQLite +events into generated ATrium messages. [`http/subscription.rs`](../src/http/subscription.rs) +owns WebSocket framing, admission, backpressure, and shutdown. A disconnected +consumer resumes from its last durable sequence cursor. A cursor older than +retention receives `OutdatedCursor` and requires repository resynchronization. +Failure of the optional event sink never turns an already committed write into +an apparent failure; doing so would encourage unsafe retries. + +[`repository/maintenance.rs`](../src/repository/maintenance.rs) separates bounded +readiness checks and garbage-collection passes from exhaustive startup storage +verification. Blob publication reserves metadata before durable byte publication. +Collection durably claims aged unreferenced objects before deletion and considers +both public and Space references. Block collection preserves current repository +graphs and blocks required by retained events. + +The process has five explicit modes: serve, verify, backup, request-crawl, and +set-account-status. Verification and operator modes require existing storage and +do not listen. Backup holds an SQLite write barrier across a native database +snapshot and CID-verified blob copy; its manifest omits passwords and signing +keys. Crawl requests deliberately do not open storage. The reversible `deleted` +hosting state suppresses access; it does not erase stored bytes or backups. + +SQLite currently uses one mutex-protected connection. This makes transaction +ownership explicit but serializes database work. Synchronous SQLite and password +hashing also remain blocking work; async function signatures and HTTP deadlines +do not by themselves preempt them. Do not infer throughput or isolation from the +router's concurrency limits. That tradeoff and the remaining operational gates +stay visible in the [production plan](production-plan.md). diff --git a/docs/atrium-evaluation.md b/docs/atrium-evaluation.md index c655117..c9cf9ec 100644 --- a/docs/atrium-evaluation.md +++ b/docs/atrium-evaluation.md @@ -6,6 +6,12 @@ ATrium-backed path is considered production-ready. It is a dated evaluation of the crates published as of 2026-08-22, not a permanent endorsement of every future release. +Implementation notes were reconciled with the working tree on 2026-09-05, +including public `applyWrites` and the shared `cpds-protocol` crate. Package +release, dependency-count, and advisory findings below remain historical +2026-08-22 evidence; rerun the dependency acceptance gate for a current claim. +For the present source layout, use the [repository guide](repository-guide.md). + The objective is a smaller PDS, not merely fewer local files. CPDS should use ATrium where it implements stable AT Protocol machinery and delete equivalent custom code after parity is proven. CPDS must still own server policy, @@ -20,7 +26,7 @@ The evaluated packages are from the | Package | Version | Intended CPDS use | Decision | | --- | ---: | --- | --- | | `atrium-api` | 0.25.8 | Generated Lexicon request, response, union, and syntax types | Adopt incrementally | -| `atrium-crypto` | 0.1.3 | AT Protocol key encoding, low-S signing, and signature verification | Adopted for service keys and Spaces; reuse for public repo commits | +| `atrium-crypto` | 0.1.3 | AT Protocol key encoding, low-S signing, and signature verification | Adopted for service keys, Spaces, and public repo commits | | `atrium-repo` | 0.1.8 | Repository-v3 commits, MST operations, block-store traits, and CAR support | Adopted behind CPDS storage adapters and fixture gates | | `atrium-identity` | 0.1.9 | DID/handle resolver composition and bidirectional identity resolution | Adopted selectively behind hardened CPDS transports | | `atrium-xrpc` | 0.12.4 | Shared XRPC HTTP and error primitives used by generated types and clients | Use transitively or in adapters; not as an Axum server framework | @@ -114,6 +120,10 @@ corruption. Per-Lexicon schema validation remains separate: `validate=true` is rejected instead of falsely claiming it, while successful unvalidated writes report `validationStatus: unknown`. +That pure adapter now lives in `cpds-protocol` and is shared directly by public +repositories and Space CAR exports. Neither feature imports the other feature's +service implementation to encode records. + The adapter accepts every valid and rejects every invalid schema-independent fixture in the official, implementation-neutral `atproto-interop-tests` data-model set at commit `056e5741bb330757205d6b16db5266fffcae937b`. Its @@ -245,14 +255,15 @@ also matches all six official firehose commit-proof fixtures: pre/post roots for single and multi-operation merge/split cases, plus the exact proof set formed by newly written blocks that remain reachable from the final root. This proves the public `Tree` API is a conformant basis for atomic `applyWrites` even -though `Repository` does not expose a batch builder; CPDS still needs to wrap -the final root in exactly one signed commit and transaction. Its block-store +though `Repository` does not expose a batch builder; CPDS wraps the final root +in exactly one signed commit and transaction. Its block-store traits are a useful composition boundary, but `DiffBlockStore` records writes after forwarding them and is not an atomic staging layer. CPDS therefore stages content-addressed writes in memory, filters a legacy backfill to ATrium's reachable export, and inserts each incremental write batch, record -index, head, operation, and outbox event in one SQLite transaction. Incremental -mutations persist ATrium's write set directly and do not walk the whole repo. +index, head, operation, and outbox event in one SQLite transaction. Commit-event +proof membership is the intersection of staged writes and the final root's +reachable blocks, excluding transient blocks from earlier operations in a batch. The source review also found boundaries that CPDS must enforce. Repository and MST open operations explicitly trust stored structure and do not verify commit @@ -504,19 +515,20 @@ tests, and federation harness before accepting one. as atomically maintained, startup-verified public and Space indexes. Schema v6 durably stages object publication and uses those indexes to transactionally claim aged orphans before bounded, crash-resumable object deletion. -3. **Complete for create/update/delete:** a CPDS SQLite staging adapter composes +3. **Complete for create/update/delete and atomic batches:** a CPDS SQLite staging adapter composes `atrium-repo` for signed repository-v3 commits, passes the pinned reference roots, returns commit metadata, enforces commit/record swaps, and atomically - publishes blocks, head, index, operation, and outbox state. Only atomic - `applyWrites` remains. Before binding, CPDS now uses ATrium's MST entry stream + publishes blocks, head, index, operation, and outbox state. Ordered public + `applyWrites` uses one final signed commit and the same transaction. Before + binding, CPDS uses ATrium's MST entry stream and repository reads to verify the signed head, every reachable record block, and exact parity with its SQL record index; routine readiness remains a bounded adapter check. ATrium 0.1.8 exposes only single-operation builders that advance the commit - when finalized; it has no public batch builder. CPDS must not simulate a - batch with hidden intermediate commits. Prefer an upstream batch API; if - that remains unavailable, any CPDS adapter must still produce exactly one - signed commit and retain ATrium's MST behavior and fixture parity. + when finalized; it has no public batch builder. CPDS composes its public MST + `Tree` API for the ordered operations and signs the final root once. The + pinned commit-proof vectors guard ATrium's root/proof behavior; CPDS service, + SQL rollback, and HTTP tests guard the surrounding batch transaction. 4. **Complete for local full and incremental export:** bounded ATrium CAR export, signed-head verification, and versioned historical block selection are live. Add a hardened bounded import adapter before enabling remote repository diff --git a/docs/cleanup-audit.md b/docs/cleanup-audit.md new file mode 100644 index 0000000..873af43 --- /dev/null +++ b/docs/cleanup-audit.md @@ -0,0 +1,99 @@ +# Cleanup audit results + +2026-09-05. Audit baseline: working-copy snapshot `04c9babe`, including the user's +pre-existing public `applyWrites` work. The [plan](cleanup-plan.md) preceded these +edits. No commit, push, deployment, or production-data migration was performed. + +## Coverage and findings + +The source review, implementation, teaching guide, and consolidated final +verification are complete. The coverage table records decisions across the +whole repository; the gates below cover the final source together. + +| Reviewed area | Implemented outcome / retained design | Behavioral evidence | +| --- | --- | --- | +| Configuration / startup / assembly | Keep injectable config and typed origins. INFO startup logging by default. Canonical DID document owned by identity, not reconstructed through Spaces. | Config, discovery and process lifecycle suites. | +| Public record service | Typed ordered batches, one commit/event path, shared record preparation. Cohesive accounts/records/commit/blobs/export/car/stream/maintenance modules. | Record flow, rollback, final-root fixtures, concurrent writes, restart verification. | +| HTTP / middleware | Mask internal errors while preserving logs. Public caching only for successful public resources; Space blobs/errors no-store. Keep typed route glue instead of generic CRUD infrastructure. | HTTP errors/security/limits/rate-limits/observability tests; private-blob regression failed before fix. | +| OAuth | Atomic code redemption/token insertion; remove unused wrappers. Reject at exact expiry, delete consumed codes, indexed expired-row cleanup on issuance/redemption. | Token-insert fault reproduced consumed-code bug; deterministic expiry/retention SQL tests. | +| Schema / account state | One ordered migration transaction replaces eight copied paths. Version 9 repairs skipped outbox rebuilds and preserves issued sequences; shared transaction-time active-account check. | Real legacy v2-v8 upgrade tests reproduced NOT NULL failure and sequence reuse (1 instead of 21). | +| Blobs / backup / filesystem | Preserve reservations, durable rename/fsync, claims, manifest identity and restore checks. Reject repeated CID with inconsistent MIME/size. Separate public-reference access from authorized Space reads. | Blob adapter contract, backup/restore, orphan recovery, corrupt/missing bytes, metadata conflict and privacy tests. | +| SQLite adapters | Separate ATrium block buffering from transaction SQL; remove duplicate head decoding. Split Space metadata/member/notice SQL from record operations. | Database rollback, block retention, account-state and adapter tests. | +| Identity / networking / relay | Shared bounded outbound HTTP policy: production public HTTPS, filtered final DNS answers, no ambient proxy, no credential redirects; local literal loopback for development. Keep relay retry policy. | Hosts/DNS filtering, streaming/declared size bounds, redirects, production local-notice denial, relay tests. | +| Space access / records | Split policy/scopes from JWT auth and record orchestration. Active-account guard for credential reads and atomic writes. Indexed blob checks/listing and filtered SQL pagination. | Read-after-deactivation regression failed before fix; >1,000-record/page cases; reference/cursor/since/isolation tests. | +| Space exports / notices | Shared native record codec; transactionally consistent bounded revision/record snapshot; bounded CAR output. Total best-effort notice deadline after commit. | Native bytes/link CAR previously returned 500; snapshot bounds/empty repo and CAR section tests; slow-notice regression. | +| Domain / syntax / clock / events | Keep ATrium primitives and persisted revision floor. Keep actively used BlobStore/EventSink/ClientRegistry seams. Bound optional event delivery once per batch. | Pinned syntax/data-model/signature fixtures, TID tests, adapter contracts, stalled-sink regression. | +| Tests / fixtures / CI | Workspace checks include protocol tests. Service tests split by responsibility. Router/SQLite tests remain distinct from loopback/process tests; fixture provenance preserved. | All CI commands inspected; final workspace run passed all 242 tests, including socket/process and protocol targets. | +| Documentation / teaching | README reconciled; architecture shortened; adoption history labeled. Guide covers vocabulary, Rust, source map, all major flows, and seven isolated exercises. | Final source map and exercise test names checked; local Markdown links and shell-block syntax validated. | + +## Crate and abstraction decisions + +- Extracted `cpds-protocol`: pure record shape, binary values and content + addressing, shared by public repositories and Spaces. Four fixture tests run + independently of HTTP, SQLite and config. The extraction requires no new + registry dependency; Cargo workspace dependencies and lints are shared. +- Kept the server in one package. A crate per adapter, error type or service would + expand APIs and dependencies without an independent use case. +- Kept migrations, legacy event decoding and blob-index backfills needed by + persisted data. Historical duplicate indexes and unused outbox scheduling + columns were not destructively removed just for a source-size metric. New + redundant Space operation-value snapshots are no longer written. +- Kept the synchronous one-connection SQLite and Argon2 execution model visible. + A worker pool changes scheduling/cancellation and needs a separate capacity + design and saturation evidence. This audit does not claim blocking work is + preemptible or that a runtime redesign has been completed. +- Optional notifications are not durable retry delivery. A total deadline bounds + waiting but cannot eliminate ambiguous outcomes from disconnects, cancellation + or crashes. Public replay durability comes from the SQL outbox, not EventSink. + +## Removed redundancy + +Copied migration chains; unused token wrappers; separate Space JSON encoding; +duplicated mutation-event construction; stringly typed batch reconstruction; +reconstructed DID documents and redundant key metadata; full-record Space blob +scans; unused operation-value copies; a single-implementation record-path trait; +duplicate HTTP input/result shapes; and mixed-purpose oversized files. + +The largest production file fell from about 2,250 to under 1,000 lines. Total source +lines have **not** decreased: safety checks and module imports offset deletions. +This is fewer competing execution paths and clearer responsibilities, not a +claim of net line-count reduction. + +## Verification gates + +Final local results on 2026-09-05: + +| Gate | Command / evidence | Result | +| --- | --- | --- | +| Formatting | `cargo fmt --all -- --check` | Passed. | +| Lints | `cargo clippy --workspace --all-targets --locked --config build.incremental=false --quiet -- -D warnings` | Passed, warnings denied. | +| Behavioral tests | `cargo test --workspace --all-targets --locked --config build.incremental=false --quiet` | 242 passed: 140 server-library unit, 1 executable unit, 97 integration, 4 protocol; none failed, ignored, or filtered out. Socket/process access enabled. | +| Minimum Rust version | `cargo check --workspace --all-targets --locked --config build.incremental=false --quiet`, with Rust 1.88.0 Cargo and rustc selected by explicit toolchain paths | Passed. Default local compiler for tests/lints is Rust 1.95.0. | +| Dependency policy | `cargo deny --workspace --locked check` refreshed advisories; final `cargo deny --workspace --locked --offline -L error check` | Advisories, bans, licenses, and sources OK. Allowed duplicate-version warnings remain; no registry dependency upgrades were introduced. | +| Doctests | `cargo test --workspace --doc --locked --config build.incremental=false --quiet` | Passed; neither package has runnable doctests. | +| Teaching artifacts | Local-link checks, `zsh -n` on shell blocks, source-map and exercise-name checks | Passed for README and all six documents under `docs/`. | +| Test preservation | Independent inventory against `04c9babe` | 216 tests became 242; 37 unchanged names moved; no silently removed tests. | + +The final run includes two corrected regressions from preceding verification: +inconsistent blob reuploads now assert the earlier 409 response while retaining +forged-record-metadata checks, and cross-account Space authorization runs before +repository lookup, preserving the existing forbidden response without weakening +its test. The complete suite was rerun after both corrections. + +Disk exhaustion interrupted an earlier build. Only the generated +`target/debug/incremental` cache was removed; it is regenerable. Source, SQLite +and blob data were untouched. Subsequent builds disabled incremental caching. + +## Completion decision + +The original goal is met: the plan preceded implementation; every subsystem and +listed finding has a disposition above; redundant execution paths were removed; +shared protocol rules have an independent crate; modules and tests follow their +responsibilities; and the source-linked beginner guide teaches the final design +through seven exercises. An independent completion review found no remaining +required cleanup implementation. + +These are local macOS results, not a claim that Linux CI or a deployed service +has run. Changes remain uncommitted and unpublished. Missing production features +and capacity work remain explicitly documented in +[production-plan.md](production-plan.md), separate from this cleanup. diff --git a/docs/cleanup-plan.md b/docs/cleanup-plan.md new file mode 100644 index 0000000..7326e3d --- /dev/null +++ b/docs/cleanup-plan.md @@ -0,0 +1,62 @@ +# CPDS audit and simplification plan + +Date: 2026-09-05. Scope: the current working tree, including the existing uncommitted public `applyWrites` implementation. Preserve that work. Do not commit or publish as part of this task. + +This file preserves the plan and initial findings established before implementation. +Current dispositions and verification are in [cleanup-audit.md](cleanup-audit.md). + +## Intended result + +Audit the whole repository for cleanliness, testability, module and file organization, and useful abstraction boundaries. Implement the resulting focused fixes. Explain the final design in a beginner-friendly repository guide that follows real code and tests. Assess separate crates explicitly; extract only a coherent boundary that earns its dependency and API overhead. + +This is not the entire production/federation roadmap. New OAuth profiles, remote ingestion, deployment infrastructure, and other unfinished product features remain in `production-plan.md`. Correctness defects in the existing implementation are in scope. + +## Working sequence + +1. **Establish evidence.** Read configuration, assembly, HTTP, identity, public repositories, SQLite, blobs/recovery, OAuth, Spaces, fixtures, CI, and documentation. Inventory responsibilities and tests. Distinguish confirmed behavior from inferred risk. +2. **Fix boundary defects first.** Write failing behavioral regressions for authorization/privacy, error disclosure, inconsistent state transitions, and pagination defects found in the review. Implement the smallest coherent fixes. +3. **Simplify shared protocol code.** Separate pure record validation/addressing from repository orchestration. Remove cross-feature dependencies on implementation modules. Preserve ATrium as the owner of standard protocol primitives. Evaluate a small protocol crate versus ordinary modules before extracting anything. +4. **Organize by responsibility.** Split large repository, identity, and Spaces modules at stable domain boundaries, keeping SQL transactions intact. Keep feature-specific HTTP code with its feature; keep shared middleware at the HTTP boundary. Do not introduce generic CRUD repositories, service containers, or a crate per layer. +5. **Improve testability where evidence warrants it.** Preserve real in-memory SQLite and shared blob contracts. Check wall-clock and network seams, failure injection, test helper duplication, and coverage of new boundaries. Keep process/socket tests separate from pure tests. +6. **Teach the final implementation.** Write `repository-guide.md` with vocabulary, source map, startup and request walkthroughs, storage/transaction model, public-versus-Space distinctions, Rust concepts, test exercises, and a recommended reading sequence. Reconcile misleading architecture text with current code. +7. **Verify completion.** Run formatting, Clippy, full locked tests, and the available dependency policy checks. Inspect the final diff for unrelated changes and unnecessary abstractions. Record exact results and remaining product limitations. Check every scope row below before marking the goal complete. + +## Coverage and completion ledger + +| Area | Initial evidence | Work planned | +| --- | --- | --- | +| Configuration / startup / assembly | Environment parsed through injectable `from_values`; canonical origin types; five process operations; `AppState` wires adapters | Review process tests and remove unnecessary coupling of identity to Spaces signing | +| HTTP / middleware | Typed public XRPC handlers; admission bounds; metadata-only request spans; blanket error conversion and cache policy | Private blob cache regression, internal-error regression, endpoint grouping | +| OAuth | Static registry, PKCE, hashed tokens, real SQLite | Review atomic code redemption/token issuance and expiry storage cleanup; do not confuse this flow with the complete AT Protocol OAuth profile | +| Public repository | About 2,250 lines: writes, signing, exports, replay, verification, maintenance, and record codec | Complete body/test review, separate cohesive modules, retain atomic batch semantics | +| SQLite | One mutex/connection; versioned migrations; transaction-staged repository changes | Complete schema/adapter review, maintain one transaction for signed state and outbox | +| Blobs / backup / private filesystem | Upload reservation and publication; transactional orphan claims | Complete recovery review; public read versus Space read access; indexed Space blob lookup | +| Identity / network / relay | Shared resolver boundary and hardened transport documented | Read full implementation and tests; assess independent transport/module boundary | +| Spaces | Separate records and alpha authorization; large service/auth/HTTP modules | Complete review; public blob escape, first-1,000-record blob lookup, filtered list pagination, module splits | +| Domain / syntax / clock / events | Small existing seams; public and Space code share record codec indirectly | Review all helpers, reduce strings where useful, assess crates | +| Tests / fixtures / CI | Formatting and Clippy pass; sandboxed test run failed to bind listeners; socket-enabled full run exited successfully | Capture exact test count; inspect representative assertions and missing behavior; dependency policy and final rerun | +| Documentation / teaching | Architecture prose has drift (e.g. three-operation BlobStore example omits readiness) | Final audit findings and source-linked beginner guide | + +## Findings to resolve or explicitly disposition + +- **Privacy:** `http/security.rs` marks authenticated Space blob responses `public, max-age=300`. Cache policy must follow access policy and response success. +- **Privacy:** `RepositoryService::get_blob` currently reads any available owner/CID; the public HTTP endpoint shares this method with Space downloads. Trace and test whether Space-only blobs escape through public sync before designing the fix. +- **Error boundary:** `ApiError` returns `self.0.to_string()` for internal failures, exposing storage/IO/upstream details. Keep full diagnostics in server logs and return stable public messages. +- **Correctness:** Space blob authorization scans only the first 1,000 records despite a maintained `space_record_blobs` index. Use the existing index. +- **Correctness:** Space listing limits the unfiltered query to 1,000 before applying requested filters. Check the existing SQL filter support and pagination contract. +- **Abstraction:** typed batch writes are converted to string actions and optional payloads, then recovered with an `unreachable!`. Preserve the enum invariant throughout planning. +- **Organization:** shared record encoding lives at the bottom of the public repository service and is imported by Spaces. It should have an explicit protocol home. +- **Testability / execution:** synchronous SQLite and Argon2 work runs on async workers. Review bounded execution as an architectural issue; a timeout alone cannot preempt blocking work. Do not claim a runtime redesign is complete without saturation evidence. +- **Lifecycle:** code redemption commits before access-token insertion. Review failure behavior and fix atomicity with one database operation if confirmed. + +## Crate decision criteria + +A separate crate must have a clear domain name, a small stable API, tests that run without the server, and no dependency back into CPDS HTTP/config/SQLite. The strongest candidate is pure AT Protocol data validation/addressing shared by public repositories and Spaces. Storage transactions, generic errors, and `AppState` are not independent reusable packages. Module extraction comes first; record the final choice and its tradeoffs in the guide. + +## Completion evidence + +All seven steps are complete. [cleanup-audit.md](cleanup-audit.md) records each +subsystem's disposition, the implemented fixes, retained design tradeoffs, and +final verification: 242 workspace tests plus formatting, Clippy, dependency +policy, and Rust 1.88 checks. The teaching deliverable is +[repository-guide.md](repository-guide.md), including seven isolated exercises. diff --git a/docs/production-plan.md b/docs/production-plan.md index 1fcb0bf..7d45222 100644 --- a/docs/production-plan.md +++ b/docs/production-plan.md @@ -136,17 +136,19 @@ Completion evidence: - [ ] Database tests cover atomic multi-write rollback, compare-and-swap races, restart persistence, corruption detection, and schema migration. -Implemented foundation: schema version 8 adds content-addressed repository -blocks, one signed head per account, ordered operation rows, a retryable outbox, +Implemented foundation: schema version 9 includes content-addressed repository +blocks, one signed head per account, ordered operation rows, a durable replay stream, ordered per-commit block membership for CAR diff/firehose recovery, and a public/Space per-record blob-reference index with durable upload and deletion states. Version 7 generalizes the outbox so lifecycle and commit events share one constrained sequence without inventing commit roots or revisions for lifecycle rows; version 8 adds a constrained durable account status with -`active` as the backward-compatible upgrade default. Bounded repository-block +`active` as the backward-compatible upgrade default. Version 9 repairs skipped +legacy stream-table upgrades while preserving issued sequence numbers, and adds +indexed OAuth credential expiry cleanup. Bounded repository-block collection uses ATrium's exported reachable graph and preserves current heads plus retained sync-event membership before deleting historical blocks. Fresh -databases create all schema generations in one transaction; version 1 through 7 +databases create all schema generations in one transaction; version 1 through 8 upgrades preserve existing rows, version 5 blobs become available in place, version 6 commit rows move into the constrained stream schema without renumbering, and version 7 accounts become active in place. Reference-index diff --git a/docs/reduction-plan.md b/docs/reduction-plan.md new file mode 100644 index 0000000..182f074 --- /dev/null +++ b/docs/reduction-plan.md @@ -0,0 +1,2348 @@ +# Behavior-preserving production-code reduction + +The goal is a net reduction of at least 2,000 production Rust lines through +better abstractions, without changing functionality. This follows, rather than +replaces, the completed cleanup audit. The user revised the original 3,000-line +target to 2,000 after the third checkpoint; the baseline remains unchanged. + +## Baseline and counting + +The starting working-copy snapshot is +`ed4a47a9e666133bfaf0bc6f444299abbcbfd64d`. It includes the prior cleanup and the +user's existing changes; none may be discarded to meet this goal. The starting +inventory is 15,395 production and 11,732 test lines across 114 Rust files. +The target is at most 13,395 production lines. + +`ruby scripts/production-lines.rb` measures the working tree; add `--revision` +and the snapshot above to reproduce the baseline. Physical lines include blanks +and comments. Dedicated test files and trailing `#[cfg(test)]` items are counted +separately. All production helpers, crates, and macro definitions remain in the +count. Nonblank/noncomment counts are also reported as a check against cosmetic +reductions. Moving code into SQL/assets/build scripts, compressing formatting, +removing documentation, or reclassifying production code is not a reduction. + +## Implementation sequence + +1. Centralize repeated SQL row decoding and transaction ownership while keeping + predicates, ordering, migrations, and durable commit boundaries unchanged. +2. Share typed HTTP assembly, response conversion, and identical wire models. + Public and Space authorization policies remain distinct and explicit. +3. Reuse cryptographic/transport validation mechanisms only where their rules + truly match; preserve error classification, ordering, and resource limits. +4. Simplify configuration parsing and record-operation planning with shared + domain values, eliminating duplicated paths rather than adding a framework. +5. Measure the net result including abstraction overhead, review the remaining + duplication, and continue until the full reduction is proved. + +## Behavior and acceptance gates + +- Preserve the baseline's 242 behavioral tests and their assertions. Internal + API changes may update test setup, but not weaken expected outcomes. +- Keep HTTP methods, wire fields, response codes/messages, validation precedence, + authentication, visibility, cache behavior, and paging unchanged. +- Preserve atomic record/block/head/outbox publication, final-root event proofs, + rollback, migration and expiry boundaries, backup/recovery, and notice bounds. +- Add focused equivalence regressions where an abstraction covers behavior the + existing tests do not distinguish. Preserve independently pinned fixtures. +- Run formatting, warning-denied workspace Clippy, locked all-target workspace + tests with socket/process access, dependency policy, and Rust 1.88 checks. +- Review the final diff against the baseline and update the teaching source map + for changed abstractions. No commit, publication, or deployment is requested. + +The goal remains incomplete until both the numerical reduction and unchanged +observable behavior are supported by the final worktree and verification. + +## First refactor slice + +The first checkpoint measured **14,806 production lines**, a net reduction of +**589**, including all shared helpers. Nonblank/noncomment production lines fell +from 13,642 to 13,041. At that checkpoint, **2,411 more production lines** were +required under the then-current 3,000-line goal; this slice was not completion. + +Implemented: a shared configuration reader and bounded-number parser; fallible +SQL query/transaction and constraint helpers; common blob-reference index +reconstruction and Space persistence primitives; XRPC route/response assembly; +one shared batch request model; one error type and contextual error mapping; +shared P-256 validation; shared signed-commit representations; Serde-derived +record encoding; and mutation-derived batch results with shared Space record +preparation. + +Preserved distinctions include single Space put versus strict batch update, +idempotent single delete versus strict batch delete, validation precedence, +typed HTTP error bodies, SQL-type-before-domain-decoding errors, and unchanged +commit/outbox transactions. Additional tests cover these abstraction boundaries. + +Validation macros were prototyped but not adopted: ordinary formatting of the +same 174 guards saved only 28 lines with a plain `ensure!` helper; typed-message +variants and `bail!` alternatives increased the count. A broad endpoint DSL or +generic public/Space repository framework has not been justified by the review. +Further structural reduction remained to be designed and verified. + +Checkpoint verification (2026-09-05): + +- `cargo fmt --all -- --check`: passed. +- `cargo clippy --workspace --all-targets --locked --config build.incremental=false --quiet -- -D warnings`: passed. +- `cargo test --workspace --all-targets --locked --config build.incremental=false --quiet`: all 250 tests passed with local socket/process access; none ignored or filtered out. +- Independent source inventory preserved all 242 baseline test names and added + eight regression tests; existing observable assertions were retained. +- Rust 1.88.0 Cargo/rustc, selected by explicit toolchain paths, passed + `cargo check --workspace --all-targets --locked --config build.incremental=false --quiet`. +- `cargo deny --workspace --locked --offline -L error check`: advisories, bans, + licenses, and sources passed against the previously refreshed cache. No + dependency was added or upgraded for this slice. +- README and all seven documents under `docs/`: 105 local links and 20 shell + blocks validated; the source guide now explains the new abstractions. + +These are local checks, not a deployment or Linux CI result. The changes remain +uncommitted. A passing checkpoint does not satisfy the outstanding line target. + +## Second refactor slice + +The second checkpoint measured **14,736 production lines**, a net reduction of +**659** from the original baseline, including **70** removed in this slice. +Nonblank/noncomment production lines were **12,972**. At that checkpoint, the +then-current 3,000-line goal required **2,341 more production lines**. + +- Share the Space record-list query between HTTP and the service, including one + implementation of the existing default and clamped limit. +- Represent borrowed Space write inputs with one `WriteRecord`: an optional + key for creation and a required key for put/update. This keeps key requirements + in the type system without duplicated fields, extra clones, or changed parsing. +- Use the common `RecordView` for public and Space records while preserving + their distinct URI construction and authorization. +- Share the identical generated create/put response representation. Serialize + Space and Space operation values directly, explicitly omitting internal + metadata and retaining existing null/omission behavior. +- Centralize identity/account lifecycle payload construction and outbox + insertion in `LifecycleEvent`, retaining explicit transaction ownership, + compare-and-set predicates, event order, timestamps, and retention. + +Seven additional regressions cover query parsing/defaults, response schema +parity, stored-value validation order, hidden fields/nulls, lifecycle payload +bytes/metadata, and rollback on lifecycle insertion failure. Existing tests and +assertions are retained. + +Second-checkpoint verification (2026-09-05): + +- `cargo fmt --all -- --check`: passed. +- Warning-denied workspace/all-target Clippy: passed. +- Locked workspace/all-target tests: all **257 passed**, none ignored or filtered + out, including local socket/process tests. +- Rust 1.88.0 workspace/all-target `cargo check --locked`: passed. +- Offline `cargo deny` checks: advisories, bans, licenses, and sources passed + against the existing advisory cache; dependencies were not changed. +- Independent source inventory preserved all 250 preceding-checkpoint test + names and added seven. README and all seven documents under `docs/`: 105 + local links and 20 shell blocks validated. + +Cargo test, Clippy, and the Rust 1.88 check used +`--config build.incremental=false --config profile.dev.debug=0 --config profile.test.debug=0` +after cleanup. These local results do not claim Linux CI or deployment. + +Disk exhaustion initially blocked edits. The user authorized removal of only +the generated Cargo `target/` artifacts, which completed successfully; source, +databases, and version-control history were not removed. Rebuilds disable +incremental compilation and debug information through command-line settings to +limit generated artifact size. Rate-limiting refactoring was separately denied +by approval review and was not applied; cleanup approval did not authorize it. + +## Third refactor slice + +The third checkpoint measured **14,615 production lines**, a net reduction of +**780** from the original baseline, with **121** removed in this slice. All +helpers and the new request module are counted. Nonblank/noncomment production +lines were **12,847**. Following the target revision, **1,220 more production +lines** remained to meet the 2,000-line goal. + +- Five native Axum extractor tuple aliases remove repeated annotations in 45 + HTTP handler signatures: **73 net lines**, including the new module and + imports. All 60 production async function bodies in the affected adapters + remain byte-identical. Extraction order is unchanged; these are type aliases, + not custom extractors. Authentication, security, and rate-limit modules are + unchanged from the second checkpoint. +- Already-validated repository operations use ATrium `RepoOpData` directly: + **17 net lines**. The durable `OutboxOperation` JSON format stays unchanged, + and CID validation still precedes wrapping links for the wire representation. +- Configuration construction uses one aggregate instead of duplicated locals + and field assembly; cross-field validation receives that aggregate, and the + settings reader owns repeated required URL parsing: **31 net lines**. Source + read order, early returns, local defaults, production requirements, bounds, + duration units, and error text remain unchanged. + +The two new configuration-order regressions passed against the preceding +implementation before the refactor and afterward. A new tuple-extractor test +compares 38 requests across all five aliases with native multi-argument handlers, +including exact rejection responses, original URIs, headers, body limits, and +no handler entry on invalid input. Existing operation tests additionally pin +durable string/null fields and generated link/omission wire fields. + +Third-checkpoint verification (2026-09-05): + +- All **260 workspace/all-target tests passed** with local socket/process access; + none were ignored or filtered out. All 242 original baseline test names remain. +- Formatting and warning-denied workspace/all-target Clippy passed. +- Rust 1.88.0 workspace/all-target locked checks passed. +- Offline dependency policy passed against the existing advisory cache; Cargo + manifests and the lockfile are unchanged from the second checkpoint. +- Independent byte comparisons verified the unchanged 60 handler bodies and + authentication, security, and rate-limit modules. +- README and all seven documents under `docs/`: 106 local links and 20 shell + blocks validated. + +Build checks used the same non-incremental, debug-info-disabled command-line +settings as the second checkpoint. No commit, publication, deployment, or Linux +CI result is claimed. The revised 2,000-line goal remains incomplete. + +A separate in-memory static-message factory prototype across 121 sites saved +only 34 net lines after formatting and helper overhead. It was not adopted: +extra error constructor wrappers did not clarify the domain model. + +## Fourth refactor slice + +The fourth checkpoint measured **14,572 production lines**, a net reduction of +**823** from the original baseline, including **43** removed in this slice. +All helpers, macros, imports, and production test-module separators are counted. +Nonblank/noncomment production lines are **12,806**, down **836** from baseline. +At that checkpoint, the revised goal required **1,177 more production lines**. + +- One explicitly indexed `sql_row!` constructor shares plain SQL field reads + across six account, OAuth, and Space metadata mappings: **10 net lines**. + Expansion retains the same `row.get` calls in declaration order, with domain + parsing afterward. SQL statements, transactions, and schema are unchanged. +- Public single/list record reads share the generated ATrium record output, + since both always include CID, URI, and value. Latest-commit responses reuse + the existing generated commit metadata conversion: **17 net lines**. Exact + JSON serialization and CID-before-value/revision error order are retained. + Concrete Rust handler return types change; HTTP wire contracts do not. +- Backup and blob storage share their byte-identical parent-path helper, and + `Result::and` retains unconditional cleanup and original-error precedence. + Private-file synchronization has one body with the existing Unix-only mode + change: **16 net lines**, including the new trailing test-module separator. + +Five new regressions cover SQL-versus-domain decoding errors, exact generated +HTTP response bytes, output validation order, and relative/root path handling. +The Space write-preparation proposals were not adopted: formatted candidates +either added seven lines or saved one while introducing an enum and another +preparation phase. Pagination sharing also added more code than it removed. + +Fourth-checkpoint verification (2026-09-05): + +- All **265 locked workspace/all-target tests passed**, none ignored or filtered + out, including local socket/process tests. All 242 original test names remain. +- Formatting, warning-denied workspace/all-target Clippy, and Rust 1.88.0 + workspace/all-target locked checks passed. +- Offline dependency policy passed against the existing advisory cache after + approved access to its lock file. Dependencies were not changed. +- Independent review confirmed generated HTTP output and error-order parity. + Source comparisons confirmed unchanged security/rate-limit implementations, + Space authorization, commit transactions, and schema from the third checkpoint. +- README and all seven documents under `docs/`: 106 local links and 20 shell + blocks validated; the source guide explains the shared output and row types. + +Build checks used the same non-incremental, debug-info-disabled command-line +settings as the preceding checkpoints. These are local macOS checks, not Linux +or non-Unix CI, deployment, publication, or an authored commit. The goal is not +complete; passing checks do not replace the remaining numerical requirement. + +## Fifth refactor slice + +The fifth checkpoint measured **14,540 production lines**, a net reduction of +**855** from the original baseline, including **32** removed in this slice. +Nonblank/noncomment production lines are **12,775**, down **867** from baseline. +All production helper and import overhead remains counted. At that checkpoint, +**1,145 production lines** remained to reach the 2,000-line goal. + +- Remove the intermediate database `Operation` representation and construct + durable `OutboxOperation` values directly alongside borrowed records: + **10 net lines**. SQL, mutation/head/operation/outbox order, timestamps, + validation, retention, and final-root publication remain unchanged. +- Delegate Space CAR varints to `unsigned-varint` 0.8.0: **14 net lines**. + The same package and features were already present transitively. The lockfile + only adds CPDS's direct dependency edge; no package is added or upgraded. + CPDS retains the saturated whole-section bound before changing output bytes. +- Share the seven HTTP-origin component checks for public and relay settings: + **5 net lines**. Per-setting parse errors, production HTTPS errors, and + component-validation short-circuit order remain unchanged. +- Use one pure successor calculation in both the atomic TID update and its + returned value: **3 net lines**. Atomic ordering, persisted floors, wall-time + handling, and saturation are unchanged. New clock regressions passed before + and after the production refactor. + +Seven new regressions cover concurrent/saturated clocks, 32 setting-specific +origin cases, varint boundaries through `usize::MAX`, multipart CAR sections, +and the intentionally different strict/tolerant blob walks. The blob walkers +remain unchanged: writable validation stops at a typed blob, while conservative +reference discovery continues into its children. An existing ordered-batch test +now pins complete durable payload bytes, including order and explicit nulls. + +A read-only exact/normalized clone scan did not establish a large overlooked +repetition outside the already-reviewed or excluded areas. A tagged-string +envelope helper added 14 formatted lines; remaining error-context substitutions +need to preserve lazy URI construction. These observations narrow the next +investigation; they do not satisfy or change the remaining numerical target. + +Fifth-checkpoint verification (2026-09-05): + +- All **272 locked workspace/all-target tests passed**, none ignored or filtered + out, including local socket/process tests. All 242 original test names remain. +- Formatting, warning-denied workspace/all-target Clippy, and Rust 1.88.0 + workspace/all-target locked checks passed. The expanded payload assertion was + extracted into a test helper to respect the existing function-length lint; + no assertion was removed or weakened, and the full suite was rerun afterward. +- Offline dependency policy passed. The lockfile diff is exactly one direct + dependency edge, and the feature tree confirms ATrium already enabled the + same `unsigned-varint` default features. +- Diff review confirmed unchanged SQL statements, transaction order, outbox + schema, error labels, and section-size enforcement. Rate-limit and security + implementation files were not changed. +- README and all seven documents under `docs/`: 106 local links and 20 shell + blocks validated; the source guide explains the changed abstractions. + +Build checks used the same non-incremental, debug-info-disabled settings. These +are local macOS results, not Linux/non-Unix CI or deployment. The work remains +uncommitted, and the full 2,000-line goal remains incomplete. + +## Sixth refactor slice + +The sixth checkpoint measured **14,472 production lines**, a net reduction of +**923** from the original baseline, including **68** removed in this slice. +Nonblank/noncomment production lines are **12,719**, down **923** from baseline. +All helper, import, and test-module separator overhead remains counted. +At that checkpoint, **1,077 production lines** remained to reach the goal. + +- Delegate fallible SQL iteration and single-row decoding to rusqlite's + `query_and_then` and `query_row_and_then`: **14 net lines**. The separate + optional-row path still distinguishes a missing row from a decoder error. + Preparation, binding, SQL stepping, and domain-decoding order are unchanged. +- Use native Unix creation options at their original call sites, share the + asynchronous permission-setting operation, and use `Result::inspect_err` + for backup cleanup: **37 net lines**. Creation flags, modes, symlink checks, + write/sync/rename order, and original-error precedence remain unchanged. + Non-Unix permission setters remain no-ops. +- Give `Record` its existing URI-only `Display` and reuse lazy contextual error + formatting: **6 net lines**, including the new implementation. Successful + operations do not format the record URI; diagnostic strings remain exact and + never include record values or CIDs. +- Represent confirmed local-handle validity directly as `bool`, removing a + redundant internal two-variant enum and its one-use adapter: **11 net lines**. + Resolver errors still return before changing the validity flag; the public + unchanged/recovered/invalidated lifecycle states and atomic ordering remain. + +Six new regressions pin SQL iteration values/errors/decoder-call counts, +private-file modes and collision behavior, URI-only formatting, and exact +stored-record verification/backfill diagnostics with failure precedence. +The ten existing identity tests passed before and after that simplification; +their validity assertions were adapted without weakening lifecycle checks. +Independent reviews confirmed identity-transition and diagnostic parity. + +Sixth-checkpoint verification (2026-09-05): + +- All **278 locked workspace/all-target tests passed**, none ignored or filtered + out, including local socket/process tests. All 242 original test names remain. +- Formatting, warning-denied workspace/all-target Clippy, and Rust 1.88.0 + workspace/all-target locked checks passed. +- Offline dependency policy passed against the existing advisory cache; + manifests and lockfile are unchanged from the fifth checkpoint. +- Security, rate-limit, Space authorization, and schema files are unchanged. + No migrations, transaction boundaries, or production features were removed. +- README and all seven documents under `docs/`: 106 local links and 20 shell + blocks validated; the source guide describes the changed abstractions. + +Build checks use the same non-incremental, debug-info-disabled settings. These +are local macOS results, not Linux/non-Unix CI, deployment, publication, or an +authored commit. Passing this checkpoint does not complete the 2,000-line goal. + +## Seventh refactor slice + +The seventh checkpoint measured **14,389 production lines**, a net reduction of +**1,006** from the original baseline, including **83** removed in this slice. +Nonblank/noncomment production lines are **12,637**, down **1,005** from baseline. +All helpers, imports, serialization annotations, and test-module separators are +counted. **994 production lines remained** at that checkpoint. + +- Use rusqlite's native `TryFrom<&Row>` tuple decoder and remove the custom + fixed-width `text_columns` helper: **31 net lines**. Native tuple conversion + reads columns in order with the same SQL types before domain validation. + Outbox pages still collect every SQL row before decoding any payload; + incremental block exports retain streaming validation and first-error order. +- Return generated ATrium `OutputData` directly for responses that previously + wrapped it in an empty `Object`: **24 net lines**. This removes only infallible + outer adapters. Nested record/commit/union objects and extension-bearing data + remain intact, as do validation order and exact JSON bytes. Internal Rust + response types change; HTTP schemas, status codes, and headers do not. +- Pass the existing Space operations/blob-list query models directly to their + services and serialize the existing `RepoOpsPage`: **28 net lines**. This + removes duplicate parameter forwarding, a response schema, and a field-copy + adapter without adding generic pagination machinery. Authorization, cursor + and revision validation, defaults, clamps, lookahead, and omission rules stay + unchanged; the exclude-values pass remains at the HTTP boundary. + +Eight new regressions cover native column failures, SQL-before-payload failure +priority, all 13 changed generated response-data types, nested extension data, +52 old/new query parser comparisons, exact Space page bytes, and service-level +authorization/pagination behavior over 1,001 records and blobs. The generated +response parity tests passed before and after removing their outer wrappers. +Independent review confirmed the unchanged conversions and nested objects. + +The installed ATrium commit builder was also inspected, but not adopted for +batch staging: its constructor is private and its mutation methods open the +tree from the latest finalized commit. It cannot directly finalize CPDS's +already-staged multi-mutation tree without a different commit sequence. + +Seventh-checkpoint verification (2026-09-05): + +- All **286 locked workspace/all-target tests passed**, none ignored or filtered + out, including local socket/process tests. All 242 original and all 278 + preceding-checkpoint test names remain. +- Formatting, warning-denied workspace/all-target Clippy, and Rust 1.88.0 + workspace/all-target locked checks passed. The initial compile required an + explicit owned `String` for an outbox tuple field; the full suite passed after + that type annotation was corrected. +- Offline dependency policy passed against the existing advisory cache; + manifests and lockfile are unchanged from the sixth checkpoint. +- Security, rate-limit, Space authorization, schema, and repository commit + implementation files are unchanged. SQL statements and transactions remain. +- README and all seven documents under `docs/`: 106 local links and 20 shell + blocks validated; the source guide describes the changed abstractions. + +Build checks use the same non-incremental, debug-info-disabled settings. These +are local macOS results, not Linux/non-Unix CI, deployment, publication, or an +authored commit. The full 2,000-line goal remains incomplete. + +## Eighth refactor slice + +The eighth checkpoint measured **14,328 production lines**, a net reduction of +**1,067** from the original baseline, including **61** removed in this slice. +Nonblank/noncomment production lines are **12,577**, down **1,065** from baseline. +All helpers, imports, serialization annotations, and test-module separators are +counted. **933 production lines remained** at that checkpoint. + +- Share `AuthorizationRequest` between the OAuth authorization query and + approval form, and serialize the issued `TokenGrant` directly: **30 net + lines**. Endpoint-specific checks retain their order; the token response keeps + its exact fields, order, and bearer type. Serde flattening can change private + parser diagnostics, but the existing HTTP normalizer preserves public error + bodies and statuses. It also buffers unmatched fields instead of immediately + discarding them; existing request limits are unchanged. `TokenGrant` gains a + required Rust `token_type` field; its sole production constructor is updated. +- Pass the existing Space creation, update, and member-list request types + directly to their services, and reuse `ReadAuthorization` for metadata reads: + **18 net lines**. This removes forwarding and duplicate read entry points. + Credential reads keep their distinct lookup rules; they do not gain OAuth + parsing, scope, membership, or account checks. Creation consumes existing owned + strings instead of copying them again. Defaults, clamps, validation order, + generated-key timing, and public responses remain unchanged. +- Put the account-active guard at the start of the private transactional Space + write/delete helpers: **10 net lines**. Every single write and every batch item + still checks activity before its first record read or mutation. Empty batches, + strict batch deletes, idempotent single deletes, SQL statements, and rollback + boundaries remain unchanged. +- Consume and replace the public write planner's per-key cache with native + `HashMap::remove`: **3 net lines**. This removes redundant record/value cloning + and a preliminary insertion, while retaining the distinction between cached + deletion and an unseen key. Every successful branch installs its next state. + +Eleven new regressions cover 98 old/new OAuth query/form comparisons, exact token +response bytes, 30 Space JSON and 16 query parser comparisons, management and +credential error ordering, member pagination, repeated-key write ordering and +final-root CAR contents, and trigger-induced account deactivation between batch +items. The two public planner regressions passed before and after the cache +change. Independent reviews confirmed guard coverage, transaction ordering, +OAuth public errors, token consumers, and wire representation parity. + +Eighth-checkpoint verification (2026-09-05): + +- All **297 locked workspace/all-target tests passed**, none ignored or filtered + out, including local socket/process tests. All 242 original and all 286 + preceding-checkpoint test names remain. +- Formatting, warning-denied workspace/all-target Clippy, and Rust 1.88.0 + workspace/all-target locked checks passed. The first Clippy run identified + test-only style issues; those were fixed without suppressions or removed + assertions, and the complete suite was rerun successfully. +- Offline dependency policy passed against the existing advisory cache; + manifests and lockfile are unchanged from the seventh checkpoint. +- Security, rate-limit, Space authorization, schema, and repository commit + implementation files are unchanged. No features or migrations were removed. +- README and all seven documents under `docs/`: 106 local links and 20 shell + blocks validated; the source guide describes the changed abstractions. + +Build checks use the same non-incremental, debug-info-disabled settings. These +are local macOS results, not Linux/non-Unix CI, deployment, publication, or an +authored commit. The full 2,000-line goal remains incomplete. + +## Ninth refactor slice + +The ninth checkpoint measured **14,246 production lines**, a net reduction of +**1,149** from the original baseline, including **82** removed in this slice. +Nonblank/noncomment production lines are **12,499**, down **1,143** from baseline. +All helpers, aliases, imports, and serialization annotations are counted. +**851 production lines remained** to reach the goal at that checkpoint. + +- Share the identical owned public/Space batch command enum: **15 net lines**. + The canonical enum keeps its existing `WriteRequest` name, Space Serde tags, + variant order, and fields; public repositories reexport it as `RecordBatchWrite`. + Both service implementations and public HTTP batch conversion remain + byte-identical. Parsing, authorization, value ownership, and planning do not move. +- Use the existing typed `LifecycleEvent` for account handle/status compare-and-set: + **22 net lines**. The closed variant selects the same literal SQL field, with + identical expanded SQL and parameters. Expected values are persisted text; + next account states remain typed. Status fallback still validates SQL type and + stored status before comparison. No-op/conflict returns retain transaction + rollback, including trigger side effects; event insertion, time sampling, + retention, and successful commit keep their original order. +- Put account/repository status error construction beside `AccountStatus`: + **15 net lines**. The callers retain their original lookup and policy ordering. + Account versus repository error names remain distinct, including the different + takedown spellings, deleted-repository mapping, and active-state error fallback. + Successful active checks do not allocate error messages. +- Reuse already-decoded repository heads and missing-blob CID/record-URI pairs: + **22 net lines**, including the retained `MissingBlob` tuple alias. This removes + duplicate summary structures and copying passes, while preserving query and + HTTP conversion order, cursor rules, and wire fields. `list_repos` becomes + crate-internal to match its existing row type; no external Rust callers exist + in the workspace. The named alias keeps the missing-blob return type readable + without reconstructing its values. +- Share the repository cursor's one malformed-input error through a local + closure and native optional-value filtering: **8 net lines**. Delimiter and + empty-field checks still precede signed-time and DID parsing, with identical + error classification, text, and collection-validation precedence. + +Five new regressions cover 49 exact old/new batch parser comparisons, all account +and repository status names/messages/HTTP responses, cursor integer and delimiter +boundaries, ignored-update trigger rollback for both lifecycle variants, and +SQL-type-before-status-validation failures. The batch parser and cursor tests +passed before and after their changes. Existing listing, blob inventory, and +lifecycle payload assertions are retained. Independent reviews confirmed the +status-policy and compare-and-set equivalence. + +A generic public/Space `Record` prototype was not adopted. Its small +reduction required new generic formatting bounds and unit context fields, and +changed derived debug output; the existing two record types remain clearer. + +Ninth-checkpoint verification (2026-09-05): + +- All **302 locked workspace/all-target tests passed**, none ignored or filtered + out, including local socket/process tests. All 242 original and all 297 + preceding-checkpoint test names remain. +- Formatting, warning-denied workspace/all-target Clippy, and Rust 1.88.0 + workspace/all-target locked checks passed. Clippy identified the nested + missing-blob return type; retaining its named alias resolved that without a + suppression, and the full suite was rerun against the final source state. +- Offline dependency policy passed against the existing advisory cache; + manifests and lockfile are unchanged from the eighth checkpoint. +- Security middleware, rate-limit, schema, and repository commit implementation + files are unchanged. Authentication and Space status checks reuse the same + policy at the same boundaries; no features or migrations were removed. +- README and all seven documents under `docs/`: 106 local links and 20 shell + blocks validated; the source guide describes the changed abstractions. + +Build checks use the same non-incremental, debug-info-disabled settings. These +are local macOS results, not Linux/non-Unix CI, deployment, publication, or an +authored commit. The full 2,000-line goal remains incomplete. + +## Tenth refactor slice + +The tenth checkpoint measured **14,190 production lines**, a net reduction of +**1,205** from the original baseline, including **56** removed in this slice. +Nonblank/noncomment production lines are **12,444**, down **1,198** from baseline. +All helpers, imports, annotations, and new test-module separators are counted. +**795 production lines remained** to reach the goal at that checkpoint. + +- Express migrations as an ordered list of thresholds, SQL batches, and optional + immediate blob backfills; use rusqlite's native `Statement::exists` for the + foreign-key check: **10 net lines**. SQL, thresholds, backfill order, version-9 + repair, validation, and transaction/version handling remain unchanged. +- Use native stable tuple-key sorting for Space CAR paths: **3 net lines**. + Paths still sort by byte length then UTF-8 bytes, including stable duplicate + paths. Section limits and encoded bytes remain unchanged. +- Reuse `SpacePageQuery` for member/writer listings and pass notification + registration inputs directly to the service; serialize the parsed writer + notice directly: **18 net lines**. Defaults, limits, credential/URI/host checks, + endpoint resolution, expiry capture, hash decoding, recording, and delivery + retain their original ordering and representations. +- Share `Error::xrpc` across 36 construction sites and remove two duplicate local + constructors: **25 net lines**, including the new constructor and test-module + separator. Error names, messages, variants, and lazy closures are preserved. + Authentication errors remain distinct; account/repository policy is unchanged. + +Six new regressions cover migration backfill failure precedence and rollback, +CAR path ordering and duplicate stability, 15 writer-query and nine registration +parser comparisons, notification JSON bytes without hash normalization, and +borrowed/owned error messages with exact HTTP response parity. Existing member +parser assertions remain. An independent review confirmed all 36 error-site +replacements preserve message expressions, guard order, and lazy allocation. + +Stream-composition candidates were not adopted: the measured alternatives added +lines or indirection, or changed serialization error behavior. Polling, shutdown, +send timeouts, backpressure, and stream framing remain explicit. + +Tenth-checkpoint verification (2026-09-05): + +- All **308 locked workspace/all-target tests passed**, none ignored or filtered + out. All 242 original and all 302 preceding-checkpoint test names remain. + One preceding full run hit the existing process readiness timeout; that test + then passed alone and the complete unchanged suite passed on rerun. The cause + of that non-reproduced timeout is not established; no assertions or timeouts + were weakened. +- Formatting, warning-denied workspace/all-target Clippy, Rust 1.88.0 + workspace/all-target locked checks, and offline dependency policy passed. +- Manifests, lockfile, security middleware, both rate-limit files, and repository + commit implementation are unchanged from the ninth checkpoint. No features + or migrations were removed. +- README and all seven documents under `docs/`: 106 local links and 20 shell + blocks validated; the source guide describes the changed abstractions. + +Build checks use the same non-incremental, debug-info-disabled settings. These +are local macOS results, not Linux/non-Unix CI, deployment, publication, or an +authored commit. The full 2,000-line goal remains incomplete. + +## Eleventh refactor slice + +That checkpoint measured **14,154 production lines**, a net reduction of +**1,241** from the original baseline, including **36** removed in this slice. +Nonblank/noncomment production lines are **12,409**, down **1,233** from baseline. +All helpers, imports, and new test-module separators are counted. +**759 production lines still need to be eliminated** to reach the goal. + +- Share durable unlink between blob deletion and temporary cleanup: **9 net + lines**. Successful removal still precedes parent-directory synchronization; + only unlink's `NotFound` becomes false. Cleanup maps that result to unit and + remains unconditionally awaited before selecting the primary/cleanup error. + Generated paths always append an owner hash and CID, proving their parent is + nonempty and allowing reuse of the existing parent helper. Leaf symlinks are + still unlinked, and directory errors, permissions, publication, stale cleanup, + and backup behavior remain unchanged. +- Pass the existing `ListSpacesQuery` directly to its service: **8 net lines**. + The model name, Serde fields, defaults, limits, cursor behavior, and + type-then-DID-then-scope validation order remain unchanged. +- Remove two pass-through Rust methods and update all workspace callers: + **14 net lines**. Public records use the already URI-only `Display` through + native `to_string()`; the memory blob store uses its existing derived + `Default`. Space record URI construction stays separate. Existing assertions + remain, with the record URI assertion now exercising the resulting view. +- Share whole-second UTC text formatting in `clock::format_utc_datetime`: + **5 net lines**. Both callers preserve their original range-error context; + repository events still parse the formatted text as a Lexicon datetime after + that step. Negative/year-zero formatting, range errors, and TID logic remain + unchanged. + +Six new regressions cover 128 path-parent vectors, native directory errors, +file/directory/dangling symlink deletion, 16 raw query parser comparisons, +discovery validation precedence and page limits, and 36 timestamp/range cases. +Independent review confirmed the parent invariant, unlink/fsync ordering, and +cleanup-error precedence, as well as formatter/error-order and wrapper equivalence. +The intentionally removed convenience methods change Rust source compatibility; +all workspace callers have been updated. Lifecycle payload composition prototypes were not +adopted: retaining version metadata changed derived debug output, and preserving +that output erased the measured reduction. + +Eleventh-checkpoint verification (2026-09-05): + +- All **314 locked workspace/all-target tests passed**, none ignored or filtered + out, including local socket/process checks. All 242 original and all 308 + preceding-checkpoint test names remain. +- Formatting, warning-denied workspace/all-target Clippy, Rust 1.88.0 + workspace/all-target locked checks, and offline dependency policy passed. +- Manifests, lockfile, security middleware, both rate-limit files, schema, and + public repository transaction/commit implementation are unchanged from the + tenth checkpoint. No features or migrations were removed. +- README and all seven documents under `docs/`: 106 local links and 20 shell + blocks validated; the source guide describes the changed abstractions. + +Build checks use the same non-incremental, debug-info-disabled settings. These +are local macOS results, not Linux/non-Unix CI, deployment, publication, or an +authored commit. The full 2,000-line goal remains incomplete. + +## Twelfth refactor slice + +That checkpoint measured **14,129 production lines**, a net reduction of +**1,266** from the original baseline, including **25** removed in this slice. +Nonblank/noncomment production lines are **12,383**, down **1,259** from baseline. +All helpers, imports, and new test-module separators are counted. +**734 production lines still need to be eliminated** to reach the goal. + +- Share native JSON-child iteration between strict blob validation and tolerant + reference discovery: **7 net lines**. Each retains its classification and + error policy; strict typed blobs still end traversal at that object, while + discovery continues into children. Array/object traversal order is unchanged. + Borrowed ATrium deserialization avoids cloning the complete JSON value first; + it still returns owned metadata and may allocate internally. +- Pass the existing concrete `TokenForm` from HTTP into the OAuth service: + **10 net lines**. The name, field order, types, and Serde rules are unchanged. + HTTP still rejects unsupported grants before service PKCE validation, then + randomness, expiry, and atomic redemption remain in their original order. + The Rust source API intentionally changes to a shared domain model and one + borrowed argument; all workspace callers are updated. +- Replace two manual byte-formatting loops with standard-library operations: + **8 net lines**. Fixed-width big-endian integer hex preserves the same JTI + bytes, lowercase text, leading zeros, and single 16-byte random fill. + Mapping each LtHash lane to little-endian bytes and flattening the array + retains the exact 2,048-byte SHA-256 input and wrapping lane arithmetic. + +Five new regressions cover 45 owned/borrowed blob decoding comparisons, +26 raw OAuth form cases, unsupported-grant validation precedence, 132 +deterministic JTI encodings plus generated-value shape checks, and set-hash +little-endian/overflow behavior across five repetition counts. Existing +traversal, rollback/replay, and crypto assertions remain. Independent review +confirmed all three changes and checked native API availability before Rust 1.88. + +Twelfth-checkpoint verification (2026-09-05): + +- All **319 locked workspace/all-target tests passed**, none ignored or filtered + out, including local socket/process checks. All 242 original and all 314 + preceding-checkpoint test names remain. +- The first run identified an incorrect new OAuth test expectation: the existing + public error message includes `the request is invalid:`. Only that new expected + string was corrected; the complete suite then passed. Production error text + and existing assertions were not changed. +- Formatting, warning-denied workspace/all-target Clippy, Rust 1.88.0 + workspace/all-target locked checks, and offline dependency policy passed. +- Manifests, lockfile, security middleware, both rate-limit files, schema, and + public repository transaction/commit implementation are unchanged from the + eleventh checkpoint. No features or migrations were removed. +- README and all seven documents under `docs/`: 106 local links and 20 shell + blocks validated; the source guide describes shared blob traversal and token + input handling. + +Build checks use the same non-incremental, debug-info-disabled settings. These +are local macOS results, not Linux/non-Unix CI, deployment, publication, or an +authored commit. The full 2,000-line goal remains incomplete. + +## Thirteenth refactor slice + +That checkpoint measured **14,079 production lines**, a net reduction of +**1,316** from the original baseline, including **50** removed in this slice. +Nonblank/noncomment production lines are **12,333**, down **1,309** from baseline. +All helpers, imports, and new test-module separators are counted. +**684 production lines still need to be eliminated** to reach the goal. + +- Open a signed repository from its root CID and expected revision directly: + **24 net lines**. Seven callers retain the same root/revision pairs; three + temporary head objects and two string clones disappear. The database's + `RepoHead` representation stays intact. Opening, revision comparison, and + signature verification retain their order and errors, and no expected-revision + borrow escapes into the returned repository. +- Share the existing concrete `RecordQuery` with Space get/delete services: + **12 net lines**. Query/JSON extraction keeps its name, four string fields, + field order, and Serde rules. Only crate-private model location, field + visibility, and two service signatures change. Authorization, validation, + repository lookup, SQL, and TID calls retain their original order. +- Convert the selected MST operation's result once after its awaited match + branch, and reuse `Error::xrpc` for `InvalidSwap`: **8 net lines**. Failure + still returns within the same loop iteration. Staging, signing, retries, + transactions, final-root proofs, and publication remain unchanged. +- Reuse `user_can_read` during credential issuance after application access and + the owner bypass: **6 net lines**. Public/member policy decisions match their + previous branches. Only non-owner member-list policy queries membership; + managing-app delegation/configuration errors and unknown-policy denial stay + separate. The stored Space view retains the exact requested URI used by SQL. + +Nine new regressions cover 18 raw queries and 45 raw JSON bodies, eight Space +read/delete precedence outcomes, three direct MST mutation failures without +head/outbox/record/record-block publication, four exact swap messages, and +revision-before-signature verification. Credential-policy coverage adds 27 +policy/principal combinations, six missing-membership-table outcomes, and five +allowlist/owner outcomes. Existing tests and their assertions remain. Independent +review confirmed the query, policy, error, and opening changes. + +Additional larger candidates were not adopted: preserving distinct JWT +verification order erased consolidation savings; shared account retry dispatch +saved only six lines while adding concepts; blob-index replacement already shares +its meaningful validation phase; exact outbox DDL composition added two lines. + +Thirteenth-checkpoint verification (2026-09-05): + +- All **328 locked workspace/all-target tests passed**, none ignored or filtered + out, including local socket/process checks. All 242 original and all 319 + preceding-checkpoint test names remain. +- Formatting, warning-denied workspace/all-target Clippy, Rust 1.88.0 + workspace/all-target locked checks, and offline dependency policy passed. + Clippy requested `let...else` in the new opening regression; only its style + changed, and the full test suite passed again with the same assertions. +- Manifests, lockfile, security middleware, both rate-limit files, schema, and + database repository transaction implementation are unchanged from the twelfth + checkpoint. The public repository service changes only its opening arguments + and error-conversion reuse described above. No features or migrations were removed. +- README and all seven documents under `docs/`: 106 local links and 20 shell + blocks validated; the source guide describes the changed abstractions. + +Build checks use the same non-incremental, debug-info-disabled settings. These +are local macOS results, not Linux/non-Unix CI, deployment, publication, or an +authored commit. The full 2,000-line goal remains incomplete. + +## Fourteenth refactor slice + +The fourteenth checkpoint measured **14,027 production lines**, a net reduction of +**1,368** from the original baseline, including **52** removed in this slice. +Nonblank/noncomment production lines are **12,284**, down **1,358** from baseline. +All helpers, imports, and new test-module separators are counted. +At that checkpoint, **632 production lines remained** to reach the goal. + +- Replace the unused stored authorization-code object with native five-string + tuple decoding, destructure it immediately, and return only DID/scope to the + service: **16 net lines**. SQL columns still decode from zero through four + before raw client/redirect/challenge comparisons. Expiry cleanup, time reads, + token insertion, code deletion, rollback, and response assembly are unchanged. +- Pass `SpacePageQuery` through the service into member/writer database + listings: **15 net lines**. Existing authorization stays in the service; + storage applies the same default 100 and clamp 1..1000 before the same query. + Explicit `i64::try_from(...).unwrap_or(i64::MAX)` preserves the supported SQL + integer binding and fallback. SQL text and row decoders remain unchanged. +- Implement concrete DNS/DID resolution directly in `IdentityNetwork`, removing + two forwarding methods, and construct resolver fields directly: **10 net + lines**. DNS, web-DID validation/request/response parsing, native non-web + resolution, error messages, initialization order, and dynamic dispatch remain + unchanged. Keep `DeploymentMode` itself instead of converting it to a boolean + and back for outbound policy: another **5 net lines**. Both mode variants retain + the same local-handle shortcut and network policy. +- Compose optional blob cursor parsing with native `Option`/`Result` operations: + **6 net lines**. Absence, successful canonicalization, inventory-specific error + context, and account/revision/cursor/SQL ordering remain unchanged. + +Six new regressions cover 16 SQL-column-error/binding-mismatch combinations with +purge rollback; 24 paging limit/cursor combinations with 48 output and 24 +owned/borrowed SQL-binding comparisons; malformed, absent, CIDv0, CIDv1, IPFS, +gateway, and alternate-base blob cursors; three web-DID pre-network rejections +and one unsupported native method; and both identity deployment modes. The +concrete identity test reads system DNS configuration but performs no DNS lookup +or HTTP request. Existing tests and assertions remain. Independent review +confirmed each change and the corrected supported SQL integer binding. + +The first build identified that the locked SQLite dependency does not enable +`usize: ToSql`. Both listing methods and the new binding test retain the original +explicit `i64` conversion instead; no dependency feature was added. Formatting +kept the measured reduction at 15 lines for that boundary. + +Fourteenth-checkpoint verification (2026-09-05): + +- All **334 locked workspace/all-target tests passed**, none ignored or filtered + out, including local socket/process checks. All 242 original and all 328 + preceding-checkpoint test names remain. +- Formatting, warning-denied workspace/all-target Clippy, Rust 1.88.0 + workspace/all-target locked checks, and offline dependency policy passed. +- Manifests, lockfile, security middleware, both rate-limit files, schema, + repository transactions, and outbound identity HTTP policy implementation + are unchanged from the thirteenth checkpoint. No features or migrations were removed. +- README and all seven documents under `docs/`: 106 local links and 20 shell + blocks validated; the source guide describes the changed abstractions. + +Build checks use the same non-incremental, debug-info-disabled settings. These +are local macOS results, not Linux/non-Unix CI, deployment, publication, or an +authored commit. The full 2,000-line goal remains incomplete. + +## Fifteenth refactor slice + +The fifteenth checkpoint measured **13,994 production lines**, a net reduction of +**1,401** from the original baseline, including **33** removed in this slice. +Nonblank/noncomment production lines are **12,255**, down **1,387** from baseline. +All helpers, imports, and test-module separators remain counted. +At that checkpoint, **599 production lines remained** to reach the goal. + +- Store ATrium's `Secp256k1Keypair` directly in `SpaceService` and make Space + commit signing a module function: **14 net lines**. The single-field + `CommitSigner` wrapper is gone. Key import remains after `TidClock::default` + in service construction, with the identical configuration error context. + Set hashing, randomness, context encoding, signing, HKDF, HMAC, and output + assembly retain their sequence and exact representations. +- Borrow the JWT's existing signed-input prefix instead of rebuilding a + `header.payload` string: **7 net lines**. Native splits retain exactly three + segments, including empty ones, and reject extra segments before decoding. + Header, claims, and signature decoding keep their error priority. All four + callers use identical signed bytes; async callers retain a valid input borrow. +- Make relay, account, and OAuth-client readers methods of their existing + `Settings` type: **4 net lines**, including shared-implementation overhead. + Environment reads, first-failure behavior, per-field validation, and final + production cross-field checks remain in the same order. +- Remove the private CLI command and user-account-transition pass-through + functions: **8 net lines**. OS arguments are still parsed immediately before + configuration. User activation/deactivation pass the same `true` policy; + operator transitions still pass `false`. Public signatures and the transition + implementation are unchanged. + +Two new regressions cover invalid zero/out-of-range signing scalars with exact +configuration error context, and 570 JWT token vectors decoded into two claim +types (1,140 old/new comparisons). JWT comparisons cover values, signed bytes, +signature bytes, exact error variants/messages, and borrowing of the original +prefix. Existing tests and assertions remain, including signature/MAC fixtures, +configuration read-order checks, CLI/process tests, and account policy tests. +Independent review confirmed the wrapper and JWT changes. Reverse JWT splitting +can scan a longer trailing suffix before rejecting an extra-dot token; it stays +linear with unchanged request limits and removes the signed-input allocation. +The bounded HTTP audit found no further substantive safe deletion this slice. + +The initial formatting/build attempt caught a raw-string delimiter collision in +the new JWT fixture containing `#atproto`. Only its Rust delimiter was corrected; +fixture bytes and expected outcomes were unchanged, then all gates were rerun. + +Fifteenth-checkpoint verification (2026-09-05): + +- All **336 locked workspace/all-target tests passed**, none ignored or filtered + out, including socket/process checks. All 242 original and all 334 + preceding-checkpoint test names remain. +- Formatting, warning-denied workspace/all-target Clippy, Rust 1.88.0 + workspace/all-target locked checks, and offline dependency policy passed. +- Manifests, lockfile, security middleware, both rate-limit files, schema, + public/Space repository transactions, and outbound identity HTTP policy + implementation are unchanged from the fourteenth checkpoint. No dependency + feature or migration was removed; no public source API changed. +- README and all seven documents under `docs/`: 106 local links and 20 shell + blocks validated; the source guide describes the changed abstractions. + +Build checks use the same non-incremental, debug-info-disabled settings. These +are local macOS results, not Linux/non-Unix CI, deployment, publication, or an +authored commit. The full 2,000-line goal remains incomplete. + +## Sixteenth refactor slice + +The sixteenth checkpoint measured **13,966 production lines**, a net reduction of +**1,429** from the original baseline, including **28** removed in this slice. +Nonblank/noncomment production lines are **12,230**, down **1,412** from baseline. +All helpers, imports, and test-module separators remain counted. +At that checkpoint, **571 production lines remained** to reach the goal. + +- Remove the commit-only projection wrapper around `SpaceService::signed_snapshot` + and let its two callers select the commit from the existing pair: **9 net + lines**. The snapshot method becomes crate-visible; its body is unchanged. + Authorization, bounded snapshot SQL, error mapping, fresh signing, the later + operation-list authorization, and the HTTP wire wrapper remain intact. +- Use the native Tokio watch sender directly for subscription and sticky + shutdown publication, removing two `AppState` forwarding methods: **8 net + lines**. Subscription still happens at the same stream entry point; shutdown + still publishes `true` after the shutdown future resolves and before drain. + The cloned state and channel lifetimes remain unchanged. +- Select present `$link`/`$bytes` values in the object encoder before calling + their decoders, removing two unnecessary optional-result layers: **6 net + lines**. Field-count errors still precede value-type and decoding errors. + Object type/blob checks, link-before-bytes priority, child/key/depth traversal, + canonical bytes, and CID validation are unchanged. +- Deserialize blob references directly from the borrowed JSON map using the + locked Serde implementation: **2 net lines**. This removes the cloned object + wrapper while still producing owned ATrium blob values. Typed/legacy fallback, + field and number decoding, error text, and subsequent canonical checks remain + unchanged. +- Use native SQLite `Statement::exists` for membership instead of an optional + unit-valued row: **3 net lines**. The exact query still prepares and steps once + under one connection lock; no column is decoded. Query/lock errors, statement + cleanup, membership policy, and transaction/timestamp behavior are unchanged. + +Two new protocol regressions compare 42 owned/borrowed blob-map vectors with +exact serialized values and error details, seven special-form error-priority +cases, and four literal CBOR outputs. Existing official data-model/CID fixtures +and all prior assertions remain. Existing membership tests cover matching and +missing members plus a dropped membership table; stream and process tests retain +the shutdown/drain checks. Independent reviews checked the encoder against locked +Serde source and the database/snapshot paths against locked SQLite code and the +previous checkpoint. The remaining ownership and record-planning candidates +either added machinery or changed meaningful raw-input/API distinctions. + +Sixteenth-checkpoint verification (2026-09-05): + +- Focused protocol tests passed, then all **338 locked workspace/all-target + tests passed**, none ignored or filtered out, including socket/process checks. + All 242 original and all 336 preceding-checkpoint test names remain. +- Formatting, warning-denied workspace/all-target Clippy, Rust 1.88.0 + workspace/all-target locked checks, and offline dependency policy passed. +- Manifests, lockfile, security middleware, both rate-limit files, schema, + public/Space repository transactions, and outbound identity HTTP policy + implementation are unchanged from the fifteenth checkpoint. No dependency + feature or migration was removed; no public source API changed. +- README and all seven documents under `docs/`: 106 local links and 20 shell + blocks validated; the source guide describes the changed abstractions. + +Build checks use the same non-incremental, debug-info-disabled settings. These +are local macOS results, not Linux/non-Unix CI, deployment, publication, or an +authored commit. The full 2,000-line goal remains incomplete. + +## Seventeenth refactor slice + +The seventeenth checkpoint measured **13,929 production lines**, a net reduction of +**1,466** from the original baseline, including **37** removed in this slice. +Nonblank/noncomment production lines are **12,195**, down **1,447** from baseline. +All helpers, imports, and test-module separators remain counted. +At that checkpoint, **534 production lines remained** to reach the goal. + +- Carry a record capability's required collection with its action: **7 net + lines**. All previous record callers supplied that collection; read and + management callers no longer pass an irrelevant `None`. Scope parsing, + defaults, duplicate-field handling, permission evaluation, and authorization + error priority are unchanged. +- Use immutable native JSON indexing and type predicates for policy fields: + **9 net lines**. The locked Serde implementation returns null for absent or + non-object fields, which the existing string/array checks reject just as + before. No mutable indexing, policy defaults, or error branches changed. +- Return service-token issuer and audience as a native pair rather than a + crate-private projection struct: **11 net lines**. All callers immediately + name both values. JWT validation, signature verification, the subsequent + issuer rate check, notification policy, and wire fields retain their order. +- Match language subtag byte shapes and share one set for duplicate detection: + **5 net lines**. Extension singletons and longer variants cannot collide. + Native parsing still happens first; private-use termination, case folding, + first-duplicate priority, and exact errors retain their existing behavior. +- Collect TID alphabet characters directly from the immutable clock value: + **2 net lines**. All thirteen output digits are identical for every `u64`; + time reads, atomic advancement, saturation, and decoding are unchanged. +- Lend reachable staged block entries to their callers instead of cloning + their bytes into an intermediate vector: **3 net lines**. Missing-block + verification remains eager. The sole production caller immediately copies + and collects only CIDs in the same staged-map iteration order. The separate + staged-write handoff and repository transactions are unchanged. + +Four new regressions cover 105 capability/scope outcomes, 120 old/new JSON field +comparisons, 57 language inputs, and 10,569 distinct TID values with the prior +encoder and round-trip decoder. Existing literal fixtures and assertions remain. +Three block-store fixture constructors explicitly recover the former owned +projection. Independent review checked all six changes, including the locked +Serde implementation, eager block verification, and service authentication order. +The additional bounded OAuth and persistence audits found no larger coherent +deletion without conflating distinct policies or transaction boundaries. + +Seventeenth-checkpoint verification (2026-09-05): + +- All **342 locked workspace/all-target tests passed**, none ignored or filtered + out, including socket/process checks. All 242 original and all 338 + preceding-checkpoint test names remain. +- Formatting, warning-denied workspace/all-target Clippy, Rust 1.88.0 + workspace/all-target locked checks, and offline dependency policy passed. +- Manifests, lockfile, security middleware, both rate-limit files, schema, + public/Space repository transactions, and outbound identity HTTP policy + implementation are unchanged from the sixteenth checkpoint. No dependency + feature or migration was removed; no public source API changed. +- README and all seven documents under `docs/`: 106 local links and 20 shell + blocks validated; the source guide describes the changed abstractions. + +Build checks use the same non-incremental, debug-info-disabled settings. These +are local macOS results, not Linux/non-Unix CI, deployment, publication, or an +authored commit. The full 2,000-line goal remains incomplete. + +## Eighteenth refactor slice + +The eighteenth checkpoint measured **13,875 production lines**, a net reduction of +**1,520** from the original baseline, including **54** removed in this slice. +Nonblank/noncomment production lines are **12,146**, down **1,496** from baseline. +All helpers, imports, and test-module separators remain counted. +At that checkpoint, **480 production lines remained** to reach the goal. + +- Share one database connection constructor and pass the schema implementation's + existing initialization flag directly: **15 net lines**. Normal and in-memory + opens still permit initialization; existing-only opens retain read/write-only + flags and reject an uninitialized file. Only forwarding methods disappear; + the migration body, SQL, pragmas, validation, and transaction order are + byte-identical. Thirty schema test calls retain all previous assertions. +- Pass existing Space metadata update and writer-notice inputs to persistence: + **11 net lines**. Policy and app-access serialization still precede the + connection lock, in that order; timestamps remain in their original SQL + binding positions. The writer store reads only space/repo/rev/hash. Service + authorization remains outside it, including the same managing-app checks. + Raw database fixtures retain invalid policy/revision/hash values and the + distinction between absent updates and an explicitly stored JSON null. +- Share one `RecordInput` between HTTP and borrowed Space write + preparation: **9 net lines**. Its owned default specialization retains the + exact concrete Serde name, fields, field order, and types. Borrowed inputs + keep optional creation keys and required update keys, without another model + or new cloning. Field evaluation, validation, TID generation, batch planning, + and transactions retain their original order. +- Remove the sole-caller Space `get_repo` forwarding method: **10 net lines**. + The HTTP handler composes the existing signed snapshot and CAR serializer. + Authentication, exclude-values defaulting, signing, serialization, response + construction, and both 64 MiB bounds remain intact. The snapshot method and + complete CAR/Space commit implementations are byte-identical. +- Use native binary TXT concatenation and borrowed percent-decoded DID path + segments: **6 net lines**. TXT bytes are joined before parsing; all `did:web` + segments still decode before path and outbound-URL validation. The resulting + URL remains owned, with no altered resolution or HTTPS policy. +- Reuse the existing optional SQL query helper for repository block reads: + **3 net lines**. The same first row decodes to bytes before content-address + verification; absence, SQL type errors, and corruption keep their errors. + One connection lock still surrounds the query. Its interval is slightly + narrower: CID string construction occurs before acquisition and the constant + missing-row error is selected after release; verification was already outside + the lock, and no shared-state operation crosses that boundary. + +Four new regressions cover 107 old/new JSON extractor cases (including duplicate +fields and sequence inputs), every split point of eight binary TXT fixtures, +73 DID URL/error vectors, and eight stored-block lookup states. Existing fixture +values and assertions remain. Independent reviews checked all six changes and +their authorization, migration, serialization, and error-priority boundaries. + +The initial Clippy run found an unnecessary owned argument in a new test-only +comparison helper. It now borrows its input; no expected outcome changed, and +the full test suite and warning-denied Clippy were rerun successfully. + +Eighteenth-checkpoint verification (2026-09-05): + +- All **346 locked workspace/all-target tests passed**, none ignored or filtered + out, including socket/process checks. All 242 original and all 342 + preceding-checkpoint test names remain. +- Formatting, warning-denied workspace/all-target Clippy, Rust 1.88.0 + workspace/all-target locked checks, and offline dependency policy passed. +- Manifests, lockfile, security middleware, both rate-limit files, public/Space + repository transactions, and outbound identity HTTP policy are unchanged + from the seventeenth checkpoint. Schema changes are exactly the removed + forwarding methods, with the full implementation body and SQL retained. + No dependency feature or migration was removed; no public source API changed. +- README and all seven documents under `docs/`: 106 local links and 20 shell + blocks validated; the source guide describes the changed abstractions. + +Build checks use the same non-incremental, debug-info-disabled settings. These +are local macOS results, not Linux/non-Unix CI, deployment, publication, or an +authored commit. The full 2,000-line goal remains incomplete. + +## Nineteenth refactor slice + +The nineteenth checkpoint measured **13,861 production lines**, a net reduction of +**1,534** from the original baseline, including **14** removed in this slice. +Nonblank/noncomment production lines are **12,132**, down **1,510** from baseline. +All helpers, imports, and test-module separators remain counted. +At that checkpoint, **466 production lines remained** to reach the goal. + +- Share one native syntax-parser error adapter and one Space-reference shape + rejection: **8 net lines**. The locked ATrium `FromStr` implementations use + the same constructors as the previous validators. Borrowed validators still + return the original text; all six error labels remain exact. Datetime minimum + handling and language duplicate checks are unchanged. Space references still + reject their prefix/segment shape before DID, collection, and record-key + validation, in that order. +- Share `SpaceWriter` between stored metadata and the HTTP writer view: + **6 net lines**. Storage retains the default raw `Vec` hash and accepts the + same unvalidated metadata. HTTP selects `LexBytes` explicitly and retains the + original eager three-field projection after cursor calculation. Field order, + exact JSON bytes, encoding timing, and raw writer Debug output are unchanged. + No custom serializer or extra response model is needed. +- Reuse `query_one` for the two outbox sequence aggregate queries: **line-neutral + after formatting**. Exact SQL, row types, empty defaults, single-lock behavior, + and database errors are preserved. The rest of `database/repo.rs`, including + all transaction and retention implementations, is byte-identical. + +Five new regressions cover 556 borrowed identifier comparisons, 90 datetime +value/error comparisons, 904 Space-reference validation-order cases, 72 writer +page JSON-byte comparisons plus an empty page, and four outbox extrema states +plus both missing-table errors. All existing test names and assertions remain. +Independent review checked each changed production boundary against the prior +checkpoint and the relevant locked parser/SQL implementation. + +The owned record-input candidate was rejected because it would weaken the +required update-key type and add a defensive missing-key branch. The first +Clippy pass found a redundant closure in the new sequence test; it now uses +`Option::unwrap_or_default` directly, with unchanged expectations. The complete +test suite and warning-denied Clippy passed after that correction. + +Nineteenth-checkpoint verification (2026-09-06): + +- All **351 locked workspace/all-target tests passed**, none ignored or filtered + out, including socket/process checks. All 242 original and all 346 + preceding-checkpoint test names remain. +- Formatting, warning-denied workspace/all-target Clippy, Rust 1.88.0 + workspace/all-target locked checks, and offline dependency policy passed. +- Manifests, lockfile, security middleware, both rate-limit files, schema, + public/Space repository transactions, Space authorization/commit/CAR code, + and outbound identity HTTP policy are unchanged from the eighteenth checkpoint. + No dependency feature or migration was removed; no public source API changed. +- README and all seven documents under `docs/`: 106 local links and 20 shell + blocks validated; the source guide describes the changed abstractions. + +Build checks use the same non-incremental, debug-info-disabled settings. These +are local macOS results, not Linux/non-Unix CI, deployment, publication, or an +authored commit. The full 2,000-line goal remains incomplete. + +## Twentieth refactor slice + +The twentieth checkpoint measured **13,841 production lines**, a net reduction of +**1,554** from the original baseline, including **20** removed in this slice. +Nonblank/noncomment production lines are **12,113**, down **1,529** from baseline. +All helpers, imports, and test-module separators remain counted. +At that checkpoint, **446 production lines remained** to reach the goal. + +- Share the existing local-Space credential gate with writer listing: + **11 net lines**. Notification registration and unregistration explicitly + retain their active-account check immediately before the shared gate. Writer + listing does not gain that check. Credential matching, Space syntax, host + authority, decoded metadata lookup, and subsequent reads or mutations retain + their exact order and errors. No active-status or membership policy changes. +- Use the HTTP library's native integer header conversion: **3 net lines**, + including the new test-module separator. Every `u64` retains the same decimal + `Retry-After` bytes, sensitivity flag, Debug output, and complete response. + The old integer string parser could not fail. Header allocation is now native; + rate-limit configuration, counters, admission policy, and both rate-limit + implementation files are unchanged. +- Use native CID parsing through the existing error-context adapter at three + storage boundaries: **3 net lines** after formatting. Public record and blob + decoding remove obsolete imports; the Space-record change is line-neutral + including its test separator. Concrete CID parsing, v0/v1 acceptance, exact + diagnostics, and SQL-type-before-CID-before-JSON ordering remain unchanged. +- Reuse that error-context adapter for three password and three Space-CAR + encoding errors: **3 net lines**. All six messages remain exact; formatting + still happens only on error. Password salt generation, hashing, SQL, and + authentication order are unchanged. CAR validation, signing, CID checks, + section ordering, and both export bounds are unchanged. + +Six new regressions cover 252 old/new credential-gate outcomes with notification +row non-mutation and expiry checks, 1,086 numeric-header and complete-response +comparisons, 100 stored-CID/JSON/SQL comparisons, and ten stored-password parser +comparisons plus a SQL-type failure. All existing test names and assertions +remain. Independent review checked all four changes against the preceding +checkpoint and the relevant locked CID/HTTP implementations. + +Larger planner candidates would conflate strict updates with upserts, move event +insertion across mutation boundaries, or change per-item validation and account +checks. A shared encoded-record helper measured line-neutral and would change +Space-CAR limits if reused there, so it was not applied. + +The initial Clippy pass identified two unnecessary owned arguments and one +overlong new regression. The helpers now borrow their arrays, and the credential +test separates setup from notification assertions without dropping any cases or +adding lint suppressions. Full tests and warning-denied Clippy passed afterward. + +Twentieth-checkpoint verification (2026-09-06): + +- All **357 locked workspace/all-target tests passed**, none ignored or filtered + out, including socket/process checks. All 242 original and all 351 + preceding-checkpoint test names remain. +- Formatting, warning-denied workspace/all-target Clippy, Rust 1.88.0 + workspace/all-target locked checks, and offline dependency policy passed. +- Manifests, lockfile, schema, all SQL and transaction implementations, security + middleware, both rate-limit files, record planning, Space signing, and outbound + identity HTTP policy are unchanged from the nineteenth checkpoint. No + dependency feature or migration was removed; no public source API changed. +- README and all seven documents under `docs/`: 106 local links and 20 shell + blocks validated; the source guide describes the changed abstractions. + +Build checks use the same non-incremental, debug-info-disabled settings. These +are local macOS results, not Linux/non-Unix CI, deployment, publication, or an +authored commit. The full 2,000-line goal remains incomplete. + +## Twenty-first refactor slice + +The twenty-first checkpoint measured **13,823 production lines**, a net reduction of +**1,572** from the original baseline, including **18** removed in this slice. +Nonblank/noncomment production lines are **12,098**, down **1,544** from baseline. +All helpers, imports, and test-module separators remain counted. +At that checkpoint, **428 production lines remained** to reach the goal. + +- Model byte-addressing as infallible: **11 net lines**. SHA-256 always produces + 32 bytes, while the locked multihash constructor fails only when its input + exceeds the 64-byte capacity. The project-owned `sha256_cid` and `dag_cbor_cid` + Rust APIs now return `Cid` directly; the unreachable `Error::Encoding` variant + and its caller-side mappings are removed. Obsolete error documentation is + replaced by the explicit fixed-width invariant. `record_cid` remains fallible + with identical validation, encoding, and size errors. Upload, maintenance, + and Space-CAR processing retain their ordering, CID bytes, and limits. +- Build all minted Space/service JWT headers in the existing signer: + **5 net lines**, including a new test-module separator. The forwarding + `sign_token` method and duplicate service header construction disappear. + Token type, key-ID presence, header field order, claims, exact signed bytes, + error mapping, and time/randomness calls remain unchanged. Only service + header string allocations move after the infallible claim construction; + no fallible, async, database, time, or RNG operation is reordered. +- Use rusqlite's native final readiness query: **2 net lines**. The same + `SELECT 1`, empty parameters, first-row callback, and database-error mapping + are retained. All other schema, migration, and connection-policy code is + byte-identical. + +Five new regressions cover 65 old/new generic CID comparisons across five +codecs and 13 byte inputs, 13 DAG-CBOR wrapper comparisons, two known SHA-256 +digests, 30 complete JWT/signature comparisons, three exact claim-serialization +errors, and seven SQL result/callback-count cases. Existing assertions remain; +test callers only discard the now-obsolete hash-result unwraps. Independent +review checked all three changes against the preceding checkpoint and the +locked hash/SQL implementations. + +A direct IPLD-model replacement was rejected because it would alter the +human-readable link representation and serializer calls, or require additional +fallible conversion and cloning. Native JWK decoding would change rejection +order and accepted fields. Shared event payloads would add a new concept and +change Debug representations for little reduction. None was applied. + +The first build found an unqualified `Result` alias in the new hash test; +qualifying the standard result type fixed it. The complete test suite and all +checks passed after that correction, without lint suppressions. + +Twenty-first-checkpoint verification (2026-09-06): + +- All **362 locked workspace/all-target tests passed**, none ignored or filtered + out, including socket/process checks. All 242 original and all 357 + preceding-checkpoint test names remain. +- Formatting, warning-denied workspace/all-target Clippy, Rust 1.88.0 + workspace/all-target locked checks, and offline dependency policy passed. +- Fifteen protected manifest, lockfile, security, rate-policy, record-planning, + transaction, authorization, commit, and outbound-HTTP files are byte-identical + to the twentieth checkpoint. Space repository production is also identical; + 61 SQL literals in four touched storage files are unchanged. No dependency + feature or migration was removed. The two hash return types and removed + unreachable error variant are intentional project-owned Rust API changes. +- README and all seven documents under `docs/`: 106 local links and 20 shell + blocks validated; the source guide describes the changed abstractions. + +Build checks use the same non-incremental, debug-info-disabled settings. These +are local macOS results, not Linux/non-Unix CI, deployment, publication, or an +authored commit. The full 2,000-line goal remains incomplete. + +## Twenty-second refactor slice + +This checkpoint measured **13,787 production lines**, a net reduction of +**1,608** from the original baseline, including **36** removed in this slice. +Nonblank/noncomment production lines were **12,068**, down **1,574** from baseline. +All helpers, imports, and test-module separators remain counted. +At that checkpoint, **392 production lines remained** to reach the goal. + +- Reuse the protocol SHA-256 constructor in the ATrium block overlay: + **4 net lines**. The existing repository-codec guard still precedes the + hash-algorithm guard. Only the proven-infallible digest/CID construction is + shared; unsupported inputs retain their exact errors and never enter staging. +- Replace per-character HTML-escape allocations with native ordered string + replacements: **5 net lines**. Ampersands are replaced first, so generated + entities are not re-escaped. Exact UTF-8/entity bytes and all five page + interpolation positions remain unchanged. The implementation makes five + whole-string passes instead of allocating a string for every character. +- Remove six single-caller private helpers: **27 net lines**. Filesystem + directory preparation and the readiness probe move into `open`, `put`, and + `check` (**13**), preserving every permission check, await, conditional fsync, + file drop, unconditional cleanup, and primary-error precedence. Space record + projection moves into its existing post-cursor map (**6**), retaining field + order and eager value exclusion. The JSON content-type predicate moves into + the existing status short-circuit (**5**), retaining case-sensitive prefix, + invalid-UTF-8, and first-header-value semantics. Notification expiry formats + directly after persistence (**3**), with identical text and error context. + Public APIs and HTTP-to-service boundaries remain intact. + +Five new regressions cover 208 old/new block-addressing outcomes and 208 +verification-context outcomes, rejected-write staging preservation, all +1,112,064 Unicode scalar values plus targeted entity/adjacency cases, eight +timestamp/JSON vectors and two timestamp errors, and 30 content-type/status +cases. Existing filesystem durability, recovery, private-permission, concurrency, +Space cursor, and exclude-values tests remain unchanged. Independent review +checked the complete moved operation sequences and wire representations. + +A broad conditional-error macro was rejected: it would hide guard syntax while +leaving the actual validation rules duplicated. Backup and blob-publication +generalization would add machinery or change transaction/cleanup boundaries. +The account-lifecycle sharing proposal was held for the following slice. + +The first new timestamp fixture incorrectly expected fractional seconds; its +expected strings now match the unchanged whole-second formatter. Clippy also +requested separators in the new Unicode-range literal. Both were test-only +corrections; final tests and all checks passed without lint suppressions. + +Twenty-second-checkpoint verification (2026-09-06): + +- All **367 locked workspace/all-target tests passed**, none ignored or filtered + out, including socket/process checks. All 242 original and all 362 + preceding-checkpoint test names remain. +- Formatting, warning-denied workspace/all-target Clippy, Rust 1.88.0 + workspace/all-target locked checks, and offline dependency policy passed. +- Twenty-two protected files are byte-identical to the twenty-first checkpoint, + including manifests, lockfile, protocol library, schema, security and rate + policies, repository planning/transactions, Space authorization/signing/CAR, + clock primitives, and outbound HTTP policy. No SQL, dependency feature, + migration, public source API, or authorization policy changed. +- README and all seven documents under `docs/`: 106 local links and 20 shell + blocks validated; the source guide describes the changed abstractions. + +Build checks use the same non-incremental, debug-info-disabled settings. These +are local macOS results, not Linux/non-Unix CI, deployment, publication, or an +authored commit. The full 2,000-line goal remains incomplete. + +## Twenty-third refactor slice + +This checkpoint measured **13,754 production lines**, a net reduction of +**1,641** from the original baseline, including **33** removed in this slice. +Nonblank/noncomment production lines were **12,038**, down **1,604** from baseline. +All helpers, imports, and test-module separators remain counted. +At that checkpoint, **359 production lines remained** to reach the goal. + +- Reuse the existing `LifecycleEvent` for the complete account-lifecycle retry + workflow: **8 net lines**. Identity and status changes now share account lookup, + idempotence, bounded compare-and-set retries, and exhaustion handling. Handle + validation remains before lookup; status equality remains before user policy. + User transitions remain restricted to active/deactivated, while operator + transitions retain their existing authority. Each event retries only its own + conflict message. Public return types, event retention, and transaction + ownership are unchanged; no new event type or retry abstraction was added. +- Pass the existing Space `RecordQuery` through persistence: **10 net lines**. + Authorization, collection/key validation, and repository existence checks + still precede the same SQL query and row decoder. Accepted Space URIs round-trip + exactly, including case and percent escapes, so the original query replaces + two URI reconstructions and a second mapping of its four string fields. + Raw database inputs are not newly validated; missing rows and SQL/CID/JSON + failures retain their existing ordering and errors. +- Remove three private single-use helpers: **15 net lines**. Blob response + construction moves into the upload handler after service success (**7**), with + the same checked size conversion, error text, and generated wire model. + Public/relay origin errors move into their existing guards (**4 each**), with + unchanged setting-specific text and URL-parse/component/HTTPS precedence. + +Four new lifecycle tests passed against the unchanged preceding implementation +before consolidation. They cover 35 status-policy cases, 25 handle cases, +lookup/validation failure priority, and ignored-update trigger rollback plus +retry exhaustion. Two additional Space tests cover 52 old/new SQL lookups, +68 exact URI round-trips, and eight invalid URI cases. The existing service +authorization/validation tests and all earlier assertions remain unchanged. +Independent review checked lifecycle equivalence and the Space lookup boundary. + +Twenty-third-checkpoint verification (2026-09-06): + +- All **373 locked workspace/all-target tests passed**, none ignored or filtered + out, including socket/process checks. All 242 original and all 367 + preceding-checkpoint test names remain; existing test content in each of the + three extended test sections was independently checked for preservation. +- Formatting, warning-denied workspace/all-target Clippy, Rust 1.88.0 + workspace/all-target locked checks, and offline dependency policy passed. +- Twenty-two protected files are byte-identical to the twenty-second checkpoint, + including manifests, lockfile, protocol library, schema, security/rate policy, + repository planning/transactions, Space authorization/signing/CAR, clock, + backup/private filesystem, URI parsing, and outbound HTTP. All 19 SQL literals + in the touched Space storage module are unchanged. No dependency feature, + migration, transaction, or public source API changed. +- README and all seven documents under `docs/`: 106 local links and 20 shell + blocks validated; the source guide describes the shared lifecycle/query paths. + +Build checks use the same non-incremental, debug-info-disabled settings. These +are local macOS results, not Linux/non-Unix CI, deployment, publication, or an +authored commit. The full 2,000-line goal remains incomplete. + +## Twenty-fourth refactor slice + +This checkpoint measured **13,730 production lines**, a net reduction of +**1,665** from the original baseline, including **24** removed in this slice. +Nonblank/noncomment production lines were **12,014**, down **1,628** from baseline. +All helpers, imports, and test-module separators remain counted. +At that checkpoint, **335 production lines remained** to reach the goal. + +- Reuse `ListBlobsQuery` and `ListSpacesQuery` through their complete inventory + workflows: **20 net lines**. The crate-private database methods now consume + the query values and own the endpoints' existing default/clamp operations. + Both methods had one production caller, whose behavior is preserved. Blob + listing retains authorization, `since` validation, and repository existence + before querying; Space discovery retains type/DID validation and scope checks. + The original Space URI replaces equivalent reconstruction. GLOB construction, + SQL, parameter order and null/text semantics, locking, and decoding remain + unchanged. Recovery's unbounded record listing and operation listing are not + migrated to these bounded queries. +- Reuse the existing DID handle-claim predicate in local document validation: + **4 net lines**. The predicate moves from the network resolver to the identity + domain module and accepts the caller's raw handle text. It still checks exact, + case-sensitive `at://{handle}` alias membership without parsing or normalizing. + Local ID/endpoint/key failures still precede the handle check; network + resolution and backlink errors retain their existing order and text. + +Four new tests add 393 blob-query and 1,153 Space-discovery old/new comparisons, +56 raw handle/alias cases, and five masked-failure priority cases. They exercise +defaults/clamps, raw empty/Unicode/NUL/percent-escaped inputs, cursors, filters, +DISTINCT/order, null handling, and SQL failures. Existing 1,001-row endpoint-limit +and authorization tests remain unchanged. Eight older raw blob fixtures now +construct the query input while retaining all their inputs and assertions. +Independent review checked both shared workflows and their failure boundaries. + +A CAR write/check helper prototype was reverted: signature and return overhead +increased production by one line. Both affected files are byte-identical to the +preceding checkpoint. A borrowed Space repository-identity type was also rejected +as net-negative; it would not eliminate a meaningful representation or workflow. +Typed-DID construction or cached authorization would additionally change accepted +inputs or remove required repeat checks. Native URL slicing and generated HTTP +output substitutions were not adopted: they changed edge-case or wire behavior, +or required extra machinery that erased their savings. + +Twenty-fourth-checkpoint verification (2026-09-06): + +- All **377 locked workspace/all-target tests passed**, none ignored or filtered + out, including socket/process checks. All 242 original and all 373 + preceding-checkpoint test names remain. Earlier inline test content in both + extended modules was independently checked for preservation; the adapted raw + blob fixtures retain their assertions. +- Formatting, warning-denied workspace/all-target Clippy, Rust 1.88.0 + workspace/all-target locked checks, and offline dependency policy passed. +- Twenty-five protected files are byte-identical to the twenty-third checkpoint, + including manifests, lockfile, protocol, schema, security/rate policy, + repository planning/transactions/CAR/export, Space authorization/signing/CAR + and recovery, clock, backup/private filesystem, URI parsing, and outbound HTTP. + All 19 SQL literals in the touched Space storage module are unchanged. No + dependency feature, migration, transaction, or public source API changed. +- README and all seven documents under `docs/`: 106 local links and 20 shell + blocks validated; the source guide describes the inventory and identity reuse. + +Build checks use the same non-incremental, debug-info-disabled settings. These +are local macOS results, not Linux/non-Unix CI, deployment, publication, or an +authored commit. The full 2,000-line goal remains incomplete. + +## Twenty-fifth refactor slice + +The twenty-fifth checkpoint measured **13,697 production lines**, a net reduction of +**1,698** from the original baseline, including **33** removed in this slice. +Nonblank/noncomment production lines were **11,985**, down **1,657** from baseline. +All helpers, imports, and test-module separators remain counted. +At that checkpoint, **302 production lines remained** to reach the goal. + +- Share OAuth's existing SHA-256/base64url hash with PKCE and fold the sole-use + verifier validator into challenge derivation: **8 net lines**. Length-first + short-circuiting, the unreserved-character predicate, digest bytes, and both + verifier/challenge errors remain unchanged. No RNG or clock call was added. +- Pass the existing authorization request through code persistence: **4 net + lines**, including the removed argument-count lint allowance. The defaulted + `AuthorizationRequest` keeps typed URL parsing in HTTP/services; + raw string URIs remain supported at persistence. Client/redirect text is + borrowed before the transaction, retaining hash/RNG/expiry and lock/clock/ + purge ordering. SQL fields, cleanup, commit/rollback, redemption, and redirect + construction are unchanged. This intentionally changes the public Rust + `store_authorization_code` signature from four scalar request fields to the + shared request; its service and existing raw-fixture callers are updated. + The request's `state` remains unused by storage, as before. +- Complete `RecordQuery` propagation for single Space deletion: **7 net lines**. + Authorization already establishes exact repository ownership, and Space URI + parsing preserves its bytes. Key validation, TID generation, transaction + timing, raw input support, and idempotent versus strict batch deletion remain + unchanged. Three earlier database fixtures now construct the shared input. +- Use native CID equality before projecting a public record response: **3 net + lines**. The complete lookup and SQL/CID/JSON decoding still precede the CID + filter. Native equality compares the version, codec, hash code, and digest + encoded in canonical CID text; successful `RecordView` bytes and missing-record + errors are unchanged. Mismatches no longer allocate discarded URI/CID strings. +- Remove redundant local plumbing: **11 net lines**. The sole-use account-status + SQL loader moves into `require_account_active` (**3**); the Retry-After value + comes directly from the existing error at its original header insertion (**4**); + and the exact blob-path predicate moves into its sole CSP caller (**4**). + SQL/parse/not-found/active-check order and all HTTP policy remain unchanged. + +Seven new tests cover 1,792 PKCE boundary/character vectors plus empty inputs, +typed/raw OAuth storage and default URL parsing, 45 account-guard outcomes, +24 repeated-delete/state comparisons, 526 getter/CID/error comparisons, and +110 complete security-header responses. The PKCE regression first passed against +unchanged production. Existing raw fixture assertions, the RFC PKCE example, +and 1,086-value Retry-After response coverage remain intact. Independent reviews +checked OAuth, HTTP, and native CID equivalence; root reviewed the storage paths. + +The broader caller audit found no dead storage feature APIs: maintenance, +backup, CLI startup, and tests still use them. Native ATrium commit builders +would change TID/encoding behavior and staging order, so they were not substituted. + +Twenty-fifth-checkpoint verification (2026-09-06): + +- All **384 locked workspace/all-target tests passed**, none ignored or filtered + out, including socket/process checks. All 242 original and all 377 + preceding-checkpoint test names remain; prior assertions were preserved. +- Formatting, warning-denied workspace/all-target Clippy, Rust 1.88.0 + workspace/all-target locked checks, and offline dependency policy passed. +- Twenty-four protected files are byte-identical to the twenty-fourth checkpoint, + including manifests, lockfile, protocol, schema, rate-policy implementations, + OAuth/repository HTTP contracts, commit/CAR/export transactions, Space policy/ + signing/recovery, clock, private filesystem, URI parsing, and outbound HTTP. + All 30 SQL literals across the three touched storage modules are unchanged. + No dependency, migration, transaction policy, or wire representation changed; + the intentional project-owned OAuth source API change is described above. +- README and all seven documents under `docs/`: 106 local links and 20 shell + blocks validated; the source guide describes the changed shared operations. + +Build checks use the same non-incremental, debug-info-disabled settings. These +are local macOS results, not Linux/non-Unix CI, deployment, publication, or an +authored commit. The full 2,000-line goal remains incomplete. + +## Twenty-sixth refactor slice + +The twenty-sixth checkpoint measured **13,681 production lines**, a net reduction of +**1,714** from the original baseline, including **16** removed in this slice. +Nonblank/noncomment production lines were **11,971**, down **1,671** from baseline. +All helpers, imports, and test-module separators remain counted. +At that checkpoint, **286 production lines remained** to reach the goal. + +- Retain the first validated DID in Space service-endpoint resolution: **6 net + lines**. The existing syntax parser returns the typed DID with its original + error mapping. Native DID parsing preserves the input bytes, so local identity + comparison and the later resolver receive the same value. Invalid-DID errors + still precede fragment checks, local service selection, and network resolution; + the second parse and its unreachable error branch disappear. +- Consolidate each directory-sync function's platform branches: **8 net lines** + across private snapshots and asynchronous blob storage. One `cfg!(unix)` I/O + guard replaces two function definitions in each module. Unix retains the same + open-then-sync operations, error propagation, and asynchronous await order; + non-Unix still returns without I/O. The referenced file APIs are cross-platform. + Unix-only permission APIs remain separately compile-time gated. +- Express temporary-file-name recognition as an Option pipeline: **2 net + lines**. UTF-8 conversion, the last pending-marker split, readiness/CID prefix + validation, sixteen-byte suffix length, and ASCII hexadecimal checks retain + their exact short-circuit order and accepted inputs. + +Five new tests cover 47 DID parser/raw-text comparisons, 50 service-endpoint +priority cases, synchronous/asynchronous directory-sync outcomes for directories, +files, missing paths, and non-directory parents, and 55 temporary-name cases on +Unix. Both directory tests compare against the previous platform-gated body and +check file-content preservation. Existing test bodies remain intact. Independent +review found no changed service-resolution, publication, cancellation, or error +boundary. A test-only parenthesis typo was corrected before the successful build. + +Removing optional outbox-field defaults was rejected: Serde still needs them to +accept truncated sequence inputs. Removing local DID document validation was +also rejected because native endpoint parsing enforces a URI-length boundary; +the apparently redundant check still affects accepted inputs and error order. + +Twenty-sixth-checkpoint verification (2026-09-06): + +- All **389 locked workspace/all-target tests passed**, none ignored or filtered + out, including socket/process checks. Multiset inventory retains all 242 + original and 384 preceding-checkpoint test occurrences, including every prior + path/name pair. The two new directory tests count separately despite sharing + a function name. +- Formatting, warning-denied workspace/all-target Clippy, Rust 1.88.0 + workspace/all-target locked checks, and offline dependency policy passed. +- All 38 independently checked protected files are byte-identical to the + preceding checkpoint, including dependency manifests/lockfile, rate policy, + transaction and signing boundaries. All production SQL/string literals across + the 13 storage modules are unchanged. No file was removed, and only the four + intended Rust files changed; dependency, wire, and transaction policy remain + unchanged. +- README and all seven documents under `docs/`: 106 local links and 20 shell + blocks validated; the source guide describes DID reuse and filesystem guards. + +Build checks use the same non-incremental, debug-info-disabled settings. These +are local macOS results. Non-Unix branches were reviewed and have parity tests, +but were not cross-compiled or executed. No Linux CI, deployment, publication, +or authored commit is claimed. The full 2,000-line goal remains incomplete. + +## Twenty-seventh refactor slice + +The working tree measures **13,653 production lines**, a net reduction of +**1,742** from the original baseline, including **28** removed in this slice. +Nonblank/noncomment production lines are **11,943**, down **1,699** from baseline. +All helpers, imports, and test-module separators remain counted. +**258 production lines still need to be eliminated** to reach the goal. + +- Remove the redundant delegation projection: **17 net lines** after formatting. + Exchange returns the already-validated `SpaceClaims` with its DPoP thumbprint; + HTTP immediately destructures the issuer and subject. The unused claim fields + survive the return boundary but are dropped before any further await, clock, + RNG, SQL, or fallible operation. Serde fields and attributes, verification, + request binding, replay consumption, and HTTP policy are unchanged. This is + a crate-private return-type change, not a token or response-format change. +- Keep record absence as data through write planning: **3 net lines**. Persistence + returns the existing optional query result directly rather than turning + absence into `NotFound` and having two write callers reverse that conversion. + Public get/delete services still require a record and produce the same + `NotFound` before CID checks. SQL/decode failures propagate unchanged. This + intentionally changes the project-owned public database API from `get_record` + returning `Record` to `find_record` returning `Option` within `Result`; + all four production callers and two direct fixture callers are updated. +- Share the complete repository-listing workflow: **4 net lines**, including an + additional explanatory documentation line. Collection-filtered discovery still + validates the collection first, then reuses the same cursor parser, head query, + and next-cursor construction as the full list. The existing optional storage + filter passes through the crate-private service method; HTTP passes `None` + for the full list. Limits, SQL, ordering, decode failures, and wire conversions + remain unchanged. +- Fold sole-caller identity HTTP dispatch into its existing timeout: **4 net + lines**. The old loop is unchanged inside an owning `async move` block. + Request ownership, timeout construction/polling, guard/URL/HTTPS precedence, + redirect handling, bounded reads, cancellation, and boxed errors are retained. + +Five additional tests cover claims/DPoP/replay and masked-failure priorities, +51 optional-record lookup outcomes, 280 repository-listing comparisons plus +three failure priorities, and 12 identity HTTP guard/parser/policy outcomes. +Existing fixture assertions remain intact. The listing regression's setup and +previous-query implementation were separated into test-only helpers, followed +by a full test rerun. Independent review checked the claims, listing, and +identity-dispatch changes; root reviewed optional lookup and all production diffs. + +A native URL host/port slice was rejected: URL serialization and Rust's IPv6 +display differ for IPv4-mapped addresses. Remaining schema/commit intermediates +were retained because they preserve SQL sequencing, raw record identity, and +mutation/head/outbox error boundaries; no SQL assembly or endpoint DSL was added. + +Twenty-seventh-checkpoint verification (2026-09-06): + +- All **394 locked workspace/all-target tests passed**, none ignored or filtered + out, including socket/process checks and the final test-helper split. A + path/name multiset retains all 242 original and 389 preceding-checkpoint test + occurrences; exactly five new tests were added. +- Formatting, warning-denied workspace/all-target Clippy, Rust 1.88.0 + workspace/all-target locked checks, and offline dependency policy passed. +- All 38 selected protected files are byte-identical to the preceding checkpoint, + including dependency manifests/lockfile, rate policy, transaction/signing/CAR + boundaries, backup and filesystem adapters. All 217 production string literals, + including every SQL literal, remain unchanged and in sequence across 13 + storage modules. Only seven intended production Rust paths and four dedicated + test paths changed; no file was added or removed. Auth signing, proof, clock, + and RNG bodies are unchanged outside the documented result projection. +- README and all seven documents under `docs/`: 106 local links and 20 shell + blocks validated; the source guide describes the changed shared workflows. + +Build checks use the same non-incremental, debug-info-disabled settings. These +are local macOS results, not Linux/non-Unix CI, deployment, publication, or an +authored commit. The full 2,000-line goal remains incomplete. + +## Twenty-eighth refactor slice + +The formatted working tree measures **13,612 production lines**, a net reduction +of **1,783** from the original baseline, including **41** removed in this slice. +Nonblank/noncomment production lines are **11,907**, down **1,735** from baseline. +All helpers, imports, documentation, and test-module separators remain counted. +**217 production lines still need to be eliminated** to reach the goal. + +- Preserve optional blob metadata through persistence: **8 net lines**. The + database returns its existing optional lookup directly, rather than creating + `NotFound` and having reference validation and byte reads translate it again. + Each caller still produces its original public absence error. The same SQL + filters, canonical CID binding, media-type-before-size decoding, negative-size + error, and metadata-before-byte-store order remain. This intentionally changes + the project-owned public database API from `blob` returning `Blob` to + `find_blob` returning `Option` within `Result`; all callers are updated. +- Remove a one-field credential projection: **5 net lines**. Verification returns + the exact verified Space string directly. Token, issuer, key binding, DPoP, + request/hash checks, and replay consumption are unchanged; HTTP still applies + the same Space rate check at the same point. The return-type change is + crate-private, with no JWT or response-format change. +- Use one owned read-authorization enum: **19 net lines**. HTTP moves the same + token or Space string into `ReadAuthorization`; services borrow that enum. + The second enum and its borrowed projection disappear, with no production + clones or new helper layers. Permission checks remain at every existing call, + including repeated snapshot authorization during operation listing. Only + crate-internal signatures change; the 26 existing test constructors are + adapted to ownership without removing assertions. +- Fold the single-use OAuth query conversion into its only handler: **7 net + lines**. Response type and challenge method are still checked in that order + before service validation and rendering. Explicitly dropping those two profile + strings preserves their previous successful conversion-return drop boundary. + Extraction fields, rejection normalization, HTML, and error text are unchanged. + The project-owned `TryFrom` implementation is removed. +- Pass notification registration input through storage: **2 net lines**. The + existing `NotifyRegistrationInput` reaches both registration and removal, + eliminating the repeated field projection. Active-account and credential + checks, expiry/time reads, locking, SQL bindings, UPSERT, and idempotent DELETE + retain their ordering and raw values. Both changed methods are crate-private. + +Five new tests cover 194 blob storage states with two comparisons each, 25 +complete OAuth status/header/body outcomes, credential binding/replay priorities, +four notification space/service pairs with three updates and two deletions each, +and reused OAuth/Space authorizations with lookup/validation failure priorities. +The old OAuth extractor vectors and assertions remain; a separate old/new actual +handler regression covers the conversion now folded into the production handler. +The first full run exposed a new notification fixture collision on the existing +`(authority, space_type, skey)` unique key. Each raw-value case now owns its own +database; no production code or expected result was changed to address it. + +Independent review passed the blob, credential, OAuth, and storage changes; +root reviewed notification passthrough and every read-authorization production +diff. An owned enum was chosen over `Cow`: production already owns both variants, +so a generic owned/borrowed representation would add concepts without benefit. +A proposed stored-event envelope was rejected because its modest net reduction +required more metadata projections and weakened the typed conversion boundary. + +Twenty-eighth-checkpoint verification (2026-09-06): + +- All **399 locked workspace/all-target tests passed**, none ignored or filtered + out, including socket/process checks and the corrected notification fixture. + Independent name and path/name multisets retain all 242 original and 394 + preceding-checkpoint test occurrences; exactly five tests were added. +- Formatting, warning-denied workspace/all-target Clippy, Rust 1.88.0 + workspace/all-target locked checks, and offline dependency policy passed. +- All 41 independently selected protected files are byte-identical to the + preceding checkpoint; root also checked its existing 38-file protection set. + Dependencies, rate policy, transaction/signing/CAR boundaries, backup, and + filesystem adapters are unchanged in this slice. Across 13 storage modules, + all 217 production string literals remain unchanged and in sequence, including + 125 SQL-bearing literals. Only the 13 intended production Rust paths and three + dedicated test paths changed; no Rust file was added or removed. +- README and all seven documents under `docs/`: 106 local links and 20 shell + blocks validated; the source guide describes the shared inputs, authorization + ownership, optional blob lookup, and handler-local OAuth guards. + +Build checks use the same non-incremental, debug-info-disabled settings. These +are local macOS results, not Linux/non-Unix CI, deployment, publication, or an +authored commit. The full 2,000-line goal remains incomplete. + +## Twenty-ninth refactor slice + +The formatted working tree measures **13,588 production lines**, a net reduction +of **1,807** from the original baseline, including **24** removed in this slice. +Nonblank/noncomment production lines are **11,885**, down **1,757** from baseline. +All helpers, imports, documentation, and test-module separators remain counted. +**193 production lines still need to be eliminated** to reach the goal. + +- Share the stream-error send operation: **8 net lines**. The existing match + selects the public name/message pair, then invokes one timed frame send. + Non-XRPC errors are still logged before selecting the sanitized response; + XRPC errors retain their exact borrowed names and messages. Encoding, timeout + creation, socket send, and cancellation order are unchanged. The adjacent + `if let` uses its expression directly, removing a function-tail temporary + without moving any fallible operation or await. +- Select deployment mode at its single configuration boundary: **6 net lines**. + Absent and exact `local` values share the local-mode branch; exact `production` + enables production behavior. All other strings retain the same error and stop + before the next setting read. This removes the private sole-use parser wrapper + and unnecessary default-string allocation. Environment access, non-Unicode + error precedence, field-read order, bounds, and cross-field checks are unchanged. +- Return the existing records/cursor pair directly: **10 net lines**. The + single-use `RecordPage` wrapper is removed, following the other listing methods' + existing tuple convention. Repository resolution, collection/cursor validation, + limit clamping, SQL ordering, cursor cloning, and record projection retain + their order. HTTP still converts every record before constructing the same + cursor/records output. This intentionally changes the project-owned public + `list_records` return type and removes `RecordPage` and its derived Debug shape; + no wire field, error, or serialization rule changes. + +Three new tests cover four complete old/new WebSocket frame comparisons, 13 +deployment-selector outcomes, and 18 direct-service pagination reads through +both DID and handle identifiers. The frame test includes escaped Unicode with +an extended-length frame, empty public details, and two sanitized internal +errors, and consumes exactly both CBOR objects. Pagination covers empty pages, +both directions, continuation/end cursors, zero and maximum requested limits, +and exact record URI/CID/value sequences. All existing assertions remain. + +Independent review passed the stream sender, configuration selector, and page +ownership/conversion analysis; root reviewed every production diff. Two other +proposals were rejected after measuring their full abstraction overhead: an +optional Space-revision return and a shared single-record commit helper each +added two production lines under normal formatting. No shortening names or +adding aliases solely to recover those formatting lines was adopted. + +Twenty-ninth-checkpoint verification (2026-09-06): + +- All **402 locked workspace/all-target tests passed**, none ignored or filtered + out, including the new live-socket regression and process checks. Independent + name and path/name multisets preserve all 242 original and 399 preceding + test occurrences; exactly three tests were added. +- Formatting, warning-denied workspace/all-target Clippy, Rust 1.88.0 + workspace/all-target locked checks, and offline dependency policy passed. +- All 13 storage production modules are byte-identical to the preceding + checkpoint. Dependency manifests/lockfile, rate policy, transactions, signing, + CAR encoding, backups, and filesystem adapters are unchanged. Only five + intended production Rust paths and two dedicated test paths changed; no Rust + file was added or removed. Existing stream frame encoders remain byte-identical. +- README and all seven documents under `docs/`: 106 local links and 20 shell + blocks validated. The source guide explains boundary-local mode selection, + direct listing pairs, and the shared stream-error send. + +Build checks use the same non-incremental, debug-info-disabled settings. These +are local macOS results, not Linux/non-Unix CI, deployment, publication, or an +authored commit. The full 2,000-line goal remains incomplete. + +## Thirtieth refactor slice + +The formatted working tree measures **13,565 production lines**, a net reduction +of **1,830** from the original baseline, including **23** removed in this slice. +Nonblank/noncomment production lines are **11,864**, down **1,778** from baseline. +All helpers, imports, documentation, and test-module separators remain counted. +**170 production lines still need to be eliminated** to reach the goal. + +- Return write results as their existing domain-value pairs: **17 net lines**. + Create/put return `(RecordView, CommitView)` and ordered batches return + `(CommitView, Vec)`, removing `RecordWrite` and `RecordBatch` + and their wrapper-only constructors. HTTP uses named destructuring, with no + tuple-index replacements or compatibility aliases. Record addressing, commit + awaits, mutation order, result projection, and fallible HTTP conversion order + are unchanged. This intentionally changes the project-owned public Rust return + types and removes `RecordWrite`'s derived Debug shape; wire schemas are unchanged. + Tuple bindings free leftover native record/commit fields in a different order + at the same synchronous return/error boundary. These values have no custom + destructors, and no await, log, I/O, or fallible operation separates their drops; + identical allocator free order is not claimed. +- Delegate SQLite error classification to its native accessor: **3 net lines**. + The locked rusqlite implementation returns an error code only for a top-level + `SqliteFailure`, exactly matching the removed variant guard. Only constraint + failures become conflicts; all other original errors, including conversion + errors wrapping nested constraints, remain database errors. Conflict-message + allocation is still lazy; SQL, locking, and transaction callers are unchanged. +- Use native fallible iteration for strict blob traversal: **3 net lines**. + The same child iterator and recursive collector now propagate their first + failure through `try_for_each`. Typed/legacy early returns, tolerant discovery, + child order, and partial-output behavior remain unchanged. No failed child is + skipped and no later child is visited after failure. + +Three new tests cover 132 old/new SQLite error classifications, six blob-walk +shapes with distinct CID/MIME/size values and competing failures, and direct +create/update/upsert result pairs. The latter checks complete record readback, +durable head CID/revision agreement, increasing revisions, and final repository +verification. Existing ordered-batch, HTTP conversion-priority, recovery, and +firehose assertions are retained through named fixture bindings. Independent +review passed all three changes, including the precise native-drop caveat; +root reviewed every production diff and checked assertion-count preservation. + +Thirtieth-checkpoint verification (2026-09-06): + +- All **405 locked workspace/all-target tests passed**, none ignored or filtered + out, including socket/process checks. Independent name and path/name multisets + preserve all 242 original and 402 preceding test occurrences; three were added. +- Formatting, warning-denied workspace/all-target Clippy, Rust 1.88.0 + workspace/all-target locked checks, and offline dependency policy passed. +- All 217 production string literals across 13 storage modules remain unchanged + and in sequence, including 125 SQL-bearing literals. All 32 independently + selected protected files are byte-identical to the preceding checkpoint. + Dependencies, rate policy, schema/transaction/signing/CAR boundaries, backup, + and filesystem adapters are unchanged. Only six intended production Rust paths + and six dedicated test paths changed; no Rust file was added or removed. +- README and all seven documents under `docs/`: 106 local links and 20 shell + blocks validated. The source guide describes result pairs, native constraint + classification, and strict fallible traversal. + +Build checks use the same non-incremental, debug-info-disabled settings. These +are local macOS results, not Linux/non-Unix CI, deployment, publication, or an +authored commit. The full 2,000-line goal remains incomplete. + +## Thirty-first refactor slice + +The formatted working tree measures **13,524 production lines**, a net reduction +of **1,871** from the original baseline, including **41** removed in this slice. +Nonblank/noncomment production lines are **11,829**, down **1,813** from baseline. +All helpers, imports, documentation, and test-module separators remain counted. +**129 production lines still need to be eliminated** to reach the goal. + +- Return OAuth authorization's existing code/redirect pair: **7 net lines**. + `OAuthService::authorize` now returns `(String, Url)`, removing the public + `AuthorizationGrant` wrapper and its derived Debug shape. HTTP uses named + bindings and retains both values through response construction. Validation, + password lookup, RNG, expiry clock, hash/storage, redirect cloning, ordered + query appends, header conversion, and 303 response construction are unchanged. + Native String/URL teardown order can differ at the same synchronous return + boundary; there is no custom destructor or intervening observable operation. +- Return four existing native results directly: **4 net lines**. Final event + pruning remains inside the transaction callback; existing-object directory + synchronization and final directory permissions keep their same awaits. + PKCE validation remains after client, redirect, and scope checks. All four + callees already return the identical application `Result<()>`, so no error + conversion, transaction decision, or durability barrier was removed. +- Let the latest-commit handler use the existing head query: **6 net lines**. + Remove the sole-use public `latest_commit` forwarding/projection method and + move all five documentation lines to `sync_head`, now crate-visible. HTTP + immediately constructs the same `CommitView` for its unchanged wire converter; + export callers are untouched. Account status, SQL decoding, CID formatting, + and HTTP CID/revision parsing retain their order. This intentionally removes + a project-owned public Rust method, without exposing a new public head API. +- Use existing JSON values for five one-string Space outputs: **24 net lines**, + including the new credential test-module separator. The keys remain `uri`, + `did`, `token`, `credential`, and `expiresAt`. Typed outer list containers keep + field order and optional-cursor omission; inputs and nested typed values are + untouched. Each service/mint/timestamp expression still runs once, in place. + This adds map/key allocation and a String copy; original temporary strings can + drop earlier during the same synchronous construction. Native allocation/free + order is not claimed identical. Only infallibly serializable strings enter + these objects, with no new reachable serialization error or changed wire bytes. + +Six new tests cover six redirect/state authorization and redemption cases, +77 old/new head-query projections, and 45 complete old/new HTTP response +comparisons (status, all headers, and body bytes). Cases include duplicate query +keys, fragments, Unicode, replay, all hosting statuses, malformed SQL/head data, +competing errors, every ASCII control character, and absent/empty/nonempty +cursors. Existing rollback, durability, timestamp-error, and wire assertions +remain intact. Independent review passed the OAuth, result-tail, and Space +output changes; root reviewed all production changes and assertion preservation. + +Thirty-first-checkpoint verification (2026-09-06): + +- All **411 locked workspace/all-target tests passed**, none ignored or filtered + out, including socket/process checks. Name and path/name multisets retain all + 242 original and 405 preceding test occurrences; six tests were added. +- Formatting, warning-denied workspace/all-target Clippy, Rust 1.88.0 + workspace/all-target locked checks, and offline dependency policy passed. +- All 217 ordered production string literals across 13 storage modules remain + unchanged, including 125 SQL-bearing literals. All 26 independently selected + protected files are byte-identical to the preceding checkpoint, including + dependencies, rate policy, authentication, signing, configuration, and backup. + Eleven production Rust paths and one dedicated test path changed; no Rust file + was added or removed. There are 117 Rust files and 21,970 test lines. +- README and all seven documents under `docs/`: 106 local links and 20 shell + blocks validated. The source guide explains head-query reuse, OAuth pairs, + direct PKCE results, and the small JSON-output allocation tradeoff. + +Build checks use the same non-incremental, debug-info-disabled settings. These +are local macOS results, not Linux/non-Unix CI, deployment, publication, or an +authored commit. The full 2,000-line goal remains incomplete. + +## Thirty-second refactor slice + +The formatted working tree measures **13,505 production lines**, a net reduction +of **1,890** from the original baseline, including **19** removed in this slice. +Nonblank/noncomment production lines are **11,809**, down **1,833** from baseline. +All helpers, imports, documentation, and test-module separators remain counted. +**110 production lines still need to be eliminated** to reach the goal. + +- Reuse the existing record key through Space deletion: **17 net lines**. + `RecordQuery` moves to the domain module and the existing Space facade exports + it from there. Batch deletes carry `SpaceWrite::Delete(RecordQuery)` instead + of repeating the same four strings; the SQL helper borrows that query directly. + The four required input fields and Deserialize order remain unchanged. + Preparation still authorizes, validates the key, constructs Space/DID strings, + and allocates its one TID before entering the same transaction. Batch absence + remains an error; single absence remains successful. SQL lookup, delete, op + insertion, timestamp, active-account guards, and rollback boundaries are intact. + String and str bindings use the same borrowed SQLite text; native field drop + order remains space, repo (formerly did), collection, then rkey. `RecordQuery` + gains Clone/Debug, and the internal delete Debug representation changes from + named variant fields to a nested query. `SpaceWrite` is explicitly crate-visible, + matching its previous effective visibility; no externally reachable API changes. +- Compose authorization-header extraction as one native optional-value pipeline: + **2 net lines**, including its new test-module separator. First-header lookup, + text decoding, exact scheme prefix, exactly one ASCII space, and nonempty + payload filtering retain their order. Intermediate conversions to the same + unit authentication error are removed. Successful tokens remain borrowed from + the same header bytes; there is no trimming, case folding, or fallback to a + second Authorization header. Authentication and rate checks above this private + parser remain byte-identical. + +Two new tests cover 80 old/new header outcomes (including duplicate headers and +pointer identity) and 16 batch-delete base fixtures, with strict repeat versus +idempotent single deletion and complete repeated-delete transaction rollback. +The existing 24 raw-key/idempotence/SQL-type comparisons retain the exact old +scalar SQL helper as their test-only oracle. Existing parsing, active-account, +trigger-rollback, and delete-result wire assertions are preserved. Independent +review passed both changes; root reviewed every production diff and checked +assertion preservation. The shared-page and owned-write-input hypotheses were +rejected: their added allocation/packing or altered projection boundaries did +not provide a worthwhile simplification. + +Thirty-second-checkpoint verification (2026-09-06): + +- All **413 locked workspace/all-target tests passed**, none ignored or filtered + out, including socket/process checks. Name and path/name multisets retain all + 242 original and 411 preceding test occurrences; two tests were added. +- Formatting, warning-denied workspace/all-target Clippy, Rust 1.88.0 + workspace/all-target locked checks, and offline dependency policy passed. +- All 217 ordered production string literals across 13 storage modules remain + unchanged, including 125 SQL-bearing literals. Twenty-five independently + selected protected files are byte-identical to the preceding checkpoint; + the sole changed member of the prior 26-file set is the reviewed HTTP parser. + Dependencies, rate policy, signing, configuration, and backup are unchanged. + Six production Rust paths and one dedicated test path changed; no Rust file + was added or removed. There are 117 Rust files and 22,131 test lines. +- README and all seven documents under `docs/`: 106 local links and 20 shell + blocks validated. The source guide explains the shared delete key and exact + authorization-header pipeline. + +Build checks use the same non-incremental, debug-info-disabled settings. These +are local macOS results, not Linux/non-Unix CI, deployment, publication, or an +authored commit. The full 2,000-line goal remains incomplete. + +## Thirty-third refactor slice + +The formatted working tree measures **13,492 production lines**, a net reduction +of **1,903** from the original baseline, including **13** removed in this slice. +Nonblank/noncomment production lines are **11,798**, down **1,844** from baseline. +All helpers, imports, documentation, and test-module separators remain counted. +**97 production lines still need to be eliminated** to reach the goal. + +- Share the HTTP and stream error payload: **6 net lines**. + `ErrorBody` replaces the duplicate borrowed CBOR payload definition. + HTTP still instantiates an owned String and streams still borrow their message. + Field declaration order, JSON response status/headers, CBOR header-before-body + serialization, frame extension, and encoding-error contexts remain unchanged. + The locked CBOR serializer ignores the changed internal struct name. There is + no added allocation, changed destructor ownership, or new public Rust API. +- Centralize the identical whole-second UTC clock helper: **6 net lines**. + `clock::now` contains the exact former database and Space-auth function body. + Database reexports it for its existing facade callers; auth children retain + their existing imported path. Backup, registration, and orphan collection use + the same helper for their direct seconds reads. No call is hoisted or cached: + separate iat/exp checks, short-circuit reads, lock-then-time ordering, manifest + field evaluation, saturating grace subtraction, and registration addition all + stay in place. OAuth's duration-before-timestamp arithmetic and TID nanoseconds + are deliberately untouched. No clock precision or bounds changed. +- Borrow relay task inputs through their awaited batch: **1 net line**. + The existing `join_all` borrows the client, configured origin, and hostname, + removing a client Arc clone, URL clone, and String clone per relay. Those + original owners already survive the entire batch; no task is spawned. Future + order, attempts, request bytes, logging, backoff, redirect policy, cancellation + scope, and complete failure aggregation remain unchanged. Avoided native + allocations/refcounts are intentional, not an assertion of allocator identity. + +Two new tests compare 45 complete old/new JSON responses and 15 concatenated +CBOR error frames, including empty/error/control/Unicode/4096-byte messages. +All existing socket/frame, expiry, backup, garbage-collection, and live-relay +tests remain intact. Independent review passed the shared payload and clock +consolidation; root reviewed all production diffs and the relay ownership change, +and checked assertion preservation. A native ATrium TID constructor was rejected +because its bit-63 mask would not preserve the current full-u64 encoding. + +Thirty-third-checkpoint verification (2026-09-06): + +- All **415 locked workspace/all-target tests passed**, none ignored or filtered + out, including socket/process checks. Name and path/name multisets retain all + 242 original and 413 preceding test occurrences; two tests were added. +- Formatting, warning-denied workspace/all-target Clippy, Rust 1.88.0 + workspace/all-target locked checks, and offline dependency policy passed. +- All 217 ordered production string literals across 13 storage modules remain + unchanged, including 125 SQL-bearing literals. Twenty-two independently + selected protected files are byte-identical to the preceding checkpoint; + four intended changed members of the 26-file core set were reviewed directly. + Dependencies, rate policy, HTTP authentication/security, signing, configuration, + and private filesystem adapters are unchanged. Exactly nine production Rust + paths changed; the other 108 Rust files are byte-identical. No Rust file was + added or removed. There are 117 Rust files and 22,206 test lines. +- README and all seven documents under `docs/`: 106 local links and 20 shell + blocks validated. The source guide explains the shared clock, generic error + body, and borrowed relay futures. + +Build checks use the same non-incremental, debug-info-disabled settings. These +are local macOS results, not Linux/non-Unix CI, deployment, publication, or an +authored commit. The full 2,000-line goal remains incomplete. + +## Thirty-fourth refactor slice + +The formatted working tree measures **13,480 production lines**, a net reduction +of **1,915** from the original baseline, including **12** removed in this slice. +Nonblank/noncomment production lines are **11,787**, down **1,855** from baseline. +All helpers, imports, documentation, and test-module separators remain counted. +**85 production lines still need to be eliminated** to reach the goal. + +- Keep native URLs in OAuth discovery metadata: **8 net lines**. + Five private output fields now retain owned `Url`/`Vec` values. The locked + URL serializer emits the same stored serialization as Display. Response struct + names, field order, service-call order, status, headers, and JSON bytes remain + unchanged. No parsing or input contract changed. Resource/issuer clones still + own their buffers; endpoint values avoid the extra URL-to-String copy. Native + allocation/deallocation differences are intentional, not allocator identity. +- Return native startup results and compose optional status parsing: **3 lines**. + Crawl and serve already return the same crate `Result<()>` as main. Their + redundant result conversions disappear without moving awaits, startup checks, + listener binding, or logging. Optional Unicode conversion and parsing now join + before the same lazy fixed configuration error. A discarded invalid-status + String drops slightly earlier, with no intervening I/O, logging, or await. +- Inline the single-use write-value predicate: **1 net line** after formatting. + The `pub(crate)` accessor disappears; its exact positive Create/Update match + stays in the same short-circuit iterator guard. Batch size, strict validation, + per-write authorization, preparation, and TID generation retain their order. + Externally reachable Rust API is unchanged; no compatibility alias is needed. + +Three new tests cover 28 complete old/new metadata responses (two endpoints, +seven origin variants, and plain/extended issuers), 15 Unix status arguments +(13 portable cases plus two non-Unicode values), and 129 batch-validation +outcomes. The batch matrix covers empty/delete-only/mixed writes and oversized +batches, including exact error precedence. All existing test bodies/assertions +are retained. Independent review passed every production change and its tests. + +Thirty-fourth-checkpoint verification (2026-09-06): + +- All **418 locked workspace/all-target tests passed**, none ignored or filtered + out, including socket/process checks. Name and path/name multisets retain all + 242 original and 415 preceding test occurrences; three tests were added. +- Formatting, warning-denied workspace/all-target Clippy, Rust 1.88.0 + workspace/all-target locked checks, and offline dependency policy passed. +- All 13 storage modules are byte-identical, preserving 217 ordered production + string literals, including 125 SQL-bearing literals. Twenty-five members of + the 26-file protected core are byte-identical; the intended main.rs change was + reviewed directly. Dependencies, rate policy, authentication/security, signing, + configuration, and private filesystem adapters are unchanged. +- Exactly four production Rust paths and one dedicated test path changed; the + other 112 Rust files are byte-identical. No Rust file was added or removed. + There are 117 Rust files and 22,415 test lines. +- README and all seven documents under `docs/`: 106 local links and 20 shell + blocks validated. The source guide explains native metadata URLs, direct + startup results, and the batch's local variant predicate. + +Build checks use the same non-incremental, debug-info-disabled settings. These +are local macOS results, not Linux/non-Unix CI, deployment, publication, or an +authored commit. The full 2,000-line goal remains incomplete. + +## Thirty-fifth refactor slice + +The formatted working tree measures **13,432 production lines**, a net reduction +of **1,963** from the original baseline, including **48** removed in this slice. +Nonblank/noncomment production lines are **11,748**, down **1,894** from baseline. +All helpers, imports, documentation, and test-module separators remain counted. +**37 production lines still need to be eliminated** to reach the goal. + +- Share the validated HTTP origin value: **28 net lines**. + `HttpOrigin` replaces `PublicOrigin` and `RelayOrigin`. Both setting-specific + parser bodies remain intact under `parse_public` and `parse_relay`, preserving + URL parsing, component/HTTPS checks, exact error labels, relay limits, and + ordered duplicate detection. The private URL representation, endpoint cloning, + Display, and hostname implementation are shared without changing their bodies. + The relay parser and its documentation move into production code in origin.rs; + the former relay file retains its existing tests, not hidden production logic. + Intentional public Rust API changes are the type/constructor names and Debug + name; cross-role equality and shared read-only methods become available. + Neither old type stored deployment mode, and the validated URL remains private. +- Remove the native Space-reference parsing facade: **4 net lines**. + Nineteen production calls now use `SpaceRef::FromStr` at the same validation + positions. The parser body is unchanged and all five facade documentation lines + move onto that implementation. Twelve test calls are mechanically updated; + their existing 904 priority vectors exercise the native parser directly. The + public `syntax::space_ref` function is intentionally removed without an alias. +- Return native notification destination pairs: **9 net lines**. + `(service, endpoint)` replaces the crate-internal registration output model + through database, service, and HTTP delivery. Native tuple decoding reads both + Strings in the original column order, preserving first-error precedence. SQL, + clock/lock order, borrowed destinations, dispatch argument order, logs, + concurrency four, and the enclosing timeout remain unchanged. Internal Debug + becomes tuple-shaped. On second-column failure the first native String may + drop before rather than after infallible error wrapping; no I/O or custom Drop + occurs between those points. Successful field ownership/order is unchanged. +- Return three private snapshot I/O results directly: **3 net lines**. + The same `From` conversion replaces `?` followed by `Ok(())` at the + directory-creation and final file-sync tails. Permissions, create-new behavior, + writes, fsync, descriptor closing, and the callers' cleanup order are unchanged. +- Use native redirect-list membership: **4 net lines**. + Slice `contains` replaces the manual equality scan with the same `Url` equality + operand order and short circuit. Registry lookup still precedes redirect, + scope, and PKCE checks; no normalization, clone, hashing, or allocation is added. + +Four new tests cover 144 old/new origin constructor outcomes plus URL operations +and equality, 256 complete OAuth validation outcomes, and 42 notification-query +comparisons including ordered/expired/raw values and both-column SQL type errors. +All existing assertions remain; fixture-only changes update the renamed APIs. +Independent reviews passed the production changes, ownership, and error order. +Native IPLD replacements were rejected because their arbitrary-Serde contract +differs in null/integer/newtype calls even where DAG-CBOR bytes can agree. + +Thirty-fifth-checkpoint verification (2026-09-06): + +- All **422 locked workspace/all-target tests passed**, none ignored or filtered + out, including socket/process checks. Name and path/name multisets retain all + 242 original and 418 preceding test occurrences; four tests were added. +- Formatting, warning-denied workspace/all-target Clippy, Rust 1.88.0 + workspace/all-target locked checks, and offline dependency policy passed. +- All 217 ordered production string literals across 13 storage modules remain + unchanged, including 125 SQL-bearing literals. Twelve storage production + bodies are byte-identical; only the intended metadata pair decoder changes. + Twenty-two of the 26 protected core files are byte-identical; four intended + configuration, private-filesystem, and Space parser-call changes were reviewed. + Rate policy, dependencies, public/Space signing and CAR code remain unchanged. +- Twenty-three Rust paths changed, including mechanical fixture updates; the + other 94 Rust files are byte-identical. No Rust file was added or removed. + There are 117 Rust files and 22,806 test lines. +- README and all seven documents under `docs/`: 106 local links and 20 shell + blocks validated. The source guide explains the shared origin, native parsing, + notification pairs, I/O results, and redirect membership. + +Build checks use the same non-incremental, debug-info-disabled settings. These +are local macOS results, not Linux/non-Unix CI, deployment, publication, or an +authored commit. The full 2,000-line goal remains incomplete. + +## Thirty-sixth refactor slice + +The formatted working tree measures **13,410 production lines**, a net reduction +of **1,985** from the original baseline, including **22** removed in this slice. +Nonblank/noncomment production lines are **11,729**, down **1,913** from baseline. +All helpers, imports, documentation, and test-module separators remain counted. +**15 production lines still need to be eliminated** to reach the goal. + +- Return hosted identity and status as a native pair: **5 net lines**. + `(Account, AccountStatus)` replaces `HostedAccount`, with named destructuring + in database, repository, and Space callers. DID and handle are decoded into + the account before reading and parsing status. SQL, lock/query positions, + status guards, lifecycle comparisons, and retries are unchanged. Named unused + account bindings preserve ownership through the existing policy checks. + This intentionally changes the public Rust return type and derived Debug + representation; no compatibility wrapper remains. +- Use typed singleton maps for two Space responses: **7 net lines**. + Native `BTreeMap` values remove the latest-commit and batch-write outer wrapper + types while retaining typed `SignedCommit`, `LexBytes`, and `AppliedWrite` + serialization. Each map has exactly one literal key, so no field ordering is + changed. Authentication, service calls, conditional publication, and nested + conversion stay in place. The private outer Rust shape is less constrained + and adds one map-node allocation; no generic JSON conversion is introduced. +- Match outbound transport policy as one tuple: **8 net lines**. + Host kind, deployment mode, and scheme share a native match. Domains still + require HTTPS; literal IPs still require local mode, loopback, and HTTP(S). + The pure scheme getter is evaluated eagerly, but credential/fragment guards, + errors, DNS/private-network denial, requests, redirects, and timeouts are + unchanged. No rate-limit policy or admission code changed. +- Use the nonempty decoded-segment invariant: **2 net lines**. + Colon splitting produces at least one segment, including for empty input; + successful fallible decoding preserves that length. Direct host/path borrowing + removes an unreachable empty-vector error branch. All decoding still precedes + the unchanged empty/path-segment checks and outbound URL validation. + +Three new tests cover 51 complete old/new HTTP responses (status, headers, and +exact bytes) and 12,104 outbound-policy comparisons. Existing SQL decoder tests +retain their five ordered success/error cases with an added exact Debug-error +comparison. The existing did:web parity test grows from 73 to 78 cases, including +leading/repeated colons and competing decoding/path errors. All prior assertions +remain, with fixture-only updates for native pair access. Independent reviews +passed all four changes and their ownership, error, and wire-order boundaries. + +Thirty-sixth-checkpoint verification (2026-09-06): + +- All **425 locked workspace/all-target tests passed**, none ignored or filtered + out, including socket/process checks. Name and path/name multisets retain all + 242 original and 422 preceding test occurrences; three tests were added. +- Formatting, warning-denied workspace/all-target Clippy, Rust 1.88.0 + workspace/all-target locked checks, and offline dependency policy passed. +- All 217 ordered production string literals across 13 storage modules remain + unchanged, including 125 SQL-bearing literals. Twelve storage production + bodies are byte-identical; only the intended hosted-account pair decoder + changes. Twenty-five of the 26 protected core files are byte-identical; + Space authorization only destructures that pair at its existing lookup. + Dependencies, rate policy, signing, CAR, and private filesystem code are intact. +- Fifteen Rust paths changed, including mechanical fixture updates; the other + 102 Rust files are byte-identical. No Rust file was added or removed. + There are 117 Rust files and 22,995 test lines. +- README and all seven documents under `docs/`: 106 local links and 20 shell + blocks validated. The source guide explains typed singleton responses, hosted + account pairs, the split invariant, and the unchanged outbound policy. + +Build checks use the same non-incremental, debug-info-disabled settings. These +are local macOS results, not Linux/non-Unix CI, deployment, publication, or an +authored commit. The full 2,000-line goal remains incomplete. + +## Thirty-seventh refactor slice and completion + +The formatted working tree measures **13,395 production lines**, a net reduction +of **2,000** from the original baseline, including **15** removed in this slice. +Nonblank/noncomment production lines are **11,716**, down **1,926** from baseline. +All helpers, imports, documentation, and test-module separators remain counted. +The production-line target is met exactly. + +- Carry native commit metadata from the service: **11 net lines**. + ATrium's `CommitMetaData` replaces the string-only `CommitView` model and the + HTTP reparsing helper. The original generated `Tid` validation remains before + staging; unsigned/signed commit values borrow that same revision while signing. + The service returns it with the native root CID only after the unchanged + durable commit and awaited event publication. No new fallible work follows + publication. Raw `RepoHead` revision text and `sync_head` remain unchanged; + latest-commit HTTP still performs its stored-revision validation after account, + head, and SQL decoding. Single-write output still validates record CID first. + Batch output conversion is now infallible because its remaining operations + already were. Nested metadata still becomes an empty-extension ATrium object + at the original projection point; latest-commit emits its data directly with + the same cid-then-rev string fields. This intentionally changes public Rust + service return types, field types, handler return type, and derived Debug, + without compatibility aliases. The native revision String now lives through + the remaining staging/persistence/publication work instead of dropping after + signing; redundant CID formatting and metadata parsing allocations are removed. + The new trailing signing-test module's separator is included in the net count. +- Use the existing CAR section writer directly: **4 net lines**. + Three callers pass their CID bytes and payload as the existing section API's + slices, removing the private `append_block` forwarding wrapper. CID encoding + remains at the same calls, and section ordering, varint framing, whole-section + bounds, failure messages, and unchanged buffers on error are preserved. + The section writer itself is byte-identical. + +Four new tests cover 18 owned/borrowed unsigned and signed JSON/DAG-CBOR/CID +vectors; 36 actual latest-commit handler comparisons of full status, headers, +bytes, and pre-masking errors; 81 CAR CID/payload/existing-prefix vectors; and +exact 64 MiB acceptance plus one-byte-overflow rejection with an unchanged +failed buffer. The HTTP matrix includes malformed SQL CID/revision values, +inactive/missing accounts and missing heads, and explicitly proves raw malformed +revision text still passes `sync_head` before failing at the HTTP boundary. +Historical synthetic invalid commit-string fixtures now check parser rejection +before native metadata can exist; they are not claimed as reachable write-service +failure coverage. The actual record-CID failure and all successful generated +schema, publication, stream, and recovery assertions remain. One missed test-only +String/Tid comparison was corrected before the final passing suite. + +Final-checkpoint verification (2026-09-06): + +- All **429 locked workspace/all-target tests passed**, none ignored or filtered + out, including socket/process checks. Name and path/name multisets retain all + 242 original and 425 preceding test occurrences; four tests were added. +- Formatting, warning-denied workspace/all-target Clippy, Rust 1.88.0 + workspace/all-target locked checks, and offline dependency policy passed. +- All 13 storage production bodies are byte-identical to the preceding + checkpoint, preserving 217 ordered literals, including 125 SQL-bearing ones. + Twenty-four of the 26 protected core files are byte-identical; the two intended + signing-representation and Space CAR adapter changes passed independent review. +- Ten Rust paths changed, including mechanical fixture updates; the other 107 + Rust files are byte-identical. No Rust file was added or removed in this slice. + There are 117 Rust files and 23,348 test lines. +- README and all seven documents under `docs/`: 106 local links and 20 shell + blocks validated. The source guide explains native metadata, its validation + boundaries, and direct CAR section assembly. + +The final independent baseline audit reproduced 15,395 production lines across +114 Rust paths. The only three added Rust paths are the counted request tuple +aliases, counted database query helpers/macro, and dedicated query tests; no +baseline Rust path was removed. Test boundaries are trailing modules, and no +production logic was hidden in tests, assets, build scripts, or ignored files. +Both rate-limit implementations remain byte-identical to the original baseline. +All 24 public and 26 Space routes retain method, order, body-limit and admission +layer arguments. Authentication, error sanitization, and security changes were +reviewed against their documented equivalence tests and preserved policy order. + +The goal is complete under the fixed counting method and documented verification +gates. These are local macOS results, not Linux/non-Unix CI, an all-feature formal +proof, deployment, publication, or an authored commit. Changes remain in the +working copy; Jujutsu snapshots only make the measurements reproducible. diff --git a/docs/repository-guide.md b/docs/repository-guide.md new file mode 100644 index 0000000..ec81da7 --- /dev/null +++ b/docs/repository-guide.md @@ -0,0 +1,820 @@ +# Understanding CPDS + +CPDS is one person's data server. An application asks it to store a record, CPDS +checks permission, and a successful write becomes durable, signed state that +another application can read or synchronize. You do not need to understand the +cryptography or every Rust feature before following that request. + +Read this guide in order the first time. Then use the source map to jump to the +question you are working on. The examples describe the current implementation, +not a promise that CPDS implements the complete AT Protocol PDS specification. +Operating commands are in [README](../README.md); design tradeoffs are in +[architecture.md](architecture.md); missing production features are tracked in +[production-plan.md](production-plan.md). + +## 1. The vocabulary you will see + +Use this as a lookup table; you do not need to memorize it before reading code. + +| Term | Meaning in this repository | +| --- | --- | +| PDS | Personal Data Server: the process hosting an identity and its data. | +| DID | The identity identifier, such as `did:web:alice.test`. It is not a password. | +| Handle | A human-readable name such as `alice.test`, resolved to a DID. Its current validity is separate from account status. | +| Record | A structured value, usually shown as JSON, stored under a collection and record key. | +| Collection / NSID | The record's type namespace, for example `app.bsky.feed.post`. A record's `$type` must match it. | +| Record key / rkey | A record's name inside a collection. Generated keys use TIDs; explicit keys need not be timestamps. | +| AT URI | A record address: `at:////`. It identifies a location, not a particular version of its contents. | +| CID | A content identifier containing a format and cryptographic hash. Changing the encoded bytes changes their CID. A CID is not an access token. | +| DAG-CBOR | The deterministic binary representation used for records and signed structures. JSON `$link` and `$bytes` forms become native binary links and bytes. | +| MST | Merkle Search Tree: ATrium's content-addressed tree mapping record paths to record CIDs. Its root summarizes the public repository state. | +| Commit / head | A signed description of a repository revision / the currently selected commit. It is unrelated to a Git or Jujutsu commit. | +| Revision / TID | A sortable timestamp identifier. CPDS keeps new public repository revisions above the persisted previous revision, including after restart. | +| CAR | Content Addressed aRchive: roots plus CID-labelled blocks, used to transfer repository data and proofs. | +| XRPC / Lexicon | AT Protocol's HTTP method surface / the schemas naming its inputs, outputs, and errors. ATrium supplies many generated Rust models. | +| Cursor | A continuation position for a list or stream. A record key, a repository-list cursor, and a firehose sequence are not interchangeable. | +| Outbox | SQLite rows recording committed public sync events so clients can replay after a disconnect or restart. | + +A useful distinction is **address versus content versus permission**. An AT URI +names a record; a CID names particular encoded content; a token authorizes an +action. None substitutes for either of the others. + +## 2. The source map + +There are two Cargo packages. The root `cpds` package builds both a library +(`src/lib.rs`) and the executable (`src/main.rs`). `cpds-protocol` is a library +under `crates/`. They share a lockfile and workspace lint policy. + +This map is an index, not a list of files to read at once. Start with `main.rs` +and `AppState::assemble`, then follow one request through the files it actually +calls. Leave the remaining features until that path makes sense. + +| Question | Start here | +| --- | --- | +| What happens when I run it? | [`src/main.rs`](../src/main.rs): command selection, startup gates, listener. | +| Where do settings come from? | [`src/config.rs`](../src/config.rs) and [`config/origin.rs`](../src/config/origin.rs): defaults, production requirements, URL policy; [`config/relay.rs`](../src/config/relay.rs) retains origin parity tests. | +| Who constructs the services? | [`src/lib.rs`](../src/lib.rs): `AppState::assemble`. | +| Which URL calls which function? | [`http/mod.rs`](../src/http/mod.rs) and [`space/routes.rs`](../src/space/routes.rs): registration; [`space/http.rs`](../src/space/http.rs) groups the feature's HTTP adapters. | +| How is a request checked or logged? | [`http/auth.rs`](../src/http/auth.rs), [`http/error.rs`](../src/http/error.rs), [`http/security.rs`](../src/http/security.rs), [`http/observability.rs`](../src/http/observability.rs), [`http/rate_limit.rs`](../src/http/rate_limit.rs), [`rate_limits.rs`](../src/rate_limits.rs). | +| What are the values passed between layers? | [`domain/`](../src/domain/mod.rs): account, record, blob, OAuth, identity, and event types; [`syntax.rs`](../src/syntax.rs): ATrium-backed syntax checks. | +| What makes record bytes valid and canonical? | [`cpds-protocol/src/lib.rs`](../crates/cpds-protocol/src/lib.rs). | +| How does a public record change? | [`repository/records.rs`](../src/repository/records.rs): reads and write planning; [`repository/commit.rs`](../src/repository/commit.rs): staged MST changes and signed commit. | +| Where are account status and public discovery? | [`repository/accounts.rs`](../src/repository/accounts.rs), [`http/server.rs`](../src/http/server.rs). | +| How do exports and events work? | [`repository/export.rs`](../src/repository/export.rs), [`repository/car.rs`](../src/repository/car.rs), [`repository/stream.rs`](../src/repository/stream.rs), [`http/subscription.rs`](../src/http/subscription.rs). | +| Where do blob bytes and metadata go? | [`repository/blobs.rs`](../src/repository/blobs.rs), [`blob_store/`](../src/blob_store/mod.rs), [`database/blob.rs`](../src/database/blob.rs). | +| Who owns SQL and migrations? | [`database/`](../src/database/mod.rs); record commit transactions in [`database/repo.rs`](../src/database/repo.rs), the ATrium block overlay in [`database/blockstore.rs`](../src/database/blockstore.rs), schema history in [`database/schema.rs`](../src/database/schema.rs). | +| How does login work? | [`oauth/`](../src/oauth/mod.rs), [`http/oauth.rs`](../src/http/oauth.rs), [`database/oauth.rs`](../src/database/oauth.rs). | +| How is identity resolved? | [`identity.rs`](../src/identity.rs): resolution; [`identity/http.rs`](../src/identity/http.rs): shared outbound transport policy; [`domain/identity.rs`](../src/domain/identity.rs): document rules. | +| What belongs only to Spaces? | [`space/mod.rs`](../src/space/mod.rs): lifecycle, membership, and notification metadata; [`space/authorization.rs`](../src/space/authorization.rs): grants/policy; [`space/records.rs`](../src/space/records.rs): record/recovery service; SQL in [`space/database/repository.rs`](../src/space/database/repository.rs), [`metadata.rs`](../src/space/database/metadata.rs), and [`auth.rs`](../src/space/database/auth.rs). | +| What checks and cleans storage? | [`repository/maintenance.rs`](../src/repository/maintenance.rs), [`http/runtime.rs`](../src/http/runtime.rs). | +| How do backup and operator actions work? | [`backup.rs`](../src/backup.rs), [`private_fs.rs`](../src/private_fs.rs), [`relay.rs`](../src/relay.rs), and the command branches in `main.rs`. | +| What remains outside those features? | [`clock.rs`](../src/clock.rs): monotonic TIDs; [`events.rs`](../src/events.rs): optional in-process observers; [`error.rs`](../src/error.rs): shared error vocabulary. | + +Several files contain `impl RepositoryService`. They add methods to the **same +type**, not separate services. [`repository/mod.rs`](../src/repository/mod.rs) +declares those modules and constructs that type. Likewise, SQL methods can live +in feature files while operating on the same `Database` connection. + +[`database/query.rs`](../src/database/query.rs) delegates fallible row iteration +to rusqlite, while `Database::query_one` uses its single-row decoder. Optional +lookups keep missing rows distinct from decoding failures. Raw tuples use +rusqlite's `TryFrom<&Row>`; the small `sql_row!` constructor handles explicitly +indexed struct fields. Both read SQL fields in order before domain validation. +Constraint mapping uses rusqlite's own error-code accessor: only a top-level +SQLite constraint becomes a conflict; conversion errors retain their original +database error and cause, even when that cause contains a constraint error. +Stored record and blob CIDs then use the native CID parser through the shared +error-context adapter; record JSON parsing still follows CID parsing. +Repository block lookup uses the same optional-query helper, then verifies the +content address after releasing the connection lock. +Block addressing checks the repository codec and hash algorithm before using +the protocol package's shared SHA-256 constructor; rejected writes never enter +the transaction-local overlay. +Outbox sequence extrema use the single-row helper with their existing aggregate +queries; an empty outbox still yields zero for the latest sequence and no earliest +sequence. +Outbox pages finish reading all SQL rows before decoding payloads, whereas +incremental block exports validate as they stream. `Database::transaction` commits only when its +callback succeeds; an error leaves the transaction to roll back. The actual SQL, validation order, +and feature-specific rules remain in the persistence modules. +Schema upgrades follow an ordered list of version thresholds, SQL batches, and +optional immediate blob-index backfills. Each step stays inside the existing +upgrade transaction; a failed backfill prevents later migrations and rolls back +earlier changes. The version-9 outbox repair remains in that same sequence. +All three database open methods share one connection constructor. It passes the +existing initialization flag to the schema implementation: normal and in-memory +opens can initialize storage; an existing-only open cannot initialize an empty +file and still uses read/write-only SQLite open flags. + +Space HTTP has the same organization principle: [`credentials`](../src/space/http/credentials.rs), +[`management`](../src/space/http/management.rs), [`records`](../src/space/http/records.rs), +[`sync`](../src/space/http/sync.rs), and [`notifications`](../src/space/http/notifications.rs) +keep related wire types beside their handlers. The small facade reexports the +route functions; it does not add another service layer. +One-string output objects use the existing JSON value representation. Their +typed list containers retain field order and optional-cursor omission; inputs +and nested structured outputs remain typed. Constructing these tiny JSON objects +adds a map/key allocation and string clone, but preserves exact response bytes. + +[`http/request.rs`](../src/http/request.rs) names recurring native Axum extractor +tuples. A handler still destructures its state, headers, URI, and parsed payload; +the alias only shares the types. Extraction order and handler authentication +remain explicit, with no custom extraction implementation. + +Public record reads share one generated ATrium output because both single and +list reads always include a CID. Latest-commit responses reuse the same generated +commit metadata as write responses. Wire fields and validation order remain the +same even where these internal Rust return types are shared. +`Error::xrpc` constructs named protocol errors for both public and Space paths; +it replaces duplicate local constructors without changing their names or +messages. Authentication errors remain a separate variant, and each caller +retains the condition and evaluation order that chooses an error. +Numeric `Retry-After` headers use the HTTP library's integer conversion, preserving +the existing decimal header bytes and response without a string-parse round trip. +Framework-error normalization checks the content-type prefix directly in its +short-circuit guard; successful and already-JSON responses remain untouched. +Outgoing responses use generated `OutputData` directly when an outer ATrium +`Object` would contain no extension fields. Nested objects and extension data +remain in their original representations. +The protocol encoder selects present `$link` and `$bytes` values before invoking +their decoders; those helpers return a decoded value, not another optional layer. +Type and blob checks still precede special forms, and special forms precede +ordinary child traversal. Blob validation borrows the JSON map through Serde's +native deserializer instead of cloning the entire object. +Repository listings retain the already-decoded `ListedRepoHead` rows until HTTP +assembles the generated response; only the existing public fields are emitted. +The collection-filtered listing validates its collection before sharing the +same cursor parsing, head query, and next-cursor construction as the full list. +Missing-blob inventories likewise retain their existing CID/record-URI pairs. +Neither path needs a second summary representation or another mapping pass. + +When representations are identical, layers share them: Space record, operation, +blob-list, and space-discovery queries reach their services directly, as do Space +creation, update, and notification-registration inputs. Member and writer listings share +the concrete `SpacePageQuery` while keeping their separate authorization rules. +Notification registration and removal pass that same input through persistence; +credential checks still precede the unchanged SQL and expiry handling. +That query reaches both database listings directly. The storage boundary applies +the existing default/clamp and converts its bounded limit to SQLite's `i64`; +SQL ordering and row decoding remain unchanged. +Single-record reads and deletes also share `RecordQuery` with their services; +query and JSON parsing retain the same four string fields and rejection rules. +Reads pass that query through persistence after authorization, collection/key +validation, and repository existence checks. `SpaceRef` uses native `FromStr` +parsing at the existing validation points and preserves the original URI bytes, +so the lookup needs no reconstructed URI or second field mapping. +Single deletes also pass this query through after ownership, write policy, and +key validation; TID generation still precedes the unchanged delete transaction. +Blob inventory and Space discovery likewise pass their query values through; +persistence applies their existing endpoint defaults and bounds before querying. +The unbounded record inventory used by recovery remains a separate path. +The parsed writer notice also serializes directly to the existing four-field +notification JSON, after its hash is decoded and the writer is recorded. +Writer listings share `SpaceWriter`: persistence keeps raw bytes, and HTTP +projects the hash to `LexBytes` after selecting the cursor. This keeps the same +eager encoding and JSON field order without a second three-field response model. +Latest-commit and batch-write responses use native single-key maps containing +typed `SignedCommit` and `AppliedWrite` values. This removes private wrapper +types at the cost of one outer map allocation, without converting nested values +to generic JSON or changing their field order. +Space record listings likewise build their four wire fields directly in the +existing post-cursor mapping, preserving the exclude-values behavior. +`RepoOpsPage` also serves +as the operation-list response; HTTP still applies exclude-values before +serializing it. Public and Space record views use +`RecordView`. Space HTTP and service writes share `RecordInput`: +HTTP owns its strings and optional key, while services borrow text and require +the key for put/update. The record value remains owned in both cases. There is +no second six-field input model. Authorization and URI rules remain separate. +Public record reads compare native CIDs after full lookup and decoding, then +construct the same `RecordView` only when the optional CID filter matches. +The database's `find_record` returns an optional record directly. Write planning +keeps absence as data; public get/delete require a record and return `NotFound`. +Both paths still receive database and decoding failures unchanged. +`find_blob` likewise preserves absence as an optional metadata result. Reference +validation and byte reads turn absence into their distinct existing public errors; +availability/deletion filters and stored-metadata decoding remain in persistence. +Space and operation serializers explicitly omit internal metadata. +`LifecycleEvent` in `database/repo.rs` constructs and inserts identity/account outbox payloads; +the calling transaction still owns state changes, retention, and rollback. +That same typed next event selects the account handle/status compare-and-set. +The expected value is persisted text; the next status remains an `AccountStatus`. +A no-op or conflict returns before commit, so trigger side effects roll back too. +The service uses the same event to share bounded lifecycle retries. Handle +validation remains before lookup; status idempotence remains before user policy, +and only the matching handle/status conflict is retried. Operator and user +permissions remain distinct, with their existing public return types. +Validated public operations reuse ATrium `RepoOpData` through the event stream; +the durable JSON payload remains a separate representation with its existing +string CIDs and null fields. The commit transaction constructs durable outbox +operations directly alongside borrowed records, retaining each SQL insertion's +original position in the transaction. + +`Record` displays only its AT URI, never its value or CID. Views and batch output +use its native `to_string()` directly. Maintenance and legacy +backfill diagnostics use that formatter through lazy error context, so successful +paths do not construct a diagnostic URI. Private snapshot and blob writes use +native file-creation options; their permission, synchronization, and cleanup +ordering remains explicit at the storage boundary. +Private snapshot helpers return their final native I/O result through the existing +error conversion, without reconstructing a redundant success result. +Each synchronous/asynchronous directory-sync function guards its I/O with +`cfg!(unix)`: Unix retains open-then-sync and other platforms return without I/O. +Filesystem opening and publication own their directory preparation directly; +readiness owns its write/sync probe and still awaits cleanup before returning +the probe result. These operations need no single-caller forwarding methods. +Blob deletion and temporary-file cleanup share the same unlink-then-parent-sync +operation. Cleanup is still awaited before choosing which error to return. +Temporary-name recognition chains UTF-8 conversion and the last pending-marker +split before checking the readiness/CID prefix and sixteen ASCII hex bytes. +Generated blob paths always contain an owner hash and CID beneath the root, +so their parent is nonempty even when the configured root is relative. + +`clock::format_utc_datetime` shares the existing whole-second timestamp text for +repository events and Space notice expiry. Callers retain their distinct range +error context and any subsequent Lexicon validation. Notification registration +formats the expiry directly after persistence, without a forwarding wrapper. +`clock::now` supplies whole Unix seconds to persistence, Space credentials, +registration, backup, and blob collection. Reads remain at their original call +sites; the helper does not cache time. OAuth's duration-before-timestamp arithmetic +and the TID clock's nanosecond reading remain separate. +The in-memory blob adapter +uses its existing derived `Default` directly; no separate constructor is needed. +TID encoding collects the thirteen alphabet digits directly from the unchanged +64-bit clock value; clock advancement, saturation, and decoding remain separate. + +Syntax validators share a small adapter around ATrium's native `FromStr` parsers. +Each caller keeps its exact error label; borrowed validators return the original +text. Datetime minimum checks and language duplicate checks remain separate. +Space service-endpoint resolution keeps the first parsed DID for lookup instead +of parsing it again; local comparisons still use its unchanged original text. +Space references check their prefix and four-segment shape before validating +the DID, collection, and record key in that order. +Language validation parses with ATrium first, then classifies subtag byte shapes +for CPDS's existing duplicate checks. Singleton extensions and longer variants +cannot collide in the shared set of case-folded subtags. +Identity resolution concatenates binary DNS TXT chunks before parsing the DID. +Local document validation and network handle verification share the exact, +case-sensitive `at://{handle}` alias-membership check. This predicate does not +parse or normalize the handle; each caller retains its earlier checks and errors. +`did:web` URL construction borrows unescaped segments and owns only decoded +ones; every segment is still decoded before path and outbound-URL validation. +Splitting on the colon always produces at least one segment, so the decoded +host and remaining path can be borrowed directly; empty segments still fail +the subsequent validation. Outbound URL policy matches host kind, deployment +mode, and scheme together, retaining HTTPS domains and local-only loopback IPs. + +Space CAR sections use `unsigned-varint`, already present through ATrium, for +length-prefix encoding. CPDS still owns the whole-section size check before any +output mutation. Headers and CID-prefixed blocks pass their byte slices directly +to the same section writer, without a separate forwarding block helper. +A stable native tuple-key sort orders record paths by byte +length, then UTF-8 bytes, retaining the input order of duplicate paths. +Public and relay configuration use the same validated `HttpOrigin` value. +Its `parse_public` and config-private `parse_relay` constructors retain their +setting-specific errors, component-check order, and production HTTPS rules. +Display, hostname formatting, and endpoint construction share the same URL +representation; endpoint construction still clones the URL and sets its path. +This intentionally replaces the former public `PublicOrigin` and `RelayOrigin` +Rust types and their derived Debug names, without compatibility aliases. +The `Settings` reader also owns relay, account, and OAuth-client parsing. These +methods use the same source and deployment mode; configuration still reads +settings in field order and stops at the first failure. +Deployment mode is selected at that boundary: absent or exact `local` uses local +mode, exact `production` enables production checks, and all other strings fail +before any further setting is read. + +### Why only one extracted library? + +Both public repositories and Spaces need exactly the same record encoding, but +neither should import the other feature's service implementation. The protocol +package expresses that shared rule and tests it without assembling CPDS. +Its byte-addressing helpers, `sha256_cid` and `dag_cbor_cid`, return a CID +directly: SHA-256 always supplies 32 bytes to a 64-byte-capacity multihash. +`record_cid` remains fallible because it validates and encodes the record and +enforces the size limit before hashing. + +Storage and HTTP are not separate packages because they are not independent +products here. Most organization needs a module, not a new public API or trait. +Keep real change points—blob storage, OAuth client lookup, event delivery, +identity resolution—explicit; do not invent a generic repository framework to +make unrelated protocols look identical. ATrium already supplies standard +protocol machinery, so CPDS composes it instead of keeping a second MST or +signature implementation. + +## 3. Follow startup before following a request + +Read `main`, then `AppState::assemble`. `main` passes the operating-system +argument iterator directly to `operation_from`; there is no separate +command-parsing wrapper. Normal startup: + +1. Initializes logging and parses the command and configuration. +2. Opens or creates SQLite and applies migrations. Opens the filesystem blob + store. `CPDS_DATABASE` defaults to `cpds.sqlite3`; bytes default to `blobs/`. + There is no database server to install and no separate migration command. + Schema version 9 preserves older records and event sequence numbers while + repairing legacy upgrades and indexing OAuth credential expiry cleanup. +3. Assembles one configured account, the signing services, identity resolver, + OAuth registry, adapters, and request/stream limits. +4. In production, verifies the configured handle and DID in both directions. + Local startup uses its derived development identity without network lookup. +5. Checks adapters, collects eligible orphan blobs, verifies durable public + repository state, reconciles the configured handle, and collects eligible + old repository blocks. +6. Binds the TCP listener, logs `cpds is listening`, and starts serving with + periodic maintenance. Ctrl-C/SIGTERM triggers bounded draining and a WAL + checkpoint. + +That is why a process can be doing startup work before it prints the listening +message. `/health` means HTTP is alive; `/ready` additionally checks dependencies +and the durable event store. Neither repeats the entire signed-graph proof on +each request. +The final database readiness probe uses rusqlite's native `query_row_and_then` +with `SELECT 1`, retaining its first-row requirement and database error mapping. +Stream handlers subscribe directly to the existing Tokio watch sender. Shutdown +publishes sticky `true` on that sender after the shutdown future resolves and +before request drain; `AppState` adds no forwarding methods around the channel. + +Handle revalidation returns confirmed validity separately from resolver errors. +A transient failure leaves the previous validity flag intact; confirmed changes +drive the existing recovery/invalidation events without changing account status. +The identity service retains its configured deployment mode directly for both +the local-handle shortcut and outbound policy. Concrete DNS and DID resolution +live in its network interface implementation; no forwarding methods are needed. +Identity HTTP dispatch likewise owns its request loop directly inside the +existing timeout, retaining its HTTPS/redirect policy and bounded body reads. + +The other commands deliberately branch away from serving. `verify` proves +existing storage, `backup` captures it, `set-account-status` changes hosting +policy offline, and `request-crawl` asks configured relays to discover the PDS +without opening storage. Do not use a fresh empty database to "repair" an +unexpectedly missing production database. +The command parser combines optional Unicode conversion and status parsing before +producing the same fixed configuration error. Crawl and serve return their native +results directly; their startup checks and logging still happen in the same order. +Relay request futures borrow the client, configured origin, and hostname through +the awaited batch. They need no per-relay clones; retries, logging, bounded delays, +and collecting every failure remain unchanged. + +## 4. Follow one public write + +Use `createRecord` first; study `applyWrites` after that. + +```text +JSON request + bearer token + → HTTP adapter: parse generated input and authenticate + → records: check owner, identifiers, record shape, blob references, swaps + → commit: stage record blocks and ATrium MST changes; sign final head + → SQLite transaction: blocks + head + indexes + operations + sync event + → commit succeeds + → optional EventSink notification; HTTP returns record/commit identifiers +``` + +The HTTP handler is not supposed to decide how to build an MST or write SQL. +The protocol encoder is not supposed to decide which account may write. SQLite +must recheck race-sensitive conditions inside the transaction, not merely trust +an earlier HTTP check. + +`swapRecord` and `swapCommit` mean "perform this only if the version I saw is +still current." They prevent silent lost updates. A generated record key and a +repository revision both use the TID clock, but they serve different purposes. + +`applyWrites` preserves an enum distinguishing create, update, and delete. The +planner walks operations in order, so a later operation sees earlier changes +in the same batch. It then commits the batch once. Failure must leave no partial +record changes, no new head, and no public event advertising uncommitted data. +The committed `RecordMutation` values also supply the HTTP results, so the +planner does not maintain a second result vector alongside its mutations. +Single writes return `(record, commit)` and batches return `(commit, mutations)`. +HTTP names those fields directly without a separate service-result wrapper. +Commit metadata uses ATrium's native `CommitMetaData`: the generated revision +is validated before staging, borrowed during signing, and returned with the +native root CID after durable publication and notification. This intentionally +replaces the string-only `CommitView` Rust type and its Debug representation. +HTTP no longer reparses these already-valid commit values; record CID validation +remains first in single-write output conversion. Stored repository heads still +carry raw revision text, which latest-commit HTTP validates at its existing +boundary. Nested wire objects and the cid-then-rev field order are unchanged. +The selected MST operation is awaited before one shared error conversion; a +failure still returns immediately before the next operation or publication. +The per-key cache is consumed and replaced for each planned operation. A cached +deletion is distinct from a key that has not been read, so later operations +cannot accidentally reload the pre-batch record from SQLite. +Public and Space batches share the owned command enum, exposed as `RecordBatchWrite` +and `WriteRequest` respectively. Public HTTP still converts its generated ATrium +inputs at the same boundary; Space JSON retains its own union tags. Their +authorization, planning, and commit rules remain separate. +Space checks Create/Update variants directly at the batch validation guard; a +single-use enum accessor is unnecessary. The size check remains first, and +strict-validation rejection still precedes per-write authorization. + +There are two related representations of a public record: the convenient SQL +index/value and the canonical content-addressed repository block. They are not +two independent truths. One commit updates both, and startup verification +checks their exact agreement. Historical schema migrations are therefore not +dead code merely because new databases start at the latest schema. +Opening a signed repository takes its root CID and expected revision directly, +without building a temporary database-head value. It still opens the graph, +checks the revision, and then verifies the signature, in that order. +The block overlay eagerly verifies that reachable blocks exist, then lends its +matching staged entries to the event-CID collector. That caller copies only CIDs, +not block bytes. The separate staged-write handoff and its ordering are unchanged. + +## 5. Follow reads, blobs, and exports + +An ordinary `getRecord` resolves a hosted active repository, validates the path, +and reads its current SQL record. An optional requested CID must match that +record. `listRecords` applies its own bounded ordering and cursor rules; do not +reuse another endpoint's cursor parser just because both cursors are strings. +Its service returns the records/cursor pair directly, matching the other listing +methods. HTTP converts each record before assembling the unchanged response; +there is no second page wrapper. + +A blob is a separate byte object, not a large JSON record: + +1. Upload validates size and MIME syntax, computes its raw-byte CID, and reserves + owner metadata in SQLite. +2. The blob adapter publishes the bytes; the filesystem adapter verifies their + CID and durably renames a private temporary file. +3. SQLite marks the upload available. A record can now reference the owner's + matching CID, MIME type, and size. +4. Public `getBlob` serves it only while a current public record references it. +A Space-only reference does not make it public. + +Strict write validation and tolerant reference discovery share only iteration +over JSON children. A valid typed blob ends strict traversal at that object; +discovery still visits its children, and their legacy/error rules stay separate. +ATrium decodes each blob from a borrowed JSON value, avoiding an initial deep +clone while still returning owned metadata. +Strict recursive traversal uses native fallible iteration, preserving child order, +partial collection, and the first error; it does not scan past a failed child. + +Optional blob cursors use native `Option`/`Result` composition: absent cursors +remain absent, parsed CIDs are canonicalized before SQL, and each inventory +retains its own malformed-cursor error context. + +Deleting a reference does not immediately delete the file. The collector checks +both public and Space reference indexes, waits through the configured grace +period, durably claims an orphan, and deletes its bytes. Claims make interrupted +cleanup resumable and prevent a new record from attaching to a claimed object. + +Exports answer different questions. `getRepo` returns a full or retained-history +incremental CAR; sync `getRecord` returns a signed proof for one record; +`getBlocks` returns requested blocks only if reachable from that repository. +They do not expose every block that happens to exist in the shared database. +ATrium reopens these locally generated archives in tests; there is no public +remote-CAR import endpoint. +The same crate-internal `sync_head` supplies exports and the latest-commit HTTP +handler. HTTP projects its CID/revision into the wire converter directly; there +is no separate service method that merely wraps that head. + +## 6. Follow OAuth without confusing its tokens + +Read [`tests/oauth_flow.rs`](../tests/oauth_flow.rs) alongside the service. The +test is a small executable client, including deliberate failure and replay. + +1. An app supplies its registered client ID, exact redirect URI, scope, state, + and S256 PKCE challenge. The challenge is a hash of a secret verifier the app + keeps for the next step. +2. The account approves with its password. CPDS checks the password using Argon2 + and returns a five-minute, one-time authorization code to that redirect. +3. The app exchanges the code with its verifier and the same client/redirect. + SQLite inserts a hashed access token and consumes the code atomically. +4. The app uses the returned one-hour opaque bearer token. CPDS hashes the + presented token, finds its account/client/scope, and applies endpoint policy. + +The authorization page escapes attributes with native string replacements, +replacing ampersands first so generated entities are not escaped a second time. +Its HTML and attribute bytes remain unchanged. +Authorization returns the code and redirect URL as a pair. HTTP retains both +through response construction and uses the URL for the unchanged redirect header. +PKCE validation returns its native result after all earlier request checks. +Registered redirect matching uses the URL list's native membership operation; +URL equality and client-before-redirect-before-scope-before-PKCE error order stay +unchanged. +Discovery metadata retains owned native URLs through response serialization. +URL serialization supplies the same canonical strings without converting the +already-owned endpoint URLs into additional String allocations. + +The account password, authorization code, and access token are different +credentials. `state` is returned for the client to check; it is not a substitute +for PKCE. A failed token insert must leave the code usable for a retry, while a +successful exchange must make replay fail. Consumed codes are deleted in that +same transaction; expired credentials are rejected at the expiration second and +cleaned up during issuance/redemption. The unit fault-injection test in +`oauth/mod.rs` checks that transaction boundary directly. +Code redemption decodes all five stored string columns before checking request +bindings, then returns only the DID and scope its service needs. This keeps +SQL-type failures ahead of binding mismatches without another stored-code model. + +The authorization page and approval form share `AuthorizationRequest`; their +endpoint-specific fields and validation remain in HTTP. Parser failures still +pass through the common public-error normalizer. The authorization handler checks +response type, then challenge method, before client/redirect/PKCE validation; +no single-use conversion wrapper is needed. The token endpoint passes its +parsed `TokenForm` directly to the service: HTTP rejects unsupported grant types +first, and the service then checks PKCE before generating credentials or redeeming +the code. The issued `TokenGrant` is also +the token response, with its bearer type and the DID serialized as `sub`. + +Authorization-code persistence also consumes the existing request. Its defaulted +URI parameter keeps `AuthorizationRequest` URL-validated in HTTP and the service, +while raw string URIs remain representable for the database boundary. Client and +redirect text are borrowed before acquiring the transaction; `state` is not stored. +PKCE challenge derivation owns its verifier guard and reuses OAuth's existing +SHA-256/base64url hash. Length and character rules still run before hashing. + +This is the implemented OAuth core, not the complete AT Protocol OAuth profile. +PAR, OAuth DPoP, refresh/revocation lifecycle, and dynamic profile-complete client +registration must not be inferred from the presence of `/oauth/token`. + +## 7. Add Spaces only after public records make sense + +A Space identifies an authority, a type, and a key. Within it, each writer has +their own repository. Space records live in separate tables from public records; +the public record endpoints and public firehose do not reveal them. + +The domain-owned `RecordQuery` carries the same four raw key strings through +HTTP parsing, single deletes, batch-delete planning, and the SQL delete helper. +It removes repeated unpacking without validating earlier or changing ownership. +Batch deletion still rejects a missing record and rolls back earlier writes; +single deletion remains idempotent. The batch result still emits only its delete +type tag, never the internal key fields. + +OAuth Space grants constrain authority/type, collection, read/write action, and +management capability. The read authorization enum distinguishes an OAuth grant +from a credential for one Space. A public CID does not bypass these checks. +HTTP owns this enum and services borrow it directly, without a second borrowed +representation. Each read still performs its own checks, including snapshot +authorization when operation listing reaches that boundary. +Record capabilities carry their required collection alongside the action; +read and management capabilities do not take an irrelevant optional collection. +Policy field reads use immutable JSON indexing followed by the expected type +check, so absent fields and wrong types still follow the existing rejection path. +Space metadata reads use that same enum while preserving each credential's own +checks; a credential read does not inherit OAuth scope or membership rules. +Blob reads consult the selected Space repository's reference index after +authorization, so they are not limited to scanning the first page of records. + +Delegation tokens are exchanged once for Space credentials bound to a DPoP +public key. DPoP is a signed proof of possession: a request proof binds its +method and URL, and `ath` binds it to the presented credential. Replay storage +rejects reused proofs. Client attestations, managing-app checks, and service +notification tokens are additional boundaries, not interchangeable credentials. +Credential verification returns the exact verified Space string directly; +HTTP applies its Space rate check before passing that scope to the service. +Authorization-header parsing is one native optional-value pipeline: first +header, valid text, exact scheme, one ASCII space, then nonempty payload. It +neither trims tokens nor falls back to a later header, and preserves the borrowed +payload or the same authentication error. +Delegation exchange returns its validated `SpaceClaims`. HTTP immediately takes +the issuer and subject from those claims, without a second delegation model; +unused claim fields are dropped before any subsequent asynchronous operation. +All three minted token kinds use one signer that builds the ES256K header from +the token type and optional key ID. Each caller still creates its own claims, +with the same time reads and random identifier generation before serialization. +JWT decoding borrows the original `header.payload` text for signature checking. +It checks the three-segment shape before decoding header, claims, and signature +in that order; no reconstructed signing-input string is needed. +Service-token verification returns the issuer/audience pair directly. HTTP names +both values and checks the issuer's service rate limit before notification policy. +Credential issuance reuses the existing public/member read-policy decision after +checking application access and the owner's bypass. Managing-app delegation and +its configuration errors remain separate, and only non-owner member-list policy +consults the membership table on this path. +That lookup uses SQLite's native first-row existence check, with the same query +and connection lock rather than an optional unit-valued row. + +Public and Space commits are deliberately different. Public repositories use +signed MST roots; the alpha Space format combines an order-independent set +hash (LtHash), fresh binding material, a signature, and a MAC. A Space CAR has a +signed view and record index as two roots. Record values still use the same +canonical codec as public records. One read transaction obtains a matching +revision and record snapshot, with a 64 MiB stored-JSON bound before loading +records and a separate 64 MiB encoded CAR bound. Metadata-only recovery omits +values, not permission checks. + +`SpaceService` holds ATrium's `Secp256k1Keypair` directly. `commit::sign` composes +the Space-specific set hash, fresh randomness, context, signature, HKDF, and MAC; +there is no second key-holder wrapper. Key import still occurs during service +construction with its Space-specific configuration error context. +Recovery and commit-only callers share `signed_snapshot`; callers needing only +the signed commit select it from the returned commit/records pair. Even those +callers still authorize, load the bounded snapshot, and sign it before discarding +the records. The commit-only HTTP wire wrapper remains unchanged. +The recovery handler passes that pair directly to the existing CAR serializer, +without another service forwarding method. + +Space record mutation helpers recheck account activity inside their transaction, +before reading or changing records. Batch writes call those helpers in order, +so each item observes the current transactional account state and any failure +rolls back the complete batch. Strict batch deletes and idempotent single deletes +retain their separate missing-record behavior. + +Space writer notifications and the public durable firehose are also different. +Space metadata persistence receives the existing update/notice inputs after +service validation. It stores only the notice's space, repository, revision, +and hash; authentication remains the service's responsibility. At the storage +boundary, `None` means no change and `Some(Value::Null)` stores JSON null. +Writer listings and notification registration share the credential-match, +local-authority, and stored-Space checks. Registration and unregistration first +check the hosted account's active status; writer listing keeps its existing +policy without that account-status check. +Stored notification destinations travel as `(service, endpoint)` pairs. The +database decodes both strings in that order, and delivery destructures the pair +into named references without another registration output model. +Space notification delivery is best-effort with one two-second budget per batch +and at most four concurrent recipients. Incoming deletion notices do +not remove a retained local replica because CPDS does not keep one. Read the +alpha feature as its own bounded protocol, not as finished general federation. + +## 8. Follow durability and recovery + +A public record transaction appends durable events before acknowledging success. +The initial sequence contains identity, account, and commit events. The optional +`EventSink` runs afterward; failure there is logged, not returned as a failed +write that might tempt a client to create the record again. The whole optional +event batch shares a two-second delivery deadline. These deadlines do not make +notifications durable or eliminate uncertain client outcomes after a disconnect. + +`subscribeRepos` rebuilds ATrium wire messages from that durable history and +sends DAG-CBOR WebSocket frames. The consumer saves its last processed sequence. +If it reconnects within retention, it can replay. If it falls behind retention, +`OutdatedCursor` means it must resynchronize from a repository export. Retained +events constrain garbage collection: blocks needed for replay must remain. +Stream errors select their public name/message before one shared timed send. +Internal failures are still logged and sanitized before encoding; XRPC errors +retain their own names and messages, with the same frame bytes and deadline. +HTTP JSON and stream CBOR share the two-field `ErrorBody` representation. +HTTP still owns its String and streams borrow their message; each keeps its +original serializer, field order, error context, and response/frame construction. + +A backup must preserve the database and blob files as one consistent pair. +CPDS verifies storage, holds an SQLite write barrier, takes a native database +snapshot, copies CID-verified blobs, and writes a non-secret manifest into a +new private directory. The barrier can delay writes; backup is not a free +background copy. The signing key and password configuration need separate +protection. `verify` checks restored storage without opening a listener and +refuses a missing or uninitialized database. + +Account hosting status is separate from identity and storage. Deactivation can +be reversed by the account; the offline operator command controls host states. +`deleted` means access suppression here, not secure erasure. Never describe +that operation as destroying all account data. +`AccountStatus` owns the existing account and repository error mappings and +active-state checks. Callers retain responsibility for looking up the right +account and choosing the appropriate policy before changing or exposing data. +The database returns a hosted identity as `(Account, AccountStatus)`, with named +destructuring in its callers. It reads DID and handle before parsing the stored +status. This replaces the former `HostedAccount` Rust wrapper and its named +`Debug` representation; protocol output and account-policy checks are unchanged. + +## 9. Enough Rust to read this code + +| Syntax or concept | How to read it here | +| --- | --- | +| `struct` | A named set of fields. `Record` holds a DID, collection, key, CID, and value. It need not be a class with inheritance. | +| `enum` / `match` | One of several valid alternatives. `RecordBatchWrite` prevents a delete from accidentally carrying a required create payload. | +| `impl Type` | Functions belonging to that type. Multiple files can contain implementations of the same service. | +| `Result` and `?` | Either a successful value or an error. `?` returns an error to the caller early; it does not swallow it. | +| `Option` | A value may be absent. `None` is not automatically an error: a list can legitimately have no next cursor. | +| `&T` / `&mut T` | Borrow a value for reading / exclusive mutation instead of transferring ownership. | +| `String` / `&str` | Owned text / a borrowed view of text. `.clone()` copies owned data unless the type explicitly shares it. | +| `Arc` | Shared ownership across tasks. Cloning an `Arc` shares the same object; it does not duplicate the database. | +| `Mutex` | Exclusive access to shared mutable state. CPDS's database lock protects one SQLite connection. A lock is not a database transaction. | +| `async`, `.await` | Work that can yield while waiting. Synchronous SQLite, hashing, or CPU work does not become nonblocking just because its caller is async. | +| `trait` / `dyn Trait` | An interface / a runtime-selected implementation. `Arc` lets tests use memory and the executable use files. | +| `impl Serialize` | A concrete but hidden return type supporting serialization. The protocol crate avoids exposing a second public data-model enum. | +| `pub`, `pub(crate)`, private | Visible to consumers, only within this crate, or within the module's privacy boundary. Module splits need not make APIs public. | +| `#[derive(...)]`, Serde | Generated routine behavior such as serialization. Wire annotations still matter: field names and omitted values are protocol contracts. | +| `#[cfg(test)]` | Code compiled for that target's unit tests, not ordinary server execution. Files in `tests/` are separate integration-test crates. | + +Do not read every `unwrap()` the same way. An assertion in a test describes the +expected setup/result; an unchecked operation on an untrusted production value +needs a very different justification. Likewise, converting every struct into a +trait does not automatically make code more testable—real SQLite often proves +transaction behavior better than a mocked database. + +The shared `Error` type receives its HTTP response implementation in +`http/error.rs`; no extra error wrapper is needed. `ErrorContext::context` adds +a diagnostic and chooses the existing error category without losing the +underlying failure text. + +## 10. Learn by running one small test at a time + +Run from the repository root. Tests create isolated memory or temporary storage; +these exercises do not use your running PDS database. Cargo may need to download +locked dependencies on its first build. Read the named test, predict the result, +then run it. The assertion following a call tells you what the author promises. + +### Exercise A: identical records have identical addresses + +Read the protocol fixture tests and look at one JSON/CBOR/CID fixture triple. +Why would encoding `$bytes` as an ordinary string change the CID? + +```sh +cargo test --locked -p cpds-protocol +``` + +### Exercise B: a batch is one transaction + +Find the create/update/delete sequence and the deliberately failing batch in +`record_flow.rs`. Which checks prove the state did not partially change? + +```sh +cargo test --locked -p cpds --test record_flow public_apply_writes_is_atomic_ordered_and_emits_one_commit +``` + +### Exercise C: uploading is not publishing + +Read the public blob test before running it. Predict the response immediately +after upload and after a public record references the returned blob. + +```sh +cargo test --locked -p cpds --test record_flow blobs_become_public_only_when_referenced_by_a_public_record +``` + +### Exercise D: a failed login exchange can be retried safely + +Follow the SQL trigger that rejects token storage. Why must the code remain +unused? Then compare that failure with replay after a successful exchange. + +```sh +cargo test --locked -p cpds --lib failed_token_storage_does_not_consume_the_authorization_code +cargo test --locked -p cpds --test oauth_flow +``` + +### Exercise E: the same interface works with files and memory + +Read `contract` in `blob_store_contract.rs` and both adapter implementations. +Find behavior a filesystem needs that an in-memory map does not. + +```sh +cargo test --locked -p cpds --test blob_store_contract +cargo test --locked -p cpds --test backup_restore +``` + +### Exercise F: permission and encoding both survive a Space export + +Read the selected Space blob-reference test and the two-root CAR test. The +former tests who can read; the latter checks what bytes are being exported. + +```sh +cargo test --locked -p cpds --test space_flow list_blobs_only_exposes_references_in_the_selected_space_repo +cargo test --locked -p cpds --test space_flow get_repo_returns_two_root_verifiable_car_layouts +``` + +### Exercise G: reconnect after missing events + +Finally read the fake-relay scenario. It receives an actual crawl request, +resolves the served identity, repairs an expired cursor, and verifies the next +live commit. This exercise requires loopback networking and child processes. + +```sh +cargo test --locked -p cpds --test subscribe_repos_flow fake_relay_crawls_resolves_identity_repairs_gap_and_resumes_live +``` + +## 11. How to choose a test and make a change + +| What changes? | Evidence to run/read | +| --- | --- | +| Pure record encoding | Protocol unit tests plus pinned fixtures under [`tests/fixtures`](../tests/fixtures/README.md). | +| A service invariant or SQL race | [`repository/tests/`](../src/repository/tests/mod.rs) groups account, blob, export, maintenance, record, and stream cases; SQL tests stay beside their adapters. These use real SQLite, rollback/failure injection, and restart checks. | +| An HTTP contract | `record_flow`, `oauth_flow`, `sync_flow`, `space_flow`, `account_lifecycle`, and `http_*` integration suites through the shared router harness. | +| An adapter | `blob_store_contract`, readiness tests, and the adapter's own tests. | +| Socket/process behavior | `subscribe_repos_flow`, `relay_crawl`, runtime unit tests, and Unix-only `process_lifecycle`. | +| Compatibility with AT Protocol | `atrium_api_interop`, `atrium_repo_interop`, `crypto_interop`, protocol/syntax fixtures, and composed export/stream tests. | + +The [shared harness](../tests/support/mod.rs) does real OAuth and real SQL. It +replaces infrastructure, not the business rules being tested. A green pure test +does not prove a process will bind, a green socket test does not prove real DNS +or TLS deployment, and fixture parity does not prove complete federation. + +For an ordinary change: + +1. Find the route or command, service rule, persistence boundary, and existing + behavioral test. Read only the additional layers needed to explain it. +2. Add a regression for the observable failure before changing behavior. +3. Prefer an existing type, helper, or mature dependency. Put new code where its + reason to change already lives. Extract a module before proposing a crate. +4. Verify failure and success, including rollback, visibility, and replay when + relevant. Remove the superseded implementation rather than keeping two paths. +5. Run the workspace gates and review the diff, keeping unrelated work intact. + +```sh +cargo fmt --all -- --check +cargo clippy --workspace --all-targets --locked -- -D warnings +cargo test --workspace --all-targets --locked +cargo deny --locked check +``` + +`--workspace` matters: plain `cargo test` selects the root package and will not +run the extracted protocol package's tests. The full suite needs permission to +open local sockets and launch processes. CI is configured for Rust 1.88 on +Linux; passing a local run is not evidence that CI, another platform, or a live +deployment has run. + +You have the central model when you can explain **who authorizes a change, what +must commit together, what can be recomputed or replayed, and which test proves +each claim**. The rest of the repository becomes a set of concrete examples of +those four questions, rather than a vocabulary test. diff --git a/scripts/production-lines.rb b/scripts/production-lines.rb new file mode 100644 index 0000000..6d0c788 --- /dev/null +++ b/scripts/production-lines.rb @@ -0,0 +1,44 @@ +#!/usr/bin/env ruby +# Physical Rust lines, with the same production/test boundary as the size audit. +require 'json' +require 'open3' + +revision = ARGV.empty? ? nil : ARGV.fetch(1) +abort 'usage: ruby scripts/production-lines.rb [--revision REV]' unless + ARGV.empty? || (ARGV.length == 2 && ARGV.first == '--revision') + +def command(*arguments) + output, status = Open3.capture2(*arguments) + abort "Command failed: #{arguments.join(' ')}" unless status.success? + output +end + +files = if revision + command('jj', '--ignore-working-copy', 'file', 'list', '-r', revision).lines.map(&:chomp) +else + command('rg', '--files', '--hidden', '-g', '*.rs', '-g', '!target/**', + '-g', '!.git/**', '-g', '!.jj/**').lines.map(&:chomp) +end + +counts = files.grep(/\.rs\z/).sort.to_h do |file| + content = revision ? command('jj', '--ignore-working-copy', 'file', 'show', '-r', revision, file) : File.read(file) + lines = content.lines + test_only = file.split('/').include?('tests') || File.basename(file) == 'tests.rs' + # In this repository cfg(test) sections are trailing items. Keep that + # convention; a production declaration after this boundary would be miscounted. + production = test_only ? [] : lines.take_while { |line| !line.match?(/^#\[cfg\(test\)\]/) } + [file, { + production: production.length, + tests: lines.length - production.length, + nonblank_noncomment_production: production.count { |line| !line.strip.empty? && !line.lstrip.start_with?('//') } + }] +end + +puts JSON.pretty_generate({ + revision: revision || 'working tree', + rust_files: counts.length, + production: counts.values.sum { |count| count[:production] }, + tests: counts.values.sum { |count| count[:tests] }, + nonblank_noncomment_production: counts.values.sum { |count| count[:nonblank_noncomment_production] }, + files: counts +}) diff --git a/src/backup.rs b/src/backup.rs index e922b83..9861941 100644 --- a/src/backup.rs +++ b/src/backup.rs @@ -3,14 +3,14 @@ use std::path::Path; use serde::{Deserialize, Serialize}; -use time::OffsetDateTime; use crate::{ blob_store::FileBlobStore, + clock::now, config::Config, database::Database, error::{Error, Result}, - private_fs, + private_fs::{self, parent_or_current}, }; const DATABASE_FILE: &str = "cpds.sqlite3"; @@ -47,13 +47,13 @@ pub fn create( ) -> Result<()> { reject_blob_descendant(&config.blob_directory, target)?; private_fs::create_directory(target)?; - let result = database + database .backup_with(&config.database_path, target.join(DATABASE_FILE), || { private_fs::secure_file(&target.join(DATABASE_FILE))?; blobs.copy_to(&target.join(BLOB_DIRECTORY))?; let manifest = Manifest { format_version: 1, - created_at: OffsetDateTime::now_utc().unix_timestamp(), + created_at: now(), public_url: config.public_origin.to_string(), account_did: config.account.did.clone(), account_handle: config.account.handle.clone(), @@ -65,12 +65,10 @@ pub fn create( )?; private_fs::sync_directory(target) }) - .and_then(|()| private_fs::sync_directory(parent_or_current(target))); - if let Err(error) = result { - let _ = std::fs::remove_dir_all(target); - return Err(error); - } - Ok(()) + .and_then(|()| private_fs::sync_directory(parent_or_current(target))) + .inspect_err(|_| { + let _ = std::fs::remove_dir_all(target); + }) } /// Verify a colocated backup manifest when the configured database is still in @@ -109,12 +107,6 @@ pub fn verify_manifest(config: &Config) -> Result<()> { Ok(()) } -fn parent_or_current(path: &Path) -> &Path { - path.parent() - .filter(|parent| !parent.as_os_str().is_empty()) - .unwrap_or_else(|| Path::new(".")) -} - fn reject_blob_descendant(blob_root: &Path, target: &Path) -> Result<()> { let blob_root = std::fs::canonicalize(blob_root)?; let parent = parent_or_current(target); diff --git a/src/blob_store/filesystem.rs b/src/blob_store/filesystem.rs index 0e580be..cfc2d01 100644 --- a/src/blob_store/filesystem.rs +++ b/src/blob_store/filesystem.rs @@ -16,7 +16,7 @@ use tokio::io::AsyncWriteExt as _; use crate::{ blob_store::BlobStore, error::{Error, Result}, - private_fs, + private_fs::{self, parent_or_current}, }; const RAW_CODEC: u64 = 0x55; @@ -37,7 +37,10 @@ impl FileBlobStore { /// or synchronized. pub async fn open(root: impl Into) -> Result { let store = Self { root: root.into() }; - store.prepare_root().await?; + if create_private_directory(&store.root).await? { + sync_directory(parent_or_current(&store.root)).await?; + } + sync_directory(&store.root).await?; store.remove_stale_temporary_files().await?; Ok(store) } @@ -115,28 +118,14 @@ impl FileBlobStore { self.root.join(owner_hash).join(cid.to_string()) } - async fn prepare_root(&self) -> Result<()> { - if create_private_directory(&self.root).await? { - sync_directory(parent_or_current(&self.root)).await?; - } - sync_directory(&self.root).await - } - - async fn prepare_owner_directory(&self, path: &Path) -> Result<()> { - let parent = required_parent(path)?; - if create_private_directory(parent).await? { - sync_directory(&self.root).await?; - } - Ok(()) - } - async fn create_temporary(&self, destination: &Path) -> Result<(PathBuf, tokio::fs::File)> { for _ in 0..16 { let suffix: u64 = rand::rng().random(); let path = destination.with_extension(format!("pending-{suffix:016x}")); let mut options = tokio::fs::OpenOptions::new(); options.write(true).create_new(true); - set_private_file_mode(&mut options); + #[cfg(unix)] + options.mode(0o600); match options.open(&path).await { Ok(file) => return Ok((path, file)), Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} @@ -174,14 +163,14 @@ impl FileBlobStore { let owner_path = owner.path(); reject_symlink(&owner_path).await?; - secure_directory(&owner_path).await?; + set_permissions(&owner_path, 0o700).await?; let mut entries = tokio::fs::read_dir(&owner_path).await?; let mut removed = false; while let Some(entry) = entries.next_entry().await? { if !entry.file_type().await?.is_file() { continue; } - secure_file(&entry.path()).await?; + set_permissions(&entry.path(), 0o600).await?; if is_temporary_name(&entry.file_name()) && is_stale(&entry.metadata().await?) { tokio::fs::remove_file(entry.path()).await?; removed = true; @@ -196,8 +185,12 @@ impl FileBlobStore { } Ok(()) } +} - async fn writable_probe(&self) -> Result<()> { +#[async_trait] +impl BlobStore for FileBlobStore { + async fn check(&self) -> Result<()> { + verify_private_directory(&self.root).await?; let destination = self.root.join(".readiness"); let (temporary, mut file) = self.create_temporary(&destination).await?; let probe = async { @@ -207,29 +200,20 @@ impl FileBlobStore { Ok(()) } .await; - let cleanup = remove_temporary(&temporary).await; - match probe { - Err(error) => Err(error), - Ok(()) => cleanup, - } - } -} - -#[async_trait] -impl BlobStore for FileBlobStore { - async fn check(&self) -> Result<()> { - verify_private_directory(&self.root).await?; - self.writable_probe().await + let cleanup = remove_file_durably(&temporary).await.map(|_| ()); + probe.and(cleanup) } async fn put(&self, owner: &str, cid: &Cid, bytes: &[u8]) -> Result<()> { verify_content(cid, bytes)?; let destination = self.path(owner, cid); - self.prepare_owner_directory(&destination).await?; + let parent = parent_or_current(&destination); + if create_private_directory(parent).await? { + sync_directory(&self.root).await?; + } if tokio::fs::try_exists(&destination).await? { Self::read_verified(&destination, cid).await?; - sync_directory(required_parent(&destination)?).await?; - return Ok(()); + return sync_directory(parent_or_current(&destination)).await; } let (temporary, mut file) = self.create_temporary(&destination).await?; @@ -245,15 +229,12 @@ impl BlobStore for FileBlobStore { return Err(rename_error.into()); } } - sync_directory(required_parent(&destination)?).await + sync_directory(parent_or_current(&destination)).await } .await; - let cleanup = remove_temporary(&temporary).await; - match publication { - Err(error) => Err(error), - Ok(()) => cleanup, - } + let cleanup = remove_file_durably(&temporary).await.map(|_| ()); + publication.and(cleanup) } async fn get(&self, owner: &str, cid: &Cid) -> Result>> { @@ -265,15 +246,7 @@ impl BlobStore for FileBlobStore { } async fn delete(&self, owner: &str, cid: &Cid) -> Result { - let path = self.path(owner, cid); - match tokio::fs::remove_file(&path).await { - Ok(()) => { - sync_directory(required_parent(&path)?).await?; - Ok(true) - } - Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(false), - Err(error) => Err(error.into()), - } + remove_file_durably(&self.path(owner, cid)).await } } @@ -294,27 +267,14 @@ fn verify_content(cid: &Cid, bytes: &[u8]) -> Result<()> { Ok(()) } -fn parent_or_current(path: &Path) -> &Path { - path.parent() - .filter(|parent| !parent.as_os_str().is_empty()) - .unwrap_or_else(|| Path::new(".")) -} - -fn required_parent(path: &Path) -> Result<&Path> { - path.parent() - .ok_or_else(|| Error::Internal("blob path does not have a parent directory".to_owned())) -} - fn is_temporary_name(name: &std::ffi::OsStr) -> bool { - let Some(name) = name.to_str() else { - return false; - }; - let Some((prefix, suffix)) = name.rsplit_once(".pending-") else { - return false; - }; - (prefix == ".readiness" || prefix.parse::().is_ok()) - && suffix.len() == 16 - && suffix.bytes().all(|byte| byte.is_ascii_hexdigit()) + name.to_str() + .and_then(|name| name.rsplit_once(".pending-")) + .is_some_and(|(prefix, suffix)| { + (prefix == ".readiness" || prefix.parse::().is_ok()) + && suffix.len() == 16 + && suffix.bytes().all(|byte| byte.is_ascii_hexdigit()) + }) } async fn reject_symlink(path: &Path) -> Result<()> { @@ -347,14 +307,16 @@ async fn verify_private_directory(path: &Path) -> Result<()> { path.display() ))); } - secure_directory(path).await?; - Ok(()) + set_permissions(path, 0o700).await } -async fn remove_temporary(path: &Path) -> Result<()> { +async fn remove_file_durably(path: &Path) -> Result { match tokio::fs::remove_file(path).await { - Ok(()) => sync_directory(parent_or_current(path)).await, - Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Ok(()) => { + sync_directory(parent_or_current(path)).await?; + Ok(true) + } + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(false), Err(error) => Err(error.into()), } } @@ -367,47 +329,187 @@ fn is_stale(metadata: &std::fs::Metadata) -> bool { .is_some_and(|age| age >= STALE_TEMPORARY_AGE) } -#[cfg(unix)] async fn sync_directory(path: &Path) -> Result<()> { - tokio::fs::File::open(path).await?.sync_all().await?; - Ok(()) -} - -#[cfg(not(unix))] -async fn sync_directory(_path: &Path) -> Result<()> { + if cfg!(unix) { + tokio::fs::File::open(path).await?.sync_all().await?; + } Ok(()) } #[cfg(unix)] -async fn secure_directory(path: &Path) -> Result<()> { +async fn set_permissions(path: &Path, mode: u32) -> Result<()> { use std::os::unix::fs::PermissionsExt as _; - tokio::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700)).await?; + tokio::fs::set_permissions(path, std::fs::Permissions::from_mode(mode)).await?; Ok(()) } #[cfg(not(unix))] -async fn secure_directory(_path: &Path) -> Result<()> { +async fn set_permissions(_path: &Path, _mode: u32) -> Result<()> { Ok(()) } -#[cfg(unix)] -async fn secure_file(path: &Path) -> Result<()> { - use std::os::unix::fs::PermissionsExt as _; +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn directory_sync_preserves_platform_io_and_errors() { + let directory = tempfile::tempdir().unwrap(); + let file = directory.path().join("file"); + std::fs::write(&file, b"contents").unwrap(); + for path in [ + directory.path().to_owned(), + file.clone(), + directory.path().join("missing"), + file.join("not-a-directory"), + ] { + let expected: Result<()> = async { + #[cfg(unix)] + tokio::fs::File::open(&path).await?.sync_all().await?; + Ok(()) + } + .await; + assert_eq!( + format!("{:?}", sync_directory(&path).await), + format!("{expected:?}") + ); + } + assert_eq!(std::fs::read(file).unwrap(), b"contents"); + } - tokio::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)).await?; - Ok(()) -} + #[test] + fn temporary_name_pipeline_preserves_utf8_prefix_and_suffix_rules() { + use std::ffi::OsStr; + + let cid = example_cid().to_string(); + for prefix in [ + "", + ".readiness", + "invalid", + cid.as_str(), + "é", + "a.pending-b", + ] { + for suffix in [ + "", + "0", + "000000000000000", + "0000000000000000", + "FEDCBA9876543210", + "00000000000000000", + "000000000000000g", + "00000000000000é", + ] { + let name = format!("{prefix}.pending-{suffix}"); + let expected = (prefix == ".readiness" || prefix.parse::().is_ok()) + && suffix.len() == 16 + && suffix.bytes().all(|byte| byte.is_ascii_hexdigit()); + assert_eq!(is_temporary_name(OsStr::new(&name)), expected, "{name:?}"); + } + } + for name in ["", ".readiness", "no separator", "é", "\0"] { + assert!(!is_temporary_name(OsStr::new(name))); + } + #[cfg(unix)] + { + use std::os::unix::ffi::OsStrExt as _; + assert!(!is_temporary_name(OsStr::from_bytes( + b"\xff.pending-0000000000000000" + ))); + assert!(!is_temporary_name(OsStr::from_bytes( + b".readiness.pending-000000000000000\xff" + ))); + } + } -#[cfg(not(unix))] -async fn secure_file(_path: &Path) -> Result<()> { - Ok(()) -} + fn example_cid() -> Cid { + Cid::new_v1( + 0x55, + cid::multihash::Multihash::wrap(0x12, &[0; 32]).unwrap(), + ) + } -#[cfg(unix)] -fn set_private_file_mode(options: &mut tokio::fs::OpenOptions) { - options.mode(0o600); -} + #[test] + fn generated_blob_paths_always_have_a_nonempty_parent() { + let hash = *example_cid().hash(); + let cids = [Cid::new_v0(hash).unwrap(), Cid::new_v1(0x55, hash)]; + for root in [ + "", + ".", + "..", + "blobs", + "./blobs", + "blobs/../other", + "/", + "/tmp/blobs", + ] { + let store = FileBlobStore { root: root.into() }; + for owner in ["", ".", "..", "/", "a/b", "\\", "\0", "did:web:例え.test"] { + for cid in cids { + let path = store.path(owner, &cid); + let parent = path.parent().unwrap(); + assert!(!parent.as_os_str().is_empty(), "{path:?}"); + assert_eq!(parent_or_current(&path), parent); + assert_eq!(path.file_name().unwrap(), cid.to_string().as_str()); + assert_eq!( + parent, + store + .root + .join(URL_SAFE_NO_PAD.encode(Sha256::digest(owner.as_bytes()))) + ); + } + } + } + } -#[cfg(not(unix))] -fn set_private_file_mode(_options: &mut tokio::fs::OpenOptions) {} + #[tokio::test] + async fn durable_blob_delete_preserves_directory_errors() { + let directory = tempfile::tempdir().unwrap(); + let store = FileBlobStore::open(directory.path()).await.unwrap(); + let cid = example_cid(); + let path = store.path("owner", &cid); + std::fs::create_dir_all(&path).unwrap(); + let child = path.join("preserved"); + std::fs::write(&child, b"content").unwrap(); + let expected = std::fs::remove_file(&path).unwrap_err(); + + let Error::Io(actual) = store.delete("owner", &cid).await.unwrap_err() else { + panic!("deleting a directory must return its I/O error"); + }; + + assert_eq!(actual.kind(), expected.kind()); + assert_eq!(actual.raw_os_error(), expected.raw_os_error()); + assert_eq!(std::fs::read(child).unwrap(), b"content"); + } + + #[cfg(unix)] + #[tokio::test] + async fn durable_blob_delete_unlinks_symlinks_without_touching_targets() { + use std::os::unix::fs::symlink; + + let directory = tempfile::tempdir().unwrap(); + let store = FileBlobStore::open(directory.path()).await.unwrap(); + let cid = example_cid(); + let path = store.path("owner", &cid); + std::fs::create_dir(path.parent().unwrap()).unwrap(); + let file = directory.path().join("target-file"); + let target_directory = directory.path().join("target-directory"); + let missing = directory.path().join("missing-target"); + std::fs::write(&file, b"preserved").unwrap(); + std::fs::create_dir(&target_directory).unwrap(); + + for target in [&file, &target_directory, &missing] { + symlink(target, &path).unwrap(); + assert!(store.delete("owner", &cid).await.unwrap()); + assert_eq!( + std::fs::symlink_metadata(&path).unwrap_err().kind(), + io::ErrorKind::NotFound + ); + assert!(!store.delete("owner", &cid).await.unwrap()); + } + assert_eq!(std::fs::read(file).unwrap(), b"preserved"); + assert!(target_directory.is_dir()); + assert!(!missing.exists()); + } +} diff --git a/src/blob_store/memory.rs b/src/blob_store/memory.rs index 8f9348c..3e3f41c 100644 --- a/src/blob_store/memory.rs +++ b/src/blob_store/memory.rs @@ -15,13 +15,6 @@ pub struct MemoryBlobStore { blobs: RwLock>>, } -impl MemoryBlobStore { - #[must_use] - pub fn new() -> Self { - Self::default() - } -} - #[async_trait] impl BlobStore for MemoryBlobStore { async fn check(&self) -> Result<()> { diff --git a/src/clock.rs b/src/clock.rs index c967986..3c26032 100644 --- a/src/clock.rs +++ b/src/clock.rs @@ -37,20 +37,17 @@ impl TidClock { .unwrap_or_default() << 10; + let successor = |previous: u64| { + wall_time + .max(floor.saturating_add(1)) + .max(previous.saturating_add(1)) + }; let value = self .last .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |previous| { - Some( - wall_time - .max(floor.saturating_add(1)) - .max(previous.saturating_add(1)), - ) + Some(successor(previous)) }) - .map_or(wall_time, |previous| { - wall_time - .max(floor.saturating_add(1)) - .max(previous.saturating_add(1)) - }); + .map_or(wall_time, successor); encode_tid(value) } @@ -70,19 +67,137 @@ fn decode_tid(encoded: &str) -> Option { }) } -fn encode_tid(mut value: u64) -> String { - let mut encoded = [TID_ALPHABET[0]; 13]; - for byte in encoded.iter_mut().rev() { - *byte = TID_ALPHABET[(value & 31) as usize]; - value >>= 5; - } - String::from_utf8(encoded.to_vec()).expect("the TID alphabet is ASCII") +fn encode_tid(value: u64) -> String { + (0..13) + .rev() + .map(|digit| char::from(TID_ALPHABET[((value >> (digit * 5)) & 31) as usize])) + .collect() +} + +pub(crate) fn now() -> i64 { + OffsetDateTime::now_utc().unix_timestamp() +} + +pub(crate) fn format_utc_datetime(timestamp: i64) -> Result { + let value = OffsetDateTime::from_unix_timestamp(timestamp)?; + Ok(format!( + "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}Z", + value.year(), + u8::from(value.month()), + value.day(), + value.hour(), + value.minute(), + value.second() + )) } #[cfg(test)] mod tests { use super::TidClock; + #[test] + fn immutable_tid_encoding_preserves_every_u64_boundary_and_digit_position() { + fn previous_encode_tid(mut value: u64) -> String { + let mut encoded = [super::TID_ALPHABET[0]; 13]; + for byte in encoded.iter_mut().rev() { + *byte = super::TID_ALPHABET[(value & 31) as usize]; + value >>= 5; + } + String::from_utf8(encoded.to_vec()).expect("the TID alphabet is ASCII") + } + + let mut values = vec![0, u64::MAX]; + for bit in 0..64 { + let boundary = 1_u64 << bit; + values.extend([boundary - 1, boundary, boundary + 1, u64::MAX ^ boundary]); + } + for position in 0..13 { + for digit in 0..32_u128 { + if let Ok(value) = u64::try_from(digit << (position * 5)) { + values.push(value); + } + } + } + let mut mixed = 0x4f70_656e_4149_u64; + for _ in 0..10_000 { + mixed = mixed + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); + values.push(mixed); + } + values.sort_unstable(); + values.dedup(); + for value in values { + let actual = super::encode_tid(value); + assert_eq!(actual, previous_encode_tid(value), "{value}"); + assert_eq!(actual.len(), 13); + assert_eq!(super::decode_tid(&actual), Some(value)); + } + assert_eq!(super::encode_tid(0), "2222222222222"); + assert_eq!(super::encode_tid(u64::MAX), "jzzzzzzzzzzzz"); + } + + #[test] + fn utc_seconds_keep_previous_formatting_and_range_errors() { + use time::{Date, Month, OffsetDateTime}; + + let minimum = Date::MIN.midnight().assume_utc().unix_timestamp(); + let maximum = Date::MAX + .with_hms(23, 59, 59) + .unwrap() + .assume_utc() + .unix_timestamp(); + let mut timestamps = vec![ + i64::MIN, + minimum - 1, + minimum, + -1, + 0, + 1, + maximum, + maximum + 1, + i64::MAX, + ]; + for year in [-999, -1, 0, 1, 999, 1000, 1900, 2000, 2026] { + for month in [Month::January, Month::February, Month::December] { + timestamps.push( + Date::from_calendar_date(year, month, 28) + .unwrap() + .with_hms(12, 34, 56) + .unwrap() + .assume_utc() + .unix_timestamp(), + ); + } + } + for timestamp in timestamps { + let previous = OffsetDateTime::from_unix_timestamp(timestamp).map(|value| { + format!( + "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}Z", + value.year(), + value.month() as u8, + value.day(), + value.hour(), + value.minute(), + value.second() + ) + }); + assert_eq!( + super::format_utc_datetime(timestamp), + previous, + "{timestamp}" + ); + } + assert_eq!( + super::format_utc_datetime(0).unwrap(), + "1970-01-01T00:00:00Z" + ); + assert_eq!( + super::format_utc_datetime(-1).unwrap(), + "1969-12-31T23:59:59Z" + ); + } + #[test] fn tids_are_fixed_width_and_monotonic() { let clock = TidClock::default(); @@ -105,4 +220,37 @@ mod tests { assert!(clock.next_after("not-a-tid").is_none()); assert!(clock.next_after("jzzzzzzzzzzzz").is_none()); } + + #[test] + fn clock_retains_saturating_floor_and_counter_behavior() { + use std::sync::atomic::Ordering; + + let clock = TidClock::default(); + let maximum = super::encode_tid(u64::MAX); + assert_eq!(clock.next_value(u64::MAX - 1), maximum); + assert_eq!(clock.last.load(Ordering::SeqCst), u64::MAX); + assert_eq!(clock.next(), maximum); + assert_eq!(clock.last.load(Ordering::SeqCst), u64::MAX); + } + + #[test] + fn concurrent_clock_updates_return_distinct_published_values() { + use std::{collections::HashSet, sync::atomic::Ordering}; + + let clock = TidClock::default(); + let values = std::thread::scope(|scope| { + let readers = (0..8) + .map(|_| scope.spawn(|| (0..100).map(|_| clock.next()).collect::>())) + .collect::>(); + readers + .into_iter() + .flat_map(|reader| reader.join().unwrap()) + .collect::>() + }); + assert_eq!(values.len(), 800); + assert_eq!( + values.iter().max().unwrap(), + &super::encode_tid(clock.last.load(Ordering::SeqCst)) + ); + } } diff --git a/src/config.rs b/src/config.rs index f354fdd..2242533 100644 --- a/src/config.rs +++ b/src/config.rs @@ -3,12 +3,7 @@ mod origin; mod relay; -use std::{ - env, - net::SocketAddr, - path::{Path, PathBuf}, - time::Duration, -}; +use std::{env, fmt::Display, net::SocketAddr, path::PathBuf, str::FromStr, time::Duration}; use atrium_api::types::string::{Did, Handle}; use atrium_crypto::keypair::Secp256k1Keypair; @@ -17,11 +12,10 @@ use url::Url; use crate::{ domain::oauth::OAuthClient, - error::{Error, Result}, + error::{Error, ErrorContext, Result}, }; -pub use origin::PublicOrigin; -pub use relay::RelayOrigin; +pub use origin::HttpOrigin; const DEVELOPMENT_PASSWORD: &str = "development-only"; const DEVELOPMENT_SIGNING_KEY: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc"; @@ -32,18 +26,6 @@ pub enum DeploymentMode { Production, } -impl DeploymentMode { - fn parse(value: &str) -> Result { - match value { - "local" => Ok(Self::Local), - "production" => Ok(Self::Production), - _ => Err(Error::Configuration( - "CPDS_ENVIRONMENT must be `local` or `production`".to_owned(), - )), - } - } -} - #[derive(Clone)] pub struct AccountConfig { pub did: String, @@ -56,7 +38,7 @@ pub struct AccountConfig { pub struct Config { pub deployment_mode: DeploymentMode, pub listen_address: SocketAddr, - pub public_origin: PublicOrigin, + pub public_origin: HttpOrigin, pub database_path: PathBuf, pub blob_directory: PathBuf, pub blob_orphan_ttl: Duration, @@ -64,7 +46,7 @@ pub struct Config { pub identity_revalidation_interval: Duration, pub repo_event_retention: usize, pub repo_stream_send_timeout: Duration, - pub relay_origins: Vec, + pub relay_origins: Vec, pub account: AccountConfig, pub oauth_clients: Vec, } @@ -97,339 +79,208 @@ impl Config { } fn from_values(mut source: impl FnMut(&str) -> Option) -> Result { - let deployment_mode = DeploymentMode::parse( - &source("CPDS_ENVIRONMENT").unwrap_or_else(|| "local".to_owned()), - )?; - let listen_address = value( - &mut source, + let deployment_mode = match source("CPDS_ENVIRONMENT").as_deref() { + None | Some("local") => DeploymentMode::Local, + Some("production") => DeploymentMode::Production, + _ => { + return Err(Error::Configuration( + "CPDS_ENVIRONMENT must be `local` or `production`".to_owned(), + )); + } + }; + let mut settings = Settings { + source, deployment_mode, - "CPDS_LISTEN", - "127.0.0.1:3000", - false, - )? - .parse() - .map_err(|error| Error::Configuration(format!("invalid CPDS_LISTEN: {error}")))?; - let public_origin = PublicOrigin::parse( - &value( - &mut source, + }; + let config = Self { + deployment_mode, + listen_address: settings + .value("CPDS_LISTEN", "127.0.0.1:3000", false)? + .parse() + .context(Error::Configuration, "invalid CPDS_LISTEN")?, + public_origin: HttpOrigin::parse_public( + &settings.value("CPDS_PUBLIC_URL", "http://127.0.0.1:3000", true)?, deployment_mode, - "CPDS_PUBLIC_URL", - "http://127.0.0.1:3000", - true, )?, - deployment_mode, - )?; - let database_path: PathBuf = value( - &mut source, - deployment_mode, - "CPDS_DATABASE", - "cpds.sqlite3", - true, - )? - .into(); - let blob_directory: PathBuf = value( - &mut source, - deployment_mode, - "CPDS_BLOB_DIRECTORY", - "blobs", - true, - )? - .into(); - let blob_orphan_ttl = duration_value( - &mut source, - deployment_mode, - "CPDS_BLOB_ORPHAN_TTL_SECONDS", - 86_400, - 60, - 31_536_000, - )?; - let blob_gc_interval = duration_value( - &mut source, - deployment_mode, - "CPDS_BLOB_GC_INTERVAL_SECONDS", - 3_600, - 1, - 86_400, - )?; - let identity_revalidation_interval = - identity_revalidation_interval(&mut source, deployment_mode)?; - let repo_event_retention = usize_value( - &mut source, - deployment_mode, - "CPDS_REPO_EVENT_RETENTION", - 100_000, - 1, - 10_000_000, - )?; - let repo_stream_send_timeout = millisecond_duration_value( - &mut source, - deployment_mode, - "CPDS_REPO_STREAM_SEND_TIMEOUT_MS", - 10_000, - 100, - 60_000, - )?; - let relay_origins = relay_origins(&mut source, deployment_mode)?; - let account = account_config(&mut source, deployment_mode)?; - let oauth_clients = oauth_clients(&mut source, deployment_mode)?; + database_path: settings + .value("CPDS_DATABASE", "cpds.sqlite3", true)? + .into(), + blob_directory: settings.value("CPDS_BLOB_DIRECTORY", "blobs", true)?.into(), + blob_orphan_ttl: Duration::from_secs(settings.bounded( + "CPDS_BLOB_ORPHAN_TTL_SECONDS", + 86_400, + 60, + 31_536_000, + )?), + blob_gc_interval: Duration::from_secs(settings.bounded( + "CPDS_BLOB_GC_INTERVAL_SECONDS", + 3_600, + 1, + 86_400, + )?), + identity_revalidation_interval: Duration::from_secs(settings.bounded( + "CPDS_IDENTITY_REVALIDATION_INTERVAL_SECONDS", + 3_600, + 60, + 86_400, + )?), + repo_event_retention: settings.bounded( + "CPDS_REPO_EVENT_RETENTION", + 100_000, + 1, + 10_000_000, + )?, + repo_stream_send_timeout: Duration::from_millis(settings.bounded( + "CPDS_REPO_STREAM_SEND_TIMEOUT_MS", + 10_000, + 100, + 60_000, + )?), + relay_origins: settings.relay_origins()?, + account: settings.account_config()?, + oauth_clients: settings.oauth_clients()?, + }; if deployment_mode == DeploymentMode::Production { - validate_production( - &public_origin, - &database_path, - &account, - &oauth_clients[0].client_id, - )?; + validate_production(&config)?; } - Ok(Self { - deployment_mode, - listen_address, - public_origin, - database_path, - blob_directory, - blob_orphan_ttl, - blob_gc_interval, - identity_revalidation_interval, - repo_event_retention, - repo_stream_send_timeout, - relay_origins, - account, - oauth_clients, - }) + Ok(config) } } -fn identity_revalidation_interval( - source: &mut impl FnMut(&str) -> Option, +struct Settings { + source: F, deployment_mode: DeploymentMode, -) -> Result { - duration_value( - source, - deployment_mode, - "CPDS_IDENTITY_REVALIDATION_INTERVAL_SECONDS", - 3_600, - 60, - 86_400, - ) } -fn relay_origins( - source: &mut impl FnMut(&str) -> Option, - deployment_mode: DeploymentMode, -) -> Result> { - let Some(value) = source("CPDS_RELAYS") else { - return Ok(Vec::new()); - }; - if value.is_empty() { - return Err(Error::Configuration( - "CPDS_RELAYS must not be empty".to_owned(), - )); - } - let origins = value - .split(',') - .map(str::trim) - .map(|value| RelayOrigin::parse(value, deployment_mode)) - .collect::>>()?; - if origins.len() > 16 { - return Err(Error::Configuration( - "CPDS_RELAYS must contain at most 16 origins".to_owned(), - )); - } - if origins - .iter() - .enumerate() - .any(|(index, origin)| origins[..index].contains(origin)) - { - return Err(Error::Configuration( - "CPDS_RELAYS must not contain duplicate origins".to_owned(), - )); +impl Option> Settings { + fn relay_origins(&mut self) -> Result> { + let Some(value) = (self.source)("CPDS_RELAYS") else { + return Ok(Vec::new()); + }; + if value.is_empty() { + return Err(Error::Configuration( + "CPDS_RELAYS must not be empty".to_owned(), + )); + } + let origins = value + .split(',') + .map(str::trim) + .map(|value| HttpOrigin::parse_relay(value, self.deployment_mode)) + .collect::>>()?; + if origins.len() > 16 { + return Err(Error::Configuration( + "CPDS_RELAYS must contain at most 16 origins".to_owned(), + )); + } + if origins + .iter() + .enumerate() + .any(|(index, origin)| origins[..index].contains(origin)) + { + return Err(Error::Configuration( + "CPDS_RELAYS must not contain duplicate origins".to_owned(), + )); + } + Ok(origins) } - Ok(origins) -} -fn duration_value( - source: &mut impl FnMut(&str) -> Option, - deployment_mode: DeploymentMode, - name: &str, - default: u64, - minimum: u64, - maximum: u64, -) -> Result { - let seconds = value(source, deployment_mode, name, &default.to_string(), false)? - .parse::() - .ok() - .filter(|seconds| (minimum..=maximum).contains(seconds)) - .ok_or_else(|| { - Error::Configuration(format!("{name} must be between {minimum} and {maximum}")) - })?; - Ok(Duration::from_secs(seconds)) -} - -fn millisecond_duration_value( - source: &mut impl FnMut(&str) -> Option, - deployment_mode: DeploymentMode, - name: &str, - default: u64, - minimum: u64, - maximum: u64, -) -> Result { - let milliseconds = value(source, deployment_mode, name, &default.to_string(), false)? - .parse::() - .ok() - .filter(|milliseconds| (minimum..=maximum).contains(milliseconds)) - .ok_or_else(|| { - Error::Configuration(format!("{name} must be between {minimum} and {maximum}")) - })?; - Ok(Duration::from_millis(milliseconds)) -} - -fn usize_value( - source: &mut impl FnMut(&str) -> Option, - deployment_mode: DeploymentMode, - name: &str, - default: usize, - minimum: usize, - maximum: usize, -) -> Result { - value(source, deployment_mode, name, &default.to_string(), false)? - .parse::() - .ok() - .filter(|value| (minimum..=maximum).contains(value)) - .ok_or_else(|| { - Error::Configuration(format!("{name} must be between {minimum} and {maximum}")) - }) -} - -fn account_config( - source: &mut impl FnMut(&str) -> Option, - deployment_mode: DeploymentMode, -) -> Result { - let signing_key: [u8; 32] = URL_SAFE_NO_PAD - .decode(value( - source, - deployment_mode, - "CPDS_ACCOUNT_SIGNING_KEY", - DEVELOPMENT_SIGNING_KEY, - true, - )?) - .map_err(|error| { - Error::Configuration(format!( - "CPDS_ACCOUNT_SIGNING_KEY must be base64url: {error}" - )) - })? - .try_into() - .map_err(|_| { + fn account_config(&mut self) -> Result { + let signing_key: [u8; 32] = URL_SAFE_NO_PAD + .decode(self.value("CPDS_ACCOUNT_SIGNING_KEY", DEVELOPMENT_SIGNING_KEY, true)?) + .context( + Error::Configuration, + "CPDS_ACCOUNT_SIGNING_KEY must be base64url", + )? + .try_into() + .map_err(|_| { + Error::Configuration( + "CPDS_ACCOUNT_SIGNING_KEY must decode to exactly 32 bytes".to_owned(), + ) + })?; + Secp256k1Keypair::import(&signing_key).map_err(|_| { Error::Configuration( - "CPDS_ACCOUNT_SIGNING_KEY must decode to exactly 32 bytes".to_owned(), + "CPDS_ACCOUNT_SIGNING_KEY is not a valid secp256k1 secret".to_owned(), ) })?; - Secp256k1Keypair::import(&signing_key).map_err(|_| { - Error::Configuration("CPDS_ACCOUNT_SIGNING_KEY is not a valid secp256k1 secret".to_owned()) - })?; - let did = value( - source, - deployment_mode, - "CPDS_ACCOUNT_DID", - "did:web:localhost", - true, - )?; - Did::new(did.clone()) - .map_err(|message| Error::Configuration(format!("invalid CPDS_ACCOUNT_DID: {message}")))?; - let handle = value( - source, - deployment_mode, - "CPDS_ACCOUNT_HANDLE", - "alice.test", - true, - )?; - Handle::new(handle.clone()).map_err(|message| { - Error::Configuration(format!("invalid CPDS_ACCOUNT_HANDLE: {message}")) - })?; + let did = self.value("CPDS_ACCOUNT_DID", "did:web:localhost", true)?; + Did::new(did.clone()).context(Error::Configuration, "invalid CPDS_ACCOUNT_DID")?; + let handle = self.value("CPDS_ACCOUNT_HANDLE", "alice.test", true)?; + Handle::new(handle.clone()).context(Error::Configuration, "invalid CPDS_ACCOUNT_HANDLE")?; - Ok(AccountConfig { - did, - handle, - password: value( - source, - deployment_mode, - "CPDS_ACCOUNT_PASSWORD", - DEVELOPMENT_PASSWORD, - true, - )?, - signing_key, - }) -} + Ok(AccountConfig { + did, + handle, + password: self.value("CPDS_ACCOUNT_PASSWORD", DEVELOPMENT_PASSWORD, true)?, + signing_key, + }) + } -fn oauth_clients( - source: &mut impl FnMut(&str) -> Option, - deployment_mode: DeploymentMode, -) -> Result> { - let client_id = parse_url( - "CPDS_OAUTH_CLIENT_ID", - &value( - source, - deployment_mode, + fn oauth_clients(&mut self) -> Result> { + let client_id = self.required_url( "CPDS_OAUTH_CLIENT_ID", "http://127.0.0.1:4000/client-metadata.json", - true, - )?, - )?; - let redirect_uri = parse_url( - "CPDS_OAUTH_REDIRECT_URI", - &value( - source, - deployment_mode, - "CPDS_OAUTH_REDIRECT_URI", - "http://127.0.0.1:4000/callback", - true, - )?, - )?; - let name = value( - source, - deployment_mode, - "CPDS_OAUTH_CLIENT_NAME", - "cpds development client", - false, - )?; - Ok(vec![OAuthClient { - client_id, - redirect_uris: vec![redirect_uri], - name, - }]) -} + )?; + let redirect_uri = + self.required_url("CPDS_OAUTH_REDIRECT_URI", "http://127.0.0.1:4000/callback")?; + let name = self.value("CPDS_OAUTH_CLIENT_NAME", "cpds development client", false)?; + Ok(vec![OAuthClient { + client_id, + redirect_uris: vec![redirect_uri], + name, + }]) + } -fn value( - source: &mut impl FnMut(&str) -> Option, - deployment_mode: DeploymentMode, - name: &str, - local_default: &str, - required_in_production: bool, -) -> Result { - match source(name) { - Some(value) if value.is_empty() => { - Err(Error::Configuration(format!("{name} must not be empty"))) - } - Some(value) => Ok(value), - None if deployment_mode == DeploymentMode::Production && required_in_production => Err( - Error::Configuration(format!("{name} is required in production")), - ), - None => Ok(local_default.to_owned()), + fn required_url(&mut self, name: &str, local_default: &str) -> Result { + Url::parse(&self.value(name, local_default, true)?) + .context(Error::Configuration, format!("invalid {name}")) + } + + fn bounded( + &mut self, + name: &str, + default: T, + minimum: T, + maximum: T, + ) -> Result { + self.value(name, &default.to_string(), false)? + .parse::() + .ok() + .filter(|value| (minimum..=maximum).contains(value)) + .ok_or_else(|| { + Error::Configuration(format!("{name} must be between {minimum} and {maximum}")) + }) } -} -fn parse_url(name: &str, value: &str) -> Result { - Url::parse(value).map_err(|error| Error::Configuration(format!("invalid {name}: {error}"))) + fn value( + &mut self, + name: &str, + local_default: &str, + required_in_production: bool, + ) -> Result { + match (self.source)(name) { + Some(value) if value.is_empty() => { + Err(Error::Configuration(format!("{name} must not be empty"))) + } + Some(value) => Ok(value), + None if self.deployment_mode == DeploymentMode::Production + && required_in_production => + { + Err(Error::Configuration(format!( + "{name} is required in production" + ))) + } + None => Ok(local_default.to_owned()), + } + } } -fn validate_production( - public_origin: &PublicOrigin, - database_path: &Path, - account: &AccountConfig, - client_id: &Url, -) -> Result<()> { - if database_path.as_os_str() == ":memory:" { +fn validate_production(config: &Config) -> Result<()> { + let public_origin = &config.public_origin; + let account = &config.account; + if config.database_path.as_os_str() == ":memory:" { return Err(Error::Configuration( "CPDS_DATABASE must be durable in production".to_owned(), )); @@ -467,7 +318,7 @@ fn validate_production( .to_owned(), )); } - if client_id.scheme() != "https" { + if config.oauth_clients[0].client_id.scheme() != "https" { return Err(Error::Configuration( "CPDS_OAUTH_CLIENT_ID must use HTTPS in production".to_owned(), )); diff --git a/src/config/origin.rs b/src/config/origin.rs index 2f70b80..dc3a0e7 100644 --- a/src/config/origin.rs +++ b/src/config/origin.rs @@ -1,3 +1,4 @@ +use crate::error::ErrorContext; use std::fmt; use url::Host; @@ -7,11 +8,11 @@ use crate::error::{Error, Result}; use super::DeploymentMode; -/// The canonical external origin used in protocol metadata and request proofs. +/// A canonical HTTP origin for public protocol metadata, request proofs, or relay requests. #[derive(Clone, Debug, Eq, PartialEq)] -pub struct PublicOrigin(Url); +pub struct HttpOrigin(Url); -impl PublicOrigin { +impl HttpOrigin { /// Parse an HTTP origin, rejecting URL components that are not part of an /// origin. Production origins must use HTTPS. /// @@ -19,19 +20,13 @@ impl PublicOrigin { /// /// Returns a configuration error for malformed, non-HTTP, non-origin, or /// non-HTTPS production values. - pub fn parse(value: &str, deployment_mode: DeploymentMode) -> Result { - let url = Url::parse(value) - .map_err(|error| Error::Configuration(format!("invalid CPDS_PUBLIC_URL: {error}")))?; - if !matches!(url.scheme(), "http" | "https") || url.host().is_none() { - return Err(invalid_origin()); - } - if !url.username().is_empty() - || url.password().is_some() - || url.path() != "/" - || url.query().is_some() - || url.fragment().is_some() - { - return Err(invalid_origin()); + pub fn parse_public(value: &str, deployment_mode: DeploymentMode) -> Result { + let url = Url::parse(value).context(Error::Configuration, "invalid CPDS_PUBLIC_URL")?; + if !is_http_origin(&url) { + return Err(Error::Configuration( + "CPDS_PUBLIC_URL must be an HTTP origin without credentials, path, query, or fragment" + .to_owned(), + )); } if deployment_mode == DeploymentMode::Production && url.scheme() != "https" { return Err(Error::Configuration( @@ -41,6 +36,23 @@ impl PublicOrigin { Ok(Self(url)) } + /// A configured relay origin eligible for explicit crawl requests. + pub(super) fn parse_relay(value: &str, deployment_mode: DeploymentMode) -> Result { + let url = Url::parse(value).context(Error::Configuration, "invalid CPDS_RELAYS entry")?; + if !is_http_origin(&url) { + return Err(Error::Configuration( + "CPDS_RELAYS entries must be HTTP origins without credentials, path, query, or fragment" + .to_owned(), + )); + } + if deployment_mode == DeploymentMode::Production && url.scheme() != "https" { + return Err(Error::Configuration( + "CPDS_RELAYS entries must use HTTPS in production".to_owned(), + )); + } + Ok(Self(url)) + } + #[must_use] pub fn as_url(&self) -> &Url { &self.0 @@ -71,17 +83,20 @@ impl PublicOrigin { } } -impl fmt::Display for PublicOrigin { +impl fmt::Display for HttpOrigin { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { self.0.fmt(formatter) } } -fn invalid_origin() -> Error { - Error::Configuration( - "CPDS_PUBLIC_URL must be an HTTP origin without credentials, path, query, or fragment" - .to_owned(), - ) +fn is_http_origin(url: &Url) -> bool { + matches!(url.scheme(), "http" | "https") + && url.host().is_some() + && url.username().is_empty() + && url.password().is_none() + && url.path() == "/" + && url.query().is_none() + && url.fragment().is_none() } #[cfg(test)] @@ -90,7 +105,8 @@ mod tests { #[test] fn accepts_canonical_local_and_production_origins() { - let local = PublicOrigin::parse("http://127.0.0.1:3000", DeploymentMode::Local).unwrap(); + let local = + HttpOrigin::parse_public("http://127.0.0.1:3000", DeploymentMode::Local).unwrap(); assert_eq!(local.to_string(), "http://127.0.0.1:3000/"); assert_eq!(local.hostname(), "127.0.0.1:3000"); assert_eq!( @@ -99,11 +115,12 @@ mod tests { ); let production = - PublicOrigin::parse("https://pds.example.com/", DeploymentMode::Production).unwrap(); + HttpOrigin::parse_public("https://pds.example.com/", DeploymentMode::Production) + .unwrap(); assert_eq!(production.to_string(), "https://pds.example.com/"); assert_eq!(production.hostname(), "pds.example.com"); assert_eq!( - PublicOrigin::parse("http://[::1]:3000", DeploymentMode::Local) + HttpOrigin::parse_public("http://[::1]:3000", DeploymentMode::Local) .unwrap() .hostname(), "[::1]:3000" @@ -119,10 +136,11 @@ mod tests { "https://pds.example.com/?query=yes", "https://pds.example.com/#fragment", ] { - assert!(PublicOrigin::parse(value, DeploymentMode::Local).is_err()); + assert!(HttpOrigin::parse_public(value, DeploymentMode::Local).is_err()); } assert!( - PublicOrigin::parse("http://pds.example.com/", DeploymentMode::Production).is_err() + HttpOrigin::parse_public("http://pds.example.com/", DeploymentMode::Production) + .is_err() ); } } diff --git a/src/config/relay.rs b/src/config/relay.rs index ea9bbd3..d908efe 100644 --- a/src/config/relay.rs +++ b/src/config/relay.rs @@ -1,73 +1,278 @@ -use std::fmt; - -use url::Url; - -use crate::error::{Error, Result}; +#[cfg(test)] +mod tests { + use crate::config::{DeploymentMode, HttpOrigin}; + use crate::error::{Error, ErrorContext, Result}; + use url::{Host, Url}; -use super::DeploymentMode; + fn previous_is_http_origin(url: &Url) -> bool { + matches!(url.scheme(), "http" | "https") + && url.host().is_some() + && url.username().is_empty() + && url.password().is_none() + && url.path() == "/" + && url.query().is_none() + && url.fragment().is_none() + } -/// A configured relay origin eligible for explicit crawl requests. -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct RelayOrigin(Url); + fn previous_public_origin(value: &str, deployment_mode: DeploymentMode) -> Result { + let url = Url::parse(value).context(Error::Configuration, "invalid CPDS_PUBLIC_URL")?; + if !previous_is_http_origin(&url) { + return Err(Error::Configuration( + "CPDS_PUBLIC_URL must be an HTTP origin without credentials, path, query, or fragment" + .to_owned(), + )); + } + if deployment_mode == DeploymentMode::Production && url.scheme() != "https" { + return Err(Error::Configuration( + "CPDS_PUBLIC_URL must use HTTPS in production".to_owned(), + )); + } + Ok(url) + } -impl RelayOrigin { - pub(super) fn parse(value: &str, deployment_mode: DeploymentMode) -> Result { - let url = Url::parse(value) - .map_err(|error| Error::Configuration(format!("invalid CPDS_RELAYS entry: {error}")))?; - if !matches!(url.scheme(), "http" | "https") - || url.host().is_none() - || !url.username().is_empty() - || url.password().is_some() - || url.path() != "/" - || url.query().is_some() - || url.fragment().is_some() - { - return Err(invalid_origin()); + fn previous_relay_origin(value: &str, deployment_mode: DeploymentMode) -> Result { + let url = Url::parse(value).context(Error::Configuration, "invalid CPDS_RELAYS entry")?; + if !previous_is_http_origin(&url) { + return Err(Error::Configuration( + "CPDS_RELAYS entries must be HTTP origins without credentials, path, query, or fragment" + .to_owned(), + )); } if deployment_mode == DeploymentMode::Production && url.scheme() != "https" { return Err(Error::Configuration( "CPDS_RELAYS entries must use HTTPS in production".to_owned(), )); } - Ok(Self(url)) + Ok(url) } - #[must_use] - pub fn endpoint(&self, path: &str) -> Url { - let mut endpoint = self.0.clone(); - endpoint.set_path(path); - endpoint + fn previous_hostname(url: &Url) -> String { + let host = match url.host().expect("validated public origin has a host") { + Host::Domain(host) => host.to_owned(), + Host::Ipv4(host) => host.to_string(), + Host::Ipv6(host) => format!("[{host}]"), + }; + url.port() + .map_or_else(|| host.clone(), |port| format!("{host}:{port}")) } -} -impl fmt::Display for RelayOrigin { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - self.0.fmt(formatter) + fn compare_origin_result( + actual: Result, + expected: Result, + value: &str, + ) -> Option<(HttpOrigin, Url)> { + match (actual, expected) { + (Ok(actual), Ok(expected)) => { + assert_eq!(actual.as_url(), &expected, "{value}"); + assert_eq!(actual.to_string(), expected.to_string(), "{value}"); + assert_eq!(actual.hostname(), previous_hostname(&expected), "{value}"); + assert_eq!(actual.clone(), actual, "{value}"); + for path in [ + "", + "/", + "xrpc/example", + "/xrpc/example", + "/a%2Fb", + "/a?b#c", + "/雪", + "//double", + "/a/../b", + ] { + let mut endpoint = expected.clone(); + endpoint.set_path(path); + assert_eq!(actual.endpoint(path), endpoint, "{value}, {path}"); + assert_eq!(actual.as_url(), &expected, "{value}, {path}"); + } + Some((actual, expected)) + } + (Err(Error::Configuration(actual)), Err(Error::Configuration(expected))) => { + assert_eq!(actual, expected, "{value}"); + None + } + (actual, expected) => { + panic!("origin result changed for {value}: {actual:?}, {expected:?}") + } + } } -} -fn invalid_origin() -> Error { - Error::Configuration( - "CPDS_RELAYS entries must be HTTP origins without credentials, path, query, or fragment" - .to_owned(), - ) -} + #[test] + fn shared_http_origin_preserves_setting_errors_and_url_operations() { + let mut accepted = Vec::new(); + for value in [ + "https://example.test", + "HTTPS://EXAMPLE.TEST:443/", + "https://example.test:444/", + "http://example.test/", + "http://example.test:80/", + "http://example.test:8080/", + "http://127.0.0.1:80/", + "http://[::1]:8080", + "https://[::ffff:192.0.2.1]/", + "https://[::ffff:c000:201]:8443/", + "https://bücher.example/", + "https://xn--bcher-kva.example/", + "https://@example.test/", + "https://:@example.test/", + "https://user@example.test/", + "https://:password@example.test/", + "https://example.test/%2F", + "https://example.test/%2e", + "https://example.test/?", + "https://example.test/?encoded=%23", + "https://example.test/#", + "https://example.test/#%2F", + "https://example.test/path", + "http://user@example.test/path?query#fragment", + "file:///tmp", + "ftp://example.test/", + "mailto:alice@example.test", + "http://", + "https://[::1", + "https://example.test:65536/", + "not a URL", + "", + " \nhttps://example.test/\t", + "https:///example.test", + "https://example.test/%20", + "https://example.test\\path", + ] { + for mode in [DeploymentMode::Local, DeploymentMode::Production] { + for (actual, expected) in [ + ( + HttpOrigin::parse_public(value, mode), + previous_public_origin(value, mode), + ), + ( + HttpOrigin::parse_relay(value, mode), + previous_relay_origin(value, mode), + ), + ] { + accepted.extend(compare_origin_result(actual, expected, value)); + } + } + } + for (left, previous_left) in &accepted { + for (right, previous_right) in &accepted { + assert_eq!(left == right, previous_left == previous_right); + } + } + } -#[cfg(test)] -mod tests { - use super::*; + #[test] + fn public_and_relay_origins_share_components_but_keep_setting_errors() { + for (value, mode, expected) in [ + ( + "https://example.test", + DeploymentMode::Production, + Ok("https://example.test/"), + ), + ( + "http://example.test:8080/", + DeploymentMode::Local, + Ok("http://example.test:8080/"), + ), + ( + "https://example.test:443/", + DeploymentMode::Production, + Ok("https://example.test/"), + ), + ( + "http://[::1]:8080", + DeploymentMode::Local, + Ok("http://[::1]:8080/"), + ), + ( + "https://@example.test/", + DeploymentMode::Production, + Ok("https://example.test/"), + ), + ( + "https://user@example.test/", + DeploymentMode::Production, + Err("origin"), + ), + ( + "https://:password@example.test/", + DeploymentMode::Production, + Err("origin"), + ), + ( + "https://example.test/?", + DeploymentMode::Production, + Err("origin"), + ), + ( + "https://example.test/#", + DeploymentMode::Production, + Err("origin"), + ), + ( + "https://example.test/path", + DeploymentMode::Production, + Err("origin"), + ), + ("file:///tmp", DeploymentMode::Local, Err("origin")), + ("ftp://example.test/", DeploymentMode::Local, Err("origin")), + ( + "http://example.test/", + DeploymentMode::Production, + Err("https"), + ), + ( + "http://user@example.test/path?query#fragment", + DeploymentMode::Production, + Err("origin"), + ), + ("http://", DeploymentMode::Production, Err("parse")), + ("not a URL", DeploymentMode::Local, Err("parse")), + ] { + for (actual, parse_context, invalid, insecure) in [ + ( + HttpOrigin::parse_public(value, mode).map(|origin| origin.to_string()), + "invalid CPDS_PUBLIC_URL", + "CPDS_PUBLIC_URL must be an HTTP origin without credentials, path, query, or fragment", + "CPDS_PUBLIC_URL must use HTTPS in production", + ), + ( + HttpOrigin::parse_relay(value, mode).map(|origin| origin.to_string()), + "invalid CPDS_RELAYS entry", + "CPDS_RELAYS entries must be HTTP origins without credentials, path, query, or fragment", + "CPDS_RELAYS entries must use HTTPS in production", + ), + ] { + match expected { + Ok(canonical) => assert_eq!(actual.unwrap(), canonical, "{value}"), + Err(kind) => { + let Error::Configuration(message) = actual.unwrap_err() else { + panic!("origin errors must remain configuration errors: {value}"); + }; + let expected = match kind { + "parse" => { + format!("{parse_context}: {}", Url::parse(value).unwrap_err()) + } + "origin" => invalid.to_owned(), + "https" => insecure.to_owned(), + _ => unreachable!(), + }; + assert_eq!(message, expected, "{value}"); + } + } + } + } + } #[test] fn accepts_local_http_and_production_https_origins() { assert_eq!( - RelayOrigin::parse("http://127.0.0.1:2470", DeploymentMode::Local) + HttpOrigin::parse_relay("http://127.0.0.1:2470", DeploymentMode::Local) .unwrap() .endpoint("/xrpc/example") .as_str(), "http://127.0.0.1:2470/xrpc/example" ); assert!( - RelayOrigin::parse("https://relay.example.com", DeploymentMode::Production).is_ok() + HttpOrigin::parse_relay("https://relay.example.com", DeploymentMode::Production) + .is_ok() ); } @@ -80,10 +285,11 @@ mod tests { "https://relay.example.com/?query=yes", "https://relay.example.com/#fragment", ] { - assert!(RelayOrigin::parse(value, DeploymentMode::Local).is_err()); + assert!(HttpOrigin::parse_relay(value, DeploymentMode::Local).is_err()); } assert!( - RelayOrigin::parse("http://relay.example.com", DeploymentMode::Production).is_err() + HttpOrigin::parse_relay("http://relay.example.com", DeploymentMode::Production) + .is_err() ); } } diff --git a/src/config/tests.rs b/src/config/tests.rs index 4aef4a5..1c2eb5f 100644 --- a/src/config/tests.rs +++ b/src/config/tests.rs @@ -2,6 +2,108 @@ use std::collections::BTreeMap; use super::*; +#[test] +fn deployment_selection_preserves_defaults_exact_errors_and_first_read() { + fn previous_mode(value: &str) -> Result { + match value { + "local" => Ok(DeploymentMode::Local), + "production" => Ok(DeploymentMode::Production), + _ => Err(Error::Configuration( + "CPDS_ENVIRONMENT must be `local` or `production`".to_owned(), + )), + } + } + for value in [ + None, + Some("local"), + Some("production"), + Some(""), + Some("LOCAL"), + Some("Production"), + Some("local "), + Some(" production"), + Some("local\n"), + Some("local\0"), + Some("production\0"), + Some("staging"), + Some("本番"), + ] { + let mut values = production_values(); + values.remove("CPDS_ENVIRONMENT"); + if let Some(value) = value { + values.insert("CPDS_ENVIRONMENT", value.to_owned()); + } + let mut reads = Vec::new(); + let actual = Config::from_values(|name| { + reads.push(name.to_owned()); + values.get(name).cloned() + }) + .map(|config| config.deployment_mode) + .map_err(|error| error.to_string()); + let expected = previous_mode(value.unwrap_or("local")).map_err(|error| error.to_string()); + assert_eq!(actual, expected, "{value:?}"); + assert_eq!(reads[0], "CPDS_ENVIRONMENT"); + if expected.is_err() { + assert_eq!(reads, ["CPDS_ENVIRONMENT"]); + } else { + assert_eq!(reads[1], "CPDS_LISTEN"); + } + } +} + +#[test] +fn settings_are_read_in_order_and_stop_at_the_first_failure() { + let expected = [ + "CPDS_ENVIRONMENT", + "CPDS_LISTEN", + "CPDS_PUBLIC_URL", + "CPDS_DATABASE", + "CPDS_BLOB_DIRECTORY", + "CPDS_BLOB_ORPHAN_TTL_SECONDS", + "CPDS_BLOB_GC_INTERVAL_SECONDS", + "CPDS_IDENTITY_REVALIDATION_INTERVAL_SECONDS", + "CPDS_REPO_EVENT_RETENTION", + "CPDS_REPO_STREAM_SEND_TIMEOUT_MS", + "CPDS_RELAYS", + "CPDS_ACCOUNT_SIGNING_KEY", + "CPDS_ACCOUNT_DID", + "CPDS_ACCOUNT_HANDLE", + "CPDS_ACCOUNT_PASSWORD", + "CPDS_OAUTH_CLIENT_ID", + "CPDS_OAUTH_REDIRECT_URI", + "CPDS_OAUTH_CLIENT_NAME", + ]; + for failure in 0..=expected.len() { + let mut reads = Vec::new(); + let result = Config::from_values(|name| { + reads.push(name.to_owned()); + (expected.get(failure).copied() == Some(name)).then(String::new) + }); + assert_eq!(reads, expected[..(failure + 1).min(expected.len())]); + if let Some(name) = expected.get(failure) { + assert!(result.err().unwrap().to_string().contains(name)); + } else { + assert!(result.is_ok()); + } + } +} + +#[test] +fn production_cross_field_validation_follows_individual_settings() { + let mut values = production_values(); + values.insert("CPDS_DATABASE", ":memory:".to_owned()); + values.insert("CPDS_OAUTH_CLIENT_NAME", String::new()); + assert_eq!( + parse(&values).err().unwrap().to_string(), + "configuration error: CPDS_OAUTH_CLIENT_NAME must not be empty" + ); + values.remove("CPDS_OAUTH_CLIENT_NAME"); + assert_eq!( + parse(&values).err().unwrap().to_string(), + "configuration error: CPDS_DATABASE must be durable in production" + ); +} + #[test] fn local_mode_uses_safe_loopback_defaults() { let config = parse(&BTreeMap::new()).unwrap(); diff --git a/src/database/account.rs b/src/database/account.rs index 68290a7..9570335 100644 --- a/src/database/account.rs +++ b/src/database/account.rs @@ -5,14 +5,14 @@ use argon2::{ password_hash::{PasswordHash, PasswordHasher, PasswordVerifier, SaltString}, }; use rand::Rng as _; -use rusqlite::{OptionalExtension, params}; +use rusqlite::{Connection, params}; use crate::{ - domain::account::{Account, AccountStatus, HostedAccount}, - error::{Error, Result}, + domain::account::{Account, AccountStatus}, + error::{Error, ErrorContext as _, Result}, }; -use super::{Database, now}; +use super::{Database, now, query_optional, sql_row}; impl Database { /// Create the hosted account once, leaving an existing account unchanged. @@ -28,10 +28,10 @@ impl Database { let mut salt_bytes = [0_u8; 16]; rand::rng().fill(&mut salt_bytes); let salt = SaltString::encode_b64(&salt_bytes) - .map_err(|error| Error::Internal(format!("could not encode password salt: {error}")))?; + .context(Error::Internal, "could not encode password salt")?; let password_hash = Argon2::default() .hash_password(password.as_bytes(), &salt) - .map_err(|error| Error::Internal(format!("could not hash password: {error}")))? + .context(Error::Internal, "could not hash password")? .to_string(); self.connection()?.execute( @@ -49,32 +49,21 @@ impl Database { /// Returns `Authorization` for unknown accounts or invalid passwords, and a /// storage error when the account cannot be read. pub fn authenticate_password(&self, identifier: &str, password: &str) -> Result { - let row = self - .connection()? - .query_row( + let (did, handle, password_hash) = self + .query_optional( "SELECT did, handle, password_hash FROM accounts WHERE did = ?1 OR handle = ?1", [identifier], - |row| { - Ok(( - Account { - did: row.get(0)?, - handle: row.get(1)?, - }, - row.get::<_, String>(2)?, - )) - }, - ) - .optional()? + |row| Ok(<(_, _, String)>::try_from(row)?), + )? .ok_or_else(|| Error::Authorization("invalid credentials".to_owned()))?; - let parsed = PasswordHash::new(&row.1).map_err(|error| { - Error::Internal(format!("stored password hash is invalid: {error}")) - })?; + let parsed = PasswordHash::new(&password_hash) + .context(Error::Internal, "stored password hash is invalid")?; Argon2::default() .verify_password(password.as_bytes(), &parsed) .map_err(|_| Error::Authorization("invalid credentials".to_owned()))?; - Ok(row.0) + Ok(Account { did, handle }) } /// Find an account by DID or handle. @@ -85,7 +74,7 @@ impl Database { pub fn account_by_identifier(&self, identifier: &str) -> Result> { Ok(self .hosted_account_by_identifier(identifier)? - .map(|hosted| hosted.account)) + .map(|(account, _status)| account)) } /// Find an account and its durable hosting status by DID or handle. @@ -94,40 +83,162 @@ impl Database { /// /// Returns an error when the database query fails or contains an invalid /// hosting status. - pub fn hosted_account_by_identifier(&self, identifier: &str) -> Result> { - self.connection()? - .query_row( - "SELECT did, handle, status FROM accounts + pub fn hosted_account_by_identifier( + &self, + identifier: &str, + ) -> Result> { + self.query_optional( + "SELECT did, handle, status FROM accounts WHERE did = ?1 OR handle = ?1", - [identifier], - |row| { - Ok(( - Account { - did: row.get(0)?, - handle: row.get(1)?, - }, - row.get::<_, String>(2)?, - )) - }, - ) - .optional() - .map_err(Error::from)? - .map(|(account, status)| { - status - .parse::() - .map(|status| HostedAccount { account, status }) - .map_err(|error| Error::Internal(format!("stored {error}"))) - }) - .transpose() + [identifier], + |row| { + Ok(( + sql_row!(row => Account { did: 0, handle: 1 }), + parse_account_status(&row.get::<_, String>(2)?)?, + )) + }, + ) } } +pub(crate) fn require_account_active(connection: &Connection, did: &str) -> Result<()> { + query_optional( + connection, + "SELECT status FROM accounts WHERE did = ?1", + [did], + |row| parse_account_status(&row.get::<_, String>(0)?), + )? + .ok_or(Error::NotFound)? + .require_active() +} + +pub(crate) fn parse_account_status(value: &str) -> Result { + value + .parse() + .map_err(|error| Error::Internal(format!("stored {error}"))) +} + #[cfg(test)] mod tests { use crate::domain::account::AccountStatus; use super::Database; + fn compare_active_guard(connection: &rusqlite::Connection, did: &str) -> super::Result<()> { + let previous = super::query_optional( + connection, + "SELECT status FROM accounts WHERE did = ?1", + [did], + |row| super::parse_account_status(&row.get::<_, String>(0)?), + ) + .and_then(|status| status.ok_or(super::Error::NotFound)) + .and_then(AccountStatus::require_active); + let actual = super::require_account_active(connection, did); + assert_eq!(format!("{actual:?}"), format!("{previous:?}")); + actual + } + + #[test] + fn active_guard_preserves_sql_status_and_missing_account_errors() { + use rusqlite::{params, types::Value}; + + let connection = rusqlite::Connection::open_in_memory().unwrap(); + connection + .execute("CREATE TABLE accounts (did, status)", []) + .unwrap(); + for did in ["did:web:alice.test", "", "not a DID", "did\0é"] { + assert!(matches!( + compare_active_guard(&connection, did), + Err(super::Error::NotFound) + )); + for status in [ + Value::Text("active".to_owned()), + Value::Text("deactivated".to_owned()), + Value::Text("suspended".to_owned()), + Value::Text("takendown".to_owned()), + Value::Text("deleted".to_owned()), + Value::Text("unknown".to_owned()), + Value::Text(String::new()), + Value::Null, + Value::Blob(vec![0]), + Value::Integer(1), + ] { + connection + .execute("INSERT INTO accounts VALUES (?1, ?2)", params![did, status]) + .unwrap(); + let actual = compare_active_guard(&connection, did); + assert_eq!(actual.is_ok(), status == Value::Text("active".to_owned())); + connection.execute("DELETE FROM accounts", []).unwrap(); + } + } + connection.execute("DROP TABLE accounts", []).unwrap(); + assert!(matches!( + compare_active_guard(&connection, "did"), + Err(super::Error::Database(_)) + )); + } + + #[test] + fn stored_password_failures_keep_parser_context_and_database_priority() { + use super::{Argon2, Error, PasswordHash, PasswordVerifier as _}; + + let database = Database::open_in_memory().unwrap(); + let did = "did:web:alice.test"; + database + .ensure_account(did, "alice.test", "password") + .unwrap(); + for value in [ + "", + "not a hash", + "$argon2id", + "$argon2id$", + "$argon2id$v=19$", + "$argon2id$v=19$m=1,t=1,p=1$c2FsdA$YmFk", + "$unknown$v=1$c2FsdA$YmFk", + "$argon2id$v=19$m=19456,t=2,p=1$!$!", + "$argon2id$v=19$m=19456,t=2,p=1$雪$雪", + "\0\n\"quotes\"", + ] { + database + .connection() + .unwrap() + .execute( + "UPDATE accounts SET password_hash = ?1 WHERE did = ?2", + [value, did], + ) + .unwrap(); + let previous = PasswordHash::new(value) + .map_err(|error| { + Error::Internal(format!("stored password hash is invalid: {error}")) + }) + .and_then(|hash| { + Argon2::default() + .verify_password(b"password", &hash) + .map_err(|_| Error::Authorization("invalid credentials".to_owned())) + }); + assert_eq!( + database + .authenticate_password(did, "password") + .map(|_| ()) + .map_err(|error| format!("{error:?}")), + previous.map_err(|error| format!("{error:?}")), + "{value:?}", + ); + } + database + .connection() + .unwrap() + .execute( + "UPDATE accounts SET password_hash = x'00' WHERE did = ?1", + [did], + ) + .unwrap(); + assert!(matches!( + database.authenticate_password(did, "password"), + Err(Error::Database(rusqlite::Error::InvalidColumnType(2, _, _))), + )); + } + #[test] fn hosted_account_reads_the_durable_status() { let database = Database::open_in_memory().unwrap(); @@ -135,14 +246,11 @@ mod tests { .ensure_account("did:web:alice.test", "alice.test", "password") .unwrap(); - assert_eq!( - database - .hosted_account_by_identifier("alice.test") - .unwrap() - .unwrap() - .status, - AccountStatus::Active - ); + let (_account, status) = database + .hosted_account_by_identifier("alice.test") + .unwrap() + .unwrap(); + assert_eq!(status, AccountStatus::Active); database .connection() @@ -152,13 +260,10 @@ mod tests { ["did:web:alice.test"], ) .unwrap(); - assert_eq!( - database - .hosted_account_by_identifier("did:web:alice.test") - .unwrap() - .unwrap() - .status, - AccountStatus::Deactivated - ); + let (_account, status) = database + .hosted_account_by_identifier("did:web:alice.test") + .unwrap() + .unwrap(); + assert_eq!(status, AccountStatus::Deactivated); } } diff --git a/src/database/blob.rs b/src/database/blob.rs index c44fc8d..d5fd343 100644 --- a/src/database/blob.rs +++ b/src/database/blob.rs @@ -1,32 +1,40 @@ //! Blob metadata persistence; opaque bytes remain in `BlobStore`. -use std::str::FromStr as _; - use cid::Cid; -use rusqlite::{OptionalExtension, Transaction, params}; +use rusqlite::{Transaction, params}; use crate::{ domain::blob::{Blob, referenced_blob_cids, writable_blob_refs}, - error::{Error, Result}, + error::{Error, ErrorContext as _, Result}, }; -use super::{Database, now}; +use super::{Database, now, query, require_account_active}; impl Database { + pub(crate) fn is_public_blob(&self, did: &str, cid: &Cid) -> Result { + self.query_one( + "SELECT EXISTS(SELECT 1 FROM record_blobs WHERE did = ?1 AND cid = ?2)", + params![did, cid.to_string()], + |row| Ok(row.get(0)?), + ) + } + /// Reserve metadata before publishing blob bytes. /// /// # Errors /// - /// Returns an error when metadata cannot be written or deletion already owns - /// this content-addressed key. + /// Returns an error when metadata cannot be written, conflicts with an + /// earlier upload, or deletion already owns this content-addressed key. pub fn begin_blob_upload(&self, did: &str, blob: &Blob) -> Result<()> { - let changed = self.connection()?.execute( + let connection = self.connection()?; + let changed = connection.execute( "INSERT INTO blobs (did, cid, media_type, size, created_at) SELECT ?1, ?2, ?3, ?4, ?5 WHERE (SELECT status FROM accounts WHERE did = ?1) = 'active' ON CONFLICT(did, cid) DO UPDATE SET created_at = excluded.created_at - WHERE blobs.delete_started_at IS NULL", + WHERE blobs.delete_started_at IS NULL + AND blobs.media_type = excluded.media_type AND blobs.size = excluded.size", params![ did, blob.cid.to_string(), @@ -36,8 +44,11 @@ impl Database { ], )?; if changed == 0 { - require_active_account(self, did)?; - return Err(Error::Conflict("blob deletion is in progress".to_owned())); + require_account_active(&connection, did)?; + return Err(Error::Conflict( + "blob metadata differs from its earlier upload or deletion is in progress" + .to_owned(), + )); } Ok(()) } @@ -48,14 +59,15 @@ impl Database { /// /// Returns an error when the reservation disappeared or entered deletion. pub fn finish_blob_upload(&self, did: &str, cid: &Cid) -> Result<()> { - let changed = self.connection()?.execute( + let connection = self.connection()?; + let changed = connection.execute( "UPDATE blobs SET available_at = COALESCE(available_at, ?3) WHERE did = ?1 AND cid = ?2 AND delete_started_at IS NULL AND (SELECT status FROM accounts WHERE did = ?1) = 'active'", params![did, cid.to_string(), now()], )?; if changed == 0 { - require_active_account(self, did)?; + require_account_active(&connection, did)?; return Err(Error::Internal(format!( "blob upload reservation disappeared for {did}/{cid}" ))); @@ -63,37 +75,28 @@ impl Database { Ok(()) } - /// Load metadata for a content-addressed blob. + /// Find available metadata for a content-addressed blob, or `None` when absent. /// /// # Errors /// - /// Returns `NotFound` when absent or a database error when lookup fails. - pub fn blob(&self, did: &str, cid: &Cid) -> Result { - self.connection()? - .query_row( - "SELECT media_type, size FROM blobs + /// Returns an error for failed lookup or invalid stored metadata. + pub fn find_blob(&self, did: &str, cid: &Cid) -> Result> { + self.query_optional( + "SELECT media_type, size FROM blobs WHERE did = ?1 AND cid = ?2 AND delete_started_at IS NULL AND available_at IS NOT NULL", - params![did, cid.to_string()], - |row| Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)), - ) - .optional()? - .map(|(media_type, size)| decode_blob(*cid, media_type, size)) - .transpose()? - .ok_or(Error::NotFound) + params![did, cid.to_string()], + |row| decode_blob(*cid, row.get(0)?, row.get(1)?), + ) } pub(crate) fn validate_blob_refs(&self, did: &str, value: &serde_json::Value) -> Result<()> { for referenced in writable_blob_refs(value).map_err(Error::InvalidRequest)? { - let stored = self.blob(did, &referenced.cid).map_err(|error| { - if matches!(error, Error::NotFound) { - Error::InvalidRequest(format!( - "record references blob {} before it is uploaded", - referenced.cid - )) - } else { - error - } + let stored = self.find_blob(did, &referenced.cid)?.ok_or_else(|| { + Error::InvalidRequest(format!( + "record references blob {} before it is uploaded", + referenced.cid + )) })?; if referenced.media_type != stored.media_type || referenced.size != stored.size { return Err(Error::InvalidRequest(format!( @@ -106,26 +109,16 @@ impl Database { } pub(crate) fn all_blobs(&self) -> Result> { - let connection = self.connection()?; - let mut statement = connection.prepare( + self.query( "SELECT did, cid, media_type, size FROM blobs WHERE delete_started_at IS NULL AND available_at IS NOT NULL ORDER BY did, cid", - )?; - statement - .query_map([], |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, String>(2)?, - row.get::<_, i64>(3)?, - )) - })? - .map(|row| { - let (did, cid, media_type, size) = row?; + [], + |row| { + let (did, cid, media_type, size) = <(_, String, _, _)>::try_from(row)?; Ok((did, decode_blob(parse_stored_cid(&cid)?, media_type, size)?)) - }) - .collect() + }, + ) } pub(crate) fn claim_orphan_blobs( @@ -133,10 +126,9 @@ impl Database { created_before: i64, limit: usize, ) -> Result> { - let mut connection = self.connection()?; - let transaction = connection.transaction()?; - let candidates = { - let mut statement = transaction.prepare( + let candidates = self.transaction(|transaction| { + let candidates = query( + transaction, "SELECT blobs.did, blobs.cid FROM blobs WHERE blobs.delete_started_at IS NOT NULL @@ -154,22 +146,18 @@ impl Database { ORDER BY blobs.delete_started_at IS NULL, blobs.created_at, blobs.did, blobs.cid LIMIT ?2", + params![created_before, i64::try_from(limit).unwrap_or(i64::MAX)], + |row| Ok(<(String, String)>::try_from(row)?), )?; - statement - .query_map( - params![created_before, i64::try_from(limit).unwrap_or(i64::MAX)], - |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), - )? - .collect::, _>>()? - }; - for (did, cid) in &candidates { - transaction.execute( - "UPDATE blobs SET delete_started_at = COALESCE(delete_started_at, ?3) + for (did, cid) in &candidates { + transaction.execute( + "UPDATE blobs SET delete_started_at = COALESCE(delete_started_at, ?3) WHERE did = ?1 AND cid = ?2", - params![did, cid, now()], - )?; - } - transaction.commit()?; + params![did, cid, now()], + )?; + } + Ok(candidates) + })?; candidates .into_iter() .map(|(did, cid)| Ok((did, parse_stored_cid(&cid)?))) @@ -197,8 +185,7 @@ impl Database { cursor: Option<&str>, limit: usize, ) -> Result> { - let connection = self.connection()?; - let mut statement = connection.prepare( + self.query( "SELECT DISTINCT record_blobs.cid FROM record_blobs WHERE record_blobs.did = ?1 @@ -214,14 +201,9 @@ impl Database { ) ORDER BY record_blobs.cid LIMIT ?4", - )?; - statement - .query_map( - params![did, cursor, since, i64::try_from(limit).unwrap_or(i64::MAX)], - |row| row.get::<_, String>(0), - )? - .map(|row| parse_stored_cid(&row?)) - .collect() + params![did, cursor, since, i64::try_from(limit).unwrap_or(i64::MAX)], + |row| parse_stored_cid(&row.get::<_, String>(0)?), + ) } pub(crate) fn missing_blob_references( @@ -230,8 +212,7 @@ impl Database { cursor: Option<&str>, limit: usize, ) -> Result> { - let connection = self.connection()?; - let mut statement = connection.prepare( + self.query( "SELECT record_blobs.cid, MIN(record_blobs.collection || '/' || record_blobs.rkey) FROM record_blobs @@ -244,39 +225,19 @@ impl Database { GROUP BY record_blobs.cid ORDER BY record_blobs.cid LIMIT ?3", - )?; - statement - .query_map( - params![did, cursor, i64::try_from(limit).unwrap_or(i64::MAX)], - |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), - )? - .map(|row| { - let (cid, path) = row?; + params![did, cursor, i64::try_from(limit).unwrap_or(i64::MAX)], + |row| { + let (cid, path) = <(String, String)>::try_from(row)?; Ok((parse_stored_cid(&cid)?, format!("at://{did}/{path}"))) - }) - .collect() - } -} - -fn require_active_account(database: &Database, did: &str) -> Result<()> { - let account = database - .hosted_account_by_identifier(did)? - .ok_or(Error::NotFound)?; - if account.status.is_active() { - return Ok(()); + }, + ) } - Err(Error::Xrpc { - name: account - .status - .account_error_name() - .unwrap_or("AccountNotActive"), - message: format!("account is {}", account.status.as_str()), - }) } fn parse_stored_cid(value: &str) -> Result { - Cid::from_str(value) - .map_err(|error| Error::Internal(format!("stored blob CID is invalid: {error}"))) + value + .parse() + .context(Error::Internal, "stored blob CID is invalid") } pub(crate) fn active_blob_ref_cids( @@ -321,6 +282,180 @@ mod tests { const DID: &str = "did:web:alice.test"; + fn previous_blob_lookup(database: &Database, cid: &Cid) -> Result { + database + .query_optional( + "SELECT media_type, size FROM blobs + WHERE did = ?1 AND cid = ?2 + AND delete_started_at IS NULL AND available_at IS NOT NULL", + params![DID, cid.to_string()], + |row| decode_blob(*cid, row.get(0)?, row.get(1)?), + )? + .ok_or(Error::NotFound) + } + + fn assert_optional_blob_parity(database: &Database, reference: &Blob) { + let previous = previous_blob_lookup(database, &reference.cid); + let actual = database + .find_blob(DID, &reference.cid) + .and_then(|blob| blob.ok_or(Error::NotFound)); + assert_eq!(format!("{actual:?}"), format!("{previous:?}")); + let expected = previous + .map_err(|error| { + if matches!(error, Error::NotFound) { + Error::InvalidRequest(format!( + "record references blob {} before it is uploaded", + reference.cid + )) + } else { + error + } + }) + .and_then(|stored| { + if reference.media_type != stored.media_type || reference.size != stored.size { + return Err(Error::InvalidRequest(format!( + "record blob {} metadata does not match the uploaded blob", + reference.cid + ))); + } + Ok(()) + }); + let value = serde_json::json!({ + "$type": "app.test.record", + "image": {"$type": "blob", "ref": {"$link": reference.cid.to_string()}, "mimeType": reference.media_type, "size": reference.size}, + }); + let actual = database.validate_blob_refs(DID, &value); + assert_eq!(format!("{actual:?}"), format!("{expected:?}")); + } + + #[test] + fn optional_blob_lookup_preserves_visibility_and_reference_validation_errors() { + use rusqlite::types::Value as SqlValue; + + let database = Database::open_in_memory().unwrap(); + let reference = Blob { + cid: cpds_protocol::sha256_cid(0x55, b"blob"), + media_type: "image/png".to_owned(), + size: 3, + }; + assert!(database.find_blob(DID, &reference.cid).unwrap().is_none()); + assert_optional_blob_parity(&database, &reference); + database.connection().unwrap().execute_batch( + "CREATE TEMP TABLE blobs (did, cid, media_type, size, available_at, delete_started_at)", + ).unwrap(); + for media_type in [ + SqlValue::Text("image/png".to_owned()), + SqlValue::Text("text/plain".to_owned()), + SqlValue::Text("not a MIME type".to_owned()), + SqlValue::Null, + SqlValue::Integer(1), + SqlValue::Blob(vec![0]), + ] { + for size in [ + SqlValue::Integer(3), + SqlValue::Integer(0), + SqlValue::Integer(-1), + SqlValue::Integer(i64::MAX), + SqlValue::Real(3.5), + SqlValue::Text("3".to_owned()), + SqlValue::Null, + SqlValue::Blob(vec![0]), + ] { + for available_at in [None, Some(1)] { + for delete_started_at in [None, Some(1)] { + { + let connection = database.connection().unwrap(); + connection.execute("DELETE FROM temp.blobs", []).unwrap(); + connection + .execute( + "INSERT INTO temp.blobs VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + params![ + DID, + reference.cid.to_string(), + media_type, + size, + available_at, + delete_started_at + ], + ) + .unwrap(); + } + assert_optional_blob_parity(&database, &reference); + } + } + } + } + database + .connection() + .unwrap() + .execute_batch("DROP TABLE temp.blobs; DROP TABLE main.blobs") + .unwrap(); + assert_optional_blob_parity(&database, &reference); + assert!(matches!( + database.find_blob(DID, &reference.cid), + Err(Error::Database(_)) + )); + } + + #[test] + fn stored_blob_cid_context_preserves_values_and_exact_errors() { + let cid = cpds_protocol::sha256_cid(0x55, b"blob"); + let v0 = Cid::new_v0(*cid.hash()).unwrap().to_string(); + let valid = cid.to_string(); + for value in [ + valid.as_str(), + v0.as_str(), + "", + "invalid-cid", + "b", + "Qm", + "é", + " bafkreia", + ] { + let old = Cid::from_str(value) + .map_err(|error| Error::Internal(format!("stored blob CID is invalid: {error}"))); + let current = parse_stored_cid(value); + assert_eq!(format!("{current:?}"), format!("{old:?}"), "{value:?}"); + } + } + + #[test] + fn repeated_uploads_cannot_change_content_addressed_metadata() { + let database = Database::open_in_memory().unwrap(); + database + .ensure_account(DID, "alice.test", "password") + .unwrap(); + let blob = Blob { + cid: Cid::from_str("bafkreibme22gw2h7y2h7tg2fhqotaqjucnbc24deqo72b6mkl2egezxhvy") + .unwrap(), + media_type: "image/png".to_owned(), + size: 3, + }; + database.begin_blob_upload(DID, &blob).unwrap(); + for available in [false, true] { + if available { + database.finish_blob_upload(DID, &blob.cid).unwrap(); + } + database.begin_blob_upload(DID, &blob).unwrap(); + for inconsistent in [ + Blob { + media_type: "text/plain".to_owned(), + ..blob.clone() + }, + Blob { + size: 4, + ..blob.clone() + }, + ] { + assert!(matches!( + database.begin_blob_upload(DID, &inconsistent), + Err(Error::Conflict(_)) + )); + } + } + assert_eq!(database.find_blob(DID, &blob.cid).unwrap().unwrap(), blob); + } + #[test] fn deactivation_prevents_blob_reservation_and_publication() { let database = Database::open_in_memory().unwrap(); @@ -351,7 +486,9 @@ mod tests { }) )); assert!(matches!( - database.blob(DID, &blob.cid), + database + .find_blob(DID, &blob.cid) + .and_then(|blob| blob.ok_or(Error::NotFound)), Err(Error::NotFound) )); diff --git a/src/database/blockstore.rs b/src/database/blockstore.rs new file mode 100644 index 0000000..96682ed --- /dev/null +++ b/src/database/blockstore.rs @@ -0,0 +1,137 @@ +//! Transaction-local `ATrium` block overlay and verified content-addressed reads. + +use std::collections::{HashMap, HashSet}; + +use atrium_repo::{ + Cid, + blockstore::{ + AsyncBlockStoreRead, AsyncBlockStoreWrite, DAG_CBOR, Error as BlockError, SHA2_256, + }, +}; + +use crate::error::{Error, Result}; + +use super::Database; + +/// `ATrium` writes into this in-memory overlay. `SQLite` is read on demand, but no +/// staged write is visible until [`Database::commit_repo`] publishes the batch. +pub(crate) struct RepoBlockStore { + database: Database, + writes: HashMap>, +} + +impl RepoBlockStore { + pub(crate) fn new(database: Database) -> Self { + Self { + database, + writes: HashMap::new(), + } + } + + pub(crate) fn reachable_writes( + &self, + reachable: impl IntoIterator, + ) -> Result)>> { + let reachable = reachable.into_iter().collect::>(); + for cid in &reachable { + if !self.writes.contains_key(cid) { + self.database.repo_block(*cid)?; + } + } + Ok(self + .writes + .iter() + .filter(move |(cid, _)| reachable.contains(cid))) + } + + pub(crate) fn writes(&self) -> Vec<(Cid, Vec)> { + self.writes + .iter() + .map(|(cid, bytes)| (*cid, bytes.clone())) + .collect() + } +} + +impl AsyncBlockStoreRead for RepoBlockStore { + fn read_block_into( + &mut self, + cid: Cid, + contents: &mut Vec, + ) -> impl Future> + Send { + let block = self + .writes + .get(&cid) + .cloned() + .map_or_else(|| self.database.repo_block(cid), Ok) + .map_err(block_error); + async move { + contents.clear(); + contents.extend_from_slice(&block?); + Ok(()) + } + } +} + +impl AsyncBlockStoreWrite for RepoBlockStore { + fn write_block( + &mut self, + codec: u64, + hash: u64, + contents: &[u8], + ) -> impl Future> + Send { + let result = block_cid(codec, hash, contents).inspect(|cid| { + self.writes.insert(*cid, contents.to_vec()); + }); + async move { result } + } +} + +impl Database { + pub(super) fn repo_block(&self, cid: Cid) -> Result> { + let bytes = self + .query_optional( + "SELECT bytes FROM repo_blocks WHERE cid = ?1", + [cid.to_string()], + |row| Ok(row.get::<_, Vec>(0)?), + )? + .ok_or(Error::NotFound)?; + verify_block(cid, &bytes)?; + Ok(bytes) + } +} + +pub(super) fn verify_block(cid: Cid, bytes: &[u8]) -> Result<()> { + let computed = block_cid(cid.codec(), cid.hash().code(), bytes).map_err(|error| { + Error::Internal(format!("could not verify repository block {cid}: {error}")) + })?; + if computed == cid { + Ok(()) + } else { + Err(Error::Internal(format!( + "repository block bytes do not match CID {cid}" + ))) + } +} + +pub(super) fn block_cid( + codec: u64, + hash: u64, + contents: &[u8], +) -> std::result::Result { + if codec != DAG_CBOR { + return Err(BlockError::Other(Box::new(std::io::Error::other(format!( + "unsupported repository codec {codec}" + ))))); + } + if hash != SHA2_256 { + return Err(BlockError::UnsupportedHash(hash)); + } + Ok(cpds_protocol::sha256_cid(codec, contents)) +} + +fn block_error(error: Error) -> BlockError { + match error { + Error::NotFound => BlockError::CidNotFound, + error => BlockError::Other(Box::new(std::io::Error::other(error.to_string()))), + } +} diff --git a/src/database/mod.rs b/src/database/mod.rs index b28f0c3..9704b06 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -2,7 +2,9 @@ mod account; mod blob; +mod blockstore; mod oauth; +mod query; mod record; mod repo; mod schema; @@ -12,16 +14,18 @@ use std::{ sync::{Arc, Mutex, MutexGuard}, }; -use rusqlite::{Connection, MAIN_DB, OpenFlags, TransactionBehavior}; -use time::OffsetDateTime; +use rusqlite::{Connection, MAIN_DB, OpenFlags, Params, Row, Transaction, TransactionBehavior}; use crate::error::{Error, Result}; +pub(crate) use crate::clock::now; +pub(crate) use account::{parse_account_status, require_account_active}; pub(crate) use blob::active_blob_ref_cids; -pub use oauth::StoredAuthorizationCode; +pub(crate) use blockstore::RepoBlockStore; +pub(crate) use query::{constraint_conflict, query, query_optional, sql_row}; pub(crate) use repo::{ - ListedRepoHead, PendingRecordMutation, PendingRepoCommit, RepoBlockStore, RepoHead, - StoredAccountEvent, StoredCommitEvent, StoredIdentityEvent, StoredRepoEvent, + LifecycleEvent, ListedRepoHead, PendingRepoCommit, RepoHead, StoredAccountEvent, + StoredCommitEvent, StoredIdentityEvent, StoredRepoEvent, }; #[derive(Clone)] @@ -30,13 +34,51 @@ pub struct Database { } impl Database { + pub(crate) fn query( + &self, + sql: &str, + params: impl Params, + decode: impl FnMut(&Row<'_>) -> Result, + ) -> Result> { + query(&*self.connection()?, sql, params, decode) + } + + pub(crate) fn query_one( + &self, + sql: &str, + params: impl Params, + decode: impl FnOnce(&Row<'_>) -> Result, + ) -> Result { + self.connection()?.query_row_and_then(sql, params, decode) + } + + pub(crate) fn query_optional( + &self, + sql: &str, + params: impl Params, + decode: impl FnOnce(&Row<'_>) -> Result, + ) -> Result> { + query_optional(&*self.connection()?, sql, params, decode) + } + + pub(crate) fn transaction( + &self, + operation: impl FnOnce(&Transaction<'_>) -> Result, + ) -> Result { + let mut connection = self.connection()?; + let transaction = connection.transaction()?; + let value = operation(&transaction)?; + transaction.commit()?; + Ok(value) + } + /// Open or create a database file and apply the current schema. /// /// # Errors /// /// Returns an error if `SQLite` cannot open or migrate the database. pub fn open(path: impl AsRef) -> Result { - Self::from_connection(Connection::open(path)?) + Self::from_connection(Connection::open(path)?, true) } /// Open and migrate an existing database without creating a replacement. @@ -47,7 +89,7 @@ impl Database { /// cannot be opened and migrated. pub fn open_existing(path: impl AsRef) -> Result { let connection = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_WRITE)?; - Self::from_existing_connection(connection) + Self::from_connection(connection, false) } /// Create an isolated database suitable for tests and ephemeral servers. @@ -56,18 +98,11 @@ impl Database { /// /// Returns an error if the in-memory database cannot be initialized. pub fn open_in_memory() -> Result { - Self::from_connection(Connection::open_in_memory()?) - } - - fn from_connection(mut connection: Connection) -> Result { - schema::apply(&mut connection)?; - Ok(Self { - connection: Arc::new(Mutex::new(connection)), - }) + Self::from_connection(Connection::open_in_memory()?, true) } - fn from_existing_connection(mut connection: Connection) -> Result { - schema::apply_existing(&mut connection)?; + fn from_connection(mut connection: Connection, initialize: bool) -> Result { + schema::apply(&mut connection, initialize)?; Ok(Self { connection: Arc::new(Mutex::new(connection)), }) @@ -91,7 +126,7 @@ impl Database { connection.query_row( "SELECT COALESCE(MAX(seq), 0), MIN(created_at) FROM repo_outbox", [], - |row| Ok((row.get::<_, i64>(0)?, row.get::<_, Option>(1)?)), + |row| <(i64, Option)>::try_from(row), )?; Ok(()) } @@ -136,7 +171,3 @@ impl Database { Ok(()) } } - -pub(crate) fn now() -> i64 { - OffsetDateTime::now_utc().unix_timestamp() -} diff --git a/src/database/oauth.rs b/src/database/oauth.rs index 13374bb..f321e6d 100644 --- a/src/database/oauth.rs +++ b/src/database/oauth.rs @@ -1,22 +1,14 @@ //! Authorization codes and access-token persistence. -use rusqlite::{OptionalExtension, params}; +use rusqlite::{Connection, OptionalExtension, params}; +use crate::domain::oauth::AuthorizationRequest; use crate::{ domain::{account::Account, oauth::AuthenticatedToken}, error::{Error, Result}, }; -use super::{Database, now}; - -#[derive(Debug)] -pub struct StoredAuthorizationCode { - pub did: String, - pub client_id: String, - pub redirect_uri: String, - pub code_challenge: String, - pub scope: String, -} +use super::{Database, now, sql_row}; impl Database { /// Store a hashed, short-lived OAuth authorization code. @@ -24,146 +16,144 @@ impl Database { /// # Errors /// /// Returns an error when the code cannot be persisted. - #[allow(clippy::too_many_arguments)] pub fn store_authorization_code( &self, code_hash: &str, did: &str, - client_id: &str, - redirect_uri: &str, - code_challenge: &str, - scope: &str, + request: &AuthorizationRequest>, expires_at: i64, ) -> Result<()> { - self.connection()?.execute( - "INSERT INTO oauth_codes + let client_id = request.client_id.as_ref(); + let redirect_uri = request.redirect_uri.as_ref(); + self.transaction(|transaction| { + purge_expired_oauth(transaction, now())?; + transaction.execute( + "INSERT INTO oauth_codes (code_hash, did, client_id, redirect_uri, code_challenge, scope, expires_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", - params![ - code_hash, - did, - client_id, - redirect_uri, - code_challenge, - scope, - expires_at, - ], - )?; - Ok(()) + params![ + code_hash, + did, + client_id, + redirect_uri, + request.code_challenge, + request.scope, + expires_at, + ], + )?; + Ok(()) + }) } - /// Atomically validate and consume an authorization code. + /// Atomically consume an authorization code and issue its access token. /// /// # Errors /// /// Returns `Authorization` for missing, expired, used, or mismatched codes, /// and a database error when the transaction fails. - pub fn redeem_authorization_code( + pub(crate) fn redeem_authorization_code( &self, code_hash: &str, client_id: &str, redirect_uri: &str, presented_challenge: &str, - ) -> Result { - let mut connection = self.connection()?; - let transaction = connection.transaction()?; - let code = transaction - .query_row( - "SELECT did, client_id, redirect_uri, code_challenge, scope, expires_at, used_at - FROM oauth_codes WHERE code_hash = ?1", - [code_hash], - |row| { - Ok(( - StoredAuthorizationCode { - did: row.get(0)?, - client_id: row.get(1)?, - redirect_uri: row.get(2)?, - code_challenge: row.get(3)?, - scope: row.get(4)?, - }, - row.get::<_, i64>(5)?, - row.get::<_, Option>(6)?, - )) - }, - ) - .optional()? - .ok_or_else(|| Error::Authorization("authorization code is invalid".to_owned()))?; - - if code.2.is_some() || code.1 < now() { - return Err(Error::Authorization( - "authorization code is expired or already used".to_owned(), - )); - } - if code.0.client_id != client_id - || code.0.redirect_uri != redirect_uri - || code.0.code_challenge != presented_challenge - { - return Err(Error::Authorization( - "authorization code is not valid for this token request".to_owned(), - )); - } - transaction.execute( - "UPDATE oauth_codes SET used_at = ?2 WHERE code_hash = ?1", - params![code_hash, now()], - )?; - transaction.commit()?; - Ok(code.0) - } - - /// Store a hashed access token for one account and client. - /// - /// # Errors - /// - /// Returns an error when the token cannot be persisted. - pub fn store_access_token( - &self, token_hash: &str, - did: &str, - client_id: &str, - scope: &str, - expires_at: i64, - ) -> Result<()> { - self.connection()?.execute( - "INSERT INTO access_tokens + token_expires_at: i64, + ) -> Result<(String, String)> { + self.transaction(|transaction| { + let timestamp = now(); + purge_expired_oauth(transaction, timestamp)?; + let (did, stored_client_id, stored_redirect_uri, code_challenge, scope) = + load_authorization_code(transaction, code_hash, timestamp)?; + if stored_client_id != client_id + || stored_redirect_uri != redirect_uri + || code_challenge != presented_challenge + { + return Err(Error::Authorization( + "authorization code is not valid for this token request".to_owned(), + )); + } + transaction.execute( + "INSERT INTO access_tokens (token_hash, did, client_id, scope, expires_at, created_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6)", - params![token_hash, did, client_id, scope, expires_at, now()], - )?; - Ok(()) - } - - /// Resolve a non-expired access token to its account. - /// - /// # Errors - /// - /// Returns `AuthenticationRequired` for unknown or expired tokens, and a - /// database error when lookup fails. - pub fn account_for_access_token(&self, token_hash: &str) -> Result { - Ok(self.access_token(token_hash)?.account) + params![ + token_hash, + did, + stored_client_id, + scope, + token_expires_at, + timestamp + ], + )?; + transaction.execute("DELETE FROM oauth_codes WHERE code_hash = ?1", [code_hash])?; + Ok((did, scope)) + }) } /// Resolve a non-expired access token with its granted client and scopes. pub(crate) fn access_token(&self, token_hash: &str) -> Result { - self.connection()? - .query_row( - "SELECT accounts.did, accounts.handle, access_tokens.client_id, + load_access_token(&*self.connection()?, token_hash, now()) + } +} + +fn load_authorization_code( + connection: &Connection, + code_hash: &str, + timestamp: i64, +) -> Result<(String, String, String, String, String)> { + connection + .query_row( + "SELECT did, client_id, redirect_uri, code_challenge, scope + FROM oauth_codes + WHERE code_hash = ?1 AND expires_at > ?2 AND used_at IS NULL", + params![code_hash, timestamp], + |row| row.try_into(), + ) + .optional()? + .ok_or_else(|| { + Error::Authorization( + "authorization code is invalid, expired, or already used".to_owned(), + ) + }) +} + +fn load_access_token( + connection: &Connection, + token_hash: &str, + timestamp: i64, +) -> Result { + connection + .query_row( + "SELECT accounts.did, accounts.handle, access_tokens.client_id, access_tokens.scope FROM access_tokens JOIN accounts ON accounts.did = access_tokens.did - WHERE access_tokens.token_hash = ?1 AND access_tokens.expires_at >= ?2", - params![token_hash, now()], - |row| { - Ok(AuthenticatedToken { - account: Account { - did: row.get(0)?, - handle: row.get(1)?, - }, - client_id: row.get(2)?, - scope: row.get(3)?, - }) - }, - ) - .optional()? - .ok_or(Error::AuthenticationRequired) - } + WHERE access_tokens.token_hash = ?1 AND access_tokens.expires_at > ?2", + params![token_hash, timestamp], + |row| { + Ok(AuthenticatedToken { + account: sql_row!(row => Account { did: 0, handle: 1 }), + client_id: row.get(2)?, + scope: row.get(3)?, + }) + }, + ) + .optional()? + .ok_or(Error::AuthenticationRequired) +} + +fn purge_expired_oauth(connection: &Connection, timestamp: i64) -> Result<()> { + connection.execute( + "DELETE FROM oauth_codes WHERE expires_at <= ?1", + [timestamp], + )?; + connection.execute( + "DELETE FROM access_tokens WHERE expires_at <= ?1", + [timestamp], + )?; + Ok(()) } + +#[cfg(test)] +mod tests; diff --git a/src/database/oauth/tests.rs b/src/database/oauth/tests.rs new file mode 100644 index 0000000..8bf2fcf --- /dev/null +++ b/src/database/oauth/tests.rs @@ -0,0 +1,306 @@ +use super::*; + +const DID: &str = "did:web:alice.test"; + +#[test] +fn authorization_request_storage_preserves_raw_and_typed_uri_bytes() { + let database = database(); + let expires_at = now() + 3_600; + for (index, (client, redirect)) in [ + ("client", "redirect"), + ("", ""), + ("raw client\0é", "raw redirect\0🦀"), + ( + "https://EXAMPLE.test:443/client", + "http://localhost:80/callback", + ), + ( + "https://café.example/path%2f", + "https://client.test/callback?x=1#part", + ), + ] + .into_iter() + .enumerate() + { + let key = format!("raw-{index}"); + let request = AuthorizationRequest { + client_id: client, + redirect_uri: redirect, + state: "state is not stored".to_owned(), + scope: "atproto\0scope".to_owned(), + code_challenge: "raw\0challenge".to_owned(), + }; + database + .store_authorization_code(&key, DID, &request, expires_at) + .unwrap(); + assert_stored_request(&database, &key, client, redirect, expires_at); + + let json = serde_json::json!({ + "client_id": client, + "redirect_uri": redirect, + "state": request.state, + "scope": request.scope, + "code_challenge": request.code_challenge, + }); + let typed: std::result::Result = serde_json::from_value(json); + let parsed = (url::Url::parse(client), url::Url::parse(redirect)); + if let (Ok(client), Ok(redirect)) = parsed { + let typed = typed.unwrap(); + let key = format!("typed-{index}"); + database + .store_authorization_code(&key, DID, &typed, expires_at) + .unwrap(); + assert_stored_request( + &database, + &key, + client.as_str(), + redirect.as_str(), + expires_at, + ); + } else { + assert!(typed.is_err()); + } + } +} + +fn assert_stored_request( + database: &Database, + key: &str, + client: &str, + redirect: &str, + expires_at: i64, +) { + let stored: (String, String, String, String, String, i64, Option) = database + .connection() + .unwrap() + .query_row( + "SELECT did, client_id, redirect_uri, code_challenge, scope, expires_at, used_at + FROM oauth_codes WHERE code_hash = ?1", + [key], + |row| row.try_into(), + ) + .unwrap(); + assert_eq!( + stored, + ( + DID.to_owned(), + client.to_owned(), + redirect.to_owned(), + "raw\0challenge".to_owned(), + "atproto\0scope".to_owned(), + expires_at, + None, + ) + ); +} + +#[test] +fn credentials_expire_at_the_expiration_second_and_never_reactivate() { + let database = database(); + let connection = database.connection().unwrap(); + seed_credentials(&connection, "credential", 100); + + assert!(load_authorization_code(&connection, "credential", 99).is_ok()); + assert!(load_access_token(&connection, "credential", 99).is_ok()); + for timestamp in [100, 101] { + assert!(matches!( + load_authorization_code(&connection, "credential", timestamp), + Err(Error::Authorization(_)) + )); + assert!(matches!( + load_access_token(&connection, "credential", timestamp), + Err(Error::AuthenticationRequired) + )); + } +} + +#[test] +fn legacy_consumed_codes_remain_unusable() { + let database = database(); + let connection = database.connection().unwrap(); + seed_credentials(&connection, "used", 100); + connection + .execute("UPDATE oauth_codes SET used_at = 1", []) + .unwrap(); + + assert!(matches!( + load_authorization_code(&connection, "used", 99), + Err(Error::Authorization(_)) + )); +} + +#[test] +fn expiration_cleanup_preserves_every_live_credential() { + let database = database(); + let connection = database.connection().unwrap(); + for (key, expires_at) in [("expired", 99), ("boundary", 100), ("live", 101)] { + seed_credentials(&connection, key, expires_at); + } + + purge_expired_oauth(&connection, 100).unwrap(); + + assert_eq!(remaining(&connection, "oauth_codes", "code_hash"), ["live"]); + assert_eq!( + remaining(&connection, "access_tokens", "token_hash"), + ["live"] + ); + assert!(load_authorization_code(&connection, "live", 100).is_ok()); + assert!(load_access_token(&connection, "live", 100).is_ok()); +} + +#[test] +fn issuing_a_code_cleans_expired_storage_and_redemption_removes_only_that_code() { + let database = database(); + let timestamp = now(); + { + let connection = database.connection().unwrap(); + seed_credentials(&connection, "expired", timestamp - 1); + seed_credentials(&connection, "live", timestamp + 3_600); + } + + database + .store_authorization_code( + "new", + DID, + &AuthorizationRequest { + client_id: "client", + redirect_uri: "redirect", + state: String::new(), + code_challenge: "challenge".to_owned(), + scope: "atproto".to_owned(), + }, + timestamp + 300, + ) + .unwrap(); + { + let connection = database.connection().unwrap(); + assert_eq!( + remaining(&connection, "oauth_codes", "code_hash"), + ["live", "new"] + ); + assert_eq!( + remaining(&connection, "access_tokens", "token_hash"), + ["live"] + ); + } + database + .redeem_authorization_code( + "new", + "client", + "redirect", + "challenge", + "issued", + timestamp + 3_600, + ) + .unwrap(); + let connection = database.connection().unwrap(); + assert_eq!(remaining(&connection, "oauth_codes", "code_hash"), ["live"]); + assert_eq!( + remaining(&connection, "access_tokens", "token_hash"), + ["issued", "live"] + ); + assert!(matches!( + load_authorization_code(&connection, "new", timestamp), + Err(Error::Authorization(_)) + )); + assert!(load_access_token(&connection, "issued", timestamp).is_ok()); +} + +#[test] +fn authorization_code_sql_types_precede_binding_mismatches_in_column_order() { + let database = database(); + let timestamp = now(); + { + let connection = database.connection().unwrap(); + seed_credentials(&connection, "credential", timestamp + 3_600); + seed_credentials(&connection, "expired", timestamp - 1); + } + + for (sql, expected_index, expected_name) in [ + ("UPDATE oauth_codes SET scope = x'00'", 4, "scope"), + ( + "UPDATE oauth_codes SET code_challenge = x'00'", + 3, + "code_challenge", + ), + ( + "UPDATE oauth_codes SET redirect_uri = x'00'", + 2, + "redirect_uri", + ), + ("UPDATE oauth_codes SET client_id = x'00'", 1, "client_id"), + ] { + database.connection().unwrap().execute(sql, []).unwrap(); + for (client_id, redirect_uri, challenge) in [ + ("different", "redirect", "challenge"), + ("client", "different", "challenge"), + ("client", "redirect", "different"), + ("client", "redirect", "challenge"), + ] { + let error = database + .redeem_authorization_code( + "credential", + client_id, + redirect_uri, + challenge, + "issued", + timestamp + 3_600, + ) + .unwrap_err(); + assert!(matches!( + error, + Error::Database(rusqlite::Error::InvalidColumnType( + index, name, rusqlite::types::Type::Blob + )) if index == expected_index && name == expected_name + )); + let connection = database.connection().unwrap(); + assert_eq!( + remaining(&connection, "oauth_codes", "code_hash"), + ["credential", "expired"] + ); + assert_eq!( + remaining(&connection, "access_tokens", "token_hash"), + ["credential", "expired"] + ); + } + } +} + +fn database() -> Database { + let database = Database::open_in_memory().unwrap(); + database + .connection() + .unwrap() + .execute( + "INSERT INTO accounts (did, handle, password_hash, created_at) + VALUES (?1, 'alice.test', 'hash', 1)", + [DID], + ) + .unwrap(); + database +} + +fn seed_credentials(connection: &Connection, key: &str, expires_at: i64) { + connection.execute( + "INSERT INTO oauth_codes (code_hash, did, client_id, redirect_uri, code_challenge, scope, expires_at) + VALUES (?1, ?2, 'client', 'redirect', 'challenge', 'atproto', ?3)", + params![key, DID, expires_at], + ).unwrap(); + connection + .execute( + "INSERT INTO access_tokens (token_hash, did, client_id, scope, expires_at, created_at) + VALUES (?1, ?2, 'client', 'atproto', ?3, 1)", + params![key, DID, expires_at], + ) + .unwrap(); +} + +fn remaining(connection: &Connection, table: &str, key: &str) -> Vec { + connection + .prepare(&format!("SELECT {key} FROM {table} ORDER BY {key}")) + .unwrap() + .query_map([], |row| row.get(0)) + .unwrap() + .collect::>() + .unwrap() +} diff --git a/src/database/query.rs b/src/database/query.rs new file mode 100644 index 0000000..6ac6516 --- /dev/null +++ b/src/database/query.rs @@ -0,0 +1,48 @@ +//! Fallible row decoding shared by connection and transaction queries. + +use rusqlite::{Connection, OptionalExtension, Params, Row}; + +use crate::error::{Error, Result}; + +// Read explicitly indexed fields in declaration order; Rust infers their types. +macro_rules! sql_row { + ($row:ident => $kind:ident { $($field:ident: $index:literal),+ $(,)? }) => { + $kind { $($field: $row.get($index)?),+ } + }; +} +pub(crate) use sql_row; + +pub(crate) fn constraint_conflict(error: rusqlite::Error, message: &str) -> Error { + if error.sqlite_error_code() == Some(rusqlite::ErrorCode::ConstraintViolation) { + Error::Conflict(message.to_owned()) + } else { + Error::Database(error) + } +} + +pub(crate) fn query( + connection: &Connection, + sql: &str, + params: impl Params, + decode: impl FnMut(&Row<'_>) -> Result, +) -> Result> { + connection + .prepare(sql)? + .query_and_then(params, decode)? + .collect() +} + +pub(crate) fn query_optional( + connection: &Connection, + sql: &str, + params: impl Params, + decode: impl FnOnce(&Row<'_>) -> Result, +) -> Result> { + connection + .query_row(sql, params, |row| Ok(decode(row))) + .optional()? + .transpose() +} + +#[cfg(test)] +mod tests; diff --git a/src/database/query/tests.rs b/src/database/query/tests.rs new file mode 100644 index 0000000..e5db88a --- /dev/null +++ b/src/database/query/tests.rs @@ -0,0 +1,365 @@ +use serde_json::Value; + +use crate::database::Database; + +use super::*; + +fn conversion_cause(error: &rusqlite::Error) -> Option<*const ()> { + match error { + rusqlite::Error::FromSqlConversionFailure(_, _, cause) + | rusqlite::Error::ToSqlConversionFailure(cause) => { + Some(std::ptr::from_ref(cause.as_ref()).cast()) + } + _ => None, + } +} + +fn assert_native_constraint_classification( + make_error: impl Fn() -> rusqlite::Error, + message: &str, +) { + let original = make_error(); + let original_kind = std::mem::discriminant(&original); + let original_debug = format!("{original:?}"); + let original_display = original.to_string(); + let original_cause = conversion_cause(&original); + let actual = constraint_conflict(original, message); + let previous = match make_error() { + rusqlite::Error::SqliteFailure(ref failure, _) + if failure.code == rusqlite::ErrorCode::ConstraintViolation => + { + Error::Conflict(message.to_owned()) + } + other => Error::Database(other), + }; + assert_eq!( + std::mem::discriminant(&actual), + std::mem::discriminant(&previous) + ); + assert_eq!(format!("{actual:?}"), format!("{previous:?}")); + assert_eq!(actual.to_string(), previous.to_string()); + if let Error::Database(retained) = actual { + assert_eq!(std::mem::discriminant(&retained), original_kind); + assert_eq!(format!("{retained:?}"), original_debug); + assert_eq!(retained.to_string(), original_display); + assert_eq!(conversion_cause(&retained), original_cause); + } +} + +#[test] +fn native_sqlite_error_codes_preserve_conflict_classification_and_payloads() { + use rusqlite::{Error as SqlError, ffi, types::Type}; + + for message in ["", "record already exists", "duplicate\0é\n\"value\""] { + for code in [ + ffi::SQLITE_OK, + ffi::SQLITE_ERROR, + ffi::SQLITE_BUSY, + ffi::SQLITE_READONLY, + ffi::SQLITE_IOERR, + ffi::SQLITE_IOERR_READ, + ffi::SQLITE_CONSTRAINT, + ffi::SQLITE_CONSTRAINT_CHECK, + ffi::SQLITE_CONSTRAINT_FOREIGNKEY, + ffi::SQLITE_CONSTRAINT_NOTNULL, + ffi::SQLITE_CONSTRAINT_PRIMARYKEY, + ffi::SQLITE_CONSTRAINT_UNIQUE, + ] { + for detail in [None, Some(""), Some("SQLite detail\0é\n")] { + assert_native_constraint_classification( + || SqlError::SqliteFailure(ffi::Error::new(code), detail.map(str::to_owned)), + message, + ); + } + } + for case in 0..8 { + assert_native_constraint_classification( + || match case { + 0 => SqlError::QueryReturnedNoRows, + 1 => SqlError::QueryReturnedMoreThanOneRow, + 2 => SqlError::InvalidColumnType(3, "scope".to_owned(), Type::Blob), + 3 => SqlError::IntegralValueOutOfRange(4, i64::MAX), + 4 => SqlError::FromSqlConversionFailure( + 5, + Type::Text, + Box::new(SqlError::SqliteFailure( + ffi::Error::new(ffi::SQLITE_CONSTRAINT_UNIQUE), + Some("nested constraint".to_owned()), + )), + ), + 5 => SqlError::ToSqlConversionFailure(Box::new(SqlError::SqliteFailure( + ffi::Error::new(ffi::SQLITE_CONSTRAINT_NOTNULL), + None, + ))), + 6 => SqlError::InvalidParameterCount(1, 2), + _ => SqlError::InvalidQuery, + }, + message, + ); + } + } +} + +#[test] +fn native_tuples_preserve_column_types_and_first_failure() { + use rusqlite::types::Value::{Blob, Integer, Null, Text}; + + type OutboxRow = ( + i64, + String, + Option, + Option, + String, + Vec, + i64, + ); + let connection = Connection::open_in_memory().unwrap(); + let sql = "SELECT ?1, ?2, ?3, ?4, ?5, ?6, ?7"; + let valid = [ + Integer(1), + Text("did:web:test".into()), + Null, + Null, + Text("identity".into()), + Blob(b"{}".to_vec()), + Integer(123), + ]; + let mut nullable = valid.clone(); + nullable[2] = Text("revision".into()); + nullable[3] = Text("root".into()); + let mut cases = vec![valid.clone(), nullable]; + for first_invalid in 0..valid.len() { + let mut values = valid.clone(); + for (column, value) in values.iter_mut().enumerate().skip(first_invalid) { + *value = if column == 5 { + Text("not a blob".into()) + } else { + Blob(vec![0]) + }; + } + cases.push(values); + } + for values in cases { + let expected = connection.query_row(sql, rusqlite::params_from_iter(&values), |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, String>(1)?, + row.get::<_, Option>(2)?, + row.get::<_, Option>(3)?, + row.get::<_, String>(4)?, + row.get::<_, Vec>(5)?, + row.get::<_, i64>(6)?, + )) + }); + let actual = connection.query_row(sql, rusqlite::params_from_iter(&values), |row| { + OutboxRow::try_from(row) + }); + match (expected, actual) { + (Ok(expected), Ok(actual)) => assert_eq!(actual, expected), + (Err(expected), Err(actual)) => { + assert_eq!( + std::mem::discriminant(&actual), + std::mem::discriminant(&expected) + ); + assert_eq!(actual.to_string(), expected.to_string()); + } + results => panic!("tuple decoding diverged for {values:?}: {results:?}"), + } + } +} + +#[test] +fn indexed_struct_rows_preserve_sql_and_domain_error_precedence() { + use crate::{ + database::parse_account_status, + domain::account::{Account, AccountStatus}, + }; + use rusqlite::types::Value::{Blob, Text}; + + #[derive(Debug)] + struct PreviousHostedAccount { + account: Account, + status: AccountStatus, + } + + let connection = Connection::open_in_memory().unwrap(); + let sql = "SELECT ?1 AS did, ?2 AS handle, ?3 AS status"; + let cases = [ + [ + Text("did:web:test".into()), + Text("test".into()), + Text("active".into()), + ], + [Blob(vec![0]), Blob(vec![0]), Text("invalid-status".into())], + [ + Text("did:web:test".into()), + Blob(vec![0]), + Text("invalid-status".into()), + ], + [ + Text("did:web:test".into()), + Text("test".into()), + Text("invalid-status".into()), + ], + [ + Text("did:web:test".into()), + Text("test".into()), + Blob(vec![0]), + ], + ]; + for values in cases { + let expected: Result = + connection.query_row_and_then(sql, rusqlite::params_from_iter(&values), |row| { + Ok(PreviousHostedAccount { + account: Account { + did: row.get(0)?, + handle: row.get(1)?, + }, + status: parse_account_status(&row.get::<_, String>(2)?)?, + }) + }); + let actual: Result<(Account, AccountStatus)> = + connection.query_row_and_then(sql, rusqlite::params_from_iter(&values), |row| { + Ok(( + sql_row!(row => Account { did: 0, handle: 1 }), + parse_account_status(&row.get::<_, String>(2)?)?, + )) + }); + match (expected, actual) { + (Ok(expected), Ok(actual)) => assert_eq!(actual, (expected.account, expected.status)), + (Err(expected), Err(actual)) => { + assert_eq!( + std::mem::discriminant(&actual), + std::mem::discriminant(&expected) + ); + assert_eq!(actual.to_string(), expected.to_string()); + assert_eq!(format!("{actual:?}"), format!("{expected:?}")); + } + results => panic!("row decoding diverged: {results:?}"), + } + } +} + +#[test] +fn optional_queries_distinguish_absence_from_decode_failures() { + let connection = Connection::open_in_memory().unwrap(); + assert_eq!( + query_optional(&connection, "SELECT 1 WHERE 0", [], |row| Ok( + row.get::<_, i64>(0)? + )) + .unwrap(), + None + ); + assert!(matches!( + Database::open_in_memory() + .unwrap() + .query_one("SELECT 1 WHERE 0", [], |_| Ok(())), + Err(Error::Database(rusqlite::Error::QueryReturnedNoRows)) + )); + assert!(matches!( + query_optional::<()>(&connection, "SELECT 1", [], |_| { + Err(Error::Database(rusqlite::Error::QueryReturnedNoRows)) + }), + Err(Error::Database(rusqlite::Error::QueryReturnedNoRows)) + )); +} + +#[test] +fn streamed_queries_preserve_row_order_and_first_decode_error() { + let connection = Connection::open_in_memory().unwrap(); + let values = query( + &connection, + "SELECT ?1 UNION ALL SELECT ?2", + ["first", "second"], + |row| Ok(row.get::<_, String>(0)?), + ) + .unwrap(); + assert_eq!(values, ["first", "second"]); + let error = query::( + &connection, + "SELECT 'invalid-json' UNION ALL SELECT x'00'", + [], + |row| Ok(serde_json::from_str(&row.get::<_, String>(0)?)?), + ) + .unwrap_err(); + assert!(matches!(error, Error::Json(_))); +} + +#[test] +fn delegated_queries_match_manual_iteration_at_each_failure_stage() { + let connection = Connection::open_in_memory().unwrap(); + for sql in [ + "SELECT '1' UNION ALL SELECT '2'", + "SELECT 'null' WHERE 0", + "SELECT 'invalid-json' UNION ALL SELECT abs(-9223372036854775808)", + "SELECT 'null' UNION ALL SELECT abs(-9223372036854775808)", + "SELECT abs(-9223372036854775808) UNION ALL SELECT 'invalid-json'", + "SELECT x'00' UNION ALL SELECT 'invalid-json'", + "SELECT 'invalid-json' UNION ALL SELECT x'00'", + "SELECT ?1", + "SELECT missing_column", + ] { + let mut expected_calls = 0; + let expected = (|| -> Result> { + let mut statement = connection.prepare(sql)?; + let mut rows = statement.query([])?; + let mut values = Vec::new(); + while let Some(row) = rows.next()? { + expected_calls += 1; + values.push(serde_json::from_str(&row.get::<_, String>(0)?)?); + } + Ok(values) + })(); + let mut actual_calls = 0; + let actual = query::(&connection, sql, [], |row| { + actual_calls += 1; + Ok(serde_json::from_str(&row.get::<_, String>(0)?)?) + }); + assert_eq!(actual_calls, expected_calls, "{sql}"); + match (expected, actual) { + (Ok(expected), Ok(actual)) => assert_eq!(actual, expected, "{sql}"), + (Err(expected), Err(actual)) => { + assert_eq!( + std::mem::discriminant(&actual), + std::mem::discriminant(&expected), + "{sql}" + ); + assert_eq!(actual.to_string(), expected.to_string(), "{sql}"); + } + results => panic!("query iteration diverged for {sql}: {results:?}"), + } + } +} + +#[test] +fn transactions_commit_success_and_rollback_failure() { + let database = Database::open_in_memory().unwrap(); + database + .connection() + .unwrap() + .execute_batch("CREATE TEMP TABLE transaction_probe (value INTEGER);") + .unwrap(); + let result: Result<()> = database.transaction(|transaction| { + transaction.execute("INSERT INTO transaction_probe VALUES (1)", [])?; + Err(Error::NotFound) + }); + assert!(matches!(result, Err(Error::NotFound))); + let count = || { + database + .query_one("SELECT COUNT(*) FROM transaction_probe", [], |row| { + Ok(row.get::<_, i64>(0)?) + }) + .unwrap() + }; + assert_eq!(count(), 0); + assert_eq!( + database + .transaction(|transaction| { + transaction.execute("INSERT INTO transaction_probe VALUES (2)", [])?; + Ok(42) + }) + .unwrap(), + 42 + ); + assert_eq!(count(), 1); +} diff --git a/src/database/record.rs b/src/database/record.rs index 555033b..9c47770 100644 --- a/src/database/record.rs +++ b/src/database/record.rs @@ -1,38 +1,36 @@ //! Record persistence indexed by repository path. -use std::str::FromStr; - -use cid::Cid; -use rusqlite::{Connection, OptionalExtension, params}; +use rusqlite::{Row, params}; use crate::{ domain::record::Record, - error::{Error, Result}, + error::{Error, ErrorContext as _, Result}, }; use super::Database; impl Database { pub(crate) fn list_collections(&self, did: &str) -> Result> { - let connection = self.connection()?; - let mut statement = connection.prepare( + self.query( "SELECT DISTINCT collection FROM records WHERE did = ?1 ORDER BY collection", - )?; - statement - .query_map([did], |row| row.get::<_, String>(0))? - .map(|row| row.map_err(Into::into)) - .collect() + [did], + |row| Ok(row.get(0)?), + ) } - /// Load one record by its complete repository path. + /// Find one record by its complete repository path, returning `None` when absent. + /// Absence remains distinct from a lookup or decoding failure. /// /// # Errors /// - /// Returns `NotFound` when absent, or an error for invalid stored data and - /// failed database access. - pub fn get_record(&self, did: &str, collection: &str, rkey: &str) -> Result { - let connection = self.connection()?; - load_record(&connection, did, collection, rkey) + /// Returns an error for invalid stored data or failed database access. + pub fn find_record(&self, did: &str, collection: &str, rkey: &str) -> Result> { + self.query_optional( + "SELECT did, collection, rkey, cid, record_json FROM records + WHERE did = ?1 AND collection = ?2 AND rkey = ?3", + params![did, collection, rkey], + decode_record, + ) } /// List the newest record keys within one collection. @@ -48,83 +46,211 @@ impl Database { reverse: bool, limit: usize, ) -> Result> { - let connection = self.connection()?; let sql = if reverse { - "SELECT rkey, cid, record_json FROM records + "SELECT did, collection, rkey, cid, record_json FROM records WHERE did = ?1 AND collection = ?2 AND (?3 IS NULL OR rkey > ?3) ORDER BY rkey ASC LIMIT ?4" } else { - "SELECT rkey, cid, record_json FROM records + "SELECT did, collection, rkey, cid, record_json FROM records WHERE did = ?1 AND collection = ?2 AND (?3 IS NULL OR rkey < ?3) ORDER BY rkey DESC LIMIT ?4" }; - let mut statement = connection.prepare(sql)?; - let rows = statement.query_map( + self.query( + sql, params![ did, collection, cursor, i64::try_from(limit).unwrap_or(i64::MAX) ], - |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, String>(2)?, - )) - }, - )?; - - rows.map(|row| { - let (rkey, cid, json) = row?; - decode(did, collection, &rkey, &cid, &json) - }) - .collect() + decode_record, + ) } pub(crate) fn all_records(&self, did: &str) -> Result> { - let connection = self.connection()?; - let mut statement = connection.prepare( - "SELECT collection, rkey, cid, record_json FROM records + self.query( + "SELECT did, collection, rkey, cid, record_json FROM records WHERE did = ?1 ORDER BY collection, rkey", - )?; - let rows = statement.query_map([did], |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, String>(2)?, - row.get::<_, String>(3)?, - )) - })?; - rows.map(|row| { - let (collection, rkey, cid, json) = row?; - decode(did, &collection, &rkey, &cid, &json) - }) - .collect() - } -} - -fn load_record(connection: &Connection, did: &str, collection: &str, rkey: &str) -> Result { - connection - .query_row( - "SELECT cid, record_json FROM records - WHERE did = ?1 AND collection = ?2 AND rkey = ?3", - params![did, collection, rkey], - |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), + [did], + decode_record, ) - .optional()? - .map(|(cid, json)| decode(did, collection, rkey, &cid, &json)) - .transpose()? - .ok_or(Error::NotFound) + } } -fn decode(did: &str, collection: &str, rkey: &str, cid: &str, json: &str) -> Result { +fn decode_record(row: &Row<'_>) -> Result { + let (did, collection, rkey, cid, json) = <(_, _, _, String, String)>::try_from(row)?; Ok(Record { - did: did.to_owned(), - collection: collection.to_owned(), - rkey: rkey.to_owned(), - cid: Cid::from_str(cid) - .map_err(|error| Error::Internal(format!("stored CID is invalid: {error}")))?, - value: serde_json::from_str(json)?, + did, + collection, + rkey, + cid: cid + .parse() + .context(Error::Internal, "stored CID is invalid")?, + value: serde_json::from_str(&json)?, }) } + +#[cfg(test)] +mod tests { + use super::*; + use rusqlite::types::Value as SqlValue; + + fn assert_optional_lookup_matches_previous_query(database: &Database) -> Result { + let previous = database + .query_optional( + "SELECT did, collection, rkey, cid, record_json FROM records + WHERE did = ?1 AND collection = ?2 AND rkey = ?3", + params!["did:web:test", "app.test.record", "rkey"], + decode_record, + ) + .and_then(|record| record.ok_or(Error::NotFound)); + let actual = database + .find_record("did:web:test", "app.test.record", "rkey") + .and_then(|record| record.ok_or(Error::NotFound)); + assert_eq!(format!("{actual:?}"), format!("{previous:?}")); + actual + } + + #[test] + fn optional_record_lookup_preserves_absence_values_and_decoding_errors() { + let database = Database::open_in_memory().unwrap(); + assert!( + database + .find_record("did:web:test", "app.test.record", "rkey") + .unwrap() + .is_none() + ); + assert!(matches!( + assert_optional_lookup_matches_previous_query(&database), + Err(Error::NotFound) + )); + database + .connection() + .unwrap() + .execute_batch("CREATE TEMP TABLE records (did, collection, rkey, cid, record_json)") + .unwrap(); + let valid = cpds_protocol::sha256_cid(0x71, b"record").to_string(); + let mut successes = 0; + for cid in [ + SqlValue::Text(valid), + SqlValue::Text(String::new()), + SqlValue::Text("invalid-cid".to_owned()), + SqlValue::Null, + SqlValue::Blob(vec![0]), + SqlValue::Integer(7), + SqlValue::Real(7.5), + ] { + for json in [ + SqlValue::Text(r#"{"text":"value"}"#.to_owned()), + SqlValue::Text("null".to_owned()), + SqlValue::Text("{".to_owned()), + SqlValue::Null, + SqlValue::Blob(vec![0]), + SqlValue::Integer(7), + SqlValue::Real(7.5), + ] { + { + let connection = database.connection().unwrap(); + connection.execute("DELETE FROM records", []).unwrap(); + connection + .execute( + "INSERT INTO records VALUES + ('did:web:test', 'app.test.record', 'rkey', ?1, ?2)", + params![cid, json], + ) + .unwrap(); + } + successes += + usize::from(assert_optional_lookup_matches_previous_query(&database).is_ok()); + } + } + assert_eq!(successes, 2); + database + .connection() + .unwrap() + .execute_batch("DROP TABLE temp.records; DROP TABLE main.records") + .unwrap(); + assert!(matches!( + assert_optional_lookup_matches_previous_query(&database), + Err(Error::Database(_)) + )); + } + + #[test] + fn stored_record_cid_context_preserves_decoding_results_and_error_order() { + let database = Database::open_in_memory().unwrap(); + let cid = cpds_protocol::sha256_cid(0x71, b"record"); + let v0 = cid::Cid::new_v0(*cid.hash()).unwrap().to_string(); + let valid = cid.to_string(); + for cid in [ + valid.as_str(), + v0.as_str(), + "", + "invalid-cid", + "b", + "Qm", + "é", + " bafyreia", + ] { + for json in [r#"{"text":"value"}"#, "null", "{"] { + assert_record_decoding( + &database, + &["did:web:test", "app.test.record", "rkey", cid, json] + .map(|value| SqlValue::Text(value.to_owned())), + ); + } + } + for index in 0..5 { + for invalid in [ + SqlValue::Null, + SqlValue::Integer(7), + SqlValue::Real(7.5), + SqlValue::Blob(vec![0]), + ] { + let mut row = [ + "did:web:test", + "app.test.record", + "rkey", + "invalid-cid", + "{", + ] + .map(|value| SqlValue::Text(value.to_owned())); + row[index] = invalid; + assert_record_decoding(&database, &row); + } + } + } + + fn assert_record_decoding(database: &Database, values: &[SqlValue; 5]) { + let sql = "SELECT ?1, ?2, ?3, ?4, ?5"; + let old = database.query_one(sql, rusqlite::params_from_iter(values), |row| { + let (did, collection, rkey, cid, json) = <(_, _, _, String, String)>::try_from(row)?; + Ok(Record { + did, + collection, + rkey, + cid: ::from_str(&cid) + .map_err(|error| Error::Internal(format!("stored CID is invalid: {error}")))?, + value: serde_json::from_str(&json)?, + }) + }); + let current = database.query_one(sql, rusqlite::params_from_iter(values), decode_record); + assert_eq!(format!("{current:?}"), format!("{old:?}"), "{values:?}"); + } + + #[test] + fn record_decoding_checks_all_sql_types_before_parsing_a_cid() { + let database = Database::open_in_memory().unwrap(); + let error = database + .query_one( + "SELECT 'did:web:test', 'app.test.record', 'rkey', 'invalid-cid', x'00'", + [], + decode_record, + ) + .unwrap_err(); + assert!(matches!( + error, + Error::Database(rusqlite::Error::InvalidColumnType(_, _, _)) + )); + } +} diff --git a/src/database/repo.rs b/src/database/repo.rs index fb41c0f..47c4e9b 100644 --- a/src/database/repo.rs +++ b/src/database/repo.rs @@ -1,26 +1,25 @@ //! Atomic repository-v3 block, head, index, operation, and outbox persistence. -use std::{ - collections::{HashMap, HashSet}, - str::FromStr, -}; +use std::{collections::HashSet, str::FromStr}; -use atrium_repo::{ - Cid, Multihash, - blockstore::{ - AsyncBlockStoreRead, AsyncBlockStoreWrite, DAG_CBOR, Error as BlockError, SHA2_256, - }, -}; -use rusqlite::{OptionalExtension, Transaction, params}; +use atrium_api::{com::atproto::sync::subscribe_repos::RepoOpData, types::CidLink}; +use atrium_repo::Cid; +use rusqlite::{Connection, OptionalExtension, Transaction, params}; use serde::{Deserialize, Serialize}; -use sha2::{Digest, Sha256}; use crate::{ - domain::{account::AccountStatus, record::Record}, - error::{Error, Result}, + domain::{ + account::AccountStatus, + record::{Record, RecordMutation}, + }, + error::{Error, ErrorContext, Result}, }; -use super::{Database, active_blob_ref_cids, now}; +use super::blockstore::verify_block; +use super::{ + Database, active_blob_ref_cids, constraint_conflict, now, parse_account_status, query, + query_optional, require_account_active, +}; #[derive(Clone, Debug, Eq, PartialEq)] pub(crate) struct RepoHead { @@ -37,16 +36,6 @@ pub(crate) struct ListedRepoHead { pub(crate) created_at_millis: i64, } -#[derive(Clone)] -pub(crate) enum PendingRecordMutation { - Create(Record), - Update { - current: Record, - replacement: Record, - }, - Delete(Record), -} - pub(crate) struct PendingRepoCommit { pub(crate) did: String, pub(crate) expected_head: Option, @@ -54,7 +43,8 @@ pub(crate) struct PendingRepoCommit { pub(crate) previous_data: Option, pub(crate) head: RepoHead, pub(crate) blocks: Vec<(Cid, Vec)>, - pub(crate) mutation: PendingRecordMutation, + pub(crate) event_blocks: Vec, + pub(crate) mutations: Vec, } #[derive(Debug, Eq, PartialEq)] @@ -83,10 +73,7 @@ pub(crate) struct StoredCommitEvent { pub(crate) root: Cid, pub(crate) since: Option, pub(crate) prev_data: Option, - pub(crate) action: String, - pub(crate) path: String, - pub(crate) cid: Option, - pub(crate) prev: Option, + pub(crate) operations: Vec, pub(crate) blocks: Vec<(Cid, Vec)>, pub(crate) created_at: i64, } @@ -108,99 +95,9 @@ pub(crate) struct StoredAccountEvent { pub(crate) created_at: i64, } -/// `ATrium` writes into this in-memory overlay. `SQLite` is read on demand, but no -/// staged write is visible until [`Database::commit_repo`] publishes the batch. -pub(crate) struct RepoBlockStore { - database: Database, - writes: HashMap>, -} - -impl RepoBlockStore { - pub(crate) fn new(database: Database) -> Self { - Self { - database, - writes: HashMap::new(), - } - } - - pub(crate) fn reachable_writes( - &self, - reachable: impl IntoIterator, - ) -> Result)>> { - let reachable = reachable.into_iter().collect::>(); - for cid in &reachable { - if !self.writes.contains_key(cid) { - self.database.repo_block(*cid)?; - } - } - Ok(self - .writes - .iter() - .filter(|(cid, _)| reachable.contains(cid)) - .map(|(cid, bytes)| (*cid, bytes.clone())) - .collect()) - } - - pub(crate) fn writes(&self) -> Vec<(Cid, Vec)> { - self.writes - .iter() - .map(|(cid, bytes)| (*cid, bytes.clone())) - .collect() - } -} - -impl AsyncBlockStoreRead for RepoBlockStore { - fn read_block_into( - &mut self, - cid: Cid, - contents: &mut Vec, - ) -> impl Future> + Send { - let block = self - .writes - .get(&cid) - .cloned() - .map_or_else(|| self.database.repo_block(cid), Ok) - .map_err(block_error); - async move { - contents.clear(); - contents.extend_from_slice(&block?); - Ok(()) - } - } -} - -impl AsyncBlockStoreWrite for RepoBlockStore { - fn write_block( - &mut self, - codec: u64, - hash: u64, - contents: &[u8], - ) -> impl Future> + Send { - let result = block_cid(codec, hash, contents).inspect(|cid| { - self.writes.insert(*cid, contents.to_vec()); - }); - async move { result } - } -} - impl Database { pub(crate) fn repo_head(&self, did: &str) -> Result> { - self.connection()? - .query_row( - "SELECT root_cid, rev FROM repo_heads WHERE did = ?1", - [did], - |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), - ) - .optional()? - .map(|(root, rev)| { - Ok(RepoHead { - root: Cid::from_str(&root).map_err(|error| { - Error::Internal(format!("stored repository root CID is invalid: {error}")) - })?, - rev, - }) - }) - .transpose() + load_head(&*self.connection()?, did) } pub(crate) fn prune_repo_blocks( @@ -208,21 +105,20 @@ impl Database { reachable: &HashSet, limit: usize, ) -> Result { - let mut connection = self.connection()?; - let transaction = connection.transaction()?; - transaction.execute_batch( - "CREATE TEMP TABLE gc_reachable_repo_blocks ( + self.transaction(|transaction| { + transaction.execute_batch( + "CREATE TEMP TABLE gc_reachable_repo_blocks ( cid TEXT PRIMARY KEY ) WITHOUT ROWID;", - )?; - for cid in reachable { - transaction.execute( - "INSERT INTO gc_reachable_repo_blocks (cid) VALUES (?1)", - [cid.to_string()], )?; - } - let deleted = transaction.execute( - "DELETE FROM repo_blocks + for cid in reachable { + transaction.execute( + "INSERT INTO gc_reachable_repo_blocks (cid) VALUES (?1)", + [cid.to_string()], + )?; + } + let deleted = transaction.execute( + "DELETE FROM repo_blocks WHERE cid IN ( SELECT repo_blocks.cid FROM repo_blocks WHERE NOT EXISTS ( @@ -244,11 +140,11 @@ impl Database { ORDER BY repo_blocks.created_at, repo_blocks.cid LIMIT ?1 )", - [i64::try_from(limit).unwrap_or(i64::MAX)], - )?; - transaction.execute_batch("DROP TABLE gc_reachable_repo_blocks;")?; - transaction.commit()?; - Ok(deleted) + [i64::try_from(limit).unwrap_or(i64::MAX)], + )?; + transaction.execute_batch("DROP TABLE gc_reachable_repo_blocks;")?; + Ok(deleted) + }) } pub(crate) fn commit_repo( @@ -256,103 +152,107 @@ impl Database { commit: &PendingRepoCommit, event_retention: usize, ) -> Result<()> { - let mut connection = self.connection()?; - let transaction = connection.transaction()?; - require_account_active(&transaction, &commit.did)?; - let current = load_head(&transaction, &commit.did)?; - if current.as_ref().map(|head| head.root) != commit.expected_head { - return Err(Error::Conflict("repository head changed".to_owned())); - } + self.transaction(|transaction| { + require_account_active(transaction, &commit.did)?; + let current = load_head(transaction, &commit.did)?; + if current.as_ref().map(|head| head.root) != commit.expected_head { + return Err(Error::Conflict("repository head changed".to_owned())); + } - let timestamp = now(); - for (cid, bytes) in &commit.blocks { - verify_block(*cid, bytes)?; - transaction.execute( - "INSERT INTO repo_blocks (cid, bytes, created_at) + let timestamp = now(); + for (cid, bytes) in &commit.blocks { + verify_block(*cid, bytes)?; + transaction.execute( + "INSERT INTO repo_blocks (cid, bytes, created_at) VALUES (?1, ?2, ?3) ON CONFLICT(cid) DO NOTHING", - params![cid.to_string(), bytes, timestamp], - )?; - let stored: Vec = transaction.query_row( - "SELECT bytes FROM repo_blocks WHERE cid = ?1", - [cid.to_string()], - |row| row.get::<_, Vec>(0), - )?; - if stored != *bytes { - return Err(Error::Internal(format!( - "stored repository block does not match CID {cid}" - ))); + params![cid.to_string(), bytes, timestamp], + )?; + let stored: Vec = transaction.query_row( + "SELECT bytes FROM repo_blocks WHERE cid = ?1", + [cid.to_string()], + |row| row.get::<_, Vec>(0), + )?; + if stored != *bytes { + return Err(Error::Internal(format!( + "stored repository block does not match CID {cid}" + ))); + } } - } - ensure_block(&transaction, commit.head.root)?; - let operation = apply_record_mutation(&transaction, &commit.mutation, timestamp)?; - transaction.execute( - "INSERT INTO repo_heads (did, root_cid, rev, updated_at) + ensure_block(transaction, commit.head.root)?; + let operations = commit + .mutations + .iter() + .map(|mutation| apply_record_mutation(transaction, mutation, timestamp)) + .collect::>>()?; + transaction.execute( + "INSERT INTO repo_heads (did, root_cid, rev, updated_at) VALUES (?1, ?2, ?3, ?4) ON CONFLICT(did) DO UPDATE SET root_cid = excluded.root_cid, rev = excluded.rev, updated_at = excluded.updated_at", - params![ - commit.did, - commit.head.root.to_string(), - commit.head.rev, - timestamp - ], - )?; - transaction.execute( - "INSERT INTO repo_ops - (did, rev, action, collection, rkey, cid, prev, created_at) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", - params![ - commit.did, - commit.head.rev, - operation.action, - operation.collection, - operation.rkey, - operation.cid, - operation.prev, - timestamp - ], - )?; - if current.is_none() { - insert_initial_repo_events(&transaction, &commit.did, timestamp)?; - } - let payload = outbox_payload(commit, &operation)?; - transaction.execute( - "INSERT INTO repo_outbox + params![ + commit.did, + commit.head.root.to_string(), + commit.head.rev, + timestamp + ], + )?; + for (record, operation) in &operations { + transaction.execute( + "INSERT INTO repo_ops + (did, rev, action, collection, rkey, cid, prev, created_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", + params![ + commit.did, + commit.head.rev, + operation.action, + record.collection, + record.rkey, + operation.cid, + operation.prev, + timestamp + ], + )?; + } + if current.is_none() { + insert_initial_repo_events(transaction, &commit.did, timestamp)?; + } + let operations = operations + .into_iter() + .map(|(_, operation)| operation) + .collect(); + let payload = outbox_payload(commit, operations)?; + transaction.execute( + "INSERT INTO repo_outbox (did, rev, root_cid, event_type, payload, next_attempt_at, created_at) VALUES (?1, ?2, ?3, 'repo.commit', ?4, ?5, ?5)", - params![ - commit.did, - commit.head.rev, - commit.head.root.to_string(), - payload, - timestamp - ], - )?; - let sequence = transaction.last_insert_rowid(); - let mut block_cids = commit - .blocks - .iter() - .map(|(cid, _)| *cid) - .collect::>(); - block_cids.sort_unstable_by_key(ToString::to_string); - block_cids.retain(|cid| cid != &commit.head.root); - block_cids.insert(0, commit.head.root); - for (position, cid) in block_cids.into_iter().enumerate() { - transaction.execute( - "INSERT INTO repo_commit_blocks (seq, cid, position) VALUES (?1, ?2, ?3)", params![ - sequence, - cid.to_string(), - i64::try_from(position).unwrap_or(i64::MAX) + commit.did, + commit.head.rev, + commit.head.root.to_string(), + payload, + timestamp ], )?; - } - prune_repo_events(&transaction, event_retention)?; - transaction.commit()?; - Ok(()) + let sequence = transaction.last_insert_rowid(); + let mut block_cids = commit.event_blocks.clone(); + block_cids.sort_unstable_by_key(ToString::to_string); + block_cids.retain(|cid| cid != &commit.head.root); + block_cids.insert(0, commit.head.root); + for (position, cid) in block_cids.into_iter().enumerate() { + transaction.execute( + "INSERT INTO repo_commit_blocks (seq, cid, position) VALUES (?1, ?2, ?3)", + params![ + sequence, + cid.to_string(), + i64::try_from(position).unwrap_or(i64::MAX) + ], + )?; + } + prune_repo_events(transaction, event_retention) + }) } pub(crate) fn repo_events_after( @@ -361,25 +261,13 @@ impl Database { limit: usize, ) -> Result> { let connection = self.connection()?; - let mut statement = connection.prepare( + let rows = query( + &connection, "SELECT seq, did, rev, root_cid, event_type, payload, created_at FROM repo_outbox WHERE seq > ?1 ORDER BY seq LIMIT ?2", - )?; - let rows = statement.query_map( params![cursor, i64::try_from(limit).unwrap_or(i64::MAX)], - |row| { - Ok(( - row.get::<_, i64>(0)?, - row.get::<_, String>(1)?, - row.get::<_, Option>(2)?, - row.get::<_, Option>(3)?, - row.get::<_, String>(4)?, - row.get::<_, Vec>(5)?, - row.get::<_, i64>(6)?, - )) - }, + |row| Ok(<(_, String, _, _, String, Vec, _)>::try_from(row)?), )?; - let rows = rows.collect::, _>>()?; rows.into_iter() .map(|(seq, did, rev, root, event_type, payload, created_at)| { match event_type.as_str() { @@ -423,95 +311,59 @@ impl Database { .collect() } - pub(crate) fn transition_account_status( - &self, - did: &str, - expected: AccountStatus, - next: AccountStatus, - event_retention: usize, - ) -> Result { - let mut connection = self.connection()?; - let transaction = connection.transaction()?; - let changed = transaction.execute( - "UPDATE accounts SET status = ?3 WHERE did = ?1 AND status = ?2", - params![did, expected.as_str(), next.as_str()], - )?; - if changed == 0 { - let actual = account_status(&transaction, did)?; - if actual == next { - return Ok(false); - } - return Err(Error::Conflict("account status changed".to_owned())); - } - - let timestamp = now(); - let payload = serde_json::to_vec(&OutboxAccount { - version: 1, - did: did.to_owned(), - active: next.is_active(), - status: next.inactive_reason().map(ToOwned::to_owned), - })?; - transaction.execute( - "INSERT INTO repo_outbox - (did, rev, root_cid, event_type, payload, next_attempt_at, created_at) - VALUES (?1, NULL, NULL, 'account', ?2, ?3, ?3)", - params![did, payload, timestamp], - )?; - prune_repo_events(&transaction, event_retention)?; - transaction.commit()?; - Ok(true) - } - - pub(crate) fn transition_account_handle( + pub(crate) fn transition_account( &self, did: &str, expected: &str, - next: &str, + event: LifecycleEvent<'_>, event_retention: usize, ) -> Result { + let (field, next) = match event { + LifecycleEvent::Identity(handle) => ("handle", handle), + LifecycleEvent::Account(status) => ("status", status.as_str()), + }; let mut connection = self.connection()?; let transaction = connection.transaction()?; let changed = transaction.execute( - "UPDATE accounts SET handle = ?3 WHERE did = ?1 AND handle = ?2", + &format!("UPDATE accounts SET {field} = ?3 WHERE did = ?1 AND {field} = ?2"), params![did, expected, next], )?; if changed == 0 { - let actual = account_handle(&transaction, did)?; + let actual: String = transaction + .query_row( + &format!("SELECT {field} FROM accounts WHERE did = ?1"), + [did], + |row| row.get(0), + ) + .optional()? + .ok_or(Error::NotFound)?; + if matches!(event, LifecycleEvent::Account(_)) { + parse_account_status(&actual)?; + } if actual == next { return Ok(false); } - return Err(Error::Conflict("account handle changed".to_owned())); + return Err(Error::Conflict(format!("account {field} changed"))); } - let timestamp = now(); - let payload = serde_json::to_vec(&OutboxIdentity { - version: 1, - did: did.to_owned(), - handle: Some(next.to_owned()), - })?; - transaction.execute( - "INSERT INTO repo_outbox - (did, rev, root_cid, event_type, payload, next_attempt_at, created_at) - VALUES (?1, NULL, NULL, 'identity', ?2, ?3, ?3)", - params![did, payload, timestamp], - )?; + event.insert(&transaction, did, now())?; prune_repo_events(&transaction, event_retention)?; transaction.commit()?; Ok(true) } pub(crate) fn latest_repo_event_sequence(&self) -> Result { - self.connection()? - .query_row("SELECT COALESCE(MAX(seq), 0) FROM repo_outbox", [], |row| { - row.get(0) - }) - .map_err(Into::into) + self.query_one("SELECT COALESCE(MAX(seq), 0) FROM repo_outbox", [], |row| { + Ok(row.get(0)?) + }) } pub(crate) fn earliest_repo_event_sequence(&self) -> Result> { - self.connection()? - .query_row("SELECT MIN(seq) FROM repo_outbox", [], |row| row.get(0)) - .map_err(Into::into) + self.query_one( + "SELECT MIN(seq) FROM repo_outbox", + [], + |row| Ok(row.get(0)?), + ) } pub(crate) fn repo_heads_after( @@ -520,8 +372,8 @@ impl Database { collection: Option<&str>, limit: usize, ) -> Result> { - let connection = self.connection()?; - let mut statement = connection.prepare( + let (created_at_millis, did) = cursor.unzip(); + self.query( "SELECT accounts.did, repo_heads.root_cid, repo_heads.rev, accounts.status, accounts.created_at * 1000 FROM accounts @@ -534,28 +386,15 @@ impl Database { ))) ORDER BY accounts.created_at, accounts.did LIMIT ?4", - )?; - let (created_at_millis, did) = cursor.unzip(); - statement - .query_map( - params![ - created_at_millis, - did, - collection, - i64::try_from(limit).unwrap_or(i64::MAX) - ], - |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, String>(2)?, - row.get::<_, String>(3)?, - row.get::<_, i64>(4)?, - )) - }, - )? - .map(|row| { - let (did, root, rev, status, created_at_millis) = row?; + params![ + created_at_millis, + did, + collection, + i64::try_from(limit).unwrap_or(i64::MAX) + ], + |row| { + let (did, root, rev, status, created_at_millis) = + <(_, String, _, String, _)>::try_from(row)?; Ok(ListedRepoHead { did, root: parse_event_cid(&root)?, @@ -563,8 +402,8 @@ impl Database { status: parse_account_status(&status)?, created_at_millis, }) - }) - .collect() + }, + ) } pub(crate) fn repo_blocks_after_revision( @@ -616,11 +455,7 @@ impl Database { ORDER BY repo_outbox.seq DESC, repo_commit_blocks.position", )?; let rows = statement.query_map(params![did, since], |row| { - Ok(( - row.get::<_, i64>(0)?, - row.get::<_, Option>(1)?, - row.get::<_, Option>>(2)?, - )) + <(i64, Option, Option>)>::try_from(row) })?; let mut blocks = Vec::new(); let mut seen = HashSet::new(); @@ -639,37 +474,15 @@ impl Database { } Ok(blocks) } - - fn repo_block(&self, cid: Cid) -> Result> { - let bytes = self - .connection()? - .query_row( - "SELECT bytes FROM repo_blocks WHERE cid = ?1", - [cid.to_string()], - |row| row.get::<_, Vec>(0), - ) - .optional()? - .ok_or(Error::NotFound)?; - verify_block(cid, &bytes)?; - Ok(bytes) - } -} - -struct Operation<'a> { - action: &'static str, - collection: &'a str, - rkey: &'a str, - cid: Option, - prev: Option, } fn apply_record_mutation<'a>( transaction: &Transaction<'_>, - mutation: &'a PendingRecordMutation, + mutation: &'a RecordMutation, timestamp: i64, -) -> Result> { - match mutation { - PendingRecordMutation::Create(record) => { +) -> Result<(&'a Record, OutboxOperation)> { + let (action, record, previous) = match mutation { + RecordMutation::Create(record) => { ensure_block(transaction, record.cid)?; let json = serde_json::to_string(&record.value)?; transaction @@ -686,17 +499,11 @@ fn apply_record_mutation<'a>( timestamp ], ) - .map_err(record_conflict)?; + .map_err(|error| constraint_conflict(error, "record already exists"))?; replace_record_blob_refs(transaction, record)?; - Ok(Operation { - action: "create", - collection: &record.collection, - rkey: &record.rkey, - cid: Some(record.cid.to_string()), - prev: None, - }) + ("create", record, None) } - PendingRecordMutation::Update { + RecordMutation::Update { current, replacement, } => { @@ -720,15 +527,9 @@ fn apply_record_mutation<'a>( return Err(Error::Conflict("record changed before update".to_owned())); } replace_record_blob_refs(transaction, replacement)?; - Ok(Operation { - action: "update", - collection: &replacement.collection, - rkey: &replacement.rkey, - cid: Some(replacement.cid.to_string()), - prev: Some(current.cid.to_string()), - }) + ("update", replacement, Some(current.cid)) } - PendingRecordMutation::Delete(record) => { + RecordMutation::Delete(record) => { let changed = transaction.execute( "DELETE FROM records WHERE did = ?1 AND collection = ?2 AND rkey = ?3 AND cid = ?4", @@ -742,15 +543,18 @@ fn apply_record_mutation<'a>( if changed == 0 { return Err(Error::Conflict("record changed before deletion".to_owned())); } - Ok(Operation { - action: "delete", - collection: &record.collection, - rkey: &record.rkey, - cid: None, - prev: Some(record.cid.to_string()), - }) + ("delete", record, Some(record.cid)) } - } + }; + Ok(( + record, + OutboxOperation { + action: action.to_owned(), + path: format!("{}/{}", record.collection, record.rkey), + cid: (!matches!(mutation, RecordMutation::Delete(_))).then(|| record.cid.to_string()), + prev: previous.map(|cid| cid.to_string()), + }, + )) } fn replace_record_blob_refs(transaction: &Transaction<'_>, record: &Record) -> Result<()> { @@ -768,23 +572,20 @@ fn replace_record_blob_refs(transaction: &Transaction<'_>, record: &Record) -> R Ok(()) } -fn load_head(transaction: &Transaction<'_>, did: &str) -> Result> { - transaction - .query_row( - "SELECT root_cid, rev FROM repo_heads WHERE did = ?1", - [did], - |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), - ) - .optional()? - .map(|(root, rev)| { +fn load_head(connection: &Connection, did: &str) -> Result> { + query_optional( + connection, + "SELECT root_cid, rev FROM repo_heads WHERE did = ?1", + [did], + |row| { + let (root, rev) = <(String, _)>::try_from(row)?; Ok(RepoHead { - root: Cid::from_str(&root).map_err(|error| { - Error::Internal(format!("stored repository root CID is invalid: {error}")) - })?, + root: Cid::from_str(&root) + .context(Error::Internal, "stored repository root CID is invalid")?, rev, }) - }) - .transpose() + }, + ) } fn ensure_block(transaction: &Transaction<'_>, cid: Cid) -> Result<()> { @@ -802,73 +603,63 @@ fn ensure_block(transaction: &Transaction<'_>, cid: Cid) -> Result<()> { } } -fn record_conflict(error: rusqlite::Error) -> Error { - match error { - rusqlite::Error::SqliteFailure(ref failure, _) - if failure.code == rusqlite::ErrorCode::ConstraintViolation => - { - Error::Conflict("record already exists".to_owned()) - } - other => Error::Database(other), - } -} - -fn verify_block(cid: Cid, bytes: &[u8]) -> Result<()> { - let computed = block_cid(cid.codec(), cid.hash().code(), bytes).map_err(|error| { - Error::Internal(format!("could not verify repository block {cid}: {error}")) - })?; - if computed == cid { - Ok(()) - } else { - Err(Error::Internal(format!( - "repository block bytes do not match CID {cid}" - ))) - } -} - -fn block_cid(codec: u64, hash: u64, contents: &[u8]) -> std::result::Result { - if codec != DAG_CBOR { - return Err(BlockError::Other(Box::new(std::io::Error::other(format!( - "unsupported repository codec {codec}" - ))))); - } - if hash != SHA2_256 { - return Err(BlockError::UnsupportedHash(hash)); - } - let digest = Sha256::digest(contents); - let multihash = Multihash::wrap(hash, digest.as_slice()) - .map_err(|error| BlockError::Other(Box::new(std::io::Error::other(error.to_string()))))?; - Ok(Cid::new_v1(codec, multihash)) -} - -fn block_error(error: Error) -> BlockError { - match error { - Error::NotFound => BlockError::CidNotFound, - error => BlockError::Other(Box::new(std::io::Error::other(error.to_string()))), - } -} - fn parse_event_cid(value: &str) -> Result { - Cid::from_str(value).map_err(|error| { - Error::Internal(format!("stored repository event CID is invalid: {error}")) - }) + Cid::from_str(value).context(Error::Internal, "stored repository event CID is invalid") } -fn outbox_payload(commit: &PendingRepoCommit, operation: &Operation<'_>) -> Result> { +fn outbox_payload(commit: &PendingRepoCommit, operations: Vec) -> Result> { Ok(serde_json::to_vec(&OutboxCommit { - version: 1, + version: 2, did: commit.did.clone(), rev: commit.head.rev.clone(), root: commit.head.root.to_string(), since: commit.previous_rev.clone(), prev_data: commit.previous_data.map(|cid| cid.to_string()), - action: operation.action.to_owned(), - path: format!("{}/{}", operation.collection, operation.rkey), - cid: operation.cid.clone(), - prev: operation.prev.clone(), + operations, + action: None, + path: None, + cid: None, + prev: None, })?) } +#[derive(Clone, Copy)] +pub(crate) enum LifecycleEvent<'a> { + Identity(&'a str), + Account(AccountStatus), +} + +impl LifecycleEvent<'_> { + fn insert(self, transaction: &Transaction<'_>, did: &str, timestamp: i64) -> Result<()> { + let (event_type, payload) = match self { + Self::Identity(handle) => ( + "identity", + serde_json::to_vec(&OutboxIdentity { + version: 1, + did: did.to_owned(), + handle: Some(handle.to_owned()), + })?, + ), + Self::Account(status) => ( + "account", + serde_json::to_vec(&OutboxAccount { + version: 1, + did: did.to_owned(), + active: status.is_active(), + status: status.inactive_reason().map(ToOwned::to_owned), + })?, + ), + }; + transaction.execute( + "INSERT INTO repo_outbox + (did, rev, root_cid, event_type, payload, next_attempt_at, created_at) + VALUES (?1, NULL, NULL, ?2, ?3, ?4, ?4)", + params![did, event_type, payload, timestamp], + )?; + Ok(()) + } +} + fn insert_initial_repo_events( transaction: &Transaction<'_>, did: &str, @@ -877,79 +668,18 @@ fn insert_initial_repo_events( let (handle, status) = transaction.query_row( "SELECT handle, status FROM accounts WHERE did = ?1", [did], - |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), + |row| <(String, String)>::try_from(row), )?; let status = parse_account_status(&status)?; - for (event_type, payload) in [ - ( - "identity", - serde_json::to_vec(&OutboxIdentity { - version: 1, - did: did.to_owned(), - handle: Some(handle), - })?, - ), - ( - "account", - serde_json::to_vec(&OutboxAccount { - version: 1, - did: did.to_owned(), - active: status.is_active(), - status: status.inactive_reason().map(ToOwned::to_owned), - })?, - ), + for event in [ + LifecycleEvent::Identity(&handle), + LifecycleEvent::Account(status), ] { - transaction.execute( - "INSERT INTO repo_outbox - (did, rev, root_cid, event_type, payload, next_attempt_at, created_at) - VALUES (?1, NULL, NULL, ?2, ?3, ?4, ?4)", - params![did, event_type, payload, timestamp], - )?; + event.insert(transaction, did, timestamp)?; } Ok(()) } -fn account_status(transaction: &Transaction<'_>, did: &str) -> Result { - let status = transaction - .query_row("SELECT status FROM accounts WHERE did = ?1", [did], |row| { - row.get::<_, String>(0) - }) - .optional()? - .ok_or(Error::NotFound)?; - parse_account_status(&status) -} - -fn account_handle(transaction: &Transaction<'_>, did: &str) -> Result { - transaction - .query_row("SELECT handle FROM accounts WHERE did = ?1", [did], |row| { - row.get(0) - }) - .optional()? - .ok_or(Error::NotFound) -} - -fn require_account_active(transaction: &Transaction<'_>, did: &str) -> Result<()> { - let status = account_status(transaction, did)?; - if status.is_active() { - Ok(()) - } else { - Err(account_write_error(status)) - } -} - -fn parse_account_status(value: &str) -> Result { - value - .parse() - .map_err(|error| Error::Internal(format!("stored {error}"))) -} - -fn account_write_error(status: AccountStatus) -> Error { - Error::Xrpc { - name: status.account_error_name().unwrap_or("AccountNotActive"), - message: format!("account is {}", status.as_str()), - } -} - fn prune_repo_events(transaction: &Transaction<'_>, event_retention: usize) -> Result<()> { transaction.execute( "DELETE FROM repo_outbox @@ -977,33 +707,54 @@ fn load_commit_event( } let incremental = match payload.version { 0 => false, - 1 => true, + 1 | 2 => true, version => { return Err(Error::Internal(format!( "stored repository event version {version} is not supported" ))); } }; - let mut block_statement = connection.prepare( + let blocks = query( + connection, "SELECT repo_commit_blocks.cid, repo_blocks.bytes FROM repo_commit_blocks JOIN repo_blocks ON repo_blocks.cid = repo_commit_blocks.cid WHERE repo_commit_blocks.seq = ?1 ORDER BY repo_commit_blocks.position", - )?; - let blocks = block_statement - .query_map([seq], |row| { - Ok((row.get::<_, String>(0)?, row.get::<_, Vec>(1)?)) - })? - .map(|row| { - let (cid, bytes) = row?; - let cid = Cid::from_str(&cid).map_err(|error| { - Error::Internal(format!( - "stored repository event block CID is invalid: {error}" - )) - })?; + [seq], + |row| { + let (cid, bytes): (String, Vec) = row.try_into()?; + let cid = Cid::from_str(&cid).context( + Error::Internal, + "stored repository event block CID is invalid", + )?; verify_block(cid, &bytes)?; Ok((cid, bytes)) + }, + )?; + let operations = if payload.version < 2 { + vec![OutboxOperation { + action: payload.action.ok_or_else(|| event_metadata_mismatch(seq))?, + path: payload.path.ok_or_else(|| event_metadata_mismatch(seq))?, + cid: payload.cid, + prev: payload.prev, + }] + } else if payload.operations.is_empty() { + return Err(event_metadata_mismatch(seq)); + } else { + payload.operations + }; + let operations = operations + .into_iter() + .map(|operation| { + let cid = operation.cid.as_deref().map(parse_event_cid).transpose()?; + let prev = operation.prev.as_deref().map(parse_event_cid).transpose()?; + Ok(RepoOpData { + action: operation.action, + path: operation.path, + cid: cid.map(CidLink), + prev: prev.map(CidLink), + }) }) .collect::>>()?; Ok(StoredRepoEvent::Commit(Box::new(StoredCommitEvent { @@ -1018,10 +769,7 @@ fn load_commit_event( .as_deref() .map(parse_event_cid) .transpose()?, - action: payload.action, - path: payload.path, - cid: payload.cid.as_deref().map(parse_event_cid).transpose()?, - prev: payload.prev.as_deref().map(parse_event_cid).transpose()?, + operations, blocks, created_at, }))) @@ -1063,6 +811,20 @@ struct OutboxCommit { since: Option, #[serde(default)] prev_data: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + operations: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + action: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + path: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + cid: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + prev: Option, +} + +#[derive(Deserialize, Serialize)] +struct OutboxOperation { action: String, path: String, cid: Option, diff --git a/src/database/repo/tests.rs b/src/database/repo/tests.rs index e1b32eb..c2c7d8e 100644 --- a/src/database/repo/tests.rs +++ b/src/database/repo/tests.rs @@ -1,29 +1,320 @@ -use atrium_repo::blockstore::{AsyncBlockStoreRead as _, AsyncBlockStoreWrite as _}; +use atrium_repo::blockstore::{ + AsyncBlockStoreRead as _, AsyncBlockStoreWrite as _, DAG_CBOR, SHA2_256, +}; use serde_json::json; +use crate::database::{RepoBlockStore, blockstore::block_cid}; + use super::*; const DID: &str = "did:web:alice.test"; +#[test] +fn sequence_extrema_preserve_empty_defaults_retention_and_database_errors() { + let database = database(); + let check = |expected| { + assert_eq!( + ( + database.latest_repo_event_sequence().unwrap(), + database.earliest_repo_event_sequence().unwrap(), + ), + expected, + ); + }; + check((0, None)); + database + .transaction(|transaction| insert_initial_repo_events(transaction, DID, 123)) + .unwrap(); + check((2, Some(1))); + database + .connection() + .unwrap() + .execute("DELETE FROM repo_outbox WHERE seq = 1", []) + .unwrap(); + check((2, Some(2))); + database + .connection() + .unwrap() + .execute("DELETE FROM repo_outbox", []) + .unwrap(); + check((0, None)); + database + .connection() + .unwrap() + .execute("DROP TABLE repo_outbox", []) + .unwrap(); + for result in [ + database.latest_repo_event_sequence(), + database + .earliest_repo_event_sequence() + .map(Option::unwrap_or_default), + ] { + assert!(matches!(result, Err(Error::Database(_)))); + } +} + +#[test] +fn outbox_checks_every_sql_row_before_decoding_any_payload() { + use rusqlite::types::Value::{Blob, Integer}; + + let database = database(); + for timestamp in [Integer(1), Blob(vec![0])] { + database + .connection() + .unwrap() + .execute( + "INSERT INTO repo_outbox + (did, event_type, payload, next_attempt_at, created_at) + VALUES (?1, 'identity', ?2, 0, ?3)", + params![DID, b"invalid-json".as_slice(), timestamp], + ) + .unwrap(); + } + assert!(matches!( + database.repo_events_after(0, 10), + Err(Error::Database(rusqlite::Error::InvalidColumnType(6, _, _))) + )); + database + .connection() + .unwrap() + .execute("UPDATE repo_outbox SET created_at = 2 WHERE seq = 2", []) + .unwrap(); + assert!(matches!( + database.repo_events_after(0, 10), + Err(Error::Json(_)) + )); +} + +#[test] +fn lifecycle_events_preserve_payload_bytes_order_and_outbox_metadata() { + let database = database(); + database + .transaction(|transaction| insert_initial_repo_events(transaction, DID, 123)) + .unwrap(); + database + .transition_account( + DID, + AccountStatus::Active.as_str(), + LifecycleEvent::Account(AccountStatus::Deactivated), + 100, + ) + .unwrap(); + database + .transition_account( + DID, + "alice.test", + LifecycleEvent::Identity("next.test"), + 100, + ) + .unwrap(); + let events = database + .query( + "SELECT event_type, payload, did, rev, root_cid, next_attempt_at, + created_at, attempts, delivered_at FROM repo_outbox ORDER BY seq", + [], + |row| { + assert_eq!(row.get::<_, String>(2)?, DID); + assert_eq!(row.get::<_, Option>(3)?, None); + assert_eq!(row.get::<_, Option>(4)?, None); + assert_eq!(row.get::<_, i64>(5)?, row.get::<_, i64>(6)?); + assert_eq!(row.get::<_, i64>(7)?, 0); + assert_eq!(row.get::<_, Option>(8)?, None); + Ok(( + row.get::<_, String>(0)?, + row.get::<_, Vec>(1)?, + row.get::<_, i64>(6)?, + )) + }, + ) + .unwrap(); + assert_eq!(events.len(), 4); + for ((event_type, payload, timestamp), (expected_type, expected_payload)) in + events.iter().zip([ + ( + "identity", + r#"{"version":1,"did":"did:web:alice.test","handle":"alice.test"}"#, + ), + ( + "account", + r#"{"version":1,"did":"did:web:alice.test","active":true,"status":null}"#, + ), + ( + "account", + r#"{"version":1,"did":"did:web:alice.test","active":false,"status":"deactivated"}"#, + ), + ( + "identity", + r#"{"version":1,"did":"did:web:alice.test","handle":"next.test"}"#, + ), + ]) + { + assert_eq!(event_type, expected_type); + assert_eq!(payload, expected_payload.as_bytes()); + assert!(*timestamp >= 123); + } + assert_eq!(events[0].2, 123); + assert_eq!(events[1].2, 123); +} + +#[test] +fn lifecycle_insert_failures_roll_back_initial_events_and_account_transitions() { + let database = database(); + database + .connection() + .unwrap() + .execute_batch( + "CREATE TRIGGER reject_account_event BEFORE INSERT ON repo_outbox + WHEN NEW.event_type = 'account' BEGIN + SELECT RAISE(ABORT, 'account event rejected'); + END;", + ) + .unwrap(); + assert!(matches!( + database.transaction(|transaction| insert_initial_repo_events(transaction, DID, 123)), + Err(Error::Database(_)) + )); + assert_eq!(count(&database, "repo_outbox"), 0); + assert!(matches!( + database.transition_account( + DID, + AccountStatus::Active.as_str(), + LifecycleEvent::Account(AccountStatus::Deactivated), + 100 + ), + Err(Error::Database(_)) + )); + database + .connection() + .unwrap() + .execute_batch( + "CREATE TRIGGER reject_identity_event BEFORE INSERT ON repo_outbox + WHEN NEW.event_type = 'identity' BEGIN + SELECT RAISE(ABORT, 'identity event rejected'); + END;", + ) + .unwrap(); + assert!(matches!( + database.transition_account( + DID, + "alice.test", + LifecycleEvent::Identity("next.test"), + 100 + ), + Err(Error::Database(_)) + )); + let (account, status) = database.hosted_account_by_identifier(DID).unwrap().unwrap(); + assert_eq!(account.handle, "alice.test"); + assert_eq!(status, AccountStatus::Active); + assert_eq!(count(&database, "repo_outbox"), 0); +} + +#[test] +fn ignored_lifecycle_updates_roll_back_side_effects_on_noop_and_conflict() { + let database = database(); + database + .connection() + .unwrap() + .execute_batch( + "CREATE TEMP TABLE transition_probe (value TEXT); + CREATE TEMP TRIGGER ignore_account_transition BEFORE UPDATE ON accounts BEGIN + INSERT INTO transition_probe VALUES ('attempted'); + SELECT RAISE(IGNORE); + END;", + ) + .unwrap(); + + for (expected, event, conflict) in [ + ("alice.test", LifecycleEvent::Identity("alice.test"), None), + ( + "alice.test", + LifecycleEvent::Identity("next.test"), + Some("account handle changed"), + ), + ( + "active", + LifecycleEvent::Account(AccountStatus::Active), + None, + ), + ( + "active", + LifecycleEvent::Account(AccountStatus::Deactivated), + Some("account status changed"), + ), + ] { + let result = database.transition_account(DID, expected, event, 100); + match conflict { + Some(expected) => { + assert!(matches!(result, Err(Error::Conflict(message)) if message == expected)); + } + None => assert!(!result.unwrap()), + } + assert_eq!(count(&database, "transition_probe"), 0); + assert_eq!(count(&database, "repo_outbox"), 0); + let (account, status) = database.hosted_account_by_identifier(DID).unwrap().unwrap(); + assert_eq!(account.handle, "alice.test"); + assert_eq!(status, AccountStatus::Active); + } +} + +#[test] +fn lifecycle_status_fallback_preserves_sql_then_domain_validation() { + use rusqlite::types::Value::{Blob, Text}; + + let database = database(); + database + .connection() + .unwrap() + .execute_batch("PRAGMA ignore_check_constraints = ON") + .unwrap(); + for status in [Blob(vec![0]), Text("invalid".to_owned())] { + database + .connection() + .unwrap() + .execute( + "UPDATE accounts SET status = ?1 WHERE did = ?2", + params![status, DID], + ) + .unwrap(); + let error = database + .transition_account( + DID, + AccountStatus::Active.as_str(), + LifecycleEvent::Account(AccountStatus::Deactivated), + 100, + ) + .unwrap_err(); + match status { + Blob(_) => assert!(matches!( + error, + Error::Database(rusqlite::Error::InvalidColumnType(0, _, _)) + )), + Text(_) => assert!(matches!(error, Error::Internal(message) + if message == "stored unknown account status invalid")), + _ => unreachable!(), + } + assert_eq!(count(&database, "repo_outbox"), 0); + } +} + #[tokio::test] async fn inactive_account_transition_is_idempotent_and_blocks_atomic_commits() { let database = database(); assert!( database - .transition_account_status( + .transition_account( DID, - AccountStatus::Active, - AccountStatus::Deactivated, + AccountStatus::Active.as_str(), + LifecycleEvent::Account(AccountStatus::Deactivated), 100_000, ) .unwrap() ); assert!( !database - .transition_account_status( + .transition_account( DID, - AccountStatus::Active, - AccountStatus::Deactivated, + AccountStatus::Active.as_str(), + LifecycleEvent::Account(AccountStatus::Deactivated), 100_000, ) .unwrap() @@ -54,7 +345,8 @@ async fn inactive_account_transition_is_idempotent_and_blocks_atomic_commits() { rev: "3jqfcqzm3fo2j".to_owned(), }, blocks: store.writes(), - mutation: PendingRecordMutation::Create(record), + event_blocks: vec![root], + mutations: vec![RecordMutation::Create(record)], }, 100_000, ) @@ -102,8 +394,13 @@ async fn staged_blocks_publish_with_head_index_operation_and_outbox() { root, rev: "3jqfcqzm3fo2j".to_owned(), }, - blocks: store.reachable_writes([root, record_cid]).unwrap(), - mutation: PendingRecordMutation::Create(record), + blocks: store + .reachable_writes([root, record_cid]) + .unwrap() + .map(|(cid, bytes)| (*cid, bytes.clone())) + .collect(), + event_blocks: vec![root, record_cid], + mutations: vec![RecordMutation::Create(record)], }, 100_000, ) @@ -154,10 +451,11 @@ async fn staged_blocks_publish_with_head_index_operation_and_outbox() { assert_eq!(commit.root, root); assert_eq!(commit.since, None); assert_eq!(commit.prev_data, None); - assert_eq!(commit.action, "create"); - assert_eq!(commit.path, "app.bsky.feed.post/one"); - assert_eq!(commit.cid, Some(record_cid)); - assert_eq!(commit.prev, None); + assert_eq!(commit.operations.len(), 1); + assert_eq!(commit.operations[0].action, "create"); + assert_eq!(commit.operations[0].path, "app.bsky.feed.post/one"); + assert_eq!(commit.operations[0].cid, Some(CidLink(record_cid))); + assert_eq!(commit.operations[0].prev, None); assert_eq!(commit.blocks.first().map(|(cid, _)| *cid), Some(root)); assert!(commit.created_at > 0); assert!(database.repo_events_after(3, 10).unwrap().is_empty()); @@ -167,6 +465,133 @@ async fn staged_blocks_publish_with_head_index_operation_and_outbox() { assert_eq!(changed.first().map(|(cid, _)| *cid), Some(root)); } +#[tokio::test] +async fn ordered_batch_persists_one_commit_and_only_final_event_blocks() { + let database = database(); + let created = record_with_text("dependent", "created"); + let updated = record_with_text("dependent", "updated"); + let survivor = record_with_text("survivor", "survivor"); + let mut store = RepoBlockStore::new(database.clone()); + for record in [&created, &updated, &survivor] { + let cid = store + .write_block( + DAG_CBOR, + SHA2_256, + &serde_ipld_dagcbor::to_vec(&record.value).unwrap(), + ) + .await + .unwrap(); + assert_eq!(cid, record.cid); + } + let root = store + .write_block(DAG_CBOR, SHA2_256, b"batch root") + .await + .unwrap(); + + database + .commit_repo( + &PendingRepoCommit { + did: DID.to_owned(), + expected_head: None, + previous_rev: None, + previous_data: None, + head: RepoHead { + root, + rev: "3jqfcqzm3fo2j".to_owned(), + }, + blocks: store.writes(), + event_blocks: vec![root, survivor.cid], + mutations: vec![ + RecordMutation::Create(created.clone()), + RecordMutation::Update { + current: created.clone(), + replacement: updated.clone(), + }, + RecordMutation::Create(survivor.clone()), + RecordMutation::Delete(updated.clone()), + ], + }, + 100_000, + ) + .unwrap(); + + assert_eq!(count(&database, "records"), 1); + assert_eq!(count(&database, "repo_ops"), 4); + assert_eq!(count(&database, "repo_outbox"), 3); + assert_eq!(count(&database, "repo_commit_blocks"), 2); + assert!(database.repo_block(created.cid).is_ok()); + assert!(database.repo_block(updated.cid).is_ok()); + let events = database.repo_events_after(0, 10).unwrap(); + let StoredRepoEvent::Commit(commit) = &events[2] else { + panic!("batch should publish one commit event"); + }; + assert_eq!( + commit + .operations + .iter() + .map(|operation| operation.action.as_str()) + .collect::>(), + ["create", "update", "create", "delete"] + ); + let payload: Vec = database + .query_one("SELECT payload FROM repo_outbox WHERE seq = 3", [], |row| { + Ok(row.get(0)?) + }) + .unwrap(); + assert_ordered_batch_payload(&payload, root, &created, &updated, &survivor); + let payload: serde_json::Value = serde_json::from_slice(&payload).unwrap(); + assert_eq!( + payload["operations"], + json!([ + {"action": "create", "path": "app.bsky.feed.post/dependent", "cid": created.cid.to_string(), "prev": null}, + {"action": "update", "path": "app.bsky.feed.post/dependent", "cid": updated.cid.to_string(), "prev": created.cid.to_string()}, + {"action": "create", "path": "app.bsky.feed.post/survivor", "cid": survivor.cid.to_string(), "prev": null}, + {"action": "delete", "path": "app.bsky.feed.post/dependent", "cid": null, "prev": updated.cid.to_string()} + ]) + ); + assert_eq!( + serde_json::to_value(&commit.operations).unwrap(), + json!([ + {"action": "create", "path": "app.bsky.feed.post/dependent", "cid": {"$link": created.cid.to_string()}}, + {"action": "update", "path": "app.bsky.feed.post/dependent", "cid": {"$link": updated.cid.to_string()}, "prev": {"$link": created.cid.to_string()}}, + {"action": "create", "path": "app.bsky.feed.post/survivor", "cid": {"$link": survivor.cid.to_string()}}, + {"action": "delete", "path": "app.bsky.feed.post/dependent", "prev": {"$link": updated.cid.to_string()}} + ]) + ); + assert_eq!( + commit + .blocks + .iter() + .map(|(cid, _)| *cid) + .collect::>(), + [root, survivor.cid] + ); +} + +fn assert_ordered_batch_payload( + payload: &[u8], + root: Cid, + created: &Record, + updated: &Record, + survivor: &Record, +) { + let expected = format!( + concat!( + r#"{{"version":2,"did":"{did}","rev":"3jqfcqzm3fo2j","root":"{root}","since":null,"prevData":null,"operations":["#, + r#"{{"action":"create","path":"app.bsky.feed.post/dependent","cid":"{created}","prev":null}},"#, + r#"{{"action":"update","path":"app.bsky.feed.post/dependent","cid":"{updated}","prev":"{created}"}},"#, + r#"{{"action":"create","path":"app.bsky.feed.post/survivor","cid":"{survivor}","prev":null}},"#, + r#"{{"action":"delete","path":"app.bsky.feed.post/dependent","cid":null,"prev":"{updated}"}}]}}"# + ), + did = DID, + root = root, + created = created.cid, + updated = updated.cid, + survivor = survivor.cid, + ); + assert_eq!(payload, expected.as_bytes()); +} + #[tokio::test] async fn incremental_export_fails_closed_across_legacy_history_gaps() { let database = database(); @@ -186,9 +611,10 @@ async fn incremental_export_fails_closed_across_legacy_history_gaps() { #[tokio::test] async fn failed_record_mutation_rolls_back_blocks_head_operation_and_outbox() { let database = database(); - let record = record("one"); - let first_root = publish(&database, record.clone(), None, b"first root").await; + let existing = record("one"); + let first_root = publish(&database, existing.clone(), None, b"first root").await; let before_blocks = count(&database, "repo_blocks"); + let fresh = record("fresh"); let mut store = RepoBlockStore::new(database.clone()); let next_root = store @@ -199,7 +625,15 @@ async fn failed_record_mutation_rolls_back_blocks_head_operation_and_outbox() { .write_block( DAG_CBOR, SHA2_256, - &serde_ipld_dagcbor::to_vec(&record.value).unwrap(), + &serde_ipld_dagcbor::to_vec(&existing.value).unwrap(), + ) + .await + .unwrap(); + let fresh_cid = store + .write_block( + DAG_CBOR, + SHA2_256, + &serde_ipld_dagcbor::to_vec(&fresh.value).unwrap(), ) .await .unwrap(); @@ -214,8 +648,16 @@ async fn failed_record_mutation_rolls_back_blocks_head_operation_and_outbox() { root: next_root, rev: "3jqfcqzm3fo2k".to_owned(), }, - blocks: store.reachable_writes([next_root, record_cid]).unwrap(), - mutation: PendingRecordMutation::Create(record), + blocks: store + .reachable_writes([next_root, record_cid, fresh_cid]) + .unwrap() + .map(|(cid, bytes)| (*cid, bytes.clone())) + .collect(), + event_blocks: vec![next_root, record_cid], + mutations: vec![ + RecordMutation::Create(fresh), + RecordMutation::Create(existing), + ], }, 100_000, ) @@ -227,6 +669,12 @@ async fn failed_record_mutation_rolls_back_blocks_head_operation_and_outbox() { database.repo_block(next_root), Err(Error::NotFound) )); + assert!(matches!( + database + .find_record(DID, "app.bsky.feed.post", "fresh") + .and_then(|record| record.ok_or(Error::NotFound)), + Err(Error::NotFound) + )); assert_eq!(count(&database, "repo_blocks"), before_blocks); assert_eq!(count(&database, "repo_ops"), 1); assert_eq!(count(&database, "repo_outbox"), 3); @@ -252,6 +700,158 @@ async fn persisted_block_corruption_fails_before_atrium_reads_it() { assert!(bytes.is_empty()); } +#[test] +fn shared_block_hash_preserves_codec_hash_error_order_and_verification_context() { + use atrium_repo::{Multihash, blockstore::Error as BlockError}; + use sha2::{Digest as _, Sha256}; + + fn previous(codec: u64, hash: u64, contents: &[u8]) -> Result { + if codec != DAG_CBOR { + return Err(BlockError::Other(Box::new(std::io::Error::other(format!( + "unsupported repository codec {codec}" + ))))); + } + if hash != SHA2_256 { + return Err(BlockError::UnsupportedHash(hash)); + } + let digest = Sha256::digest(contents); + let multihash = Multihash::wrap(hash, digest.as_slice()).map_err(|error| { + BlockError::Other(Box::new(std::io::Error::other(error.to_string()))) + })?; + Ok(Cid::new_v1(codec, multihash)) + } + + for length in [0, 1, 31, 32, 55, 56, 63, 64, 65, 127, 128, 129, 1_024] { + let bytes: Vec = (0..=255).cycle().take(length).collect(); + for codec in [0, 0x55, DAG_CBOR, u64::MAX] { + for hash in [0, SHA2_256, 0x13, u64::MAX] { + let expected = previous(codec, hash, &bytes); + let actual = block_cid(codec, hash, &bytes); + let case = format!("length={length}/codec={codec}/hash={hash}"); + assert_eq!(format!("{actual:?}"), format!("{expected:?}"), "{case}"); + if codec != DAG_CBOR { + assert_eq!( + actual.unwrap_err().to_string(), + format!("unsupported repository codec {codec}"), + "{case}" + ); + } else if hash != SHA2_256 { + assert!( + matches!(actual, Err(BlockError::UnsupportedHash(value)) if value == hash) + ); + } else { + assert_eq!( + actual.unwrap().to_bytes(), + expected.as_ref().unwrap().to_bytes() + ); + } + let cid = Cid::new_v1( + codec, + Multihash::wrap(hash, &Sha256::digest(&bytes)).unwrap(), + ); + let expected = expected.map(|_| ()).map_err(|error| { + Error::Internal(format!("could not verify repository block {cid}: {error}")) + }); + assert_eq!( + format!("{:?}", verify_block(cid, &bytes)), + format!("{expected:?}"), + "verify: {case}" + ); + } + } + } +} + +#[tokio::test] +async fn rejected_block_codecs_and_hashes_leave_staged_writes_unchanged() { + let mut store = RepoBlockStore::new(Database::open_in_memory().unwrap()); + let initial = store + .write_block(DAG_CBOR, SHA2_256, b"initial") + .await + .unwrap(); + let before = store.writes(); + assert_eq!(before, [(initial, b"initial".to_vec())]); + for (codec, hash) in [(0x55, SHA2_256), (0x55, 0), (DAG_CBOR, 0)] { + let expected = block_cid(codec, hash, b"rejected"); + let actual = store.write_block(codec, hash, b"rejected").await; + assert!(actual.is_err()); + assert_eq!(format!("{actual:?}"), format!("{expected:?}")); + assert_eq!(store.writes(), before); + } + let accepted = store + .write_block(DAG_CBOR, SHA2_256, b"accepted") + .await + .unwrap(); + assert_eq!(accepted, cpds_protocol::sha256_cid(DAG_CBOR, b"accepted")); + let writes = store.writes(); + assert_eq!(writes.len(), 2); + assert!(writes.contains(&(initial, b"initial".to_vec()))); + assert!(writes.contains(&(accepted, b"accepted".to_vec()))); +} + +#[test] +fn shared_block_lookup_preserves_missing_rows_sql_types_and_content_errors() { + use crate::database::blockstore::verify_block; + use rusqlite::{OptionalExtension as _, types::Value}; + + fn previous(database: &Database, cid: Cid) -> Result> { + let bytes = database + .connection()? + .query_row( + "SELECT bytes FROM repo_blocks WHERE cid = ?1", + [cid.to_string()], + |row| row.get::<_, Vec>(0), + ) + .optional()? + .ok_or(Error::NotFound)?; + verify_block(cid, &bytes)?; + Ok(bytes) + } + + let database = Database::open_in_memory().unwrap(); + let bytes = b"stored block"; + let cid = block_cid(DAG_CBOR, SHA2_256, bytes).unwrap(); + assert!(matches!(database.repo_block(cid), Err(Error::NotFound))); + let compare = || { + let shape = |result: Result>| result.map_err(|error| format!("{error:?}")); + assert_eq!( + shape(database.repo_block(cid)), + shape(previous(&database, cid)) + ); + }; + compare(); + for value in [ + Value::Blob(bytes.to_vec()), + Value::Blob(Vec::new()), + Value::Blob(b"corrupt".to_vec()), + Value::Integer(1), + Value::Real(1.5), + Value::Text("not a blob".to_owned()), + ] { + database + .connection() + .unwrap() + .execute( + "INSERT OR REPLACE INTO repo_blocks (cid, bytes, created_at) VALUES (?1, ?2, 0)", + params![cid.to_string(), value], + ) + .unwrap(); + compare(); + if value == Value::Blob(bytes.to_vec()) { + assert_eq!(database.repo_block(cid).unwrap(), bytes); + } else { + assert!(database.repo_block(cid).is_err()); + } + } + database + .connection() + .unwrap() + .execute("DROP TABLE repo_blocks", []) + .unwrap(); + compare(); + assert!(matches!(database.repo_block(cid), Err(Error::Database(_)))); +} + async fn publish( database: &Database, record: Record, @@ -282,8 +882,13 @@ async fn publish( root, rev: "3jqfcqzm3fo2j".to_owned(), }, - blocks: store.reachable_writes([root, record_cid]).unwrap(), - mutation: PendingRecordMutation::Create(record), + blocks: store + .reachable_writes([root, record_cid]) + .unwrap() + .map(|(cid, bytes)| (*cid, bytes.clone())) + .collect(), + event_blocks: vec![root, record_cid], + mutations: vec![RecordMutation::Create(record)], }, 100_000, ) @@ -300,9 +905,13 @@ fn database() -> Database { } fn record(rkey: &str) -> Record { + record_with_text(rkey, "hello") +} + +fn record_with_text(rkey: &str, text: &str) -> Record { let value = json!({ "$type": "app.bsky.feed.post", - "text": "hello", + "text": text, "createdAt": "2026-08-22T00:00:00Z" }); let bytes = serde_ipld_dagcbor::to_vec(&value).unwrap(); diff --git a/src/database/schema.rs b/src/database/schema.rs index c9c428e..d614089 100644 --- a/src/database/schema.rs +++ b/src/database/schema.rs @@ -10,14 +10,11 @@ use crate::{ error::{Error, Result}, }; -const CURRENT_SCHEMA_VERSION: i64 = 8; +const CURRENT_SCHEMA_VERSION: i64 = 9; const BUSY_TIMEOUT_MS: i64 = 5_000; const WAL_AUTOCHECKPOINT_PAGES: i64 = 1_000; const JOURNAL_SIZE_LIMIT_BYTES: i64 = 64 * 1024 * 1024; -type RecordBlobReference = (String, String, String, String); -type SpaceRecordBlobReference = (String, String, String, String, String); - const CONNECTION_PRAGMAS: &str = "PRAGMA foreign_keys = ON; PRAGMA journal_mode = WAL; PRAGMA synchronous = FULL; @@ -315,6 +312,10 @@ SELECT seq, did, rev, root_cid, event_type, payload, attempts, next_attempt_at, delivered_at, created_at FROM repo_outbox; +DELETE FROM sqlite_sequence WHERE name = 'repo_outbox_v7'; +INSERT INTO sqlite_sequence (name, seq) +SELECT 'repo_outbox_v7', seq FROM sqlite_sequence WHERE name = 'repo_outbox'; + CREATE TABLE repo_commit_blocks_v7 ( seq INTEGER NOT NULL REFERENCES repo_outbox_v7(seq) ON DELETE CASCADE, cid TEXT NOT NULL REFERENCES repo_blocks(cid), @@ -341,45 +342,33 @@ ALTER TABLE accounts ADD COLUMN status TEXT NOT NULL DEFAULT 'active' const VALIDATE_ACCOUNT_STATUS_SCHEMA: &str = " SELECT status FROM accounts LIMIT 0;"; -pub(super) fn apply(connection: &mut Connection) -> Result<()> { - apply_with_initialization(connection, true) -} - -pub(super) fn apply_existing(connection: &mut Connection) -> Result<()> { - apply_with_initialization(connection, false) -} +const OAUTH_EXPIRY_SCHEMA: &str = " +CREATE INDEX oauth_codes_by_expiry ON oauth_codes(expires_at); +CREATE INDEX access_tokens_by_expiry ON access_tokens(expires_at);"; -fn apply_with_initialization(connection: &mut Connection, initialize: bool) -> Result<()> { +pub(super) fn apply(connection: &mut Connection, initialize: bool) -> Result<()> { connection.execute_batch(CONNECTION_PRAGMAS)?; verify_connection_policy(connection)?; let version: i64 = connection.pragma_query_value(None, "user_version", |row| row.get(0))?; - match version { - CURRENT_SCHEMA_VERSION => { - connection.execute_batch(VALIDATE_INITIAL_SCHEMA)?; - connection.execute_batch(VALIDATE_ACCOUNT_STATUS_SCHEMA)?; - connection.execute_batch(VALIDATE_REPOSITORY_SCHEMA)?; - connection.execute_batch(VALIDATE_COMMIT_BLOCK_SCHEMA)?; - connection.execute_batch(VALIDATE_RECORD_BLOB_SCHEMA)?; - connection.execute_batch(VALIDATE_BLOB_LIFECYCLE_SCHEMA)?; - connection.execute_batch(VALIDATE_BLOB_PUBLICATION_SCHEMA)?; - verify_integrity(connection)?; - verify_record_blob_indexes(connection) - } - 0 if initialize => migrate_initial_schema(connection), - 0 => Err(Error::Configuration( - "existing database is not initialized as CPDS storage".to_owned(), - )), - 1 => migrate_repository_schema(connection), - 2 => migrate_commit_block_schema(connection), - 3 => migrate_record_blob_schema(connection), - 4 => migrate_blob_lifecycle_schema(connection), - 5 => migrate_blob_publication_schema(connection), - 6 => migrate_stream_outbox_schema(connection), - 7 => migrate_account_status_schema(connection), - version => Err(Error::Configuration(format!( + if !(0..=CURRENT_SCHEMA_VERSION).contains(&version) { + return Err(Error::Configuration(format!( "database schema version {version} is not supported; this binary supports {CURRENT_SCHEMA_VERSION}" - ))), + ))); } + if version == 0 && !initialize { + return Err(Error::Configuration( + "existing database is not initialized as CPDS storage".to_owned(), + )); + } + if version < CURRENT_SCHEMA_VERSION { + let transaction = connection.transaction()?; + migrate(&transaction, version)?; + validate_schema(&transaction)?; + transaction.pragma_update(None, "user_version", CURRENT_SCHEMA_VERSION)?; + transaction.commit()?; + return Ok(()); + } + validate_schema(connection) } pub(super) fn check(connection: &Connection) -> Result<()> { @@ -390,9 +379,7 @@ pub(super) fn check(connection: &Connection) -> Result<()> { "database schema version {version} is not supported; this binary supports {CURRENT_SCHEMA_VERSION}" ))); } - connection - .query_row("SELECT 1", [], |_| Ok(())) - .map_err(Into::into) + connection.query_row_and_then("SELECT 1", [], |_| Ok(())) } fn verify_connection_policy(connection: &Connection) -> Result<()> { @@ -433,8 +420,7 @@ fn verify_integrity(connection: &Connection) -> Result<()> { } let mut statement = connection.prepare("PRAGMA foreign_key_check")?; - let mut rows = statement.query([])?; - if rows.next()?.is_some() { + if statement.exists([])? { return Err(Error::Configuration( "SQLite foreign_key_check found an invalid reference".to_owned(), )); @@ -442,300 +428,115 @@ fn verify_integrity(connection: &Connection) -> Result<()> { Ok(()) } -fn migrate_initial_schema(connection: &mut Connection) -> Result<()> { - let transaction = connection.transaction()?; - transaction.execute_batch(INITIAL_SCHEMA)?; - transaction.execute_batch(VALIDATE_INITIAL_SCHEMA)?; - transaction.execute_batch(REPOSITORY_SCHEMA)?; - transaction.execute_batch(VALIDATE_REPOSITORY_SCHEMA)?; - transaction.execute_batch(COMMIT_BLOCK_SCHEMA)?; - transaction.execute_batch(VALIDATE_COMMIT_BLOCK_SCHEMA)?; - transaction.execute_batch(RECORD_BLOB_SCHEMA)?; - transaction.execute_batch(VALIDATE_RECORD_BLOB_SCHEMA)?; - transaction.execute_batch(BLOB_LIFECYCLE_SCHEMA)?; - transaction.execute_batch(VALIDATE_BLOB_LIFECYCLE_SCHEMA)?; - transaction.execute_batch(BLOB_PUBLICATION_SCHEMA)?; - transaction.execute_batch(VALIDATE_BLOB_PUBLICATION_SCHEMA)?; - add_account_status(&transaction)?; - backfill_record_blobs(&transaction)?; - backfill_space_record_blobs(&transaction)?; - verify_integrity(&transaction)?; - transaction.pragma_update(None, "user_version", CURRENT_SCHEMA_VERSION)?; - transaction.commit()?; - Ok(()) -} - -fn migrate_repository_schema(connection: &mut Connection) -> Result<()> { - let transaction = connection.transaction()?; - transaction.execute_batch(VALIDATE_INITIAL_SCHEMA)?; - transaction.execute_batch(REPOSITORY_SCHEMA)?; - transaction.execute_batch(VALIDATE_REPOSITORY_SCHEMA)?; - transaction.execute_batch(COMMIT_BLOCK_SCHEMA)?; - transaction.execute_batch(VALIDATE_COMMIT_BLOCK_SCHEMA)?; - transaction.execute_batch(RECORD_BLOB_SCHEMA)?; - transaction.execute_batch(VALIDATE_RECORD_BLOB_SCHEMA)?; - transaction.execute_batch(BLOB_LIFECYCLE_SCHEMA)?; - transaction.execute_batch(VALIDATE_BLOB_LIFECYCLE_SCHEMA)?; - transaction.execute_batch(BLOB_PUBLICATION_SCHEMA)?; - transaction.execute_batch(VALIDATE_BLOB_PUBLICATION_SCHEMA)?; - add_account_status(&transaction)?; - backfill_record_blobs(&transaction)?; - backfill_space_record_blobs(&transaction)?; - verify_integrity(&transaction)?; - transaction.pragma_update(None, "user_version", CURRENT_SCHEMA_VERSION)?; - transaction.commit()?; - Ok(()) -} - -fn migrate_commit_block_schema(connection: &mut Connection) -> Result<()> { - let transaction = connection.transaction()?; - transaction.execute_batch(VALIDATE_INITIAL_SCHEMA)?; - transaction.execute_batch(VALIDATE_REPOSITORY_SCHEMA)?; - transaction.execute_batch(COMMIT_BLOCK_SCHEMA)?; - transaction.execute_batch(VALIDATE_COMMIT_BLOCK_SCHEMA)?; - transaction.execute_batch(RECORD_BLOB_SCHEMA)?; - transaction.execute_batch(VALIDATE_RECORD_BLOB_SCHEMA)?; - transaction.execute_batch(BLOB_LIFECYCLE_SCHEMA)?; - transaction.execute_batch(VALIDATE_BLOB_LIFECYCLE_SCHEMA)?; - transaction.execute_batch(BLOB_PUBLICATION_SCHEMA)?; - transaction.execute_batch(VALIDATE_BLOB_PUBLICATION_SCHEMA)?; - add_account_status(&transaction)?; - backfill_record_blobs(&transaction)?; - backfill_space_record_blobs(&transaction)?; - verify_integrity(&transaction)?; - transaction.pragma_update(None, "user_version", CURRENT_SCHEMA_VERSION)?; - transaction.commit()?; - Ok(()) -} - -fn migrate_record_blob_schema(connection: &mut Connection) -> Result<()> { - let transaction = connection.transaction()?; - transaction.execute_batch(VALIDATE_INITIAL_SCHEMA)?; - transaction.execute_batch(VALIDATE_REPOSITORY_SCHEMA)?; - transaction.execute_batch(VALIDATE_COMMIT_BLOCK_SCHEMA)?; - transaction.execute_batch(RECORD_BLOB_SCHEMA)?; - transaction.execute_batch(VALIDATE_RECORD_BLOB_SCHEMA)?; - transaction.execute_batch(BLOB_LIFECYCLE_SCHEMA)?; - transaction.execute_batch(VALIDATE_BLOB_LIFECYCLE_SCHEMA)?; - transaction.execute_batch(BLOB_PUBLICATION_SCHEMA)?; - transaction.execute_batch(VALIDATE_BLOB_PUBLICATION_SCHEMA)?; - add_account_status(&transaction)?; - backfill_record_blobs(&transaction)?; - backfill_space_record_blobs(&transaction)?; - verify_integrity(&transaction)?; - transaction.pragma_update(None, "user_version", CURRENT_SCHEMA_VERSION)?; - transaction.commit()?; - Ok(()) -} - -fn migrate_blob_lifecycle_schema(connection: &mut Connection) -> Result<()> { - let transaction = connection.transaction()?; - transaction.execute_batch(VALIDATE_INITIAL_SCHEMA)?; - transaction.execute_batch(VALIDATE_REPOSITORY_SCHEMA)?; - transaction.execute_batch(VALIDATE_COMMIT_BLOCK_SCHEMA)?; - transaction.execute_batch(VALIDATE_RECORD_BLOB_SCHEMA)?; - transaction.execute_batch(BLOB_LIFECYCLE_SCHEMA)?; - transaction.execute_batch(VALIDATE_BLOB_LIFECYCLE_SCHEMA)?; - transaction.execute_batch(BLOB_PUBLICATION_SCHEMA)?; - transaction.execute_batch(VALIDATE_BLOB_PUBLICATION_SCHEMA)?; - add_account_status(&transaction)?; - backfill_space_record_blobs(&transaction)?; - verify_integrity(&transaction)?; - transaction.pragma_update(None, "user_version", CURRENT_SCHEMA_VERSION)?; - transaction.commit()?; - Ok(()) -} - -fn migrate_blob_publication_schema(connection: &mut Connection) -> Result<()> { - let transaction = connection.transaction()?; - transaction.execute_batch(VALIDATE_INITIAL_SCHEMA)?; - transaction.execute_batch(VALIDATE_REPOSITORY_SCHEMA)?; - transaction.execute_batch(VALIDATE_COMMIT_BLOCK_SCHEMA)?; - transaction.execute_batch(VALIDATE_RECORD_BLOB_SCHEMA)?; - transaction.execute_batch(VALIDATE_BLOB_LIFECYCLE_SCHEMA)?; - transaction.execute_batch(BLOB_PUBLICATION_SCHEMA)?; - transaction.execute_batch(VALIDATE_BLOB_PUBLICATION_SCHEMA)?; - add_account_status(&transaction)?; - verify_integrity(&transaction)?; - transaction.pragma_update(None, "user_version", CURRENT_SCHEMA_VERSION)?; - transaction.commit()?; - Ok(()) -} - -fn migrate_stream_outbox_schema(connection: &mut Connection) -> Result<()> { - let transaction = connection.transaction()?; - transaction.execute_batch(VALIDATE_INITIAL_SCHEMA)?; - transaction.execute_batch(VALIDATE_REPOSITORY_SCHEMA)?; - transaction.execute_batch(VALIDATE_COMMIT_BLOCK_SCHEMA)?; - transaction.execute_batch(VALIDATE_RECORD_BLOB_SCHEMA)?; - transaction.execute_batch(VALIDATE_BLOB_LIFECYCLE_SCHEMA)?; - transaction.execute_batch(VALIDATE_BLOB_PUBLICATION_SCHEMA)?; - transaction.execute_batch(STREAM_OUTBOX_SCHEMA)?; - transaction.execute_batch(VALIDATE_REPOSITORY_SCHEMA)?; - transaction.execute_batch(VALIDATE_COMMIT_BLOCK_SCHEMA)?; - add_account_status(&transaction)?; - verify_integrity(&transaction)?; - transaction.pragma_update(None, "user_version", CURRENT_SCHEMA_VERSION)?; - transaction.commit()?; +fn migrate(transaction: &rusqlite::Transaction<'_>, version: i64) -> Result<()> { + for (threshold, schema, backfill) in [ + (1, INITIAL_SCHEMA, None), + (2, REPOSITORY_SCHEMA, None), + (3, COMMIT_BLOCK_SCHEMA, None), + (4, RECORD_BLOB_SCHEMA, Some(&PUBLIC_BLOBS)), + (5, BLOB_LIFECYCLE_SCHEMA, Some(&SPACE_BLOBS)), + (6, BLOB_PUBLICATION_SCHEMA, None), + (8, ACCOUNT_STATUS_SCHEMA, None), + // Earlier direct upgrades could skip the v7 outbox rebuild. Repair them + // as well, retaining rows, block memberships, and the sequence counter. + (9, STREAM_OUTBOX_SCHEMA, None), + (9, OAUTH_EXPIRY_SCHEMA, None), + ] { + if version < threshold { + transaction.execute_batch(schema)?; + if let Some(index) = backfill { + backfill_blob_index(transaction, index)?; + } + } + } Ok(()) } -fn migrate_account_status_schema(connection: &mut Connection) -> Result<()> { - let transaction = connection.transaction()?; - transaction.execute_batch(VALIDATE_INITIAL_SCHEMA)?; - transaction.execute_batch(VALIDATE_REPOSITORY_SCHEMA)?; - transaction.execute_batch(VALIDATE_COMMIT_BLOCK_SCHEMA)?; - transaction.execute_batch(VALIDATE_RECORD_BLOB_SCHEMA)?; - transaction.execute_batch(VALIDATE_BLOB_LIFECYCLE_SCHEMA)?; - transaction.execute_batch(VALIDATE_BLOB_PUBLICATION_SCHEMA)?; - add_account_status(&transaction)?; - verify_integrity(&transaction)?; - transaction.pragma_update(None, "user_version", CURRENT_SCHEMA_VERSION)?; - transaction.commit()?; - Ok(()) +fn validate_schema(connection: &Connection) -> Result<()> { + for schema in [ + VALIDATE_INITIAL_SCHEMA, + VALIDATE_REPOSITORY_SCHEMA, + VALIDATE_COMMIT_BLOCK_SCHEMA, + VALIDATE_RECORD_BLOB_SCHEMA, + VALIDATE_BLOB_LIFECYCLE_SCHEMA, + VALIDATE_BLOB_PUBLICATION_SCHEMA, + VALIDATE_ACCOUNT_STATUS_SCHEMA, + ] { + connection.execute_batch(schema)?; + } + verify_integrity(connection)?; + verify_record_blob_indexes(connection) } -fn add_account_status(transaction: &rusqlite::Transaction<'_>) -> Result<()> { - transaction.execute_batch(ACCOUNT_STATUS_SCHEMA)?; - transaction.execute_batch(VALIDATE_ACCOUNT_STATUS_SCHEMA)?; - Ok(()) +struct BlobIndex { + records: &'static str, + entries: &'static str, + insert: &'static str, + mismatch: &'static str, } -fn backfill_record_blobs(transaction: &rusqlite::Transaction<'_>) -> Result<()> { - for (did, collection, rkey, cid) in expected_record_blob_index(transaction)? { - transaction.execute( - "INSERT INTO record_blobs (did, collection, rkey, cid) +const PUBLIC_BLOBS: BlobIndex = BlobIndex { + records: "SELECT did, collection, rkey, record_json FROM records + ORDER BY did, collection, rkey", + entries: "SELECT did, collection, rkey, cid FROM record_blobs + ORDER BY did, collection, rkey, cid", + insert: "INSERT INTO record_blobs (did, collection, rkey, cid) VALUES (?1, ?2, ?3, ?4)", - rusqlite::params![did, collection, rkey, cid], - )?; - } - Ok(()) -} - -fn backfill_space_record_blobs(transaction: &rusqlite::Transaction<'_>) -> Result<()> { - for (space, did, collection, rkey, cid) in expected_space_record_blob_index(transaction)? { - transaction.execute( - "INSERT INTO space_record_blobs (space_uri, did, collection, rkey, cid) + mismatch: "record blob reference index does not match stored records", +}; +const SPACE_BLOBS: BlobIndex = BlobIndex { + records: "SELECT space_uri, did, collection, rkey, record_json FROM space_records + ORDER BY space_uri, did, collection, rkey", + entries: "SELECT space_uri, did, collection, rkey, cid FROM space_record_blobs + ORDER BY space_uri, did, collection, rkey, cid", + insert: "INSERT INTO space_record_blobs (space_uri, did, collection, rkey, cid) VALUES (?1, ?2, ?3, ?4, ?5)", - rusqlite::params![space, did, collection, rkey, cid], - )?; + mismatch: "space record blob reference index does not match stored records", +}; + +fn backfill_blob_index(connection: &Connection, index: &BlobIndex) -> Result<()> { + for reference in expected_blob_index(connection, index)? { + connection.execute(index.insert, rusqlite::params_from_iter(reference))?; } Ok(()) } fn verify_record_blob_indexes(connection: &Connection) -> Result<()> { - let expected = expected_record_blob_index(connection)?; - let actual = { - let mut statement = connection.prepare( - "SELECT did, collection, rkey, cid FROM record_blobs - ORDER BY did, collection, rkey, cid", - )?; - statement - .query_map([], |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, String>(2)?, - row.get::<_, String>(3)?, - )) - })? - .collect::, _>>()? - }; - if actual != expected { - return Err(Error::Configuration( - "record blob reference index does not match stored records".to_owned(), - )); - } - - let expected = expected_space_record_blob_index(connection)?; - let actual = { - let mut statement = connection.prepare( - "SELECT space_uri, did, collection, rkey, cid FROM space_record_blobs - ORDER BY space_uri, did, collection, rkey, cid", - )?; - statement - .query_map([], |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, String>(2)?, - row.get::<_, String>(3)?, - row.get::<_, String>(4)?, - )) - })? - .collect::, _>>()? - }; - if actual != expected { - return Err(Error::Configuration( - "space record blob reference index does not match stored records".to_owned(), - )); + for index in [&PUBLIC_BLOBS, &SPACE_BLOBS] { + let expected = expected_blob_index(connection, index)?; + let actual = index_rows(connection, index.entries)?.into_iter().collect(); + if expected != actual { + return Err(Error::Configuration(index.mismatch.to_owned())); + } } Ok(()) } -fn expected_record_blob_index(connection: &Connection) -> Result> { - let records = { - let mut statement = connection.prepare( - "SELECT did, collection, rkey, record_json FROM records - ORDER BY did, collection, rkey", - )?; - statement - .query_map([], |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, String>(2)?, - row.get::<_, String>(3)?, - )) - })? - .collect::, _>>()? - }; +fn expected_blob_index( + connection: &Connection, + index: &BlobIndex, +) -> Result>> { let mut references = BTreeSet::new(); - for (did, collection, rkey, json) in records { + for mut key in index_rows(connection, index.records)? { + let json = key + .pop() + .expect("record queries select JSON after the composite key"); let value: Value = serde_json::from_str(&json)?; - references.extend( - referenced_blob_cids(&value) - .into_iter() - .map(|cid| (did.clone(), collection.clone(), rkey.clone(), cid)), - ); + for cid in referenced_blob_cids(&value) { + let mut reference = key.clone(); + reference.push(cid); + references.insert(reference); + } } Ok(references) } -fn expected_space_record_blob_index( - connection: &Connection, -) -> Result> { - let records = { - let mut statement = connection.prepare( - "SELECT space_uri, did, collection, rkey, record_json FROM space_records - ORDER BY space_uri, did, collection, rkey", - )?; - statement - .query_map([], |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, String>(2)?, - row.get::<_, String>(3)?, - row.get::<_, String>(4)?, - )) - })? - .collect::, _>>()? - }; - let mut references = BTreeSet::new(); - for (space, did, collection, rkey, json) in records { - let value: Value = serde_json::from_str(&json)?; - references.extend(referenced_blob_cids(&value).into_iter().map(|cid| { - ( - space.clone(), - did.clone(), - collection.clone(), - rkey.clone(), - cid, - ) - })); - } - Ok(references) +fn index_rows(connection: &Connection, sql: &str) -> Result>> { + super::query(connection, sql, [], |row| { + (0..row.as_ref().column_count()) + .map(|column| Ok(row.get(column)?)) + .collect() + }) } #[cfg(test)] diff --git a/src/database/schema/tests.rs b/src/database/schema/tests.rs index af1ce47..ada2be0 100644 --- a/src/database/schema/tests.rs +++ b/src/database/schema/tests.rs @@ -8,7 +8,7 @@ use super::*; fn fresh_database_receives_the_current_schema_version() { let mut connection = Connection::open_in_memory().unwrap(); - apply(&mut connection).unwrap(); + apply(&mut connection, true).unwrap(); assert_eq!(version(&connection), CURRENT_SCHEMA_VERSION); assert!(table_exists(&connection, "accounts")); @@ -23,6 +23,17 @@ fn fresh_database_receives_the_current_schema_version() { assert!(column_exists(&connection, "accounts", "status")); assert!(column_exists(&connection, "blobs", "delete_started_at")); assert!(column_exists(&connection, "blobs", "available_at")); + for index in ["oauth_codes_by_expiry", "access_tokens_by_expiry"] { + assert!( + connection + .query_row( + "SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE type = 'index' AND name = ?1)", + [index], + |row| row.get::<_, bool>(0), + ) + .unwrap() + ); + } connection .execute( "INSERT INTO accounts (did, handle, password_hash, created_at) @@ -48,7 +59,7 @@ fn fresh_database_receives_the_current_schema_version() { fn connection_policy_is_applied_and_verified() { let mut connection = Connection::open_in_memory().unwrap(); - apply(&mut connection).unwrap(); + apply(&mut connection, true).unwrap(); assert!(rusqlite::version_number() >= 3_051_003); assert_eq!(pragma_i64(&connection, "foreign_keys"), 1); @@ -75,7 +86,7 @@ fn connection_policy_is_applied_and_verified() { #[test] fn readiness_rechecks_connection_policy_and_schema_version() { let mut connection = Connection::open_in_memory().unwrap(); - apply(&mut connection).unwrap(); + apply(&mut connection, true).unwrap(); check(&connection).unwrap(); connection @@ -100,10 +111,39 @@ fn readiness_rechecks_connection_policy_and_schema_version() { ); } +#[test] +fn native_readiness_query_preserves_sql_diagnostics_and_first_row_behavior() { + let connection = Connection::open_in_memory().unwrap(); + for sql in [ + "SELECT 1", + "SELECT NULL", + "SELECT 1 WHERE 0", + "SELECT 1 UNION ALL SELECT abs(-9223372036854775808)", + "SELECT abs(-9223372036854775808)", + "SELECT ?1", + "SELECT missing_column", + ] { + let mut old_calls = 0; + let old: Result<()> = connection + .query_row(sql, [], |_| { + old_calls += 1; + Ok(()) + }) + .map_err(Into::into); + let mut native_calls = 0; + let native: Result<()> = connection.query_row_and_then(sql, [], |_| { + native_calls += 1; + Ok(()) + }); + assert_eq!(native_calls, old_calls, "{sql}"); + assert_eq!(format!("{native:?}"), format!("{old:?}"), "{sql}"); + } +} + #[test] fn foreign_keys_are_enforced() { let mut connection = Connection::open_in_memory().unwrap(); - apply(&mut connection).unwrap(); + apply(&mut connection, true).unwrap(); assert!( connection @@ -132,7 +172,7 @@ fn unversioned_database_is_migrated_without_losing_existing_data() { ) .unwrap(); - apply(&mut connection).unwrap(); + apply(&mut connection, true).unwrap(); assert_eq!(version(&connection), CURRENT_SCHEMA_VERSION); assert_eq!( @@ -151,7 +191,7 @@ fn incompatible_unversioned_schema_rolls_back() { .execute_batch("CREATE TABLE accounts (did TEXT PRIMARY KEY);") .unwrap(); - assert!(apply(&mut connection).is_err()); + assert!(apply(&mut connection, true).is_err()); assert_eq!(version(&connection), 0); assert!(!table_exists(&connection, "records")); @@ -171,7 +211,7 @@ fn version_one_database_gains_repository_tables_without_losing_records() { ) .unwrap(); - apply(&mut connection).unwrap(); + apply(&mut connection, true).unwrap(); assert_eq!(version(&connection), CURRENT_SCHEMA_VERSION); assert_eq!( @@ -196,7 +236,7 @@ fn incompatible_version_one_repository_schema_rolls_back() { ) .unwrap(); - assert!(apply(&mut connection).is_err()); + assert!(apply(&mut connection, true).is_err()); assert_eq!(version(&connection), 1); assert!(!table_exists(&connection, "repo_heads")); @@ -216,12 +256,98 @@ fn version_two_database_gains_commit_block_membership() { connection.execute_batch(REPOSITORY_SCHEMA).unwrap(); connection.pragma_update(None, "user_version", 2).unwrap(); - apply(&mut connection).unwrap(); + apply(&mut connection, true).unwrap(); assert_eq!(version(&connection), CURRENT_SCHEMA_VERSION); assert!(table_exists(&connection, "repo_commit_blocks")); } +#[test] +fn every_legacy_repository_version_supports_identity_events_after_upgrade() { + for old_version in 2..=8 { + let mut connection = Connection::open_in_memory().unwrap(); + connection.execute_batch(INITIAL_SCHEMA).unwrap(); + connection + .execute_batch(&legacy_repository_schema()) + .unwrap(); + if old_version >= 3 { + connection.execute_batch(COMMIT_BLOCK_SCHEMA).unwrap(); + } + if old_version >= 4 { + connection.execute_batch(RECORD_BLOB_SCHEMA).unwrap(); + } + if old_version >= 5 { + connection.execute_batch(BLOB_LIFECYCLE_SCHEMA).unwrap(); + } + if old_version >= 6 { + connection.execute_batch(BLOB_PUBLICATION_SCHEMA).unwrap(); + } + if old_version >= 8 { + connection.execute_batch(ACCOUNT_STATUS_SCHEMA).unwrap(); + } + connection + .execute_batch( + "INSERT INTO accounts (did, handle, password_hash, created_at) + VALUES ('did:web:alice.test', 'alice.test', 'hash', 1); + INSERT INTO repo_outbox ( + seq, did, rev, root_cid, event_type, payload, next_attempt_at, created_at + ) VALUES (7, 'did:web:alice.test', '2222222222222', 'root', 'repo.commit', X'01', 1, 1);", + ) + .unwrap(); + connection + .pragma_update(None, "user_version", old_version) + .unwrap(); + + apply(&mut connection, true).unwrap(); + + connection + .execute( + "INSERT INTO repo_outbox ( + did, event_type, payload, next_attempt_at, created_at + ) VALUES ('did:web:alice.test', 'identity', X'02', 2, 2)", + [], + ) + .unwrap_or_else(|error| panic!("upgraded version {old_version}: {error}")); + assert_eq!(connection.last_insert_rowid(), 8); + assert_eq!(version(&connection), CURRENT_SCHEMA_VERSION); + } +} + +#[test] +fn outbox_migration_never_reuses_a_previously_issued_sequence() { + let mut connection = Connection::open_in_memory().unwrap(); + for schema in [ + INITIAL_SCHEMA, + REPOSITORY_SCHEMA, + COMMIT_BLOCK_SCHEMA, + RECORD_BLOB_SCHEMA, + BLOB_LIFECYCLE_SCHEMA, + BLOB_PUBLICATION_SCHEMA, + ] { + connection.execute_batch(schema).unwrap(); + } + connection + .execute_batch( + "INSERT INTO accounts VALUES ('did:web:alice.test', 'alice.test', 'hash', 1); + INSERT INTO repo_outbox (seq, did, event_type, payload, next_attempt_at, created_at) + VALUES (20, 'did:web:alice.test', 'identity', X'01', 1, 1); + DELETE FROM repo_outbox; + PRAGMA user_version = 6;", + ) + .unwrap(); + + apply(&mut connection, true).unwrap(); + + connection + .execute( + "INSERT INTO repo_outbox (did, event_type, payload, next_attempt_at, created_at) + VALUES ('did:web:alice.test', 'identity', X'02', 2, 2)", + [], + ) + .unwrap(); + assert_eq!(connection.last_insert_rowid(), 21); +} + #[test] fn incompatible_version_two_commit_block_schema_rolls_back() { let mut connection = Connection::open_in_memory().unwrap(); @@ -234,7 +360,7 @@ fn incompatible_version_two_commit_block_schema_rolls_back() { ) .unwrap(); - assert!(apply(&mut connection).is_err()); + assert!(apply(&mut connection, true).is_err()); assert_eq!(version(&connection), 2); assert_eq!( @@ -246,6 +372,55 @@ fn incompatible_version_two_commit_block_schema_rolls_back() { ); } +#[test] +fn blob_backfill_failures_precede_later_migrations_and_roll_back() { + for (old_version, record, later_conflict, index_table) in [ + ( + 3, + "INSERT INTO records VALUES ( + 'did:web:alice.test', 'com.example.record', 'one', 'cid', 'invalid', 1, 1 + );", + "ALTER TABLE blobs ADD COLUMN delete_started_at INTEGER;", + "record_blobs", + ), + ( + 4, + "INSERT INTO space_records VALUES ( + 'at://did:web:alice.test/com.example.space/one', 'did:web:alice.test', + 'com.example.record', 'one', 'cid', 'invalid', 1, 1 + );", + "ALTER TABLE blobs ADD COLUMN available_at INTEGER;", + "space_record_blobs", + ), + ] { + let mut connection = Connection::open_in_memory().unwrap(); + for schema in [INITIAL_SCHEMA, REPOSITORY_SCHEMA, COMMIT_BLOCK_SCHEMA] { + connection.execute_batch(schema).unwrap(); + } + if old_version == 4 { + connection.execute_batch(RECORD_BLOB_SCHEMA).unwrap(); + } + connection + .execute_batch( + "INSERT INTO accounts VALUES ('did:web:alice.test', 'alice.test', 'hash', 1);", + ) + .unwrap(); + connection.execute_batch(record).unwrap(); + // Later DDL also fails, so the JSON error pins immediate backfill order. + connection.execute_batch(later_conflict).unwrap(); + connection + .pragma_update(None, "user_version", old_version) + .unwrap(); + + assert!(matches!(apply(&mut connection, true), Err(Error::Json(_)))); + assert_eq!(version(&connection), old_version); + assert!(!table_exists(&connection, index_table)); + if old_version == 4 { + assert!(!column_exists(&connection, "blobs", "delete_started_at")); + } + } +} + #[test] fn version_three_database_indexes_existing_atrium_blob_references() { const BLOB_CID: &str = "bafkreibme22gw2h7y2h7tg2fhqotaqjucnbc24deqo72b6mkl2egezxhvy"; @@ -265,7 +440,7 @@ fn version_three_database_indexes_existing_atrium_blob_references() { )) .unwrap(); - apply(&mut connection).unwrap(); + apply(&mut connection, true).unwrap(); assert_eq!(version(&connection), CURRENT_SCHEMA_VERSION); assert_eq!( @@ -290,7 +465,7 @@ fn incompatible_version_three_record_blob_schema_rolls_back() { ) .unwrap(); - assert!(apply(&mut connection).is_err()); + assert!(apply(&mut connection, true).is_err()); assert_eq!(version(&connection), 3); assert_eq!( @@ -323,7 +498,7 @@ fn version_four_database_indexes_existing_space_blob_references() { )) .unwrap(); - apply(&mut connection).unwrap(); + apply(&mut connection, true).unwrap(); assert_eq!(version(&connection), CURRENT_SCHEMA_VERSION); assert!(column_exists(&connection, "blobs", "delete_started_at")); @@ -350,7 +525,7 @@ fn incompatible_version_four_blob_lifecycle_schema_rolls_back() { ) .unwrap(); - assert!(apply(&mut connection).is_err()); + assert!(apply(&mut connection, true).is_err()); assert_eq!(version(&connection), 4); assert!(!column_exists(&connection, "blobs", "delete_started_at")); @@ -380,7 +555,7 @@ fn version_five_database_marks_existing_blobs_available() { ) .unwrap(); - apply(&mut connection).unwrap(); + apply(&mut connection, true).unwrap(); assert_eq!(version(&connection), CURRENT_SCHEMA_VERSION); assert_eq!( @@ -407,7 +582,7 @@ fn incompatible_version_five_blob_publication_schema_rolls_back() { ) .unwrap(); - assert!(apply(&mut connection).is_err()); + assert!(apply(&mut connection, true).is_err()); assert_eq!(version(&connection), 5); assert_eq!( @@ -426,24 +601,7 @@ fn incompatible_version_five_blob_publication_schema_rolls_back() { fn version_six_database_generalizes_the_outbox_without_losing_commits() { let mut connection = Connection::open_in_memory().unwrap(); connection.execute_batch(INITIAL_SCHEMA).unwrap(); - let legacy_repository_schema = REPOSITORY_SCHEMA - .replace("rev TEXT,", "rev TEXT NOT NULL,") - .replace("root_cid TEXT,", "root_cid TEXT NOT NULL,") - .replace( - "event_type TEXT NOT NULL CHECK (event_type IN ('repo.commit', 'identity', 'account')),", - "event_type TEXT NOT NULL,", - ) - .replace( - ", - CHECK ( - (event_type = 'repo.commit' AND rev IS NOT NULL AND root_cid IS NOT NULL) - OR - (event_type IN ('identity', 'account') AND rev IS NULL AND root_cid IS NULL) - ) -);", - " -);", - ); + let legacy_repository_schema = legacy_repository_schema(); connection.execute_batch(&legacy_repository_schema).unwrap(); connection.execute_batch(COMMIT_BLOCK_SCHEMA).unwrap(); connection.execute_batch(RECORD_BLOB_SCHEMA).unwrap(); @@ -465,7 +623,7 @@ fn version_six_database_generalizes_the_outbox_without_losing_commits() { ) .unwrap(); - apply(&mut connection).unwrap(); + apply(&mut connection, true).unwrap(); assert_eq!(version(&connection), CURRENT_SCHEMA_VERSION); assert_eq!( @@ -525,7 +683,7 @@ fn version_seven_database_gains_durable_account_status() { ) .unwrap(); - apply(&mut connection).unwrap(); + apply(&mut connection, true).unwrap(); assert_eq!(version(&connection), CURRENT_SCHEMA_VERSION); assert_eq!( @@ -540,7 +698,7 @@ fn version_seven_database_gains_durable_account_status() { #[test] fn stream_outbox_rejects_inconsistent_event_shapes() { let mut connection = Connection::open_in_memory().unwrap(); - apply(&mut connection).unwrap(); + apply(&mut connection, true).unwrap(); connection .execute_batch( "INSERT INTO accounts (did, handle, password_hash, created_at) @@ -574,7 +732,7 @@ fn stream_outbox_rejects_inconsistent_event_shapes() { fn current_schema_rejects_a_drifted_record_blob_index() { const BLOB_CID: &str = "bafkreibme22gw2h7y2h7tg2fhqotaqjucnbc24deqo72b6mkl2egezxhvy"; let mut connection = Connection::open_in_memory().unwrap(); - apply(&mut connection).unwrap(); + apply(&mut connection, true).unwrap(); connection .execute_batch(&format!( "INSERT INTO accounts (did, handle, password_hash, created_at) @@ -587,7 +745,7 @@ fn current_schema_rejects_a_drifted_record_blob_index() { )) .unwrap(); - let error = apply(&mut connection).unwrap_err().to_string(); + let error = apply(&mut connection, true).unwrap_err().to_string(); assert!(error.contains("blob reference index")); assert_eq!(version(&connection), CURRENT_SCHEMA_VERSION); @@ -597,7 +755,7 @@ fn current_schema_rejects_a_drifted_record_blob_index() { fn current_schema_rejects_a_drifted_space_blob_index() { const BLOB_CID: &str = "bafkreibme22gw2h7y2h7tg2fhqotaqjucnbc24deqo72b6mkl2egezxhvy"; let mut connection = Connection::open_in_memory().unwrap(); - apply(&mut connection).unwrap(); + apply(&mut connection, true).unwrap(); connection .execute_batch(&format!( "INSERT INTO accounts (did, handle, password_hash, created_at) @@ -611,7 +769,7 @@ fn current_schema_rejects_a_drifted_space_blob_index() { )) .unwrap(); - let error = apply(&mut connection).unwrap_err().to_string(); + let error = apply(&mut connection, true).unwrap_err().to_string(); assert!(error.contains("space record blob reference index")); assert_eq!(version(&connection), CURRENT_SCHEMA_VERSION); @@ -645,7 +803,7 @@ fn invalid_foreign_keys_fail_and_roll_back_migration() { ) .unwrap(); - let error = apply(&mut connection).unwrap_err().to_string(); + let error = apply(&mut connection, true).unwrap_err().to_string(); assert!(error.contains("foreign_key_check")); assert_eq!(version(&connection), 0); @@ -659,7 +817,7 @@ fn newer_schema_is_rejected_without_modification() { .pragma_update(None, "user_version", CURRENT_SCHEMA_VERSION + 1) .unwrap(); - let error = apply(&mut connection).err().unwrap().to_string(); + let error = apply(&mut connection, true).err().unwrap().to_string(); assert!(error.contains("is not supported")); assert_eq!(version(&connection), CURRENT_SCHEMA_VERSION + 1); @@ -672,7 +830,7 @@ fn schema_and_data_survive_an_on_disk_reopen() { let path = directory.path().join("cpds.sqlite3"); { let mut connection = Connection::open(&path).unwrap(); - apply(&mut connection).unwrap(); + apply(&mut connection, true).unwrap(); connection .execute( "INSERT INTO accounts (did, handle, password_hash, created_at) @@ -683,7 +841,7 @@ fn schema_and_data_survive_an_on_disk_reopen() { } let mut reopened = Connection::open(path).unwrap(); - apply(&mut reopened).unwrap(); + apply(&mut reopened, true).unwrap(); assert_eq!(version(&reopened), CURRENT_SCHEMA_VERSION); assert_eq!(pragma_string(&reopened, "journal_mode"), "wal"); @@ -746,6 +904,27 @@ fn readiness_check_queries_the_durable_repository_outbox() { assert!(database.check().is_err()); } +fn legacy_repository_schema() -> String { + REPOSITORY_SCHEMA + .replace("rev TEXT,", "rev TEXT NOT NULL,") + .replace("root_cid TEXT,", "root_cid TEXT NOT NULL,") + .replace( + "event_type TEXT NOT NULL CHECK (event_type IN ('repo.commit', 'identity', 'account')),", + "event_type TEXT NOT NULL,", + ) + .replace( + ", + CHECK ( + (event_type = 'repo.commit' AND rev IS NOT NULL AND root_cid IS NOT NULL) + OR + (event_type IN ('identity', 'account') AND rev IS NULL AND root_cid IS NULL) + ) +);", + " +);", + ) +} + fn version(connection: &Connection) -> i64 { connection .pragma_query_value(None, "user_version", |row| row.get(0)) diff --git a/src/domain/account.rs b/src/domain/account.rs index 9f4c5d9..2d9bd48 100644 --- a/src/domain/account.rs +++ b/src/domain/account.rs @@ -1,5 +1,7 @@ //! The authenticated identity hosted by this PDS. +use crate::error::{Error, Result}; + #[derive(Clone, Debug, Eq, PartialEq)] pub struct Account { pub did: String, @@ -42,27 +44,40 @@ impl AccountStatus { } } - #[must_use] - pub const fn account_error_name(self) -> Option<&'static str> { - match self { - Self::Active => None, - Self::Deactivated => Some("AccountDeactivated"), - Self::Suspended => Some("AccountSuspended"), - Self::TakenDown => Some("AccountTakedown"), - Self::Deleted => Some("AccountDeleted"), + pub(crate) fn require_active(self) -> Result<()> { + if self.is_active() { + Ok(()) + } else { + Err(self.account_error()) } } - #[must_use] - pub const fn repo_error_name(self) -> Option<&'static str> { - match self { - Self::Active => None, - Self::Deactivated => Some("RepoDeactivated"), - Self::Suspended => Some("RepoSuspended"), - Self::TakenDown => Some("RepoTakendown"), - Self::Deleted => Some("RepoNotFound"), + pub(crate) fn account_error(self) -> Error { + Error::Xrpc { + name: match self { + Self::Active => "AccountNotActive", + Self::Deactivated => "AccountDeactivated", + Self::Suspended => "AccountSuspended", + Self::TakenDown => "AccountTakedown", + Self::Deleted => "AccountDeleted", + }, + message: format!("account is {}", self.as_str()), } } + + pub(crate) fn require_repo_active(self) -> Result<()> { + let name = match self { + Self::Active => return Ok(()), + Self::Deactivated => "RepoDeactivated", + Self::Suspended => "RepoSuspended", + Self::TakenDown => "RepoTakendown", + Self::Deleted => "RepoNotFound", + }; + Err(Error::xrpc( + name, + format!("repository is {}", self.as_str()), + )) + } } impl std::str::FromStr for AccountStatus { @@ -80,15 +95,15 @@ impl std::str::FromStr for AccountStatus { } } -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct HostedAccount { - pub account: Account, - pub status: AccountStatus, -} - #[cfg(test)] mod tests { use super::AccountStatus; + use crate::error::Error; + use axum::{ + body::to_bytes, + http::{StatusCode, header::CONTENT_TYPE}, + response::IntoResponse as _, + }; #[test] fn account_status_storage_values_are_exact_and_closed() { @@ -105,10 +120,91 @@ mod tests { status.inactive_reason(), (!status.is_active()).then(|| status.as_str()) ); - assert_eq!(status.account_error_name().is_none(), status.is_active()); - assert_eq!(status.repo_error_name().is_none(), status.is_active()); + assert_eq!(status.require_active().is_ok(), status.is_active()); + assert_eq!(status.require_repo_active().is_ok(), status.is_active()); } assert!("takedown".parse::().is_err()); } + + #[tokio::test] + async fn account_and_repository_status_errors_keep_exact_names_and_responses() { + for (status, storage, account_name, repo_name) in [ + (AccountStatus::Active, "active", "AccountNotActive", None), + ( + AccountStatus::Deactivated, + "deactivated", + "AccountDeactivated", + Some("RepoDeactivated"), + ), + ( + AccountStatus::Suspended, + "suspended", + "AccountSuspended", + Some("RepoSuspended"), + ), + ( + AccountStatus::TakenDown, + "takendown", + "AccountTakedown", + Some("RepoTakendown"), + ), + ( + AccountStatus::Deleted, + "deleted", + "AccountDeleted", + Some("RepoNotFound"), + ), + ] { + assert_eq!(status.as_str(), storage); + let account_message = format!("account is {storage}"); + assert_status_error( + status.account_error(), + account_name, + &account_message, + StatusCode::FORBIDDEN, + ) + .await; + if let Some(repo_name) = repo_name { + assert_status_error( + status.require_active().unwrap_err(), + account_name, + &account_message, + StatusCode::FORBIDDEN, + ) + .await; + assert_status_error( + status.require_repo_active().unwrap_err(), + repo_name, + &format!("repository is {storage}"), + StatusCode::BAD_REQUEST, + ) + .await; + } else { + status.require_active().unwrap(); + status.require_repo_active().unwrap(); + } + } + } + + async fn assert_status_error( + error: Error, + expected_name: &str, + expected_message: &str, + expected_status: StatusCode, + ) { + let Error::Xrpc { name, message } = &error else { + panic!("account state errors must keep their XRPC classification"); + }; + assert_eq!(*name, expected_name); + assert_eq!(message, expected_message); + assert_eq!(error.to_string(), expected_message); + let response = error.into_response(); + assert_eq!(response.status(), expected_status); + assert_eq!(response.headers()[CONTENT_TYPE], "application/json"); + assert_eq!( + to_bytes(response.into_body(), usize::MAX).await.unwrap(), + format!(r#"{{"error":"{expected_name}","message":"{expected_message}"}}"#), + ); + } } diff --git a/src/domain/blob.rs b/src/domain/blob.rs index 4c6d4f8..c093b04 100644 --- a/src/domain/blob.rs +++ b/src/domain/blob.rs @@ -4,6 +4,7 @@ use std::{collections::BTreeSet, str::FromStr as _}; use atrium_api::types::{BlobRef, TypedBlobRef}; use cid::Cid; +use serde::Deserialize as _; use serde_json::Value; #[derive(Clone, Debug, Eq, PartialEq)] @@ -26,71 +27,60 @@ pub(crate) fn writable_blob_refs(value: &Value) -> Result, String> { } fn collect_writable_blobs(value: &Value, output: &mut Vec) -> Result<(), String> { - match value { - Value::Object(object) => { - if object.get("$type").and_then(Value::as_str) == Some("blob") { - let blob = serde_json::from_value::(value.clone()).map_err(|error| { - format!("record contains an invalid blob reference: {error}") - })?; - let BlobRef::Typed(TypedBlobRef::Blob(blob)) = blob else { - return Err("record cannot write a legacy blob reference".to_owned()); - }; - output.push(Blob { - cid: blob.r#ref.0, - media_type: blob.mime_type, - size: u64::try_from(blob.size).unwrap_or(u64::MAX), - }); - return Ok(()); - } - if !object.contains_key("$type") - && object.contains_key("cid") - && object.contains_key("mimeType") - { + if let Value::Object(object) = value { + if object.get("$type").and_then(Value::as_str) == Some("blob") { + let blob = BlobRef::deserialize(value) + .map_err(|error| format!("record contains an invalid blob reference: {error}"))?; + let BlobRef::Typed(TypedBlobRef::Blob(blob)) = blob else { return Err("record cannot write a legacy blob reference".to_owned()); - } - for value in object.values() { - collect_writable_blobs(value, output)?; - } + }; + output.push(Blob { + cid: blob.r#ref.0, + media_type: blob.mime_type, + size: u64::try_from(blob.size).unwrap_or(u64::MAX), + }); + return Ok(()); } - Value::Array(values) => { - for value in values { - collect_writable_blobs(value, output)?; - } + if !object.contains_key("$type") + && object.contains_key("cid") + && object.contains_key("mimeType") + { + return Err("record cannot write a legacy blob reference".to_owned()); } - _ => {} } - Ok(()) + children(value).try_for_each(|value| collect_writable_blobs(value, output)) } fn collect_blob_cids(value: &Value, output: &mut BTreeSet) { - match value { - Value::Object(object) => { - let looks_typed = object.get("$type").and_then(Value::as_str) == Some("blob"); - let looks_legacy = !object.contains_key("$type") - && object.contains_key("cid") - && object.contains_key("mimeType"); - if (looks_typed || looks_legacy) - && let Ok(blob) = serde_json::from_value::(value.clone()) - { - let cid = match blob { - BlobRef::Typed(TypedBlobRef::Blob(blob)) => Some(blob.r#ref.0), - BlobRef::Untyped(blob) => Cid::from_str(&blob.cid).ok(), - }; - if let Some(cid) = cid { - output.insert(cid.to_string()); - } - } - for value in object.values() { - collect_blob_cids(value, output); - } - } - Value::Array(values) => { - for value in values { - collect_blob_cids(value, output); + if let Value::Object(object) = value { + let looks_typed = object.get("$type").and_then(Value::as_str) == Some("blob"); + let looks_legacy = !object.contains_key("$type") + && object.contains_key("cid") + && object.contains_key("mimeType"); + if (looks_typed || looks_legacy) + && let Ok(blob) = BlobRef::deserialize(value) + { + let cid = match blob { + BlobRef::Typed(TypedBlobRef::Blob(blob)) => Some(blob.r#ref.0), + BlobRef::Untyped(blob) => Cid::from_str(&blob.cid).ok(), + }; + if let Some(cid) = cid { + output.insert(cid.to_string()); } } - _ => {} } + for value in children(value) { + collect_blob_cids(value, output); + } +} + +fn children(value: &Value) -> impl Iterator { + value.as_array().into_iter().flatten().chain( + value + .as_object() + .into_iter() + .flat_map(|object| object.values()), + ) } #[cfg(test)] @@ -101,6 +91,117 @@ mod tests { const CID: &str = "bafkreibme22gw2h7y2h7tg2fhqotaqjucnbc24deqo72b6mkl2egezxhvy"; + #[test] + fn fallible_blob_walk_keeps_order_partial_output_and_first_failure() { + let blob = json!({ + "$type": "blob", "ref": {"$link": CID}, "mimeType": "image/png", "size": 3, + }); + let legacy = json!({"cid": CID, "mimeType": "image/png"}); + let malformed = json!({"$type": "blob", "ref": {"$link": "invalid"}}); + let malformed_error = format!( + "record contains an invalid blob reference: {}", + BlobRef::deserialize(&malformed).unwrap_err() + ); + let expected = Blob { + cid: CID.parse().unwrap(), + media_type: "image/png".to_owned(), + size: 3, + }; + let other = Blob { + cid: cpds_protocol::sha256_cid(0x55, b"second"), + media_type: "image/jpeg".to_owned(), + size: 6, + }; + let second = json!({ + "$type": "blob", "ref": {"$link": other.cid.to_string()}, + "mimeType": other.media_type, "size": other.size, + }); + for (value, expected_output, error) in [ + (json!([]), vec![], None), + (json!([null, false, 1, "text"]), vec![], None), + ( + json!([blob, [second]]), + vec![expected.clone(), other.clone()], + None, + ), + ( + json!([blob, legacy, malformed]), + vec![expected.clone()], + Some("record cannot write a legacy blob reference".to_owned()), + ), + ( + json!([blob, malformed, legacy]), + vec![expected.clone()], + Some(malformed_error), + ), + ( + json!([[second], {"nested": [blob, second]}]), + vec![other.clone(), expected, other], + None, + ), + ] { + let mut output = Vec::new(); + let result = collect_writable_blobs(&value, &mut output); + let expected_result = error.map_or(Ok(()), Err); + assert_eq!(result, expected_result, "{value}"); + assert_eq!(output, expected_output, "{value}"); + assert_eq!( + writable_blob_refs(&value), + expected_result.map(|()| expected_output), + "{value}" + ); + } + } + + #[test] + fn borrowed_blob_decoding_matches_owned_values_and_errors() { + let typed = json!({ + "$type": "blob", "ref": {"$link": CID}, "mimeType": "image/png", "size": 3, + }); + let mut values = vec![ + Value::Null, + json!(false), + json!(1), + json!("blob"), + json!([]), + json!({}), + typed.clone(), + json!({"cid": CID, "mimeType": "image/png"}), + json!({"$type": "blob", "cid": CID, "mimeType": "image/png"}), + json!({"$type": "other", "cid": CID, "mimeType": "image/png"}), + json!({"$type": "blob", "ref": {"$link": CID}, "mimeType": "image/png", "size": u64::MAX}), + serde_json::from_str(r#"{"$type":"blob","ref":{"$link":"invalid"},"mimeType":"image/png","size":18446744073709551616}"#).unwrap(), + ]; + for field in ["$type", "ref", "mimeType", "size"] { + let mut missing = typed.clone(); + missing.as_object_mut().unwrap().remove(field); + values.push(missing); + for replacement in [ + Value::Null, + json!(false), + json!(-1), + json!(1.5), + json!(""), + json!([]), + json!({}), + ] { + let mut value = typed.clone(); + value[field] = replacement; + values.push(value); + } + } + let mut extended = typed; + extended["unknown"] = json!({"array": [null, true, 1.5, {"nested": "value"}]}); + values.push(extended); + + for value in values { + let previous = + serde_json::from_value::(value.clone()).map_err(|error| error.to_string()); + let actual = BlobRef::deserialize(&value).map_err(|error| error.to_string()); + assert_eq!(actual, previous, "{value}"); + } + } + #[test] fn atrium_blob_refs_are_found_recursively_and_canonicalized() { let value = json!({ @@ -146,4 +247,49 @@ mod tests { .is_err() ); } + + #[test] + fn blob_walkers_preserve_typed_subtree_and_array_distinctions() { + let nested_cid = cpds_protocol::sha256_cid(0x55, b"nested blob").to_string(); + let mut typed = json!({ + "$type": "blob", "ref": {"$link": CID}, "mimeType": "image/png", "size": 3, + }); + let nested = json!({ + "$type": "blob", "ref": {"$link": nested_cid}, "mimeType": "image/png", "size": 4, + }); + typed["nested"] = json!({"array": [[nested]]}); + assert_eq!( + referenced_blob_cids(&typed), + BTreeSet::from([CID.to_owned(), nested_cid.clone()]) + ); + assert_eq!(writable_blob_refs(&typed).unwrap().len(), 1); + + let array = json!([typed, [nested, nested]]); + let blobs = writable_blob_refs(&array).unwrap(); + assert_eq!( + blobs + .iter() + .map(|blob| blob.cid.to_string()) + .collect::>(), + [CID.to_owned(), nested_cid.clone(), nested_cid] + ); + } + + #[test] + fn blob_walkers_keep_tolerant_reads_and_first_write_failure() { + let nested = json!({ + "$type": "blob", "ref": {"$link": CID}, "mimeType": "image/png", "size": 1, + }); + let malformed = json!({"$type": "blob", "nested": nested}); + assert_eq!( + referenced_blob_cids(&malformed), + BTreeSet::from([CID.to_owned()]) + ); + let expected = writable_blob_refs(&malformed).unwrap_err(); + let value = json!({ + "first": [malformed], + "second": {"cid": CID, "mimeType": "image/png"}, + }); + assert_eq!(writable_blob_refs(&value).unwrap_err(), expected); + } } diff --git a/src/domain/identity.rs b/src/domain/identity.rs index 6401a1b..ee2795b 100644 --- a/src/domain/identity.rs +++ b/src/domain/identity.rs @@ -66,12 +66,7 @@ pub fn validate_local_did_document( "#atproto signing key does not match the configured repository key", )); } - let handle = format!("at://{handle}"); - if document - .also_known_as - .as_ref() - .is_none_or(|aliases| !aliases.contains(&handle)) - { + if !document_claims_handle(document, handle) { return Err(invalid_document( "alsoKnownAs does not contain the configured handle", )); @@ -79,6 +74,14 @@ pub fn validate_local_did_document( Ok(()) } +pub(crate) fn document_claims_handle(document: &DidDocument, handle: &str) -> bool { + let alias = format!("at://{handle}"); + document + .also_known_as + .as_ref() + .is_some_and(|aliases| aliases.contains(&alias)) +} + fn invalid_document(message: &str) -> Error { Error::Configuration(format!("local DID document is invalid: {message}")) } @@ -87,6 +90,112 @@ fn invalid_document(message: &str) -> Error { mod tests { use super::*; + #[test] + fn shared_handle_claim_keeps_exact_raw_alias_membership() { + for handle in [ + "alice.example.com", + "ALICE.example.com", + "café.example", + "", + "literal space", + "at://nested", + "a\0b", + "a/b", + ] { + let claim = format!("at://{handle}"); + let mut document = local_did_document( + "did:example:alice", + handle, + "https://pds.example.com/", + "zPublicKey", + ); + for aliases in [ + None, + Some(vec![]), + Some(vec![claim.clone()]), + Some(vec!["unrelated".to_owned(), claim.clone(), claim.clone()]), + Some(vec![format!("{claim}/")]), + Some(vec![format!("AT://{handle}")]), + Some(vec![handle.to_owned()]), + ] { + document.also_known_as = aliases; + let previous_missing = document + .also_known_as + .as_ref() + .is_none_or(|aliases| !aliases.contains(&claim)); + assert_eq!( + document_claims_handle(&document, handle), + !previous_missing, + "{handle:?}: {:?}", + document.also_known_as, + ); + let result = validate_local_did_document( + &document, + "did:example:alice", + handle, + "https://pds.example.com/", + "zPublicKey", + ); + if previous_missing { + assert!(matches!( + result, + Err(Error::Configuration(message)) + if message == "local DID document is invalid: alsoKnownAs does not contain the configured handle" + )); + } else { + result.unwrap(); + } + } + } + } + + #[test] + fn handle_claim_validation_remains_after_document_endpoint_and_key_checks() { + let original = local_did_document( + "did:example:alice", + "alice.example.com", + "https://pds.example.com/", + "zPublicKey", + ); + for (stage, expected) in [ + (0, "document ID does not match the account DID"), + (1, "#atproto_pds does not match the public origin"), + (2, "#atproto signing key is missing"), + ( + 3, + "#atproto signing key does not match the configured repository key", + ), + (4, "alsoKnownAs does not contain the configured handle"), + ] { + let mut document = original.clone(); + document.also_known_as = None; + if stage == 0 { + document.id = "did:example:other".to_owned(); + } + if stage <= 1 { + document.service = None; + } + if stage <= 2 { + document.verification_method = None; + } else if stage == 3 { + document.verification_method.as_mut().unwrap()[0].controller = + "did:example:other".to_owned(); + } + let result = validate_local_did_document( + &document, + "did:example:alice", + "alice.example.com", + "https://pds.example.com/", + "zPublicKey", + ); + assert!(matches!( + result, + Err(Error::Configuration(message)) + if message == format!("local DID document is invalid: {expected}") + )); + } + } + #[test] fn atrium_document_helpers_find_the_validated_service_and_key() { let mut document = local_did_document( diff --git a/src/domain/oauth.rs b/src/domain/oauth.rs index 5a79c42..8ba799f 100644 --- a/src/domain/oauth.rs +++ b/src/domain/oauth.rs @@ -1,5 +1,6 @@ //! OAuth values shared by the service and HTTP adapter. +use serde::{Deserialize, Serialize}; use url::Url; use super::account::Account; @@ -11,26 +12,31 @@ pub struct OAuthClient { pub name: String, } -#[derive(Clone, Debug)] -pub struct AuthorizationRequest { - pub client_id: Url, - pub redirect_uri: Url, +#[derive(Clone, Debug, Deserialize)] +pub struct AuthorizationRequest { + pub client_id: Uri, + pub redirect_uri: Uri, pub state: String, pub scope: String, pub code_challenge: String, } -#[derive(Clone, Debug)] -pub struct AuthorizationGrant { +#[derive(Deserialize)] +pub struct TokenForm { + pub grant_type: String, pub code: String, - pub redirect_to: Url, + pub client_id: Url, + pub redirect_uri: Url, + pub code_verifier: String, } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Serialize)] pub struct TokenGrant { pub access_token: String, + pub token_type: &'static str, pub expires_in: u64, pub scope: String, + #[serde(rename = "sub")] pub did: String, } diff --git a/src/domain/record.rs b/src/domain/record.rs index 6cd6cfc..d8afd1b 100644 --- a/src/domain/record.rs +++ b/src/domain/record.rs @@ -1,8 +1,9 @@ //! AT repository records and their stable identity. use cid::Cid; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use serde_json::Value; +use std::fmt; #[derive(Clone, Debug)] pub struct Record { @@ -13,10 +14,38 @@ pub struct Record { pub value: Value, } -impl Record { - #[must_use] - pub fn uri(&self) -> String { - format!("at://{}/{}/{}", self.did, self.collection, self.rkey) +#[derive(Deserialize)] +#[serde(tag = "$type")] +pub enum WriteRequest { + #[serde(rename = "com.atproto.space.applyWrites#create")] + Create { + collection: String, + rkey: Option, + value: Value, + }, + #[serde(rename = "com.atproto.space.applyWrites#update")] + Update { + collection: String, + rkey: String, + value: Value, + }, + #[serde(rename = "com.atproto.space.applyWrites#delete")] + Delete { collection: String, rkey: String }, +} + +#[derive(Clone)] +pub enum RecordMutation { + Create(Record), + Update { + current: Record, + replacement: Record, + }, + Delete(Record), +} + +impl fmt::Display for Record { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "at://{}/{}/{}", self.did, self.collection, self.rkey) } } @@ -27,30 +56,159 @@ pub struct RecordView { pub value: Value, } -#[derive(Clone, Debug)] -pub struct CommitView { - pub cid: String, - pub rev: String, -} - -#[derive(Debug)] -pub struct RecordWrite { - pub record: RecordView, - pub commit: CommitView, -} - -#[derive(Debug)] -pub struct RecordPage { - pub records: Vec, - pub cursor: Option, -} - impl From for RecordView { fn from(record: Record) -> Self { Self { - uri: record.uri(), + uri: record.to_string(), cid: record.cid.to_string(), value: record.value, } } } + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn shared_batch_commands_preserve_space_deserialization_and_errors() { + let mut inputs = [ + "null", + "true", + "1", + "\"command\"", + "[]", + "[1]", + "{}", + "{\"$type\":null}", + "{\"$type\":0}", + "{\"$type\":", + r#"{"$type":"unknown"}"#, + r#"{"$type":"com.atproto.repo.applyWrites#create"}"#, + r#"{"$type":"com.atproto.space.applyWrites#create","$type":"com.atproto.space.applyWrites#delete"}"#, + ] + .map(str::to_owned) + .to_vec(); + for kind in ["create", "update", "delete"] { + for fields in [ + r#""collection":"app.test.record","rkey":"key","value":{"$type":"app.test.record"}"#, + r#""collection":"app.test.record","value":null"#, + r#""collection":"app.test.record","rkey":null,"value":{}"#, + r#""collection":null,"rkey":"key","value":{}"#, + r#""rkey":"key","value":{}"#, + r#""collection":"app.test.record","rkey":"key""#, + r#""collection":"app.test.record","rkey":false,"value":{}"#, + r#""collection":"app.test.record","rkey":"key","value":[],"ignored":true"#, + r#""collection":"app.test.record","collection":"other","rkey":"key","value":{}"#, + r#""collection":"app.test.record","rkey":"key","rkey":null,"value":{}"#, + r#""collection":"app.test.record","rkey":null,"rkey":"key","value":{}"#, + r#""collection":"app.test.record","rkey":"key","value":{},"value":null"#, + ] { + inputs.push(format!( + r#"{{"$type":"com.atproto.space.applyWrites#{kind}",{fields}}}"# + )); + } + } + for input in inputs { + let old = previous::parse(&input) + .map(write_shape) + .map_err(|error| error.to_string()); + let current = serde_json::from_str::(&input) + .map(write_shape) + .map_err(|error| error.to_string()); + assert_eq!(current, old, "{input}"); + } + } + + fn write_shape(write: crate::space::WriteRequest) -> Value { + use crate::space::WriteRequest; + + match write { + WriteRequest::Create { + collection, + rkey, + value, + } => json!(["create", collection, rkey, value]), + WriteRequest::Update { + collection, + rkey, + value, + } => json!(["update", collection, rkey, value]), + WriteRequest::Delete { collection, rkey } => json!(["delete", collection, rkey]), + } + } + + mod previous { + use serde::Deserialize; + use serde_json::Value; + + #[derive(Deserialize)] + #[serde(tag = "$type")] + enum WriteRequest { + #[serde(rename = "com.atproto.space.applyWrites#create")] + Create { + collection: String, + rkey: Option, + value: Value, + }, + #[serde(rename = "com.atproto.space.applyWrites#update")] + Update { + collection: String, + rkey: String, + value: Value, + }, + #[serde(rename = "com.atproto.space.applyWrites#delete")] + Delete { collection: String, rkey: String }, + } + + pub(super) fn parse(input: &str) -> serde_json::Result { + use crate::space::WriteRequest as Current; + + serde_json::from_str::(input).map(|write| match write { + WriteRequest::Create { + collection, + rkey, + value, + } => Current::Create { + collection, + rkey, + value, + }, + WriteRequest::Update { + collection, + rkey, + value, + } => Current::Update { + collection, + rkey, + value, + }, + WriteRequest::Delete { collection, rkey } => Current::Delete { collection, rkey }, + }) + } + } + + #[test] + fn display_is_exactly_the_record_uri_without_private_data() { + for (did, collection, rkey) in [ + ("did:web:alice.test", "app.bsky.feed.post", "record"), + ("did:example:alice%2Fbob", "com.example.record", "a:b"), + ("", "", ""), + ] { + let mut record = Record { + did: did.to_owned(), + collection: collection.to_owned(), + rkey: rkey.to_owned(), + cid: cpds_protocol::record_cid(&json!({})).unwrap(), + value: json!({"secret": "private record value"}), + }; + let expected = format!("at://{did}/{collection}/{rkey}"); + assert_eq!(record.to_string(), expected); + assert_eq!(RecordView::from(record.clone()).uri, expected); + record.cid = cpds_protocol::record_cid(&json!({"different": true})).unwrap(); + record.value = json!({"secret": "different private record value"}); + assert_eq!(record.to_string(), expected); + } + } +} diff --git a/src/error.rs b/src/error.rs index c9fa4fe..79adb39 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,9 +1,28 @@ //! Errors shared by the application layers. +use std::fmt::Display; use thiserror::Error; pub type Result = std::result::Result; +/// Classify a boundary failure while retaining its original diagnostic. +pub(crate) trait ErrorContext { + fn context(self, kind: impl FnOnce(String) -> Error, message: impl Display) -> Result; +} + +impl ErrorContext for Result { + fn context(self, kind: impl FnOnce(String) -> Error, message: impl Display) -> Result { + self.map_err(|error| kind(format!("{message}: {error}"))) + } +} + +impl From for Error { + fn from(error: cpds_protocol::Error) -> Self { + let cpds_protocol::Error::InvalidRecord(message) = error; + Self::InvalidRequest(message) + } +} + #[derive(Debug, Error)] pub enum Error { #[error("authentication is required")] @@ -37,3 +56,55 @@ pub enum Error { #[error("internal error: {0}")] Internal(String), } + +impl Error { + pub(crate) fn xrpc(name: &'static str, message: impl Into) -> Self { + Self::Xrpc { + name, + message: message.into(), + } + } +} + +#[cfg(test)] +mod tests { + use axum::{body::to_bytes, http::StatusCode, response::IntoResponse as _}; + + use super::Error; + + #[tokio::test] + async fn xrpc_constructor_preserves_borrowed_and_owned_message_responses() { + let message = "details with \"quotes\", a newline\n and café"; + for (name, status) in [ + ("SpaceNotFound", StatusCode::BAD_REQUEST), + ("AccountSuspended", StatusCode::FORBIDDEN), + ] { + let original = Error::Xrpc { + name, + message: message.to_owned(), + } + .into_response(); + assert_eq!(original.status(), status); + let headers = original.headers().clone(); + let body = to_bytes(original.into_body(), usize::MAX).await.unwrap(); + + for error in [ + Error::xrpc(name, message), + Error::xrpc(name, message.to_owned()), + ] { + assert!( + matches!(&error, Error::Xrpc { name: actual_name, message: actual_message } + if *actual_name == name && actual_message == message) + ); + assert_eq!(error.to_string(), message); + let response = error.into_response(); + assert_eq!(response.status(), status); + assert_eq!(response.headers(), &headers); + assert_eq!( + to_bytes(response.into_body(), usize::MAX).await.unwrap(), + body + ); + } + } + } +} diff --git a/src/events.rs b/src/events.rs index dabdc4b..6d0a6f3 100644 --- a/src/events.rs +++ b/src/events.rs @@ -1,6 +1,6 @@ //! A narrow extension seam for repository changes. -use std::sync::Mutex; +use std::{future::Future, sync::Mutex, time::Duration}; use async_trait::async_trait; @@ -9,6 +9,19 @@ use crate::{ error::{Error, Result}, }; +/// Notifications are optional once durable state has committed. Bound the whole +/// batch, including resolution, so slow listeners do not hold a write response. +pub(crate) async fn best_effort_notice( + label: &'static str, + delivery: impl Future>, +) { + match tokio::time::timeout(Duration::from_secs(2), delivery).await { + Ok(Ok(())) => {} + Ok(Err(error)) => tracing::warn!(%error, label, "post-commit notification failed"), + Err(_) => tracing::warn!(label, "post-commit notification deadline reached"), + } +} + #[async_trait] pub trait EventSink: Send + Sync { async fn check(&self) -> Result<()>; diff --git a/src/http/auth.rs b/src/http/auth.rs index 6ed0d4f..c6296de 100644 --- a/src/http/auth.rs +++ b/src/http/auth.rs @@ -10,13 +10,10 @@ use crate::{ oauth::AuthenticatedToken, }, error::Error, - space::{ - ReadAuthorization, - auth::{Credential, ServiceIdentity}, - }, + space::ReadAuthorization, }; -use super::{ApiError, ApiResult}; +use super::ApiResult; pub fn account(headers: &HeaderMap, state: &AppState) -> ApiResult { Ok(token(headers, state)?.account) @@ -24,12 +21,7 @@ pub fn account(headers: &HeaderMap, state: &AppState) -> ApiResult { pub fn token(headers: &HeaderMap, state: &AppState) -> ApiResult { let (authenticated, status) = token_with_status(headers, state)?; - if let Some(name) = status.account_error_name() { - return Err(ApiError(Error::Xrpc { - name, - message: format!("account is {}", status.as_str()), - })); - } + status.require_active()?; Ok(authenticated) } @@ -61,52 +53,31 @@ pub fn dpop_proof(headers: &HeaderMap) -> ApiResult<&str> { .get("dpop") .and_then(|value| value.to_str().ok()) .filter(|value| !value.is_empty()) - .ok_or(ApiError(Error::Authentication { + .ok_or(Error::Authentication { name: "MissingDpopProof", message: "request requires a DPoP proof".to_owned(), - })) + }) } pub fn request_url(state: &AppState, uri: &Uri) -> Url { state.config.public_origin.endpoint(uri.path()) } -pub enum SpaceReadToken { - OAuth(AuthenticatedToken), - Credential(Credential), -} - -impl SpaceReadToken { - pub fn authorization(&self) -> ReadAuthorization<'_> { - match self { - Self::OAuth(token) => ReadAuthorization::OAuth(token), - Self::Credential(credential) => ReadAuthorization::Credential(&credential.space), - } - } -} - pub async fn space_read_token( headers: &HeaderMap, state: &AppState, uri: &Uri, method: &str, -) -> ApiResult { +) -> ApiResult { let uses_dpop = headers .get(AUTHORIZATION) .and_then(|value| value.to_str().ok()) .is_some_and(|value| value.starts_with("DPoP ")); if uses_dpop { - let credential = dpop_credential(headers)?; - let proof = dpop_proof(headers)?; - let htu = request_url(state, uri); - let credential = state - .space_auth - .verify_credential(credential, proof, method, &htu) - .await?; - state.rate_limits.check_space(&credential.space)?; - Ok(SpaceReadToken::Credential(credential)) + let credential = space_credential(headers, state, uri, method).await?; + Ok(ReadAuthorization::Credential(credential)) } else { - Ok(SpaceReadToken::OAuth(token(headers, state)?)) + Ok(ReadAuthorization::OAuth(token(headers, state)?)) } } @@ -115,42 +86,96 @@ pub async fn space_credential( state: &AppState, uri: &Uri, method: &str, -) -> ApiResult { +) -> ApiResult { let credential = dpop_credential(headers)?; let proof = dpop_proof(headers)?; let htu = request_url(state, uri); - let credential = state + let space = state .space_auth .verify_credential(credential, proof, method, &htu) - .await - .map_err(ApiError::from)?; - state.rate_limits.check_space(&credential.space)?; - Ok(credential) + .await?; + state.rate_limits.check_space(&space)?; + Ok(space) } pub async fn service( headers: &HeaderMap, state: &AppState, lxm: &str, -) -> ApiResult { - let service = state +) -> ApiResult<(String, String)> { + let (issuer, audience) = state .space_auth .verify_service_token(bearer(headers)?, lxm) - .await - .map_err(ApiError::from)?; - state.rate_limits.check_service(&service.iss)?; - Ok(service) + .await?; + state.rate_limits.check_service(&issuer)?; + Ok((issuer, audience)) } fn authorization<'a>(headers: &'a HeaderMap, scheme: &str) -> ApiResult<&'a str> { - let authorization = headers + headers .get(AUTHORIZATION) .and_then(|value| value.to_str().ok()) - .ok_or(ApiError(Error::AuthenticationRequired))?; - let token = authorization - .strip_prefix(scheme) + .and_then(|value| value.strip_prefix(scheme)) .and_then(|value| value.strip_prefix(' ')) .filter(|token| !token.is_empty()) - .ok_or(ApiError(Error::AuthenticationRequired))?; - Ok(token) + .ok_or(Error::AuthenticationRequired) +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::http::HeaderValue; + + #[test] + fn authorization_pipeline_preserves_exact_header_values_and_borrowing() { + fn previous<'a>(headers: &'a HeaderMap, scheme: &str) -> ApiResult<&'a str> { + let authorization = headers + .get(AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .ok_or(Error::AuthenticationRequired)?; + let token = authorization + .strip_prefix(scheme) + .and_then(|value| value.strip_prefix(' ')) + .filter(|token| !token.is_empty()) + .ok_or(Error::AuthenticationRequired)?; + Ok(token) + } + + let mut values = vec![None]; + values.extend( + [ + "", + "Bearer", + "Bearer ", + "Bearer token", + "Bearer token ", + "Bearer\ttoken", + "bearer token", + "DPoP credential", + "DPoP ", + "DPoP ", + "BearerX token", + " token", + " ", + "Bearer token, DPoP other", + ] + .map(|value| Some(HeaderValue::from_str(value).unwrap())), + ); + values.push(Some(HeaderValue::from_bytes(b"Bearer \x80").unwrap())); + for value in values { + let mut headers = HeaderMap::new(); + if let Some(value) = value { + headers.append(AUTHORIZATION, value); + headers.append(AUTHORIZATION, HeaderValue::from_static("Bearer second")); + } + for scheme in ["Bearer", "DPoP", "", "bearer", "é"] { + let expected = previous(&headers, scheme); + let actual = authorization(&headers, scheme); + assert_eq!(format!("{actual:?}"), format!("{expected:?}")); + if let Ok(actual) = actual { + assert!(std::ptr::eq(actual, expected.unwrap())); + } + } + } + } } diff --git a/src/http/error.rs b/src/http/error.rs index f90ca59..4169f8f 100644 --- a/src/http/error.rs +++ b/src/http/error.rs @@ -2,7 +2,6 @@ use axum::{ Json, - body::Body, extract::Request, http::StatusCode, http::header::{ALLOW, CONTENT_TYPE, RETRY_AFTER}, @@ -13,25 +12,15 @@ use serde::Serialize; use crate::error::Error; -pub type ApiResult = Result; - -pub struct ApiError(pub Error); - -impl From for ApiError { - fn from(error: Error) -> Self { - Self(error) - } -} - #[derive(Serialize)] -struct ErrorBody { - error: &'static str, - message: String, +pub(super) struct ErrorBody { + pub(super) error: &'static str, + pub(super) message: Message, } -impl IntoResponse for ApiError { +impl IntoResponse for Error { fn into_response(self) -> Response { - let (status, name) = match &self.0 { + let (status, name) = match &self { Error::AuthenticationRequired => (StatusCode::UNAUTHORIZED, "AuthenticationRequired"), Error::Authentication { name, .. } => (StatusCode::UNAUTHORIZED, *name), Error::Authorization(_) => (StatusCode::FORBIDDEN, "Forbidden"), @@ -51,19 +40,20 @@ impl IntoResponse for ApiError { | Error::Internal(_) => (StatusCode::INTERNAL_SERVER_ERROR, "InternalServerError"), }; - let retry_after = match &self.0 { - Error::RateLimited { retry_after } => Some(*retry_after), - _ => None, + let message = if status.is_server_error() { + tracing::error!(error = %self, "request failed"); + match status { + StatusCode::BAD_GATEWAY => "upstream service failed".to_owned(), + _ => "internal server error".to_owned(), + } + } else { + self.to_string() }; - let mut response = response(status, name, self.0.to_string()); - if let Some(retry_after) = retry_after { - response.headers_mut().insert( - RETRY_AFTER, - retry_after - .to_string() - .parse() - .expect("integer Retry-After is valid"), - ); + let mut response = response(status, name, message); + if let Error::RateLimited { retry_after } = &self { + response + .headers_mut() + .insert(RETRY_AFTER, (*retry_after).into()); } response } @@ -88,7 +78,11 @@ pub(super) async fn normalize_framework_rejections(request: Request, next: Next) let is_xrpc = request.uri().path().starts_with("/xrpc/"); let response = next.run(request).await; if (!response.status().is_client_error() && !response.status().is_server_error()) - || has_json_content_type(&response) + || response + .headers() + .get(CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.starts_with("application/json")) { return response; } @@ -134,10 +128,258 @@ pub(super) async fn normalize_framework_rejections(request: Request, next: Next) normalized } -fn has_json_content_type(response: &Response) -> bool { - response - .headers() - .get(CONTENT_TYPE) - .and_then(|value| value.to_str().ok()) - .is_some_and(|value| value.starts_with("application/json")) +#[cfg(test)] +mod tests { + use axum::{ + Router, + body::{Body, to_bytes}, + http::HeaderValue, + middleware::from_fn, + }; + use rand::{Rng as _, SeedableRng as _, rngs::StdRng}; + use tower::ServiceExt as _; + + use super::*; + + #[tokio::test] + async fn shared_error_body_keeps_owned_json_response_bytes() { + #[derive(Serialize)] + struct PreviousErrorBody { + error: &'static str, + message: String, + } + let messages = [ + String::new(), + "cursor is in the future".to_owned(), + "quote-\"-backslash-\\-雪-🦀".to_owned(), + (0u8..=31).map(char::from).collect(), + "a".repeat(4096), + ]; + for name in ["", "FutureCursor", "Error-\"\\\n-雪"] { + for message in &messages { + for status in [ + StatusCode::OK, + StatusCode::BAD_REQUEST, + StatusCode::BAD_GATEWAY, + ] { + let previous = ( + status, + Json(PreviousErrorBody { + error: name, + message: message.clone(), + }), + ) + .into_response(); + let actual = response(status, name, message.clone()); + assert_eq!(actual.status(), previous.status()); + assert_eq!(actual.headers(), previous.headers()); + assert_eq!( + to_bytes(actual.into_body(), usize::MAX).await.unwrap(), + to_bytes(previous.into_body(), usize::MAX).await.unwrap(), + ); + } + } + } + } + + #[tokio::test] + async fn security_headers_preserve_exact_paths_statuses_and_header_replacement() { + use axum::http::{HeaderMap, header::CONTENT_LENGTH}; + + for (path, blob, cacheable) in [ + ("/xrpc/com.atproto.sync.getBlob", true, true), + ("/xrpc/com.atproto.sync.getBlob?ignored=1", true, true), + ("/xrpc/com.atproto.space.getBlob", true, false), + ("/xrpc/com.atproto.space.getBlob?ignored=1", true, false), + ("/xrpc/com.atproto.sync.getBlob/", false, false), + ("/xrpc/com.atproto.sync.getBlobOther", false, false), + ("/xrpc/com.atproto.sync.getblob", false, false), + ("/xrpc/com.atproto.sync.get%42lob", false, false), + ("/.well-known/did.json", false, true), + ("/.well-known", false, false), + ("/oauth/token", false, false), + ] { + for status in + [200, 206, 302, 400, 500].map(|status| StatusCode::from_u16(status).unwrap()) + { + for preexisting in [false, true] { + let policy = [ + ("x-content-type-options", "nosniff"), + ("referrer-policy", "no-referrer"), + ("x-frame-options", "DENY"), + ( + "content-security-policy", + if blob { + "default-src 'none'; sandbox" + } else { + "default-src 'none'; base-uri 'none'; form-action 'self'; frame-ancestors 'none'" + }, + ), + ( + "cache-control", + if cacheable && status.is_success() { + "public, max-age=300" + } else { + "no-store" + }, + ), + ]; + let mut original = HeaderMap::new(); + original.insert(CONTENT_TYPE, HeaderValue::from_static("text/plain")); + original.insert(CONTENT_LENGTH, HeaderValue::from_static("13")); + original.insert("x-original", HeaderValue::from_static("untouched")); + if preexisting { + for (name, _) in policy { + let mut old = HeaderValue::from_static("previous policy"); + old.set_sensitive(true); + original.append(name, old.clone()); + original.append(name, old); + } + } + let mut expected = original.clone(); + for (name, value) in policy { + expected.insert(name, HeaderValue::from_static(value)); + } + let app = Router::new() + .fallback(move || { + let headers = original.clone(); + async move { + let mut response = Response::new(Body::from("original body")); + *response.status_mut() = status; + *response.headers_mut() = headers; + response + } + }) + .layer(from_fn(crate::http::security::response_headers)); + let response = app + .oneshot(axum::http::Request::get(path).body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(response.status(), status, "{path}"); + assert_eq!(response.headers(), &expected, "{path}: {status}"); + for (name, _) in policy { + assert_eq!(response.headers().get_all(name).iter().count(), 1); + assert!(!response.headers()[name].is_sensitive()); + } + assert_eq!( + to_bytes(response.into_body(), usize::MAX) + .await + .unwrap() + .as_ref(), + b"original body", + ); + } + } + } + } + + #[tokio::test] + async fn framework_normalization_preserves_exact_content_type_prefix_semantics() { + let cases: &[(&[&[u8]], bool)] = &[ + (&[], false), + (&[b"application/json"], true), + (&[b"application/json; charset=utf-8"], true), + (&[b"application/json-not-a-mime-type"], true), + (&[b"Application/JSON"], false), + (&[b" application/json"], false), + (&[b"text/plain"], false), + (&[b"application/json\x80"], false), + (&[b"application/json", b"text/plain"], true), + (&[b"text/plain", b"application/json"], false), + ]; + for (content_types, is_json) in cases { + for status in [ + StatusCode::OK, + StatusCode::BAD_REQUEST, + StatusCode::INTERNAL_SERVER_ERROR, + ] { + let mut headers = axum::http::HeaderMap::new(); + headers.insert(ALLOW, HeaderValue::from_static("GET,HEAD")); + headers.insert("x-original", HeaderValue::from_static("kept")); + headers.insert( + axum::http::header::CONTENT_LENGTH, + HeaderValue::from("original body\0not JSON".len()), + ); + for content_type in *content_types { + headers.append(CONTENT_TYPE, HeaderValue::from_bytes(content_type).unwrap()); + } + let expected_headers = headers.clone(); + let app = Router::new() + .fallback(move || { + let headers = headers.clone(); + async move { + let mut response = Response::new(Body::from("original body\0not JSON")); + *response.status_mut() = status; + *response.headers_mut() = headers; + response + } + }) + .layer(from_fn(normalize_framework_rejections)); + let actual = app + .oneshot( + axum::http::Request::get("/xrpc/test") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(actual.status(), status); + assert_eq!(actual.headers()[ALLOW], expected_headers[ALLOW]); + let preserved = status == StatusCode::OK || *is_json; + if preserved { + assert_eq!(actual.headers(), &expected_headers); + } else { + assert_eq!(actual.headers()[CONTENT_TYPE], "application/json"); + assert!(!actual.headers().contains_key("x-original")); + } + let body = to_bytes(actual.into_body(), usize::MAX).await.unwrap(); + assert_eq!(body.as_ref() == b"original body\0not JSON", preserved); + } + } + } + + #[tokio::test] + async fn native_retry_after_preserves_header_and_complete_error_response() { + let mut values = vec![0, u64::MAX]; + for exponent in 0..20 { + let boundary = 10_u64.pow(exponent); + values.extend([boundary - 1, boundary, boundary + 1]); + } + let mut random = StdRng::seed_from_u64(0x5f4d_893c_127a_6e20); + values.extend((0..1024).map(|_| random.random::())); + + for retry_after in values { + let previous: HeaderValue = retry_after + .to_string() + .parse() + .expect("integer Retry-After is valid"); + let native = HeaderValue::from(retry_after); + assert_eq!(native.as_bytes(), previous.as_bytes()); + assert_eq!(native.to_str().unwrap(), retry_after.to_string()); + assert_eq!(native.is_sensitive(), previous.is_sensitive()); + assert!(!native.is_sensitive()); + assert_eq!(format!("{native:?}"), format!("{previous:?}")); + + let mut expected = response( + StatusCode::TOO_MANY_REQUESTS, + "RateLimitExceeded", + "authenticated principal request rate exceeded", + ); + expected.headers_mut().insert(RETRY_AFTER, previous); + let actual = Error::RateLimited { retry_after }.into_response(); + assert_eq!(actual.status(), expected.status()); + assert_eq!(actual.version(), expected.version()); + assert_eq!(actual.headers(), expected.headers()); + assert!(!actual.headers()[RETRY_AFTER].is_sensitive()); + let expected = to_bytes(expected.into_body(), usize::MAX).await.unwrap(); + assert_eq!( + expected.as_ref(), + br#"{"error":"RateLimitExceeded","message":"authenticated principal request rate exceeded"}"#, + ); + assert_eq!( + to_bytes(actual.into_body(), usize::MAX).await.unwrap(), + expected, + ); + } + } } diff --git a/src/http/mod.rs b/src/http/mod.rs index d23629b..8bc4812 100644 --- a/src/http/mod.rs +++ b/src/http/mod.rs @@ -4,8 +4,9 @@ pub(crate) mod auth; mod error; mod oauth; mod observability; -mod rate_limit; +pub(crate) mod rate_limit; mod repository; +pub(crate) mod request; mod runtime; mod security; mod server; @@ -17,17 +18,25 @@ use axum::{ Router, error_handling::HandleErrorLayer, extract::DefaultBodyLimit, - http::{StatusCode, header::RETRY_AFTER}, - routing::{get, post}, + http::{ + HeaderValue, StatusCode, + header::{CONTENT_TYPE, RETRY_AFTER}, + }, + response::{IntoResponse, Response}, + routing::{MethodRouter, get, post}, }; use tower::{ BoxError, ServiceBuilder, limit::GlobalConcurrencyLimitLayer, load_shed::LoadShedLayer, timeout::TimeoutLayer, }; -use crate::{AppState, repository::MAX_BLOB_BYTES, space::http as space}; +use crate::{ + AppState, + error::{Error, ErrorContext as _}, + repository::MAX_BLOB_BYTES, +}; -pub use error::{ApiError, ApiResult}; +pub use crate::error::Result as ApiResult; pub use runtime::{serve, shutdown_signal}; const STRUCTURED_BODY_LIMIT: usize = 2 * 1024 * 1024; @@ -104,7 +113,7 @@ pub fn router(state: AppState) -> Router { ) .merge(rate_limit::oauth_routes()) .merge(atproto_routes()) - .merge(space_routes()) + .merge(crate::space::routes::routes()) .layer(DefaultBodyLimit::max(STRUCTURED_BODY_LIMIT)) .with_state(state); let router = AdmissionPolicy::production() @@ -118,198 +127,84 @@ pub fn router(state: AppState) -> Router { } fn atproto_routes() -> Router { - Router::new() - .route( - "/xrpc/com.atproto.server.describeServer", - get(server::describe_server), - ) - .route( - "/xrpc/com.atproto.server.getSession", - rate_limit::protected(get(server::get_session)), - ) - .route( - "/xrpc/com.atproto.server.activateAccount", - rate_limit::protected(post(server::activate_account)), - ) - .route( - "/xrpc/com.atproto.server.deactivateAccount", - rate_limit::protected(post(server::deactivate_account)), - ) - .route( - "/xrpc/com.atproto.identity.resolveHandle", - get(server::resolve_handle), - ) - .route( - "/xrpc/com.atproto.repo.createRecord", - rate_limit::protected(post(repository::create_record)), - ) - .route( - "/xrpc/com.atproto.repo.putRecord", - rate_limit::protected(post(repository::put_record)), - ) - .route( - "/xrpc/com.atproto.repo.getRecord", - get(repository::get_record), - ) - .route( - "/xrpc/com.atproto.repo.describeRepo", - get(repository::describe_repo), - ) - .route( - "/xrpc/com.atproto.repo.listRecords", - get(repository::list_records), - ) - .route( - "/xrpc/com.atproto.repo.listMissingBlobs", - rate_limit::protected(get(repository::list_missing_blobs)), - ) - .route( - "/xrpc/com.atproto.repo.deleteRecord", - rate_limit::protected(post(repository::delete_record)), - ) - .route( - "/xrpc/com.atproto.repo.uploadBlob", - rate_limit::protected( - post(repository::upload_blob).layer(DefaultBodyLimit::max(MAX_BLOB_BYTES)), - ), - ) - .route( - "/xrpc/com.atproto.sync.getBlob", - rate_limit::expensive(get(repository::get_blob)), - ) - .route( - "/xrpc/com.atproto.sync.getLatestCommit", - get(repository::get_latest_commit), - ) - .route( - "/xrpc/com.atproto.sync.getRepoStatus", - get(repository::get_repo_status), - ) - .route( - "/xrpc/com.atproto.sync.getRepo", - rate_limit::expensive(get(repository::get_repo)), - ) - .route( - "/xrpc/com.atproto.sync.getRecord", - rate_limit::expensive(get(repository::get_sync_record)), - ) - .route( - "/xrpc/com.atproto.sync.getBlocks", - rate_limit::expensive(get(repository::get_blocks)), - ) - .route( - "/xrpc/com.atproto.sync.listBlobs", - rate_limit::expensive(get(repository::list_blobs)), - ) - .route( - "/xrpc/com.atproto.sync.listRepos", - get(repository::list_repos), - ) - .route( - "/xrpc/com.atproto.sync.listReposByCollection", + use rate_limit::{expensive, protected}; + + xrpc_routes([ + ("server.describeServer", get(server::describe_server)), + ("server.getSession", protected(get(server::get_session))), + ( + "server.activateAccount", + protected(post(server::activate_account)), + ), + ( + "server.deactivateAccount", + protected(post(server::deactivate_account)), + ), + ("identity.resolveHandle", get(server::resolve_handle)), + ( + "repo.createRecord", + protected(post(repository::create_record)), + ), + ("repo.putRecord", protected(post(repository::put_record))), + ( + "repo.applyWrites", + protected(post(repository::apply_writes)), + ), + ("repo.getRecord", get(repository::get_record)), + ("repo.describeRepo", get(repository::describe_repo)), + ("repo.listRecords", get(repository::list_records)), + ( + "repo.listMissingBlobs", + protected(get(repository::list_missing_blobs)), + ), + ( + "repo.deleteRecord", + protected(post(repository::delete_record)), + ), + ( + "repo.uploadBlob", + protected(post(repository::upload_blob).layer(DefaultBodyLimit::max(MAX_BLOB_BYTES))), + ), + ("sync.getBlob", expensive(get(repository::get_blob))), + ("sync.getLatestCommit", get(repository::get_latest_commit)), + ("sync.getRepoStatus", get(repository::get_repo_status)), + ("sync.getRepo", expensive(get(repository::get_repo))), + ( + "sync.getRecord", + expensive(get(repository::get_sync_record)), + ), + ("sync.getBlocks", expensive(get(repository::get_blocks))), + ("sync.listBlobs", expensive(get(repository::list_blobs))), + ("sync.listRepos", get(repository::list_repos)), + ( + "sync.listReposByCollection", get(repository::list_repos_by_collection), - ) - .route( - "/xrpc/com.atproto.sync.subscribeRepos", - rate_limit::expensive(get(subscription::subscribe_repos)), - ) + ), + ( + "sync.subscribeRepos", + expensive(get(subscription::subscribe_repos)), + ), + ]) } -fn space_routes() -> Router { - let router = Router::new() - .route( - "/xrpc/com.atproto.simplespace.createSpace", - post(space::create_space), - ) - .route( - "/xrpc/com.atproto.simplespace.updateSpace", - post(space::update_space), - ) - .route( - "/xrpc/com.atproto.simplespace.deleteSpace", - post(space::delete_space), - ) - .route( - "/xrpc/com.atproto.simplespace.getSpace", - get(space::get_space), - ) - .route( - "/xrpc/com.atproto.simplespace.addMember", - post(space::add_member), - ) - .route( - "/xrpc/com.atproto.simplespace.removeMember", - post(space::remove_member), - ) - .route( - "/xrpc/com.atproto.simplespace.listMembers", - get(space::list_members), - ) - .route( - "/xrpc/com.atproto.space.createRecord", - post(space::create_record), - ) - .route( - "/xrpc/com.atproto.space.getDelegationToken", - get(space::get_delegation_token), - ) - .route( - "/xrpc/com.atproto.space.getSpaceCredential", - rate_limit::expensive(post(space::get_space_credential)), - ) - .route("/xrpc/com.atproto.space.putRecord", post(space::put_record)) - .route( - "/xrpc/com.atproto.space.deleteRecord", - post(space::delete_record), - ) - .route( - "/xrpc/com.atproto.space.applyWrites", - post(space::apply_writes), - ) - .route("/xrpc/com.atproto.space.getRecord", get(space::get_record)) - .route( - "/xrpc/com.atproto.space.getRepo", - rate_limit::expensive(get(space::get_repo)), - ) - .route( - "/xrpc/com.atproto.space.notifyWrite", - post(space::notify_write), - ) - .route("/xrpc/com.atproto.space.listRepos", get(space::list_repos)) - .route( - "/xrpc/com.atproto.space.registerNotify", - post(space::register_notify), - ) - .route( - "/xrpc/com.atproto.space.unregisterNotify", - post(space::unregister_notify), - ) - .route( - "/xrpc/com.atproto.space.notifySpaceDeleted", - post(space::notify_space_deleted), - ) - .route( - "/xrpc/com.atproto.space.listRecords", - get(space::list_records), - ) - .route( - "/xrpc/com.atproto.space.listSpaces", - get(space::list_spaces), - ) - .route( - "/xrpc/com.atproto.space.listRepoOps", - get(space::list_repo_ops), - ) - .route( - "/xrpc/com.atproto.space.getLatestCommit", - get(space::get_latest_commit), - ) - .route( - "/xrpc/com.atproto.space.getBlob", - rate_limit::expensive(get(space::get_blob)), - ) - .route("/xrpc/com.atproto.space.listBlobs", get(space::list_blobs)); - rate_limit::protected_routes(router) +pub(crate) fn xrpc_routes( + routes: impl IntoIterator)>, +) -> Router { + routes + .into_iter() + .fold(Router::new(), |router, (name, method)| { + router.route(&format!("/xrpc/com.atproto.{name}"), method) + }) +} + +pub(crate) fn car_response(bytes: Vec) -> Response { + ([(CONTENT_TYPE, "application/vnd.ipld.car")], bytes).into_response() +} + +pub(crate) fn blob_response(media_type: &str, bytes: Vec) -> ApiResult { + let content_type = + HeaderValue::from_str(media_type).context(Error::Internal, "invalid stored MIME type")?; + Ok(([(CONTENT_TYPE, content_type)], bytes).into_response()) } #[cfg(test)] diff --git a/src/http/oauth.rs b/src/http/oauth.rs index b34551b..c889ac9 100644 --- a/src/http/oauth.rs +++ b/src/http/oauth.rs @@ -1,5 +1,6 @@ //! Browser authorization and token endpoints. +use crate::http::request::QueryRequest; use axum::{ Form, Json, extract::{Query, State}, @@ -9,14 +10,18 @@ use axum::{ use serde::{Deserialize, Serialize}; use url::Url; -use crate::{AppState, domain::oauth::AuthorizationRequest, error::Error}; +use crate::{ + AppState, + domain::oauth::{AuthorizationRequest, TokenForm, TokenGrant}, + error::{Error, ErrorContext as _}, +}; -use super::{ApiError, ApiResult}; +use super::ApiResult; #[derive(Serialize)] pub struct ProtectedResourceMetadata { - resource: String, - authorization_servers: Vec, + resource: Url, + authorization_servers: Vec, scopes_supported: Vec<&'static str>, bearer_methods_supported: Vec<&'static str>, } @@ -25,8 +30,8 @@ pub async fn protected_resource_metadata( State(state): State, ) -> Json { Json(ProtectedResourceMetadata { - resource: state.config.public_origin.to_string(), - authorization_servers: vec![state.oauth.issuer().to_string()], + resource: state.config.public_origin.as_url().clone(), + authorization_servers: vec![state.oauth.issuer().clone()], scopes_supported: vec!["atproto"], bearer_methods_supported: vec!["header"], }) @@ -34,9 +39,9 @@ pub async fn protected_resource_metadata( #[derive(Serialize)] pub struct AuthorizationServerMetadata { - issuer: String, - authorization_endpoint: String, - token_endpoint: String, + issuer: Url, + authorization_endpoint: Url, + token_endpoint: Url, scopes_supported: Vec<&'static str>, response_types_supported: Vec<&'static str>, grant_types_supported: Vec<&'static str>, @@ -47,19 +52,10 @@ pub struct AuthorizationServerMetadata { pub async fn authorization_server_metadata( State(state): State, ) -> Json { - let issuer = state.oauth.issuer(); Json(AuthorizationServerMetadata { - issuer: issuer.to_string(), - authorization_endpoint: state - .config - .public_origin - .endpoint("oauth/authorize") - .to_string(), - token_endpoint: state - .config - .public_origin - .endpoint("oauth/token") - .to_string(), + issuer: state.oauth.issuer().clone(), + authorization_endpoint: state.config.public_origin.endpoint("oauth/authorize"), + token_endpoint: state.config.public_origin.endpoint("oauth/token"), scopes_supported: vec!["atproto"], response_types_supported: vec!["code"], grant_types_supported: vec!["authorization_code"], @@ -71,54 +67,34 @@ pub async fn authorization_server_metadata( #[derive(Clone, Deserialize)] pub struct AuthorizeQuery { response_type: String, - client_id: Url, - redirect_uri: Url, - state: String, - scope: String, - code_challenge: String, + #[serde(flatten)] + request: AuthorizationRequest, code_challenge_method: String, } -impl TryFrom for AuthorizationRequest { - type Error = Error; - - fn try_from(query: AuthorizeQuery) -> Result { - if query.response_type != "code" { - return Err(Error::InvalidRequest( - "response_type must be `code`".to_owned(), - )); - } - if query.code_challenge_method != "S256" { - return Err(Error::InvalidRequest( - "code_challenge_method must be `S256`".to_owned(), - )); - } - Ok(Self { - client_id: query.client_id, - redirect_uri: query.redirect_uri, - state: query.state, - scope: query.scope, - code_challenge: query.code_challenge, - }) - } -} - pub async fn authorization_page( - State(state): State, - Query(query): Query, + (State(state), Query(query)): QueryRequest, ) -> ApiResult> { - let request = AuthorizationRequest::try_from(query)?; + if query.response_type != "code" { + return Err(Error::InvalidRequest( + "response_type must be `code`".to_owned(), + )); + } + if query.code_challenge_method != "S256" { + return Err(Error::InvalidRequest( + "code_challenge_method must be `S256`".to_owned(), + )); + } + let request = query.request; + drop((query.response_type, query.code_challenge_method)); state.oauth.validate_authorization_request(&request)?; Ok(Html(render_authorization_page(&request))) } #[derive(Deserialize)] pub struct AuthorizationForm { - client_id: Url, - redirect_uri: Url, - state: String, - scope: String, - code_challenge: String, + #[serde(flatten)] + request: AuthorizationRequest, identifier: String, password: String, } @@ -127,61 +103,25 @@ pub async fn approve_authorization( State(state): State, Form(form): Form, ) -> ApiResult { - let request = AuthorizationRequest { - client_id: form.client_id, - redirect_uri: form.redirect_uri, - state: form.state, - scope: form.scope, - code_challenge: form.code_challenge, - }; - let grant = state - .oauth - .authorize(&request, &form.identifier, &form.password)?; - let location = HeaderValue::from_str(grant.redirect_to.as_str()) - .map_err(|error| ApiError(Error::Internal(format!("invalid redirect header: {error}"))))?; + let (_code, redirect_to) = + state + .oauth + .authorize(&form.request, &form.identifier, &form.password)?; + let location = HeaderValue::from_str(redirect_to.as_str()) + .context(Error::Internal, "invalid redirect header")?; Ok((StatusCode::SEE_OTHER, [(LOCATION, location)]).into_response()) } -#[derive(Deserialize)] -pub struct TokenForm { - grant_type: String, - code: String, - client_id: Url, - redirect_uri: Url, - code_verifier: String, -} - -#[derive(Serialize)] -pub struct TokenResponse { - access_token: String, - token_type: &'static str, - expires_in: u64, - scope: String, - sub: String, -} - pub async fn token( State(state): State, Form(form): Form, -) -> ApiResult> { +) -> ApiResult> { if form.grant_type != "authorization_code" { - return Err(ApiError(Error::InvalidRequest( + return Err(Error::InvalidRequest( "grant_type must be `authorization_code`".to_owned(), - ))); - } - let grant = state.oauth.exchange_authorization_code( - &form.code, - &form.client_id, - &form.redirect_uri, - &form.code_verifier, - )?; - Ok(Json(TokenResponse { - access_token: grant.access_token, - token_type: "Bearer", - expires_in: grant.expires_in, - scope: grant.scope, - sub: grant.did, - })) + )); + } + Ok(Json(state.oauth.exchange_authorization_code(&form)?)) } fn render_authorization_page(request: &AuthorizationRequest) -> String { @@ -208,14 +148,620 @@ fn render_authorization_page(request: &AuthorizationRequest) -> String { fn escape_attribute(value: &str) -> String { value - .chars() - .map(|character| match character { - '&' => "&".to_owned(), - '<' => "<".to_owned(), - '>' => ">".to_owned(), - '"' => """.to_owned(), - '\'' => "'".to_owned(), - other => other.to_string(), + .replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('"', """) + .replace('\'', "'") +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::{ + Router, + body::{Body, to_bytes}, + extract::FromRequest, + http::{Request, header::CONTENT_TYPE}, + middleware::from_fn, + routing::{get, post}, + }; + use serde_json::{Value, json}; + use tower::ServiceExt as _; + + #[tokio::test] + async fn native_url_metadata_keeps_full_response_bytes() { + use crate::{ + config::{DeploymentMode, HttpOrigin}, + database::Database, + oauth::{OAuthService, StaticClientRegistry}, + }; + use std::sync::Arc; + + #[derive(Serialize)] + struct PreviousProtectedResourceMetadata { + resource: String, + authorization_servers: Vec, + scopes_supported: Vec<&'static str>, + bearer_methods_supported: Vec<&'static str>, + } + #[derive(Serialize)] + struct PreviousAuthorizationServerMetadata { + issuer: String, + authorization_endpoint: String, + token_endpoint: String, + scopes_supported: Vec<&'static str>, + response_types_supported: Vec<&'static str>, + grant_types_supported: Vec<&'static str>, + code_challenge_methods_supported: Vec<&'static str>, + token_endpoint_auth_methods_supported: Vec<&'static str>, + } + let base = authorization_state(); + for raw in [ + "http://pds.test", + "HTTPS://EXAMPLE.COM:443/", + "http://127.0.0.1:4321/", + "http://[::ffff:192.0.2.1]:4321/", + "https://BÜCHER.example:8443/", + "https://[2001:db8::1]/", + "https://xn--bcher-kva.example/", + ] { + let origin = HttpOrigin::parse_public(raw, DeploymentMode::Local).unwrap(); + for extended_issuer in [false, true] { + let mut state = base.clone(); + Arc::make_mut(&mut state.config).public_origin = origin.clone(); + let mut issuer = origin.as_url().clone(); + if extended_issuer { + issuer.set_path("/issuer/quoted-\"雪\"/%2f"); + issuer.set_query(Some("value=雪&escaped=%26")); + issuer.set_fragment(Some("fragment-\"quoted\"")); + } + state.oauth = Arc::new(OAuthService::new( + Database::open_in_memory().unwrap(), + Arc::new(StaticClientRegistry::new(Vec::new())), + issuer, + )); + let protected = Json(PreviousProtectedResourceMetadata { + resource: state.config.public_origin.to_string(), + authorization_servers: vec![state.oauth.issuer().to_string()], + scopes_supported: vec!["atproto"], + bearer_methods_supported: vec!["header"], + }); + let authorization = Json(PreviousAuthorizationServerMetadata { + issuer: state.oauth.issuer().to_string(), + authorization_endpoint: origin.endpoint("oauth/authorize").to_string(), + token_endpoint: origin.endpoint("oauth/token").to_string(), + scopes_supported: vec!["atproto"], + response_types_supported: vec!["code"], + grant_types_supported: vec!["authorization_code"], + code_challenge_methods_supported: vec!["S256"], + token_endpoint_auth_methods_supported: vec!["none"], + }); + for (previous, actual) in [ + ( + protected.into_response(), + protected_resource_metadata(State(state.clone())) + .await + .into_response(), + ), + ( + authorization.into_response(), + authorization_server_metadata(State(state.clone())) + .await + .into_response(), + ), + ] { + assert_eq!(actual.status(), previous.status()); + assert_eq!(actual.headers(), previous.headers()); + assert_eq!( + to_bytes(actual.into_body(), usize::MAX).await.unwrap(), + to_bytes(previous.into_body(), usize::MAX).await.unwrap(), + "{raw}, extended issuer: {extended_issuer}", + ); + } + } + } + } + + #[test] + fn native_attribute_replacements_preserve_all_unicode_and_entity_bytes() { + fn previous(value: &str) -> String { + value + .chars() + .map(|character| match character { + '&' => "&".to_owned(), + '<' => "<".to_owned(), + '>' => ">".to_owned(), + '"' => """.to_owned(), + '\'' => "'".to_owned(), + other => other.to_string(), + }) + .collect() + } + + let unicode: String = (0..=0x0010_ffff).filter_map(char::from_u32).collect(); + for value in [ + "", + "plain attribute", + "&<>\"'", + "&<>"''", + "'>\0\r\n\t", + "café e\u{301} 中文 🦀 \u{202e}", + &unicode, + ] { + assert_eq!( + escape_attribute(value).as_bytes(), + previous(value).as_bytes() + ); + } + assert_eq!(escape_attribute("&<>\"'"), "&<>"'"); + for entity in ["&", "<", ">", """, "'"] { + for character in ['&', '<', '>', '"', '\''] { + let value = format!("{entity}{character}{entity}{character}"); + assert_eq!(escape_attribute(&value), previous(&value)); + } + } + } + + async fn parse_token_form( + body: &str, + ) -> Result { + let request = Request::post("/oauth/token") + .header(CONTENT_TYPE, "application/x-www-form-urlencoded") + .body(Body::from(body.to_owned())) + .unwrap(); + Form::::from_request(request, &()) + .await + .map(|Form(form)| form) + .map_err(|error| (error.status(), error.body_text())) + } + + #[tokio::test] + async fn shared_token_form_preserves_values_and_raw_body_rejections() { + #[derive(Deserialize)] + struct PreviousTokenForm { + grant_type: String, + code: String, + client_id: Url, + redirect_uri: Url, + code_verifier: String, + } + let valid = vec![ + ("grant_type", "authorization_code"), + ("code", "quotes '\" & café"), + ("client_id", "http://client.test/metadata.json"), + ("redirect_uri", "http://client.test/callback?existing=1"), + ("code_verifier", "verifier"), + ]; + let mut cases = vec![valid.clone(), Vec::new()]; + for index in 0..valid.len() { + let mut missing = valid.clone(); + missing.remove(index); + cases.push(missing); + let mut duplicate = valid.clone(); + duplicate.push(valid[index]); + cases.push(duplicate); + for value in ["", "invalid"] { + let mut changed = valid.clone(); + changed[index].1 = value; + cases.push(changed); + } + } + let mut unknown = valid.clone(); + unknown.extend([("unknown", "one"), ("unknown", "two")]); + cases.push(unknown); + let mut bodies: Vec<_> = cases + .into_iter() + .map(|values| { + url::form_urlencoded::Serializer::new(String::new()) + .extend_pairs(values) + .finish() + }) + .collect(); + bodies.extend(["%", "client_id=%FF", "client_id=a&redirect_uri=%00"].map(str::to_owned)); + for body in bodies { + let previous = parse_token_form::(&body) + .await + .map(|form| { + ( + form.grant_type, + form.code, + form.client_id, + form.redirect_uri, + form.code_verifier, + ) + }); + let actual = parse_token_form::(&body).await.map(|form| { + ( + form.grant_type, + form.code, + form.client_id, + form.redirect_uri, + form.code_verifier, + ) + }); + assert_eq!(actual, previous, "token form {body}"); + } + } + + #[test] + fn issued_token_grants_preserve_response_bytes() { + #[derive(Serialize)] + struct PreviousResponse<'a> { + access_token: &'a str, + token_type: &'static str, + expires_in: u64, + scope: &'a str, + sub: &'a str, + } + + for expires_in in [0, 3600, u64::MAX] { + let grant = TokenGrant { + access_token: "token-\"\\\n-café".to_owned(), + token_type: "Bearer", + expires_in, + scope: "atproto space:com.example.test".to_owned(), + did: "did:web:alice.test".to_owned(), + }; + let previous = PreviousResponse { + access_token: &grant.access_token, + token_type: "Bearer", + expires_in, + scope: &grant.scope, + sub: &grant.did, + }; + assert_eq!( + serde_json::to_vec(&grant).unwrap(), + serde_json::to_vec(&previous).unwrap(), + ); + } + } + + #[derive(Deserialize)] + struct PreviousQuery { + response_type: String, + client_id: Url, + redirect_uri: Url, + state: String, + scope: String, + code_challenge: String, + code_challenge_method: String, + } + + #[derive(Deserialize)] + struct PreviousForm { + client_id: Url, + redirect_uri: Url, + state: String, + scope: String, + code_challenge: String, + identifier: String, + password: String, + } + + fn request_value(request: &AuthorizationRequest) -> Value { + json!({ + "client_id": request.client_id, "redirect_uri": request.redirect_uri, + "state": request.state, "scope": request.scope, + "code_challenge": request.code_challenge, }) - .collect() + } + + fn query_value(query: PreviousQuery) -> ApiResult> { + if query.response_type != "code" { + return Err(Error::InvalidRequest( + "response_type must be `code`".to_owned(), + )); + } + if query.code_challenge_method != "S256" { + return Err(Error::InvalidRequest( + "code_challenge_method must be `S256`".to_owned(), + )); + } + Ok(Json(request_value(&AuthorizationRequest { + client_id: query.client_id, + redirect_uri: query.redirect_uri, + state: query.state, + scope: query.scope, + code_challenge: query.code_challenge, + }))) + } + + fn parser_router(previous: bool) -> Router { + let router = if previous { + Router::new() + .route( + "/query", + get(|Query(query): Query| async { query_value(query) }), + ) + .route( + "/form", + post(|Form(form): Form| async move { + Json(json!({"request": request_value(&AuthorizationRequest { + client_id: form.client_id, redirect_uri: form.redirect_uri, + state: form.state, scope: form.scope, code_challenge: form.code_challenge, + }), "identifier": form.identifier, "password": form.password})) + }), + ) + } else { + Router::new() + .route( + "/query", + get(|Query(query): Query| async { + query_value(PreviousQuery { + response_type: query.response_type, + client_id: query.request.client_id, + redirect_uri: query.request.redirect_uri, + state: query.request.state, + scope: query.request.scope, + code_challenge: query.request.code_challenge, + code_challenge_method: query.code_challenge_method, + }) + }), + ) + .route( + "/form", + post(|Form(form): Form| async move { + Json(json!({"request": request_value(&form.request), + "identifier": form.identifier, "password": form.password})) + }), + ) + }; + router.layer(from_fn(crate::http::error::normalize_framework_rejections)) + } + + fn authorization_state() -> AppState { + use crate::{ + blob_store::MemoryBlobStore, + config::{AccountConfig, Config, DeploymentMode, HttpOrigin}, + database::Database, + domain::oauth::OAuthClient, + events::NoopEventSink, + }; + use std::{sync::Arc, time::Duration}; + + AppState::assemble( + Config { + deployment_mode: DeploymentMode::Local, + listen_address: "127.0.0.1:3000".parse().unwrap(), + public_origin: HttpOrigin::parse_public("http://pds.test/", DeploymentMode::Local) + .unwrap(), + database_path: ":memory:".into(), + blob_directory: "unused".into(), + blob_orphan_ttl: Duration::from_secs(86_400), + blob_gc_interval: Duration::from_secs(3_600), + identity_revalidation_interval: Duration::from_secs(3_600), + repo_event_retention: 100_000, + repo_stream_send_timeout: Duration::from_secs(10), + relay_origins: Vec::new(), + account: AccountConfig { + did: "did:web:alice.test".to_owned(), + handle: "alice.test".to_owned(), + password: "test password".to_owned(), + signing_key: [7; 32], + }, + oauth_clients: vec![OAuthClient { + client_id: "http://client.test/metadata.json".parse().unwrap(), + redirect_uris: vec!["http://client.test/callback?existing=1".parse().unwrap()], + name: "test client".to_owned(), + }], + }, + Database::open_in_memory().unwrap(), + Arc::new(MemoryBlobStore::default()), + Arc::new(NoopEventSink), + ) + .unwrap() + } + + async fn previous_authorization_page( + (State(state), Query(query)): QueryRequest, + ) -> ApiResult> { + fn previous_request(query: AuthorizeQuery) -> Result { + if query.response_type != "code" { + return Err(Error::InvalidRequest( + "response_type must be `code`".to_owned(), + )); + } + if query.code_challenge_method != "S256" { + return Err(Error::InvalidRequest( + "code_challenge_method must be `S256`".to_owned(), + )); + } + Ok(query.request) + } + let request = previous_request(query)?; + state.oauth.validate_authorization_request(&request)?; + Ok(Html(render_authorization_page(&request))) + } + + #[tokio::test] + async fn authorization_page_preserves_profile_priority_and_complete_responses() { + let state = authorization_state(); + let routers = [ + Router::new().route("/query", get(previous_authorization_page)), + Router::new().route("/query", get(authorization_page)), + ] + .map(|router| { + router + .layer(from_fn(crate::http::error::normalize_framework_rejections)) + .with_state(state.clone()) + }); + let challenge = crate::oauth::challenge_for_verifier(&"a".repeat(43)).unwrap(); + let valid = vec![ + ("response_type", "code"), + ("client_id", "http://client.test/metadata.json"), + ("redirect_uri", "http://client.test/callback?existing=1"), + ("state", "quotes '\" & café"), + ("scope", "atproto space:com.example.test"), + ("code_challenge", challenge.as_str()), + ("code_challenge_method", "S256"), + ]; + let mut cases = vec![valid.clone(), Vec::new()]; + for index in 0..valid.len() { + for value in ["", "invalid"] { + let mut changed = valid.clone(); + changed[index].1 = value; + cases.push(changed); + } + let mut duplicate = valid.clone(); + duplicate.push(valid[index]); + cases.push(duplicate); + } + for response_type in ["code", "token"] { + let mut invalid = valid.clone(); + invalid[0].1 = response_type; + invalid[1].1 = "http://unregistered.test/"; + invalid[6].1 = "plain"; + cases.push(invalid); + } + for (index, values) in cases.into_iter().enumerate() { + let encoded = url::form_urlencoded::Serializer::new(String::new()) + .extend_pairs(values) + .finish(); + let mut responses = Vec::new(); + for router in &routers { + let response = router + .clone() + .oneshot( + Request::get(format!("/query?{encoded}")) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + responses.push(( + response.status(), + response.headers().clone(), + to_bytes(response.into_body(), usize::MAX).await.unwrap(), + )); + } + assert_eq!(responses[0], responses[1], "{encoded}"); + if index == 0 { + assert_eq!(responses[1].0, StatusCode::OK); + assert_eq!(responses[1].1[CONTENT_TYPE], "text/html; charset=utf-8"); + let html = std::str::from_utf8(&responses[1].2).unwrap(); + assert!(html.contains("quotes '" & <unicode> café")); + assert!(html.contains(&format!("value=\"{challenge}\""))); + } + } + } + + async fn parser_response(router: Router, path: &str, values: &str) -> (StatusCode, Vec) { + let request = if path == "/query" { + Request::get(format!("{path}?{values}")) + .body(Body::empty()) + .unwrap() + } else { + Request::post(path) + .header(CONTENT_TYPE, "application/x-www-form-urlencoded") + .body(Body::from(values.to_owned())) + .unwrap() + }; + let response = router.oneshot(request).await.unwrap(); + let status = response.status(); + ( + status, + to_bytes(response.into_body(), usize::MAX) + .await + .unwrap() + .to_vec(), + ) + } + + #[tokio::test] + async fn shared_authorization_fields_preserve_values_and_public_rejections() { + let common = [ + ("client_id", "http://client.test/metadata.json"), + ("redirect_uri", "http://client.test/callback?existing=1"), + ("state", "quotes '\" & café"), + ("scope", "atproto space:com.example.test"), + ("code_challenge", "test-challenge"), + ]; + let previous = parser_router(true); + let current = parser_router(false); + for (path, extras) in [ + ( + "/query", + [("response_type", "code"), ("code_challenge_method", "S256")], + ), + ( + "/form", + [("identifier", "alice.test"), ("password", "test-password")], + ), + ] { + let valid: Vec<_> = common.into_iter().chain(extras).collect(); + let mut cases = vec![valid.clone(), Vec::new()]; + let mut reversed = valid.clone(); + reversed.reverse(); + cases.push(reversed); + for index in 0..valid.len() { + let mut missing = valid.clone(); + missing.remove(index); + cases.push(missing); + let mut duplicate = valid.clone(); + duplicate.push(valid[index]); + cases.push(duplicate); + for first in [false, true] { + let mut conflicting = valid.clone(); + let entry = (valid[index].0, "different"); + if first { + conflicting.insert(0, entry); + } else { + conflicting.push(entry); + } + cases.push(conflicting); + } + let mut empty = valid.clone(); + empty[index].1 = ""; + cases.push(empty); + } + for addition in [ + ("unknown", "ignored"), + ("request", "ignored"), + ("client_id", "not a URL"), + ("redirect_uri", "not a URL"), + ("response_type", "token"), + ("code_challenge_method", "plain"), + ] { + let mut changed = valid.clone(); + changed.retain(|(key, _)| key != &addition.0); + changed.push(addition); + cases.push(changed); + } + let mut both_invalid = valid.clone(); + both_invalid + .retain(|(key, _)| !["response_type", "code_challenge_method"].contains(key)); + both_invalid.extend([ + ("response_type", "token"), + ("code_challenge_method", "plain"), + ]); + cases.push(both_invalid); + for values in cases { + let encoded = url::form_urlencoded::Serializer::new(String::new()) + .extend_pairs(values) + .finish(); + assert_eq!( + parser_response(current.clone(), path, &encoded).await, + parser_response(previous.clone(), path, &encoded).await, + "{path}: {encoded}", + ); + } + for malformed in ["%", "client_id=%FF", "client_id=a&redirect_uri=%00"] { + assert_eq!( + parser_response(current.clone(), path, malformed).await, + parser_response(previous.clone(), path, malformed).await, + ); + } + let mut many_fields = url::form_urlencoded::Serializer::new(String::new()); + many_fields.extend_pairs(valid); + for index in 0..128 { + many_fields.append_pair(&format!("ignored{index}"), "ignored-value"); + } + let encoded = many_fields.finish(); + assert_eq!( + parser_response(current.clone(), path, &encoded).await, + parser_response(previous.clone(), path, &encoded).await, + ); + } + } } diff --git a/src/http/rate_limit.rs b/src/http/rate_limit.rs index 36fa644..688d8a5 100644 --- a/src/http/rate_limit.rs +++ b/src/http/rate_limit.rs @@ -149,7 +149,7 @@ pub(super) fn protected(route: MethodRouter) -> MethodRouter govern_method(route, PeerIpKeyExtractor, PEER_BURST, PEER_PERIOD, None) } -pub(super) fn protected_routes(router: Router) -> Router { +pub(crate) fn protected_routes(router: Router) -> Router { let router = govern_router( router, CredentialKeyExtractor, @@ -159,7 +159,7 @@ pub(super) fn protected_routes(router: Router) -> Router { govern_router(router, PeerIpKeyExtractor, PEER_BURST, PEER_PERIOD) } -pub(super) fn expensive(route: MethodRouter) -> MethodRouter { +pub(crate) fn expensive(route: MethodRouter) -> MethodRouter { let route = govern_method( route, CredentialKeyExtractor, diff --git a/src/http/repository.rs b/src/http/repository.rs index bcac3b5..3d12019 100644 --- a/src/http/repository.rs +++ b/src/http/repository.rs @@ -1,20 +1,22 @@ //! XRPC endpoints for records and blobs. +use crate::http::request::{HeaderQuery, JsonRequest, QueryRequest}; +use std::{fmt::Display, str::FromStr}; + use crate::{ AppState, - domain::{ - blob::Blob, - identity::local_did_document, - record::{CommitView, RecordView, RecordWrite}, - }, - error::Error, - repository::RecordSwap, + domain::record::{RecordMutation, RecordView}, + error::{Error, ErrorContext as _}, + repository::{RecordBatchWrite, RecordSwap}, }; use atrium_api::{ com::atproto::{ repo::{ - create_record, defs, delete_record, describe_repo, get_record, list_missing_blobs, - list_records, put_record, upload_blob, + apply_writes, + create_record::{self, OutputData as RecordWriteOutput}, + defs, delete_record, describe_repo, get_record, list_missing_blobs, + list_records::{self, Record as RecordOutput}, + put_record, upload_blob, }, sync::{ get_blob, get_blocks, get_latest_commit, get_record as sync_get_record, get_repo, @@ -29,28 +31,18 @@ use axum::{ Json, body::Bytes, extract::{Query, RawQuery, State}, - http::{HeaderMap, HeaderValue, header::CONTENT_TYPE}, - response::{IntoResponse, Response}, + http::{HeaderMap, header::CONTENT_TYPE}, + response::Response, }; -use super::{ApiError, ApiResult, auth}; +use super::{ApiResult, auth, blob_response, car_response}; pub async fn create_record( - State(state): State, - headers: HeaderMap, - Json(input): Json, -) -> ApiResult> { + (State(state), headers, Json(input)): JsonRequest, +) -> ApiResult> { let account = auth::account(&headers, &state)?; - if input.validate == Some(true) { - return Err(ApiError(Error::InvalidRequest( - "required Lexicon validation is not available".to_owned(), - ))); - } - let value = serde_json::to_value(input.record).map_err(|error| { - ApiError(Error::InvalidRequest(format!( - "record is not valid AT Protocol data: {error}" - ))) - })?; + reject_required_validation(input.validate)?; + let value = record_value(&input.record)?; let record = state .repository .create_record( @@ -62,25 +54,15 @@ pub async fn create_record( input.swap_commit.as_ref().map(AsRef::as_ref), ) .await?; - Ok(Json(create_record_output(record)?)) + Ok(Json(record_write_output(record)?)) } pub async fn put_record( - State(state): State, - headers: HeaderMap, - Json(input): Json, -) -> ApiResult> { + (State(state), headers, Json(input)): JsonRequest, +) -> ApiResult> { let account = auth::account(&headers, &state)?; - if input.validate == Some(true) { - return Err(ApiError(Error::InvalidRequest( - "required Lexicon validation is not available".to_owned(), - ))); - } - let value = serde_json::to_value(input.record).map_err(|error| { - ApiError(Error::InvalidRequest(format!( - "record is not valid AT Protocol data: {error}" - ))) - })?; + reject_required_validation(input.validate)?; + let value = record_value(&input.record)?; let record = state .repository .put_record( @@ -95,103 +77,106 @@ pub async fn put_record( }, ) .await?; - Ok(Json(put_record_output(record)?)) + Ok(Json(record_write_output(record)?)) +} + +pub async fn apply_writes( + (State(state), headers, Json(input)): JsonRequest, +) -> ApiResult> { + let account = auth::account(&headers, &state)?; + reject_required_validation(input.validate)?; + let writes = input + .writes + .into_iter() + .map(|write| { + Ok(match write { + apply_writes::InputWritesItem::Create(write) => RecordBatchWrite::Create { + collection: write.collection.as_str().to_owned(), + rkey: write.rkey.as_ref().map(|rkey| rkey.as_str().to_owned()), + value: record_value(&write.value)?, + }, + apply_writes::InputWritesItem::Update(write) => RecordBatchWrite::Update { + collection: write.collection.as_str().to_owned(), + rkey: write.rkey.as_str().to_owned(), + value: record_value(&write.value)?, + }, + apply_writes::InputWritesItem::Delete(write) => RecordBatchWrite::Delete { + collection: write.collection.as_str().to_owned(), + rkey: write.rkey.as_str().to_owned(), + }, + }) + }) + .collect::>>()?; + let result = state + .repository + .apply_writes( + &account, + input.repo.as_ref(), + writes, + input.swap_commit.as_ref().map(AsRef::as_ref), + ) + .await?; + Ok(Json(apply_writes_output(result))) } pub async fn get_record( - State(state): State, - Query(query): Query, -) -> ApiResult> { + (State(state), Query(query)): QueryRequest, +) -> ApiResult> { let record = state.repository.get_record( query.repo.as_ref(), query.collection.as_str(), query.rkey.as_str(), query.cid.as_ref().map(AsRef::as_ref), )?; - Ok(Json(get_record_output(record)?)) + Ok(Json(record_output(record)?)) } pub async fn describe_repo( - State(state): State, - Query(query): Query, -) -> ApiResult> { + (State(state), Query(query)): QueryRequest, +) -> ApiResult> { let (account, collections) = state.repository.describe_repo(query.repo.as_ref())?; - let document = local_did_document( - &account.did, - &state.config.account.handle, - &state.config.public_origin.to_string(), - &state.spaces.public_key_multibase(), - ); - let did_doc = serde_json::from_value(serde_json::to_value(document).map_err(|error| { - ApiError(Error::Internal(format!( - "could not encode local DID document: {error}" - ))) - })?) - .map_err(|error| { - ApiError(Error::Internal(format!( - "local DID document is not valid AT Protocol data: {error}" - ))) - })?; + let document = state.identity.local_document(); + let did_doc = serde_json::from_value( + serde_json::to_value(document) + .context(Error::Internal, "could not encode local DID document")?, + ) + .context( + Error::Internal, + "local DID document is not valid AT Protocol data", + )?; let collections = collections .into_iter() - .map(|collection| { - collection.parse().map_err(|error| { - ApiError(Error::Internal(format!( - "invalid stored repository collection: {error}" - ))) - }) - }) - .collect::>>()?; - Ok(Json( - describe_repo::OutputData { - collections, - did: account.did.parse().map_err(|error| { - ApiError(Error::Internal(format!( - "invalid stored repository DID: {error}" - ))) - })?, - did_doc, - handle: account.handle.parse().map_err(|error| { - ApiError(Error::Internal(format!( - "invalid stored repository handle: {error}" - ))) - })?, - handle_is_correct: state.identity.local_handle_verified(), - } - .into(), - )) + .map(|collection| stored(&collection, "repository collection")) + .collect::, _>>()?; + Ok(Json(describe_repo::OutputData { + collections, + did: stored(&account.did, "repository DID")?, + did_doc, + handle: stored(&account.handle, "repository handle")?, + handle_is_correct: state.identity.local_handle_verified(), + })) } pub async fn list_records( - State(state): State, - Query(query): Query, -) -> ApiResult> { - let page = state.repository.list_records( + (State(state), Query(query)): QueryRequest, +) -> ApiResult> { + let (records, cursor) = state.repository.list_records( query.repo.as_ref(), query.collection.as_str(), query.cursor.as_deref(), query.reverse.unwrap_or(false), usize::from(query.limit.map_or(50, u8::from)), )?; - let records = page - .records + let records = records .into_iter() - .map(list_record_output) + .map(record_output) .collect::>()?; - Ok(Json( - list_records::OutputData { - cursor: page.cursor, - records, - } - .into(), - )) + Ok(Json(list_records::OutputData { cursor, records })) } pub async fn list_missing_blobs( - State(state): State, - headers: HeaderMap, - Query(query): Query, -) -> ApiResult> { + (State(state), headers, Query(query)): HeaderQuery, +) -> ApiResult> { let account = auth::account(&headers, &state)?; let limit = usize::from(query.limit.map_or(500, u16::from)); let (blobs, cursor) = @@ -200,24 +185,20 @@ pub async fn list_missing_blobs( .list_missing_blobs(&account, query.cursor.as_deref(), limit)?; let blobs = blobs .into_iter() - .map(|blob| { + .map(|(cid, record_uri)| { list_missing_blobs::RecordBlobData { - cid: AtriumCid::new(blob.cid), - record_uri: blob.record_uri, + cid: AtriumCid::new(cid), + record_uri, } .into() }) .collect(); - Ok(Json( - list_missing_blobs::OutputData { blobs, cursor }.into(), - )) + Ok(Json(list_missing_blobs::OutputData { blobs, cursor })) } pub async fn delete_record( - State(state): State, - headers: HeaderMap, - Json(input): Json, -) -> ApiResult> { + (State(state), headers, Json(input)): JsonRequest, +) -> ApiResult> { let account = auth::account(&headers, &state)?; let commit = state .repository @@ -230,130 +211,97 @@ pub async fn delete_record( input.swap_commit.as_ref().map(AsRef::as_ref), ) .await?; - Ok(Json( - delete_record::OutputData { - commit: Some(commit_output(&commit)?), - } - .into(), - )) + Ok(Json(delete_record::OutputData { + commit: Some(commit.into()), + })) } pub async fn upload_blob( State(state): State, headers: HeaderMap, bytes: Bytes, -) -> ApiResult> { +) -> ApiResult> { let account = auth::account(&headers, &state)?; let media_type = headers .get(CONTENT_TYPE) .and_then(|value| value.to_str().ok()) - .ok_or_else(|| ApiError(Error::InvalidRequest("Content-Type is required".to_owned())))?; + .ok_or_else(|| Error::InvalidRequest("Content-Type is required".to_owned()))?; let blob = state .repository .upload_blob(&account, media_type, &bytes) .await?; - Ok(Json(blob_output(blob)?)) + let size = usize::try_from(blob.size) + .map_err(|_| Error::Internal("stored blob size does not fit this platform".to_owned()))?; + Ok(Json(upload_blob::OutputData { + blob: BlobRef::Typed(TypedBlobRef::Blob(AtriumBlob { + r#ref: CidLink(blob.cid), + mime_type: blob.media_type, + size, + })), + })) } pub async fn get_blob( - State(state): State, - Query(query): Query, + (State(state), Query(query)): QueryRequest, ) -> ApiResult { let (blob, bytes) = state .repository .get_blob(query.did.as_str(), query.cid.as_ref()) .await?; - let content_type = HeaderValue::from_str(&blob.media_type).map_err(|error| { - ApiError(Error::Internal(format!( - "invalid stored MIME type: {error}" - ))) - })?; - Ok(([(CONTENT_TYPE, content_type)], bytes).into_response()) + blob_response(&blob.media_type, bytes) } pub async fn get_latest_commit( - State(state): State, - Query(query): Query, -) -> ApiResult> { - let commit = state.repository.latest_commit(query.did.as_str())?; - Ok(Json( - get_latest_commit::OutputData { - cid: commit.cid.parse().map_err(|error| { - ApiError(Error::Internal(format!( - "invalid stored repository commit CID: {error}" - ))) - })?, - rev: commit.rev.parse().map_err(|error| { - ApiError(Error::Internal(format!( - "invalid stored repository revision: {error}" - ))) - })?, - } - .into(), - )) + (State(state), Query(query)): QueryRequest, +) -> ApiResult> { + let head = state.repository.sync_head(query.did.as_str())?; + Ok(Json(defs::CommitMetaData { + cid: AtriumCid::new(head.root), + rev: stored(&head.rev, "repository revision")?, + })) } pub async fn get_repo_status( - State(state): State, - Query(query): Query, -) -> ApiResult> { + (State(state), Query(query)): QueryRequest, +) -> ApiResult> { let (status, rev) = state.repository.repo_status(query.did.as_str())?; let rev = rev - .map(|rev| { - rev.parse().map_err(|error| { - ApiError(Error::Internal(format!( - "invalid stored repository revision: {error}" - ))) - }) - }) + .map(|rev| stored(&rev, "repository revision")) .transpose()?; - Ok(Json( - get_repo_status::OutputData { - active: status.is_active(), - did: query.did, - rev, - status: status.inactive_reason().map(ToOwned::to_owned), - } - .into(), - )) + Ok(Json(get_repo_status::OutputData { + active: status.is_active(), + did: query.did, + rev, + status: status.inactive_reason().map(ToOwned::to_owned), + })) } pub async fn list_repos( - State(state): State, - Query(query): Query, -) -> ApiResult> { + (State(state), Query(query)): QueryRequest, +) -> ApiResult> { let limit = usize::from(query.limit.map_or(500, u16::from)); let (repos, cursor) = state .repository - .list_repos(query.cursor.as_deref(), limit)?; + .list_repos(query.cursor.as_deref(), None, limit)?; let repos = repos .into_iter() .map(|repo| { Ok(list_repos::RepoData { active: Some(repo.status.is_active()), - did: repo.did.parse().map_err(|error| { - ApiError(Error::Internal(format!( - "invalid stored repository DID: {error}" - ))) - })?, - head: AtriumCid::new(repo.head), - rev: repo.rev.parse().map_err(|error| { - ApiError(Error::Internal(format!( - "invalid stored repository revision: {error}" - ))) - })?, + did: stored(&repo.did, "repository DID")?, + head: AtriumCid::new(repo.root), + rev: stored(&repo.rev, "repository revision")?, status: repo.status.inactive_reason().map(ToOwned::to_owned), } .into()) }) .collect::>>()?; - Ok(Json(list_repos::OutputData { cursor, repos }.into())) + Ok(Json(list_repos::OutputData { cursor, repos })) } pub async fn list_repos_by_collection( - State(state): State, - Query(query): Query, -) -> ApiResult> { + (State(state), Query(query)): QueryRequest, +) -> ApiResult> { let limit = usize::from(query.limit.map_or(500, u16::from)); let (repos, cursor) = state.repository.list_repos_by_collection( query.collection.as_str(), @@ -364,41 +312,26 @@ pub async fn list_repos_by_collection( .into_iter() .map(|did| { Ok(list_repos_by_collection::RepoData { - did: did.parse().map_err(|error| { - ApiError(Error::Internal(format!( - "invalid stored repository DID: {error}" - ))) - })?, + did: stored(&did, "repository DID")?, } .into()) }) .collect::>>()?; - Ok(Json( - list_repos_by_collection::OutputData { cursor, repos }.into(), - )) + Ok(Json(list_repos_by_collection::OutputData { cursor, repos })) } pub async fn get_repo( - State(state): State, - Query(query): Query, + (State(state), Query(query)): QueryRequest, ) -> ApiResult { let bytes = state .repository .export_repo(query.did.as_str(), query.since.as_deref()) .await?; - Ok(( - [( - CONTENT_TYPE, - HeaderValue::from_static("application/vnd.ipld.car"), - )], - bytes, - ) - .into_response()) + Ok(car_response(bytes)) } pub async fn get_sync_record( - State(state): State, - Query(query): Query, + (State(state), Query(query)): QueryRequest, ) -> ApiResult { let bytes = state .repository @@ -408,14 +341,7 @@ pub async fn get_sync_record( query.rkey.as_str(), ) .await?; - Ok(( - [( - CONTENT_TYPE, - HeaderValue::from_static("application/vnd.ipld.car"), - )], - bytes, - ) - .into_response()) + Ok(car_response(bytes)) } pub async fn get_blocks( @@ -423,11 +349,8 @@ pub async fn get_blocks( RawQuery(query): RawQuery, ) -> ApiResult { let query: get_blocks::ParametersData = - serde_html_form::from_str(query.as_deref().unwrap_or_default()).map_err(|error| { - ApiError(Error::InvalidRequest(format!( - "invalid getBlocks query: {error}" - ))) - })?; + serde_html_form::from_str(query.as_deref().unwrap_or_default()) + .context(Error::InvalidRequest, "invalid getBlocks query")?; let cids = query .cids .iter() @@ -437,20 +360,12 @@ pub async fn get_blocks( .repository .export_blocks(query.did.as_str(), &cids) .await?; - Ok(( - [( - CONTENT_TYPE, - HeaderValue::from_static("application/vnd.ipld.car"), - )], - bytes, - ) - .into_response()) + Ok(car_response(bytes)) } pub async fn list_blobs( - State(state): State, - Query(query): Query, -) -> ApiResult> { + (State(state), Query(query)): QueryRequest, +) -> ApiResult> { let limit = usize::from(query.limit.map_or(500, u16::from)); let (cids, cursor) = state.repository.list_blobs( query.did.as_str(), @@ -458,94 +373,553 @@ pub async fn list_blobs( query.cursor.as_deref(), limit, )?; - Ok(Json( - list_blobs::OutputData { - cids: cids.into_iter().map(AtriumCid::new).collect(), - cursor, - } - .into(), - )) + Ok(Json(list_blobs::OutputData { + cids: cids.into_iter().map(AtriumCid::new).collect(), + cursor, + })) } -fn blob_output(blob: Blob) -> Result { - let size = usize::try_from(blob.size) - .map_err(|_| Error::Internal("stored blob size does not fit this platform".to_owned()))?; - Ok(upload_blob::OutputData { - blob: BlobRef::Typed(TypedBlobRef::Blob(AtriumBlob { - r#ref: CidLink(blob.cid), - mime_type: blob.media_type, - size, - })), +fn reject_required_validation(validate: Option) -> Result<(), Error> { + if validate == Some(true) { + return Err(Error::InvalidRequest( + "required Lexicon validation is not available".to_owned(), + )); } - .into()) + Ok(()) } -fn create_record_output(write: RecordWrite) -> Result { - Ok(create_record::OutputData { - cid: write - .record - .cid - .parse() - .map_err(|error| Error::Internal(format!("invalid stored record CID: {error}")))?, - commit: Some(commit_output(&write.commit)?), - uri: write.record.uri, - validation_status: Some("unknown".to_owned()), - } - .into()) +fn record_value(record: &Unknown) -> Result { + serde_json::to_value(record).context( + Error::InvalidRequest, + "record is not valid AT Protocol data", + ) } -fn put_record_output(write: RecordWrite) -> Result { - Ok(put_record::OutputData { - cid: write - .record - .cid - .parse() - .map_err(|error| Error::Internal(format!("invalid stored record CID: {error}")))?, - commit: Some(commit_output(&write.commit)?), - uri: write.record.uri, +fn record_write_output( + (record, commit): (RecordView, defs::CommitMetaData), +) -> Result { + Ok(create_record::OutputData { + cid: stored(&record.cid, "record CID")?, + commit: Some(commit.into()), + uri: record.uri, validation_status: Some("unknown".to_owned()), - } - .into()) + }) } -fn commit_output(commit: &CommitView) -> Result { - Ok(defs::CommitMetaData { - cid: commit.cid.parse().map_err(|error| { - Error::Internal(format!("invalid stored repository commit CID: {error}")) - })?, - rev: commit.rev.parse().map_err(|error| { - Error::Internal(format!("invalid stored repository revision: {error}")) - })?, +fn apply_writes_output( + (commit, results): (defs::CommitMetaData, Vec), +) -> apply_writes::OutputData { + let results = results + .into_iter() + .map(|result| match result { + RecordMutation::Create(record) => { + apply_writes::OutputResultsItem::CreateResult(Box::new( + apply_writes::CreateResultData { + cid: AtriumCid::new(record.cid), + uri: record.to_string(), + validation_status: Some("unknown".to_owned()), + } + .into(), + )) + } + RecordMutation::Update { + replacement: record, + .. + } => apply_writes::OutputResultsItem::UpdateResult(Box::new( + apply_writes::UpdateResultData { + cid: AtriumCid::new(record.cid), + uri: record.to_string(), + validation_status: Some("unknown".to_owned()), + } + .into(), + )), + RecordMutation::Delete(_) => apply_writes::OutputResultsItem::DeleteResult(Box::new( + apply_writes::DeleteResultData {}.into(), + )), + }) + .collect(); + apply_writes::OutputData { + commit: Some(commit.into()), + results: Some(results), } - .into()) } -fn get_record_output(record: RecordView) -> Result { - let (uri, cid, value) = wire_record(record)?; - Ok(get_record::OutputData { - cid: Some(cid), - uri, +fn record_output(record: RecordView) -> Result { + let cid = stored(&record.cid, "record CID")?; + let value = serde_json::from_value(record.value).context( + Error::Internal, + "stored record is not valid AT Protocol data", + )?; + Ok(list_records::RecordData { + cid, + uri: record.uri, value, } .into()) } -fn list_record_output(record: RecordView) -> Result { - let (uri, cid, value) = wire_record(record)?; - Ok(list_records::RecordData { cid, uri, value }.into()) +fn stored(value: &str, kind: &str) -> Result +where + T::Err: Display, +{ + value + .parse() + .context(Error::Internal, format_args!("invalid stored {kind}")) } -fn wire_record( - record: RecordView, -) -> Result<(String, atrium_api::types::string::Cid, Unknown), Error> { - let cid = record - .cid - .parse() - .map_err(|error| Error::Internal(format!("invalid stored record CID: {error}")))?; - let value = serde_json::from_value(record.value).map_err(|error| { - Error::Internal(format!( - "stored record is not valid AT Protocol data: {error}" +#[cfg(test)] +mod tests { + use super::{ + Error, RecordMutation, RecordView, apply_writes_output, create_record, defs, + get_latest_commit, get_record, list_records, put_record, record_output, + record_write_output, stored, + }; + use crate::domain::record::Record; + use serde_json::json; + + #[test] + fn generated_response_data_matches_empty_extension_objects() { + use atrium_api::com::atproto::{repo, sync}; + + fn check( + value: serde_json::Value, + ) { + let data: T = serde_json::from_value(value).unwrap(); + let wrapped = atrium_api::types::Object::from(data.clone()); + assert_eq!( + serde_json::to_vec(&data).unwrap(), + serde_json::to_vec(&wrapped).unwrap(), + ); + } + + let (record, _) = record_write(); + let cid = record.cid; + let did = "did:web:alice.test"; + let rev = "3jqfcqzm3fo2j"; + let uri = "at://did:web:alice.test/app.bsky.feed.post/key"; + for commit in [ + json!(null), + json!({"cid": cid, "rev": rev}), + json!({"cid": cid, "rev": rev, "extension": {"values": [null, 42, "kept"]}}), + ] { + for validation_status in [json!(null), json!("unknown")] { + check::(json!({ + "cid": cid, "uri": uri, "commit": commit, + "validationStatus": validation_status, + })); + } + check::(json!({"commit": commit})); + for results in [ + json!(null), + json!([]), + json!([ + {"$type": "com.atproto.repo.applyWrites#createResult", "cid": cid, "uri": uri}, + {"$type": "com.atproto.repo.applyWrites#updateResult", "cid": cid, "uri": uri}, + {"$type": "com.atproto.repo.applyWrites#deleteResult"}, + ]), + ] { + check::( + json!({"commit": commit, "results": results}), + ); + } + } + check::(json!({ + "collections": ["app.bsky.feed.post"], "did": did, "handle": "alice.test", + "didDoc": {"id": did}, "handleIsCorrect": false, + })); + check::(json!({"blob": { + "$type": "blob", "ref": {"$link": cid}, "mimeType": "image/png", "size": 42, + }})); + for cursor in [json!(null), json!("next")] { + for populated in [false, true] { + check::(json!({ + "cursor": cursor, "records": if populated { + json!([{"uri": uri, "cid": cid, "value": {"text": "post"}}]) + } else { json!([]) }, + })); + check::(json!({ + "cursor": cursor, "blobs": if populated { + json!([{"cid": cid, "recordUri": uri}]) + } else { json!([]) }, + })); + check::(json!({ + "cursor": cursor, "cids": if populated { json!([cid]) } else { json!([]) }, + })); + check::(json!({ + "cursor": cursor, "repos": if populated { json!([{"did": did}]) } else { json!([]) }, + })); + check::(json!({ + "cursor": cursor, "repos": if populated { + json!([{"did": did, "head": cid, "rev": rev, "active": false, "status": "deactivated"}]) + } else { json!([]) }, + })); + } + } + for (active, rev, status) in [ + (true, json!(rev), json!(null)), + (false, json!(null), json!("deactivated")), + ] { + check::( + json!({"did": did, "active": active, "rev": rev, "status": status}), + ); + } + } + + #[test] + fn record_output_matches_generated_get_and_list_schemas() { + let (mut record, _) = record_write(); + record.value["link"] = json!({"$link": record.cid}); + record.value["bytes"] = json!({"$bytes": "AQID"}); + let expected = json!({"cid": record.cid, "uri": record.uri, "value": record.value}); + let output = record_output(record).unwrap(); + assert_eq!(serde_json::to_value(&output).unwrap(), expected); + let get: get_record::Output = serde_json::from_value(expected.clone()).unwrap(); + let listed: list_records::Record = serde_json::from_value(expected).unwrap(); + assert_eq!( + serde_json::to_vec(&output).unwrap(), + serde_json::to_vec(&get).unwrap() + ); + assert_eq!( + serde_json::to_vec(&output).unwrap(), + serde_json::to_vec(&listed).unwrap() + ); + } + + #[test] + fn record_output_checks_the_cid_before_decoding_record_data() { + for invalid_cid in [true, false] { + let (mut record, _) = record_write(); + record.value = json!({"float": 1.5}); + if invalid_cid { + record.cid = "invalid".to_owned(); + } + let Error::Internal(message) = record_output(record).unwrap_err() else { + panic!("invalid stored records must be internal errors"); + }; + let expected = if invalid_cid { + "invalid stored record CID:" + } else { + "stored record is not valid AT Protocol data:" + }; + assert!(message.starts_with(expected), "{message}"); + } + } + + #[test] + fn commit_output_matches_generated_latest_commit_schema() { + let (_, commit) = record_write(); + let expected = json!({"cid": commit.cid, "rev": commit.rev}); + let output = commit; + assert_eq!(serde_json::to_value(&output).unwrap(), expected); + let latest: get_latest_commit::Output = serde_json::from_value(expected).unwrap(); + assert_eq!( + serde_json::to_vec(&output).unwrap(), + serde_json::to_vec(&latest).unwrap() + ); + } + + #[test] + fn record_write_output_matches_both_generated_endpoint_schemas() { + let (record, commit) = record_write(); + let expected = json!({ + "cid": record.cid, + "commit": {"cid": commit.cid, "rev": commit.rev}, + "uri": record.uri, + "validationStatus": "unknown", + }); + let output = record_write_output((record, commit)).unwrap(); + assert_eq!(serde_json::to_value(&output).unwrap(), expected); + let create: create_record::Output = serde_json::from_value(expected.clone()).unwrap(); + let put: put_record::Output = serde_json::from_value(expected).unwrap(); + assert_eq!( + serde_json::to_vec(&output).unwrap(), + serde_json::to_vec(&create).unwrap() + ); + assert_eq!( + serde_json::to_vec(&output).unwrap(), + serde_json::to_vec(&put).unwrap() + ); + } + + #[test] + fn record_write_output_preserves_stored_value_validation_order() { + for (invalid_record, invalid_commit, expected) in [ + (true, true, "invalid stored record CID:"), + (false, true, "invalid stored repository commit CID:"), + (false, false, "invalid stored repository revision:"), + ] { + let (mut record, commit) = record_write(); + if invalid_record { + record.cid = "invalid".to_owned(); + } + let raw_cid = if invalid_commit { + "invalid".to_owned() + } else { + commit.cid.as_ref().to_string() + }; + let previous: Result<(), Error> = (|| { + stored::(&record.cid, "record CID")?; + stored::(&raw_cid, "repository commit CID")?; + stored::("invalid", "repository revision")?; + Ok(()) + })(); + let actual = if invalid_record { + record_write_output((record, commit)).map(|_| ()) + } else { + // These historical raw fixtures cannot construct native metadata. + (|| { + let _metadata = defs::CommitMetaData { + cid: stored(&raw_cid, "repository commit CID")?, + rev: stored("invalid", "repository revision")?, + }; + Ok(()) + })() + }; + assert_eq!(format!("{actual:?}"), format!("{previous:?}")); + let Error::Internal(message) = actual.unwrap_err() else { + panic!("invalid stored write metadata must be an internal error"); + }; + assert!(message.starts_with(expected), "{message}"); + } + } + + fn record_write() -> (RecordView, defs::CommitMetaData) { + let value = json!({"$type": "app.test.record", "text": "record"}); + let cid = cpds_protocol::record_cid(&value).unwrap().to_string(); + ( + RecordView { + uri: "at://did:web:alice.test/app.test.record/key".to_owned(), + cid: cid.clone(), + value, + }, + defs::CommitMetaData { + cid: cid.parse().unwrap(), + rev: "3jqfcqzm3fo2j".parse().unwrap(), + }, + ) + } + + fn latest_commit_state() -> (crate::AppState, crate::database::Database) { + use crate::{ + blob_store::MemoryBlobStore, + config::{AccountConfig, Config, DeploymentMode, HttpOrigin}, + database::Database, + events::NoopEventSink, + }; + use std::{sync::Arc, time::Duration}; + + let config = Config { + deployment_mode: DeploymentMode::Local, + listen_address: "127.0.0.1:3000".parse().unwrap(), + public_origin: HttpOrigin::parse_public("http://pds.test/", DeploymentMode::Local) + .unwrap(), + database_path: ":memory:".into(), + blob_directory: "unused".into(), + blob_orphan_ttl: Duration::from_secs(86_400), + blob_gc_interval: Duration::from_secs(3_600), + identity_revalidation_interval: Duration::from_secs(3_600), + repo_event_retention: 100_000, + repo_stream_send_timeout: Duration::from_secs(10), + relay_origins: Vec::new(), + account: AccountConfig { + did: "did:web:alice.test".to_owned(), + handle: "alice.test".to_owned(), + password: "password".to_owned(), + signing_key: [7; 32], + }, + oauth_clients: Vec::new(), + }; + let database = Database::open_in_memory().unwrap(); + let state = crate::AppState::assemble( + config, + database.clone(), + Arc::new(MemoryBlobStore::default()), + Arc::new(NoopEventSink), + ) + .unwrap(); + (state, database) + } + + async fn compare_latest_commit_response(state: &crate::AppState, did: &str) -> u16 { + use axum::{ + Json, + body::to_bytes, + extract::{Query, State}, + response::IntoResponse as _, + }; + + let previous: Result, Error> = (|| { + let head = state.repository.sync_head(did)?; + let cid = head.root.to_string(); + let rev = head.rev; + Ok(Json( + defs::CommitMetaData { + cid: stored(&cid, "repository commit CID")?, + rev: stored(&rev, "repository revision")?, + } + .into(), + )) + })(); + let actual = get_latest_commit(( + State(state.clone()), + Query(get_latest_commit::ParametersData { + did: did.parse().unwrap(), + }), )) - })?; - Ok((record.uri, cid, value)) + .await; + match (&actual, &previous) { + (Ok(actual), Ok(previous)) => assert_eq!( + serde_json::to_vec(&actual.0).unwrap(), + serde_json::to_vec(&previous.0).unwrap() + ), + (Err(actual), Err(previous)) => { + assert_eq!(format!("{actual:?}"), format!("{previous:?}")); + } + results => panic!("latest-commit result changed: {results:?}"), + } + let actual = actual.into_response(); + let previous = previous.into_response(); + let status = actual.status().as_u16(); + assert_eq!(actual.status(), previous.status()); + assert_eq!(actual.headers(), previous.headers()); + assert_eq!( + to_bytes(actual.into_body(), usize::MAX).await.unwrap(), + to_bytes(previous.into_body(), usize::MAX).await.unwrap() + ); + status + } + + #[tokio::test] + async fn native_latest_commit_preserves_malformed_storage_and_account_http_failures() { + use rusqlite::{ + params, + types::Value::{Blob, Null, Text}, + }; + + let (state, database) = latest_commit_state(); + database + .connection() + .unwrap() + .execute_batch("CREATE TEMP TABLE repo_heads (did, root_cid, rev)") + .unwrap(); + let root = cpds_protocol::dag_cbor_cid(b"head").to_string(); + for status in ["active", "deactivated"] { + database + .connection() + .unwrap() + .execute("UPDATE accounts SET status = ?1", [status]) + .unwrap(); + for cid in [ + Text(root.clone()), + Text("invalid CID".to_owned()), + Null, + Blob(vec![0]), + ] { + for rev in [ + Text("3jqfcqzm3fo2j".to_owned()), + Text("invalid revision".to_owned()), + Null, + Blob(vec![0]), + ] { + { + let connection = database.connection().unwrap(); + connection.execute("DELETE FROM repo_heads", []).unwrap(); + connection + .execute( + "INSERT INTO repo_heads VALUES (?1, ?2, ?3)", + params!["did:web:alice.test", cid, rev], + ) + .unwrap(); + } + let response = + compare_latest_commit_response(&state, "did:web:alice.test").await; + let valid = + cid == Text(root.clone()) && rev == Text("3jqfcqzm3fo2j".to_owned()); + assert_eq!( + response, + if status != "active" { + 400 + } else if valid { + 200 + } else { + 500 + } + ); + if status == "active" + && cid == Text(root.clone()) + && rev == Text("invalid revision".to_owned()) + { + assert_eq!( + state + .repository + .sync_head("did:web:alice.test") + .unwrap() + .rev, + "invalid revision" + ); + } + } + } + database + .connection() + .unwrap() + .execute("DELETE FROM repo_heads", []) + .unwrap(); + assert_eq!( + compare_latest_commit_response(&state, "did:web:alice.test").await, + 400 + ); + assert_eq!( + compare_latest_commit_response(&state, "did:web:missing.test").await, + 400 + ); + } + } + + #[test] + fn batch_results_preserve_tags_and_expose_replacements_not_previous_records() { + let record = |text| { + let value = json!({"$type": "app.test.record", "text": text}); + Record { + did: "did:web:alice.test".to_owned(), + collection: "app.test.record".to_owned(), + rkey: "key".to_owned(), + cid: cpds_protocol::record_cid(&value).unwrap(), + value, + } + }; + let original = record("original"); + let replacement = record("replacement"); + let original_cid = original.cid.to_string(); + let replacement_cid = replacement.cid.to_string(); + let output = apply_writes_output(( + defs::CommitMetaData { + cid: replacement_cid.parse().unwrap(), + rev: "3jqfcqzm3fo2j".parse().unwrap(), + }, + vec![ + RecordMutation::Create(original.clone()), + RecordMutation::Update { + current: original, + replacement: replacement.clone(), + }, + RecordMutation::Delete(replacement), + ], + )); + assert_eq!( + serde_json::to_value(output).unwrap(), + json!({ + "commit": {"cid": replacement_cid, "rev": "3jqfcqzm3fo2j"}, + "results": [ + {"$type": "com.atproto.repo.applyWrites#createResult", + "uri": "at://did:web:alice.test/app.test.record/key", + "cid": original_cid, "validationStatus": "unknown"}, + {"$type": "com.atproto.repo.applyWrites#updateResult", + "uri": "at://did:web:alice.test/app.test.record/key", + "cid": replacement_cid, "validationStatus": "unknown"}, + {"$type": "com.atproto.repo.applyWrites#deleteResult"}, + ], + }) + ); + } } diff --git a/src/http/request.rs b/src/http/request.rs new file mode 100644 index 0000000..1ac29e2 --- /dev/null +++ b/src/http/request.rs @@ -0,0 +1,293 @@ +//! Standard extractor tuples keep request parsing in Axum's declared order. + +use axum::{ + Json, + extract::{OriginalUri, Query, State}, + http::HeaderMap, +}; + +use crate::AppState; + +pub(crate) type JsonRequest = (State, HeaderMap, Json); +pub(crate) type HeaderQuery = (State, HeaderMap, Query); +pub(crate) type QueryRequest = (State, Query); +pub(crate) type SpaceQuery = (State, OriginalUri, HeaderMap, Query); +pub(crate) type SpaceJson = (State, OriginalUri, HeaderMap, Json); + +#[cfg(test)] +mod tests { + use std::{ + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + time::Duration, + }; + + use axum::{ + Extension, Router, + body::{Body, to_bytes}, + extract::DefaultBodyLimit, + http::{Request, StatusCode, header::CONTENT_TYPE}, + routing::{get, post}, + }; + use serde::Deserialize; + use serde_json::{Value, json}; + use tower::ServiceExt as _; + + use super::*; + use crate::{ + blob_store::MemoryBlobStore, + config::{AccountConfig, Config, DeploymentMode, HttpOrigin}, + database::Database, + events::NoopEventSink, + }; + + type Calls = Arc; + + #[derive(Deserialize)] + struct Input { + number: u8, + } + + #[tokio::test] + async fn extractor_aliases_preserve_native_values_rejections_and_handler_entry() { + let calls = Arc::new(AtomicUsize::new(0)); + let state = state(); + let routers = [ + Router::new() + .route("/json", post(native_json)) + .route("/header-query", get(native_header_query)) + .route("/query", get(native_query)) + .route("/space-query", get(native_space_query)) + .route("/space-json", post(native_space_json)), + Router::new() + .route("/json", post(aliased_json)) + .route("/header-query", get(aliased_header_query)) + .route("/query", get(aliased_query)) + .route("/space-query", get(aliased_space_query)) + .route("/space-json", post(aliased_space_json)), + ] + .map(|router| { + Router::new() + .nest("/nested", router) + .layer(DefaultBodyLimit::max(32)) + .layer(Extension(calls.clone())) + .with_state(state.clone()) + }); + + for path in ["json", "space-json"] { + for (body, content_type, status) in [ + (r#"{"number":7}"#, Some("application/json"), StatusCode::OK), + ("{", Some("application/json"), StatusCode::BAD_REQUEST), + ( + r#"{"number":"wrong"}"#, + Some("application/json"), + StatusCode::UNPROCESSABLE_ENTITY, + ), + (r#"{"number":7}"#, None, StatusCode::UNSUPPORTED_MEDIA_TYPE), + ( + r#"{"number":7,"padding":"abcdefghijklmnopqrstuvwxyz"}"#, + Some("application/json"), + StatusCode::PAYLOAD_TOO_LARGE, + ), + ] { + let mut request = Request::post(format!("/nested/{path}?trace=original")) + .header("x-test", "preserved"); + if let Some(content_type) = content_type { + request = request.header(CONTENT_TYPE, content_type); + } + assert_equivalent(&routers, &calls, request.body(body).unwrap(), status).await; + } + } + for path in ["query", "header-query", "space-query"] { + for (query, status) in [ + ("number=7", StatusCode::OK), + ("number=wrong", StatusCode::BAD_REQUEST), + ("other=7", StatusCode::BAD_REQUEST), + ] { + let request = Request::get(format!("/nested/{path}?{query}")) + .header("x-test", "preserved") + .body("") + .unwrap(); + assert_equivalent(&routers, &calls, request, status).await; + } + } + } + + async fn assert_equivalent( + routers: &[Router; 2], + calls: &AtomicUsize, + request: Request<&'static str>, + status: StatusCode, + ) { + let before = calls.load(Ordering::SeqCst); + let mut responses = Vec::new(); + for (index, router) in routers.iter().enumerate() { + let response = router + .clone() + .oneshot(request.clone().map(Body::from)) + .await + .unwrap(); + assert_eq!(response.status(), status); + let content_type = response.headers().get(CONTENT_TYPE).cloned(); + let bytes = to_bytes(response.into_body(), 1024).await.unwrap(); + assert_eq!( + calls.load(Ordering::SeqCst), + before + (index + 1) * usize::from(status.is_success()) + ); + responses.push((content_type, bytes)); + } + assert_eq!(responses[0], responses[1]); + if !status.is_success() { + return; + } + let body: Value = serde_json::from_slice(&responses[0].1).unwrap(); + assert_eq!(body["did"], "did:web:alice.test"); + assert_eq!(body["number"], 7); + assert_eq!( + body["header"], + if request.uri().path() == "/nested/query" { + Value::Null + } else { + json!("preserved") + } + ); + assert_eq!( + body["uri"], + if request.uri().path().contains("space-") { + json!(request.uri().to_string()) + } else { + Value::Null + } + ); + } + + fn observed( + state: &AppState, + headers: Option, + uri: Option, + number: u8, + calls: &AtomicUsize, + ) -> Json { + calls.fetch_add(1, Ordering::SeqCst); + Json(json!({ + "did": state.config.account.did, + "header": headers.and_then(|headers| headers.get("x-test").and_then(|header| header.to_str().ok()).map(str::to_owned)), + "uri": uri.map(|uri| uri.0.to_string()), + "number": number, + })) + } + + async fn native_json( + Extension(calls): Extension, + State(state): State, + headers: HeaderMap, + Json(input): Json, + ) -> Json { + observed(&state, Some(headers), None, input.number, &calls) + } + + async fn aliased_json( + Extension(calls): Extension, + (State(state), headers, Json(input)): JsonRequest, + ) -> Json { + observed(&state, Some(headers), None, input.number, &calls) + } + + async fn native_header_query( + Extension(calls): Extension, + State(state): State, + headers: HeaderMap, + Query(input): Query, + ) -> Json { + observed(&state, Some(headers), None, input.number, &calls) + } + + async fn aliased_header_query( + Extension(calls): Extension, + (State(state), headers, Query(input)): HeaderQuery, + ) -> Json { + observed(&state, Some(headers), None, input.number, &calls) + } + + async fn native_query( + Extension(calls): Extension, + State(state): State, + Query(input): Query, + ) -> Json { + observed(&state, None, None, input.number, &calls) + } + + async fn aliased_query( + Extension(calls): Extension, + (State(state), Query(input)): QueryRequest, + ) -> Json { + observed(&state, None, None, input.number, &calls) + } + + async fn native_space_query( + Extension(calls): Extension, + State(state): State, + uri: OriginalUri, + headers: HeaderMap, + Query(input): Query, + ) -> Json { + observed(&state, Some(headers), Some(uri), input.number, &calls) + } + + async fn aliased_space_query( + Extension(calls): Extension, + (State(state), uri, headers, Query(input)): SpaceQuery, + ) -> Json { + observed(&state, Some(headers), Some(uri), input.number, &calls) + } + + async fn native_space_json( + Extension(calls): Extension, + State(state): State, + uri: OriginalUri, + headers: HeaderMap, + Json(input): Json, + ) -> Json { + observed(&state, Some(headers), Some(uri), input.number, &calls) + } + + async fn aliased_space_json( + Extension(calls): Extension, + (State(state), uri, headers, Json(input)): SpaceJson, + ) -> Json { + observed(&state, Some(headers), Some(uri), input.number, &calls) + } + + fn state() -> AppState { + let config = Config { + deployment_mode: DeploymentMode::Local, + listen_address: "127.0.0.1:3000".parse().unwrap(), + public_origin: HttpOrigin::parse_public("http://pds.test/", DeploymentMode::Local) + .unwrap(), + database_path: ":memory:".into(), + blob_directory: "unused".into(), + blob_orphan_ttl: Duration::from_secs(86_400), + blob_gc_interval: Duration::from_secs(3_600), + identity_revalidation_interval: Duration::from_secs(3_600), + repo_event_retention: 100_000, + repo_stream_send_timeout: Duration::from_secs(10), + relay_origins: Vec::new(), + account: AccountConfig { + did: "did:web:alice.test".to_owned(), + handle: "alice.test".to_owned(), + password: "correct horse battery staple".to_owned(), + signing_key: [7; 32], + }, + oauth_clients: Vec::new(), + }; + AppState::assemble( + config, + Database::open_in_memory().unwrap(), + Arc::new(MemoryBlobStore::default()), + Arc::new(NoopEventSink), + ) + .unwrap() + } +} diff --git a/src/http/runtime.rs b/src/http/runtime.rs index 0b7d7cd..32424bb 100644 --- a/src/http/runtime.rs +++ b/src/http/runtime.rs @@ -27,7 +27,7 @@ pub async fn serve( ) -> Result<()> { let checkpoint = state.repository.clone(); let collector = state.repository.clone(); - let stream_shutdown = state.clone(); + let shutdown_state = state.clone(); let orphan_ttl = state.config.blob_orphan_ttl; let gc_interval = state.config.blob_gc_interval; let identity_interval = state.config.identity_revalidation_interval; @@ -69,7 +69,7 @@ pub async fn serve( }); let shutdown = async move { shutdown.await; - stream_shutdown.stop_streams(); + shutdown_state.stream_shutdown.send_replace(true); }; let server_result = serve_router(listener, super::router(state), shutdown, DRAIN_TIMEOUT).await; maintenance.abort(); diff --git a/src/http/security.rs b/src/http/security.rs index 2ff9c84..5cac529 100644 --- a/src/http/security.rs +++ b/src/http/security.rs @@ -29,36 +29,37 @@ pub(super) fn cors() -> CorsLayer { pub(super) async fn response_headers(request: Request, next: Next) -> Response { let path = request.uri().path().to_owned(); let mut response = next.run(request).await; + let publicly_cacheable = response.status().is_success() + && (path == "/xrpc/com.atproto.sync.getBlob" || path.starts_with("/.well-known/")); let headers = response.headers_mut(); insert(headers, "x-content-type-options", "nosniff"); insert(headers, "referrer-policy", "no-referrer"); insert(headers, "x-frame-options", "DENY"); - if is_blob_path(&path) { - insert(headers, "content-security-policy", BLOB_CSP); - insert(headers, "cache-control", PUBLIC_CACHE); - } else { - insert(headers, "content-security-policy", DEFAULT_CSP); - insert( - headers, - "cache-control", - if path.starts_with("/.well-known/") { - PUBLIC_CACHE - } else { - NO_STORE - }, - ); - } + insert( + headers, + "content-security-policy", + if matches!( + path.as_str(), + "/xrpc/com.atproto.sync.getBlob" | "/xrpc/com.atproto.space.getBlob" + ) { + BLOB_CSP + } else { + DEFAULT_CSP + }, + ); + insert( + headers, + "cache-control", + if publicly_cacheable { + PUBLIC_CACHE + } else { + NO_STORE + }, + ); response } -fn is_blob_path(path: &str) -> bool { - matches!( - path, - "/xrpc/com.atproto.sync.getBlob" | "/xrpc/com.atproto.space.getBlob" - ) -} - fn insert(headers: &mut HeaderMap, name: &'static str, value: &'static str) { headers.insert( HeaderName::from_static(name), diff --git a/src/http/server.rs b/src/http/server.rs index 17f87ad..e8c57b8 100644 --- a/src/http/server.rs +++ b/src/http/server.rs @@ -1,5 +1,6 @@ //! Discovery, health, and session endpoints. +use crate::http::request::{JsonRequest, QueryRequest}; use atrium_api::{ com::atproto::{ identity::resolve_handle, @@ -16,7 +17,10 @@ use axum::{ }; use serde::Serialize; -use crate::{AppState, domain::identity::local_did_document, error::Error}; +use crate::{ + AppState, + error::{Error, ErrorContext as _}, +}; use super::{ApiResult, auth}; @@ -46,12 +50,7 @@ pub async fn ready(State(state): State) -> axum::response::Response { } pub async fn did_document(State(state): State) -> Json { - Json(local_did_document( - &state.config.account.did, - &state.config.account.handle, - &state.config.public_origin.to_string(), - &state.spaces.public_key_multibase(), - )) + Json(state.identity.local_document().clone()) } #[derive(Serialize)] @@ -73,26 +72,22 @@ pub async fn describe_server(State(state): State) -> Json, headers: HeaderMap, -) -> ApiResult> { +) -> ApiResult> { let (token, status) = auth::token_with_status(&headers, &state)?; let account = token.account; - let did = Did::new(account.did) - .map_err(|message| Error::Internal(format!("invalid stored account DID: {message}")))?; - let handle = Handle::new(account.handle) - .map_err(|message| Error::Internal(format!("invalid stored account handle: {message}")))?; - Ok(Json( - get_session::OutputData { - active: Some(status.is_active()), - did, - did_doc: None, - email: None, - email_auth_factor: None, - email_confirmed: None, - handle, - status: status.inactive_reason().map(ToOwned::to_owned), - } - .into(), - )) + let did = Did::new(account.did).context(Error::Internal, "invalid stored account DID")?; + let handle = + Handle::new(account.handle).context(Error::Internal, "invalid stored account handle")?; + Ok(Json(get_session::OutputData { + active: Some(status.is_active()), + did, + did_doc: None, + email: None, + email_auth_factor: None, + email_confirmed: None, + handle, + status: status.inactive_reason().map(ToOwned::to_owned), + })) } pub async fn activate_account( @@ -105,9 +100,7 @@ pub async fn activate_account( } pub async fn deactivate_account( - State(state): State, - headers: HeaderMap, - Json(_input): Json, + (State(state), headers, Json(_input)): JsonRequest, ) -> ApiResult { let (token, _) = auth::token_with_status(&headers, &state)?; state.repository.deactivate_account(&token.account.did)?; @@ -115,9 +108,37 @@ pub async fn deactivate_account( } pub async fn resolve_handle( - State(state): State, - Query(query): Query, -) -> ApiResult> { + (State(state), Query(query)): QueryRequest, +) -> ApiResult> { let did = state.identity.resolve_handle(&query.handle).await?; - Ok(Json(resolve_handle::OutputData { did }.into())) + Ok(Json(resolve_handle::OutputData { did })) +} + +#[cfg(test)] +mod tests { + use super::{get_session, resolve_handle}; + use atrium_api::types::Object; + use serde_json::json; + + #[test] + fn server_response_data_matches_empty_extension_objects() { + let did = "did:web:alice.test"; + let handle: resolve_handle::OutputData = + serde_json::from_value(json!({"did": did})).unwrap(); + assert_eq!( + serde_json::to_vec(&handle).unwrap(), + serde_json::to_vec(&Object::from(handle.clone())).unwrap(), + ); + for (active, status) in [(true, None), (false, Some("deactivated"))] { + let session: get_session::OutputData = serde_json::from_value(json!({ + "active": active, "did": did, "handle": "alice.test", "status": status, + "didDoc": null, "email": null, "emailAuthFactor": null, "emailConfirmed": null, + })) + .unwrap(); + assert_eq!( + serde_json::to_vec(&session).unwrap(), + serde_json::to_vec(&Object::from(session.clone())).unwrap(), + ); + } + } } diff --git a/src/http/subscription.rs b/src/http/subscription.rs index 2237017..6c110ae 100644 --- a/src/http/subscription.rs +++ b/src/http/subscription.rs @@ -16,9 +16,13 @@ use tokio::time::{sleep, timeout}; use tower_http::request_id::RequestId; use tracing::Instrument as _; -use crate::{AppState, error::Error, rate_limits::SubscriptionPermit}; +use crate::{ + AppState, + error::{Error, ErrorContext as _}, + rate_limits::SubscriptionPermit, +}; -use super::{ApiError, ApiResult}; +use super::{ApiResult, error::ErrorBody}; const EVENT_PAGE_SIZE: usize = 100; const POLL_INTERVAL: Duration = Duration::from_millis(250); @@ -32,9 +36,9 @@ pub async fn subscribe_repos( websocket: WebSocketUpgrade, ) -> ApiResult { if query.cursor.is_some_and(|cursor| cursor < 0) { - return Err(ApiError(Error::InvalidRequest( + return Err(Error::InvalidRequest( "repository stream cursor cannot be negative".to_owned(), - ))); + )); } let cursor = query .cursor @@ -61,7 +65,7 @@ async fn stream_repositories( mut cursor: i64, _permit: SubscriptionPermit, ) { - let mut shutdown = state.stream_shutdown(); + let mut shutdown = state.stream_shutdown.subscribe(); let send_timeout = state.config.repo_stream_send_timeout; tracing::info!(cursor, "repository stream opened"); loop { @@ -184,30 +188,23 @@ fn encode_message(message: &subscribe_repos::Message) -> Result, Error> fn encode_frame(kind: &'static str, payload: &impl Serialize) -> Result, Error> { let mut frame = serde_ipld_dagcbor::to_vec(&MessageHeader { op: 1, t: kind }) - .map_err(|error| Error::Internal(format!("could not encode event header: {error}")))?; - frame - .extend(serde_ipld_dagcbor::to_vec(payload).map_err(|error| { - Error::Internal(format!("could not encode event payload: {error}")) - })?); + .context(Error::Internal, "could not encode event header")?; + frame.extend( + serde_ipld_dagcbor::to_vec(payload) + .context(Error::Internal, "could not encode event payload")?, + ); Ok(frame) } async fn send_stream_error(socket: &mut ws::WebSocket, error: &Error, send_timeout: Duration) { - match error { - Error::Xrpc { name, message } => { - send_error_frame(socket, name, message, send_timeout).await; - } + let (name, message) = match error { + Error::Xrpc { name, message } => (*name, message.as_str()), error => { tracing::error!(%error, "repository stream failed"); - send_error_frame( - socket, - "InternalServerError", - "repository stream failed", - send_timeout, - ) - .await; + ("InternalServerError", "repository stream failed") } - } + }; + send_error_frame(socket, name, message, send_timeout).await; } async fn send_error_frame( @@ -216,21 +213,20 @@ async fn send_error_frame( message: &str, send_timeout: Duration, ) { - let frame = encode_error(name, message); - if let Ok(frame) = frame { + if let Ok(frame) = encode_error(name, message) { let _ = timeout(send_timeout, socket.send(ws::Message::Binary(frame.into()))).await; } } fn encode_error(name: &'static str, message: &str) -> Result, Error> { let mut frame = serde_ipld_dagcbor::to_vec(&ErrorHeader { op: -1 }) - .map_err(|error| Error::Internal(format!("could not encode error header: {error}")))?; + .context(Error::Internal, "could not encode error header")?; frame.extend( - serde_ipld_dagcbor::to_vec(&ErrorPayload { + serde_ipld_dagcbor::to_vec(&ErrorBody { error: name, message, }) - .map_err(|error| Error::Internal(format!("could not encode error payload: {error}")))?, + .context(Error::Internal, "could not encode error payload")?, ); Ok(frame) } @@ -246,22 +242,21 @@ struct ErrorHeader { op: i8, } -#[derive(Serialize)] -struct ErrorPayload<'a> { - error: &'static str, - message: &'a str, -} - #[cfg(test)] mod tests { - use std::io::Cursor; + use std::{io::Cursor, sync::Arc}; use super::*; use atrium_api::{ com::atproto::sync::subscribe_repos, types::string::{Datetime, Did}, }; + use axum::{Router, extract::Path, routing::get}; use serde::Deserialize; + use tokio::{ + io::{AsyncBufReadExt as _, AsyncReadExt as _, AsyncWriteExt as _, BufReader}, + net::{TcpListener, TcpStream}, + }; #[derive(Deserialize)] struct MessageHeader { @@ -332,4 +327,190 @@ mod tests { assert_eq!(payload.message, "cursor is in the future"); assert_eq!(usize::try_from(reader.position()).unwrap(), frame_len); } + + #[test] + fn shared_error_body_keeps_borrowed_cbor_frame_bytes() { + #[derive(Serialize)] + struct PreviousErrorHeader { + op: i8, + } + #[derive(Serialize)] + struct PreviousErrorPayload<'a> { + error: &'static str, + message: &'a str, + } + let messages = [ + String::new(), + "cursor is in the future".to_owned(), + "quote-\"-backslash-\\-雪-🦀".to_owned(), + (0u8..=31).map(char::from).collect(), + "a".repeat(4096), + ]; + for name in ["", "FutureCursor", "Error-\"\\\n-雪"] { + for message in &messages { + let mut previous = + serde_ipld_dagcbor::to_vec(&PreviousErrorHeader { op: -1 }).unwrap(); + previous.extend( + serde_ipld_dagcbor::to_vec(&PreviousErrorPayload { + error: name, + message, + }) + .unwrap(), + ); + assert_eq!(encode_error(name, message).unwrap(), previous); + } + } + } + + async fn previous_send_stream_error( + socket: &mut ws::WebSocket, + error: &Error, + send_timeout: Duration, + ) { + async fn previous_send_error_frame( + socket: &mut ws::WebSocket, + name: &'static str, + message: &str, + send_timeout: Duration, + ) { + let frame = encode_error(name, message); + if let Ok(frame) = frame { + let _ = timeout(send_timeout, socket.send(ws::Message::Binary(frame.into()))).await; + } + } + match error { + Error::Xrpc { name, message } => { + previous_send_error_frame(socket, name, message, send_timeout).await; + } + error => { + tracing::error!(%error, "repository stream failed"); + previous_send_error_frame( + socket, + "InternalServerError", + "repository stream failed", + send_timeout, + ) + .await; + } + } + } + + async fn receive_error_frame( + address: SocketAddr, + previous: bool, + index: usize, + ) -> (Vec, Vec) { + let mut stream = TcpStream::connect(address).await.unwrap(); + stream + .write_all( + format!( + "GET /{previous}/{index} HTTP/1.1\r\nHost: {address}\r\n\ + Upgrade: websocket\r\nConnection: upgrade\r\n\ + Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\nSec-WebSocket-Version: 13\r\n\r\n" + ) + .as_bytes(), + ) + .await + .unwrap(); + let mut reader = BufReader::new(stream); + let mut line = String::new(); + reader.read_line(&mut line).await.unwrap(); + assert!(line.starts_with("HTTP/1.1 101 "), "{line}"); + loop { + line.clear(); + assert_ne!(reader.read_line(&mut line).await.unwrap(), 0); + if line == "\r\n" { + break; + } + } + let mut header = [0_u8; 2]; + reader.read_exact(&mut header).await.unwrap(); + assert_eq!(header[0], 0x82, "one final binary frame"); + assert_eq!(header[1] & 0x80, 0, "server frames must not be masked"); + let mut frame = header.to_vec(); + let length = match header[1] { + 126 => { + let mut length = [0; 2]; + reader.read_exact(&mut length).await.unwrap(); + frame.extend_from_slice(&length); + usize::from(u16::from_be_bytes(length)) + } + 127 => panic!("test error frames must stay below the 64-bit length threshold"), + length => usize::from(length), + }; + let mut payload = vec![0; length]; + reader.read_exact(&mut payload).await.unwrap(); + frame.extend_from_slice(&payload); + (frame, payload) + } + + #[tokio::test] + async fn shared_error_send_preserves_transport_bytes_and_internal_sanitization() { + let public_message = format!("cursor \"quoted\"\n café: {}", "x".repeat(128)); + let errors = Arc::new(vec![ + Error::xrpc("FutureCursor", &public_message), + Error::xrpc("EmptyDetails", ""), + Error::Internal("private stored record content must not escape".to_owned()), + Error::Upstream("https://private-service.test/?secret=credential".to_owned()), + ]); + let router = Router::new() + .route( + "/{previous}/{index}", + get( + |State(errors): State>>, + Path((previous, index)): Path<(bool, usize)>, + upgrade: WebSocketUpgrade| async move { + upgrade.on_upgrade(move |mut socket| async move { + let send_timeout = Duration::from_secs(1); + if previous { + previous_send_stream_error( + &mut socket, + &errors[index], + send_timeout, + ) + .await; + } else { + send_stream_error(&mut socket, &errors[index], send_timeout).await; + } + }) + }, + ), + ) + .with_state(errors); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + for (index, name, message) in [ + (0, "FutureCursor", public_message.as_str()), + (1, "EmptyDetails", ""), + (2, "InternalServerError", "repository stream failed"), + (3, "InternalServerError", "repository stream failed"), + ] { + let previous = timeout( + Duration::from_secs(2), + receive_error_frame(address, true, index), + ) + .await + .unwrap(); + let current = timeout( + Duration::from_secs(2), + receive_error_frame(address, false, index), + ) + .await + .unwrap(); + assert_eq!(current, previous); + assert_eq!(current.1, encode_error(name, message).unwrap()); + let length = current.1.len(); + let mut reader = Cursor::new(current.1); + let header: ErrorHeader = + serde_ipld_dagcbor::de::from_reader_once(&mut reader).unwrap(); + let payload: ErrorPayload = + serde_ipld_dagcbor::de::from_reader_once(&mut reader).unwrap(); + assert_eq!(header.op, -1); + assert_eq!(payload.error, name); + assert_eq!(payload.message, message); + assert_eq!(usize::try_from(reader.position()).unwrap(), length); + } + server.abort(); + } } diff --git a/src/identity.rs b/src/identity.rs index 8148840..9928375 100644 --- a/src/identity.rs +++ b/src/identity.rs @@ -1,14 +1,13 @@ //! Bounded AT Protocol identity resolution over deployment-controlled DNS. +pub(crate) mod http; + use std::{ - collections::BTreeSet, - error::Error as StdError, - net::SocketAddr, + collections::HashSet, sync::{ Arc, Mutex, atomic::{AtomicBool, Ordering}, }, - time::Duration, }; use atrium_api::{ @@ -20,75 +19,34 @@ use atrium_identity::{ did::{CommonDidResolver, CommonDidResolverConfig, DEFAULT_PLC_DIRECTORY_URL}, handle::{WellKnownHandleResolver, WellKnownHandleResolverConfig}, }; -use atrium_xrpc::{ - HttpClient, - http::{Request, Response}, -}; +use atrium_xrpc::{HttpClient, http::Request}; use hickory_resolver::{TokioResolver, proto::rr::RData}; use percent_encoding::percent_decode_str; -use reqwest::dns::{Addrs, Name, Resolve, Resolving}; -use url::{Host, Url}; +use url::Url; + +use self::http::{OutboundHttp, dns_resolver, validate_https_url}; use crate::{ config::DeploymentMode, - error::{Error, Result}, + domain::identity::document_claims_handle, + error::{Error, ErrorContext as _, Result}, }; -const DNS_TIMEOUT: Duration = Duration::from_secs(3); -const HTTP_CONNECT_TIMEOUT: Duration = Duration::from_secs(5); -const HTTP_TIMEOUT: Duration = Duration::from_secs(10); -const MAX_REDIRECTS: usize = 3; -const MAX_DOCUMENT_BYTES: usize = 256 * 1024; -const DNS_CACHE_ENTRIES: u64 = 256; -const DNS_POSITIVE_TTL: Duration = Duration::from_secs(300); -const DNS_NEGATIVE_TTL: Duration = Duration::from_secs(60); - -const BLOCKED_ADDRESS_RANGES: &[&str] = &[ - "0.0.0.0/8", - "10.0.0.0/8", - "100.64.0.0/10", - "127.0.0.0/8", - "169.254.0.0/16", - "172.16.0.0/12", - "192.0.0.0/24", - "192.0.2.0/24", - "192.168.0.0/16", - "198.18.0.0/15", - "198.51.100.0/24", - "203.0.113.0/24", - "224.0.0.0/4", - "240.0.0.0/4", - "::/128", - "::1/128", - "::ffff:0:0/96", - "100::/64", - "2001:db8::/32", - "fc00::/7", - "fe80::/10", - "ff00::/8", -]; - /// Network identity resolver plus the canonical identity hosted by this PDS. pub(crate) struct IdentityService { local_document: DidDocument, local_handle: Handle, local_did: Did, - local_resolution: bool, + mode: DeploymentMode, local_handle_verified: AtomicBool, network: Mutex>>, } struct NetworkIdentity { dns: TokioResolver, - did: CommonDidResolver, - https_handle: WellKnownHandleResolver, - http: Arc, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) enum LocalHandleStatus { - Valid, - Invalid, + did: CommonDidResolver, + https_handle: WellKnownHandleResolver, + http: Arc, } #[async_trait::async_trait] @@ -99,18 +57,26 @@ trait IdentityNetwork: Send + Sync { } impl IdentityService { + pub(crate) fn local_document(&self) -> &DidDocument { + &self.local_document + } + + pub(crate) fn outbound_http(&self) -> Result { + OutboundHttp::new(None, self.mode) + } + pub(crate) fn new(local_document: DidDocument, mode: DeploymentMode) -> Result { let local_did = local_document .id .parse::() - .map_err(|error| Error::Configuration(format!("invalid local DID: {error}")))?; + .context(Error::Configuration, "invalid local DID")?; let local_handle = local_document .also_known_as .as_ref() .and_then(|aliases| aliases.iter().find_map(|alias| alias.strip_prefix("at://"))) .ok_or_else(|| Error::Configuration("local DID document has no handle".to_owned()))? .parse::() - .map_err(|error| Error::Configuration(format!("invalid local handle: {error}")))?; + .context(Error::Configuration, "invalid local handle")?; let network: Option> = (mode == DeploymentMode::Production) .then(NetworkIdentity::new) .transpose()? @@ -119,7 +85,7 @@ impl IdentityService { local_document, local_handle, local_did, - local_resolution: mode == DeploymentMode::Local, + mode, local_handle_verified: AtomicBool::new(mode == DeploymentMode::Local), network: Mutex::new(network), }) @@ -127,7 +93,7 @@ impl IdentityService { /// Resolve a handle using the preferred DNS TXT method, then HTTPS well-known. pub(crate) async fn resolve_handle(&self, handle: &Handle) -> Result { - if self.local_resolution && handle == &self.local_handle { + if self.mode == DeploymentMode::Local && handle == &self.local_handle { return Ok(self.local_did.clone()); } let network = self.network()?; @@ -161,7 +127,7 @@ impl IdentityService { pub(crate) async fn verify_handle(&self, handle: &Handle) -> Result { let did = self.resolve_handle(handle).await?; let document = self.resolve_did(&did).await?; - if !document_claims_handle(&document, handle) { + if !document_claims_handle(&document, handle.as_str()) { return Err(handle_not_found( "resolved DID document does not link back to the handle", )); @@ -171,29 +137,29 @@ impl IdentityService { /// Verify the configured public handle before production begins serving. pub(crate) async fn verify_local_handle(&self) -> Result<()> { - match self.local_handle_status().await? { - LocalHandleStatus::Valid => { - self.local_handle_verified.store(true, Ordering::Release); - Ok(()) - } - LocalHandleStatus::Invalid => { - self.local_handle_verified.store(false, Ordering::Release); - Err(Error::Configuration(format!( - "configured handle {} does not resolve bidirectionally to {}", - self.local_handle.as_str(), - self.local_did.as_str() - ))) - } + if self.revalidate_local_handle().await? { + return Ok(()); } + Err(Error::Configuration(format!( + "configured handle {} does not resolve bidirectionally to {}", + self.local_handle.as_str(), + self.local_did.as_str() + ))) } /// Recheck the configured handle, preserving the prior result on a transient /// resolver failure and returning a confirmed valid/invalid result. - pub(crate) async fn revalidate_local_handle(&self) -> Result { - let status = self.local_handle_status().await?; - self.local_handle_verified - .store(status == LocalHandleStatus::Valid, Ordering::Release); - Ok(status) + pub(crate) async fn revalidate_local_handle(&self) -> Result { + let valid = match self.verify_handle(&self.local_handle).await { + Ok(did) => did == self.local_did, + Err(Error::Xrpc { + name: "HandleNotFound", + .. + }) => false, + Err(error) => return Err(error), + }; + self.local_handle_verified.store(valid, Ordering::Release); + Ok(valid) } pub(crate) fn local_handle_verified(&self) -> bool { @@ -208,101 +174,54 @@ impl IdentityService { if let Some(network) = network.as_ref() { return Ok(network.clone()); } - let initialized: Arc = - Arc::new(NetworkIdentity::new().map_err(|error| { - Error::Upstream(format!( - "network identity resolver initialization failed: {error}" - )) - })?); + let initialized: Arc = Arc::new(NetworkIdentity::new().context( + Error::Upstream, + "network identity resolver initialization failed", + )?); *network = Some(initialized.clone()); Ok(initialized) } - - async fn local_handle_status(&self) -> Result { - match self.verify_handle(&self.local_handle).await { - Ok(did) if did == self.local_did => Ok(LocalHandleStatus::Valid), - Ok(_) - | Err(Error::Xrpc { - name: "HandleNotFound", - .. - }) => Ok(LocalHandleStatus::Invalid), - Err(error) => Err(error), - } - } } impl NetworkIdentity { fn new() -> Result { let dns = dns_resolver()?; - let http = Arc::new(IdentityHttpClient::new(dns.clone())?); - let did = CommonDidResolver::new(CommonDidResolverConfig { - plc_directory_url: DEFAULT_PLC_DIRECTORY_URL.to_owned(), - http_client: http.clone(), - }); - let https_handle = WellKnownHandleResolver::new(WellKnownHandleResolverConfig { - http_client: http.clone(), - }); + let http = Arc::new(OutboundHttp::new( + Some(dns.clone()), + DeploymentMode::Production, + )?); Ok(Self { dns, - did, - https_handle, + did: CommonDidResolver::new(CommonDidResolverConfig { + plc_directory_url: DEFAULT_PLC_DIRECTORY_URL.to_owned(), + http_client: http.clone(), + }), + https_handle: WellKnownHandleResolver::new(WellKnownHandleResolverConfig { + http_client: http.clone(), + }), http, }) } +} - async fn dns_handle(&self, handle: &Handle) -> Result> { +#[async_trait::async_trait] +impl IdentityNetwork for NetworkIdentity { + async fn resolve_dns_handle(&self, handle: &Handle) -> Result> { let query = format!("_atproto.{}.", handle.as_str()); let lookup = self .dns .txt_lookup(query) .await - .map_err(|error| Error::Upstream(format!("DNS TXT lookup failed: {error}")))?; + .context(Error::Upstream, "DNS TXT lookup failed")?; let records = lookup.answers().iter().filter_map(|answer| { let RData::TXT(txt) = &answer.data else { return None; }; - Some( - txt.txt_data - .iter() - .flat_map(|part| part.iter().copied()) - .collect::>(), - ) + Some(txt.txt_data.concat()) }); select_dns_handle(records) } - async fn resolve_web_did(&self, did: &Did) -> Result { - let url = did_web_url(did)?; - let request = Request::builder() - .header( - atrium_xrpc::http::header::ACCEPT, - "application/did+ld+json,application/json", - ) - .uri(url.as_str()) - .body(Vec::new()) - .map_err(|error| Error::Internal(format!("DID request: {error}")))?; - let response = self - .http - .send_http(request) - .await - .map_err(|error| Error::Upstream(format!("DID resolution failed: {error}")))?; - if !response.status().is_success() { - return Err(Error::Upstream(format!( - "DID resolution returned {}", - response.status() - ))); - } - serde_json::from_slice(response.body()) - .map_err(|error| Error::Upstream(format!("invalid DID document: {error}"))) - } -} - -#[async_trait::async_trait] -impl IdentityNetwork for NetworkIdentity { - async fn resolve_dns_handle(&self, handle: &Handle) -> Result> { - self.dns_handle(handle).await - } - async fn resolve_https_handle(&self, handle: &Handle) -> Result { self.https_handle.resolve(handle).await.map_err(|error| { if matches!( @@ -318,14 +237,34 @@ impl IdentityNetwork for NetworkIdentity { } async fn resolve_did(&self, did: &Did) -> Result { - if did.as_str().starts_with("did:web:") { - self.resolve_web_did(did).await - } else { - self.did + if !did.as_str().starts_with("did:web:") { + return self + .did .resolve(did) .await - .map_err(|error| Error::Upstream(format!("DID resolution failed: {error}"))) + .context(Error::Upstream, "DID resolution failed"); } + let url = did_web_url(did)?; + let request = Request::builder() + .header( + atrium_xrpc::http::header::ACCEPT, + "application/did+ld+json,application/json", + ) + .uri(url.as_str()) + .body(Vec::new()) + .context(Error::Internal, "DID request")?; + let response = self + .http + .send_http(request) + .await + .context(Error::Upstream, "DID resolution failed")?; + if !response.status().is_success() { + return Err(Error::Upstream(format!( + "DID resolution returned {}", + response.status() + ))); + } + serde_json::from_slice(response.body()).context(Error::Upstream, "invalid DID document") } } @@ -339,13 +278,10 @@ fn did_web_url(did: &Did) -> Result { .map(|segment| { percent_decode_str(segment) .decode_utf8() - .map(String::from) .map_err(|_| Error::Upstream("invalid percent encoding in did:web".to_owned())) }) .collect::>>()?; - let (host, path) = segments - .split_first() - .ok_or_else(|| Error::Upstream("did:web has no host".to_owned()))?; + let (host, path) = (&segments[0], &segments[1..]); if segments .iter() .any(|segment| segment.is_empty() || segment.contains(['/', '?', '#'])) @@ -360,602 +296,33 @@ fn did_web_url(did: &Did) -> Result { format!("/{}/did.json", path.join("/")) }; let url = Url::parse(&format!("https://{host}{suffix}")) - .map_err(|error| Error::Upstream(format!("invalid did:web URL: {error}")))?; + .context(Error::Upstream, "invalid did:web URL")?; validate_https_url(&url).map_err(|error| Error::Upstream(error.to_owned()))?; Ok(url) } fn select_dns_handle(records: impl IntoIterator>) -> Result> { - let mut dids = BTreeSet::new(); - for record in records { - let Ok(record) = std::str::from_utf8(&record) else { - continue; - }; - let Some(did) = record.strip_prefix("did=") else { - continue; - }; - if let Ok(did) = did.parse::() { - dids.insert(did.as_str().to_owned()); - } - } + let dids: HashSet = records + .into_iter() + .filter_map(|record| { + std::str::from_utf8(&record) + .ok()? + .strip_prefix("did=")? + .parse() + .ok() + }) + .collect(); if dids.len() > 1 { return Err(handle_not_found( "DNS publishes conflicting valid DID records for the handle", )); } - dids.into_iter() - .next() - .map(|did| { - did.parse::().map_err(|error| { - Error::Internal(format!("validated DNS DID could not be parsed: {error}")) - }) - }) - .transpose() -} - -fn document_claims_handle(document: &DidDocument, handle: &Handle) -> bool { - let alias = format!("at://{}", handle.as_str()); - document - .also_known_as - .as_ref() - .is_some_and(|aliases| aliases.contains(&alias)) -} - -fn dns_resolver() -> Result { - let mut builder = TokioResolver::builder_tokio() - .map_err(|error| Error::Configuration(format!("system DNS configuration: {error}")))?; - let options = builder.options_mut(); - options.timeout = DNS_TIMEOUT; - options.attempts = 2; - options.cache_size = DNS_CACHE_ENTRIES; - options.positive_max_ttl = Some(DNS_POSITIVE_TTL); - options.negative_max_ttl = Some(DNS_NEGATIVE_TTL); - options.deny_answers = BLOCKED_ADDRESS_RANGES - .iter() - .map(|network| network.parse()) - .collect::, _>>() - .map_err(|error| Error::Internal(format!("invalid blocked address range: {error}")))?; - builder - .build() - .map_err(|error| Error::Configuration(format!("DNS resolver: {error}"))) -} - -#[derive(Clone)] -struct PublicDnsResolver(TokioResolver); - -impl Resolve for PublicDnsResolver { - fn resolve(&self, name: Name) -> Resolving { - let resolver = self.0.clone(); - let name = format!("{}.", name.as_str().trim_end_matches('.')); - Box::pin(async move { - let lookup = resolver.lookup_ip(name).await.map_err(boxed_error)?; - let addresses = lookup - .iter() - .map(|ip| SocketAddr::new(ip, 0)) - .collect::>(); - if addresses.is_empty() { - return Err(message_error("DNS returned no public addresses")); - } - Ok(Box::new(addresses.into_iter()) as Addrs) - }) - } -} - -struct IdentityHttpClient { - client: reqwest::Client, -} - -impl IdentityHttpClient { - fn new(dns: TokioResolver) -> Result { - let client = reqwest::Client::builder() - .connect_timeout(HTTP_CONNECT_TIMEOUT) - .timeout(HTTP_TIMEOUT) - .redirect(reqwest::redirect::Policy::custom(|attempt| { - if attempt.previous().len() >= MAX_REDIRECTS - || validate_https_url(attempt.url()).is_err() - { - attempt.stop() - } else { - attempt.follow() - } - })) - .dns_resolver(std::sync::Arc::new(PublicDnsResolver(dns))) - .user_agent(concat!("cpds/", env!("CARGO_PKG_VERSION"))) - .build() - .map_err(|error| Error::Configuration(format!("identity HTTP client: {error}")))?; - Ok(Self { client }) - } -} - -impl HttpClient for IdentityHttpClient { - async fn send_http( - &self, - request: Request>, - ) -> std::result::Result>, Box> { - if request.method() != atrium_xrpc::http::Method::GET || !request.body().is_empty() { - return Err(message_error( - "identity HTTP transport accepts only empty GET requests", - )); - } - let url = Url::parse(&request.uri().to_string()).map_err(boxed_error)?; - validate_https_url(&url).map_err(message_error)?; - let mut response = self - .client - .get(url) - .headers(request.headers().clone()) - .send() - .await - .map_err(boxed_error)?; - if response - .content_length() - .is_some_and(|length| length > MAX_DOCUMENT_BYTES as u64) - { - return Err(message_error("identity response exceeds the byte limit")); - } - let status = response.status(); - let headers = response.headers().clone(); - let mut body = Vec::new(); - while let Some(chunk) = response.chunk().await.map_err(boxed_error)? { - if body.len().saturating_add(chunk.len()) > MAX_DOCUMENT_BYTES { - return Err(message_error("identity response exceeds the byte limit")); - } - body.extend_from_slice(&chunk); - } - let mut response = Response::new(body); - *response.status_mut() = status; - *response.headers_mut() = headers; - Ok(response) - } -} - -fn validate_https_url(url: &Url) -> std::result::Result<(), &'static str> { - if url.scheme() != "https" - || !url.username().is_empty() - || url.password().is_some() - || url.fragment().is_some() - || !matches!(url.host(), Some(Host::Domain(_))) - { - return Err("identity URL must be credential-free public HTTPS"); - } - Ok(()) + Ok(dids.into_iter().next()) } fn handle_not_found(message: &str) -> Error { - Error::Xrpc { - name: "HandleNotFound", - message: message.to_owned(), - } -} - -fn boxed_error(error: impl StdError + Send + Sync + 'static) -> Box { - Box::new(error) -} - -fn message_error(message: impl Into) -> Box { - Box::new(std::io::Error::other(message.into())) + Error::xrpc("HandleNotFound", message) } #[cfg(test)] -mod tests { - use std::sync::atomic::AtomicUsize; - - use crate::{ - AppState, IdentityRevalidation, - blob_store::MemoryBlobStore, - config::{AccountConfig, Config, PublicOrigin}, - database::Database, - events::NoopEventSink, - }; - use axum::{ - body::{Body, to_bytes}, - extract::ConnectInfo, - http::Request as AxumRequest, - }; - use tower::ServiceExt as _; - - use super::*; - - #[derive(Clone, Copy)] - enum DnsAnswer { - Found, - Missing, - Failure, - Unavailable, - Conflict, - } - - struct FakeNetwork { - dns: DnsAnswer, - did: Did, - document: DidDocument, - https_calls: AtomicUsize, - } - - #[async_trait::async_trait] - impl IdentityNetwork for FakeNetwork { - async fn resolve_dns_handle(&self, _handle: &Handle) -> Result> { - match self.dns { - DnsAnswer::Found => Ok(Some(self.did.clone())), - DnsAnswer::Missing => Ok(None), - DnsAnswer::Failure | DnsAnswer::Unavailable => { - Err(Error::Upstream("DNS unavailable".to_owned())) - } - DnsAnswer::Conflict => Err(handle_not_found("conflicting DNS records")), - } - } - - async fn resolve_https_handle(&self, _handle: &Handle) -> Result { - self.https_calls.fetch_add(1, Ordering::SeqCst); - if matches!(self.dns, DnsAnswer::Unavailable) { - return Err(Error::Upstream("HTTPS unavailable".to_owned())); - } - Ok(self.did.clone()) - } - - async fn resolve_did(&self, _did: &Did) -> Result { - Ok(self.document.clone()) - } - } - - #[test] - fn dns_handle_selection_ignores_noise_and_rejects_conflicts() { - let selected = select_dns_handle([ - b"unrelated=value".to_vec(), - b"did=not-a-did".to_vec(), - b"did=did:plc:ewvi7nxzyoun6zhxrhs64oiz".to_vec(), - b"did=did:plc:ewvi7nxzyoun6zhxrhs64oiz".to_vec(), - ]) - .unwrap() - .unwrap(); - assert_eq!(selected.as_str(), "did:plc:ewvi7nxzyoun6zhxrhs64oiz"); - - assert!( - select_dns_handle([ - b"did=did:plc:ewvi7nxzyoun6zhxrhs64oiz".to_vec(), - b"did=did:web:other.example".to_vec(), - ]) - .is_err() - ); - } - - #[test] - fn identity_urls_exclude_ssrf_and_redirect_ambiguity() { - assert!(validate_https_url(&Url::parse("https://example.com/did.json").unwrap()).is_ok()); - for url in [ - "http://example.com/did.json", - "https://user@example.com/did.json", - "https://127.0.0.1/did.json", - "https://[::1]/did.json", - "https://example.com/did.json#fragment", - ] { - assert!( - validate_https_url(&Url::parse(url).unwrap()).is_err(), - "{url}" - ); - } - } - - #[test] - fn web_dids_map_to_their_authoritative_documents() { - assert_eq!( - did_web_url(&"did:web:example.com".parse().unwrap()) - .unwrap() - .as_str(), - "https://example.com/.well-known/did.json" - ); - assert_eq!( - did_web_url(&"did:web:localhost%3A8443:users:alice".parse().unwrap()) - .unwrap() - .as_str(), - "https://localhost:8443/users/alice/did.json" - ); - } - - #[test] - fn bidirectional_validation_requires_an_explicit_handle_claim() { - let handle = "alice.example.com".parse::().unwrap(); - let mut document = DidDocument { - context: None, - id: "did:web:alice.example.com".to_owned(), - also_known_as: None, - verification_method: None, - service: None, - }; - assert!(!document_claims_handle(&document, &handle)); - document.also_known_as = Some(vec!["at://other.example.com".to_owned()]); - assert!(!document_claims_handle(&document, &handle)); - document.also_known_as = Some(vec!["at://alice.example.com".to_owned()]); - assert!(document_claims_handle(&document, &handle)); - } - - #[tokio::test] - async fn resolver_prefers_dns_and_falls_back_only_for_absence_or_failure() { - for (dns, expected_https_calls) in [ - (DnsAnswer::Found, 0), - (DnsAnswer::Missing, 1), - (DnsAnswer::Failure, 1), - ] { - let network = fake_network(dns, "did:web:bob.example.com", None); - let service = service_with_network(network.clone()); - let resolved = service - .resolve_handle(&"bob.example.com".parse().unwrap()) - .await - .unwrap(); - assert_eq!(resolved.as_str(), "did:web:bob.example.com"); - assert_eq!( - network.https_calls.load(Ordering::SeqCst), - expected_https_calls - ); - } - - let network = fake_network(DnsAnswer::Conflict, "did:web:bob.example.com", None); - let service = service_with_network(network.clone()); - assert!( - service - .resolve_handle(&"bob.example.com".parse().unwrap()) - .await - .is_err() - ); - assert_eq!(network.https_calls.load(Ordering::SeqCst), 0); - } - - #[tokio::test] - async fn bidirectional_resolution_rejects_missing_backlinks_and_mismatched_documents() { - let handle = "bob.example.com".parse::().unwrap(); - let missing = service_with_network(fake_network( - DnsAnswer::Found, - "did:web:bob.example.com", - None, - )); - assert!(missing.verify_handle(&handle).await.is_err()); - - let valid = service_with_network(fake_network( - DnsAnswer::Found, - "did:web:bob.example.com", - Some("at://bob.example.com"), - )); - assert_eq!( - valid.verify_handle(&handle).await.unwrap().as_str(), - "did:web:bob.example.com" - ); - - let mismatched = service_with_network(fake_network( - DnsAnswer::Found, - "did:web:bob.example.com", - Some("at://bob.example.com"), - )); - *mismatched.network.lock().unwrap() = Some(fake_network( - DnsAnswer::Found, - "did:web:mallory.example.com", - Some("at://bob.example.com"), - )); - assert!( - mismatched - .resolve_did(&"did:web:bob.example.com".parse().unwrap()) - .await - .is_err() - ); - } - - #[tokio::test] - async fn production_gate_marks_only_the_configured_bidirectional_identity_verified() { - let valid = production_service_with_network(fake_network( - DnsAnswer::Found, - "did:web:alice.example.com", - Some("at://alice.example.com"), - )); - assert!(!valid.local_handle_verified()); - valid.verify_local_handle().await.unwrap(); - assert!(valid.local_handle_verified()); - - let wrong_did = production_service_with_network(fake_network( - DnsAnswer::Found, - "did:web:mallory.example.com", - Some("at://alice.example.com"), - )); - assert!(wrong_did.verify_local_handle().await.is_err()); - assert!(!wrong_did.local_handle_verified()); - } - - #[tokio::test] - async fn revalidation_distinguishes_confirmed_invalidity_from_transient_failure() { - let service = production_service_with_network(fake_network( - DnsAnswer::Found, - "did:web:alice.example.com", - Some("at://alice.example.com"), - )); - assert_eq!( - service.revalidate_local_handle().await.unwrap(), - LocalHandleStatus::Valid - ); - assert!(service.local_handle_verified()); - - *service.network.lock().unwrap() = Some(fake_network( - DnsAnswer::Unavailable, - "did:web:alice.example.com", - Some("at://alice.example.com"), - )); - assert!(service.revalidate_local_handle().await.is_err()); - assert!(service.local_handle_verified()); - - *service.network.lock().unwrap() = Some(fake_network( - DnsAnswer::Found, - "did:web:mallory.example.com", - Some("at://alice.example.com"), - )); - assert_eq!( - service.revalidate_local_handle().await.unwrap(), - LocalHandleStatus::Invalid - ); - assert!(!service.local_handle_verified()); - - *service.network.lock().unwrap() = Some(fake_network( - DnsAnswer::Found, - "did:web:alice.example.com", - Some("at://alice.example.com"), - )); - assert_eq!( - service.revalidate_local_handle().await.unwrap(), - LocalHandleStatus::Valid - ); - assert!(service.local_handle_verified()); - } - - #[tokio::test] - async fn hosted_identity_revalidation_emits_only_confirmed_state_changes() { - let mut state = AppState::assemble( - production_config(), - Database::open_in_memory().unwrap(), - Arc::new(MemoryBlobStore::new()), - Arc::new(NoopEventSink), - ) - .unwrap(); - Arc::get_mut(&mut state.config).unwrap().deployment_mode = DeploymentMode::Production; - state.identity = Arc::new(production_service_with_network(fake_network( - DnsAnswer::Found, - "did:web:alice.example.com", - Some("at://alice.example.com"), - ))); - - assert_eq!( - state.revalidate_network_identity().await.unwrap(), - IdentityRevalidation::Unchanged - ); - *state.identity.network.lock().unwrap() = Some(fake_network( - DnsAnswer::Found, - "did:web:mallory.example.com", - Some("at://alice.example.com"), - )); - assert_eq!( - state.revalidate_network_identity().await.unwrap(), - IdentityRevalidation::Invalidated - ); - assert_eq!( - state - .repository - .account_status("did:web:alice.example.com") - .unwrap(), - crate::domain::account::AccountStatus::Active - ); - let mut request = AxumRequest::get( - "/xrpc/com.atproto.repo.describeRepo?repo=did%3Aweb%3Aalice.example.com", - ) - .body(Body::empty()) - .unwrap(); - request.extensions_mut().insert(ConnectInfo( - "192.0.2.1:12345".parse::().unwrap(), - )); - let response = crate::http::router(state.clone()) - .oneshot(request) - .await - .unwrap(); - assert!(response.status().is_success()); - let body: serde_json::Value = - serde_json::from_slice(&to_bytes(response.into_body(), usize::MAX).await.unwrap()) - .unwrap(); - assert_eq!(body["handle"], "handle.invalid"); - assert_eq!(body["didDoc"]["alsoKnownAs"][0], "at://alice.example.com"); - assert_eq!(body["handleIsCorrect"], false); - - *state.identity.network.lock().unwrap() = Some(fake_network( - DnsAnswer::Unavailable, - "did:web:alice.example.com", - Some("at://alice.example.com"), - )); - assert!(state.revalidate_network_identity().await.is_err()); - assert_eq!( - state - .repository - .repo_messages_after(0, 10) - .await - .unwrap() - .len(), - 1 - ); - - *state.identity.network.lock().unwrap() = Some(fake_network( - DnsAnswer::Found, - "did:web:alice.example.com", - Some("at://alice.example.com"), - )); - assert_eq!( - state.revalidate_network_identity().await.unwrap(), - IdentityRevalidation::Recovered - ); - let messages = state.repository.repo_messages_after(0, 10).await.unwrap(); - assert!(matches!( - messages.as_slice(), - [ - atrium_api::com::atproto::sync::subscribe_repos::Message::Identity(invalid), - atrium_api::com::atproto::sync::subscribe_repos::Message::Identity(recovered), - ] if invalid.handle.as_ref().map(Handle::as_str) == Some("handle.invalid") - && recovered.handle.as_ref().map(Handle::as_str) == Some("alice.example.com") - )); - } - - fn service_with_network(network: Arc) -> IdentityService { - let mut service = IdentityService::new( - document("did:web:alice.example.com", Some("at://alice.example.com")), - DeploymentMode::Local, - ) - .unwrap(); - service.network = Mutex::new(Some(network)); - service - } - - fn production_service_with_network(network: Arc) -> IdentityService { - let mut service = service_with_network(network); - service.local_resolution = false; - service - .local_handle_verified - .store(false, Ordering::Release); - service - } - - fn fake_network(dns: DnsAnswer, did: &str, alias: Option<&str>) -> Arc { - let did = did.parse::().unwrap(); - Arc::new(FakeNetwork { - dns, - document: document(did.as_str(), alias), - did, - https_calls: AtomicUsize::new(0), - }) - } - - fn production_config() -> Config { - Config { - deployment_mode: DeploymentMode::Local, - listen_address: "127.0.0.1:3000".parse().unwrap(), - public_origin: PublicOrigin::parse( - "https://alice.example.com", - DeploymentMode::Production, - ) - .unwrap(), - database_path: ":memory:".into(), - blob_directory: "unused".into(), - blob_orphan_ttl: Duration::from_secs(86_400), - blob_gc_interval: Duration::from_secs(3_600), - identity_revalidation_interval: Duration::from_secs(3_600), - repo_event_retention: 100_000, - repo_stream_send_timeout: Duration::from_secs(10), - relay_origins: Vec::new(), - account: AccountConfig { - did: "did:web:alice.example.com".to_owned(), - handle: "alice.example.com".to_owned(), - password: "correct horse battery staple".to_owned(), - signing_key: [7; 32], - }, - oauth_clients: Vec::new(), - } - } - - fn document(did: &str, alias: Option<&str>) -> DidDocument { - DidDocument { - context: None, - id: did.to_owned(), - also_known_as: alias.map(|alias| vec![alias.to_owned()]), - verification_method: None, - service: None, - } - } -} +mod tests; diff --git a/src/identity/http.rs b/src/identity/http.rs new file mode 100644 index 0000000..808103e --- /dev/null +++ b/src/identity/http.rs @@ -0,0 +1,295 @@ +//! One outbound policy for identity documents and authenticated Space requests. + +use std::{error::Error as StdError, net::SocketAddr, sync::Arc, time::Duration}; + +use atrium_xrpc::{ + HttpClient, + http::{Method, Request, Response, StatusCode, header}, +}; +use hickory_resolver::{ + TokioResolver, + proto::access_control::{AccessControlSet, AccessControlSetBuilder}, +}; +use reqwest::dns::{Addrs, Name, Resolve, Resolving}; +use serde::de::DeserializeOwned; +use tokio::sync::OnceCell; +use url::{Host, Url}; + +use crate::{ + config::DeploymentMode, + error::{Error, ErrorContext as _, Result}, +}; + +const MAX_DOCUMENT_BYTES: usize = 256 * 1024; +const MAX_REDIRECTS: usize = 3; +const HTTP_TIMEOUT: Duration = Duration::from_secs(10); +// Exclude private/special-purpose destinations and IP translation routes that +// could reach a denied IPv4 address through an otherwise acceptable IPv6 address. +const BLOCKED_ADDRESS_RANGES: &[&str] = &[ + "0.0.0.0/8", + "10.0.0.0/8", + "100.64.0.0/10", + "127.0.0.0/8", + "169.254.0.0/16", + "172.16.0.0/12", + "192.0.0.0/24", + "192.0.2.0/24", + "192.88.99.0/24", + "192.168.0.0/16", + "198.18.0.0/15", + "198.51.100.0/24", + "203.0.113.0/24", + "224.0.0.0/4", + "240.0.0.0/4", + "::/96", + "::ffff:0:0/96", + "64:ff9b::/96", + "64:ff9b:1::/48", + "100::/64", + "100:0:0:1::/64", + "2001::/32", + "2001:2::/48", + "2001:10::/28", + "2001:db8::/32", + "2002::/16", + "3fff::/20", + "5f00::/16", + "fc00::/7", + "fe80::/10", + "fec0::/10", + "ff00::/8", +]; + +pub(crate) struct OutboundHttp { + client: reqwest::Client, + mode: DeploymentMode, +} + +impl OutboundHttp { + pub(super) fn new(dns: Option, mode: DeploymentMode) -> Result { + let client = reqwest::Client::builder() + .connect_timeout(Duration::from_secs(5)) + .timeout(HTTP_TIMEOUT) + .redirect(reqwest::redirect::Policy::none()) + // An ambient proxy would resolve destinations outside our address policy. + .no_proxy() + .dns_resolver(Arc::new(PublicDnsResolver::new(dns)?)) + .user_agent(concat!("cpds/", env!("CARGO_PKG_VERSION"))) + .build() + .context(Error::Configuration, "outbound HTTP client")?; + Ok(Self { client, mode }) + } + + pub(crate) fn validate_url(&self, url: &Url) -> Result<()> { + validate_url(url, self.mode).map_err(|message| Error::Upstream(message.to_owned())) + } + + pub(crate) async fn get_json( + &self, + url: Url, + bearer: Option<&str>, + ) -> Result { + self.validate_url(&url)?; + let mut request = self + .client + .get(url) + .header(header::ACCEPT, "application/json"); + if let Some(token) = bearer { + request = request.bearer_auth(token); + } + let response = request.send().await.map_err(upstream)?; + require_success(&response)?; + let response = read_document(response).await?; + serde_json::from_slice(response.body()).map_err(upstream) + } + + pub(crate) async fn post_json( + &self, + url: Url, + bearer: &str, + body: &serde_json::Value, + ) -> Result<()> { + self.validate_url(&url)?; + let response = self + .client + .post(url) + .bearer_auth(bearer) + .json(body) + .send() + .await + .map_err(upstream)?; + require_success(&response) + } +} + +impl HttpClient for OutboundHttp { + async fn send_http( + &self, + request: Request>, + ) -> std::result::Result>, Box> { + tokio::time::timeout(HTTP_TIMEOUT, async move { + if request.method() != Method::GET + || !request.body().is_empty() + || request.headers().contains_key(header::AUTHORIZATION) + || request.headers().contains_key(header::COOKIE) + { + return Err(Error::Upstream( + "identity transport accepts only unauthenticated empty GET requests".to_owned(), + ) + .into()); + } + let mut url = Url::parse(&request.uri().to_string())?; + // Only public identity discovery follows redirects, never authenticated requests. + for _ in 0..=MAX_REDIRECTS { + validate_https_url(&url).map_err(|message| Error::Upstream(message.to_owned()))?; + let response = self + .client + .get(url.clone()) + .headers(request.headers().clone()) + .send() + .await?; + if matches!( + response.status(), + StatusCode::MOVED_PERMANENTLY + | StatusCode::FOUND + | StatusCode::SEE_OTHER + | StatusCode::TEMPORARY_REDIRECT + | StatusCode::PERMANENT_REDIRECT + ) && let Some(location) = response.headers().get(header::LOCATION) + { + url = url.join(location.to_str()?)?; + continue; + } + return read_document(response).await.map_err(Into::into); + } + Err(Error::Upstream("identity redirect limit exceeded".to_owned()).into()) + }) + .await + .map_err(|_| Error::Upstream("identity request timed out".to_owned()))? + } +} + +async fn read_document(mut response: reqwest::Response) -> Result>> { + if response + .content_length() + .is_some_and(|length| length > MAX_DOCUMENT_BYTES as u64) + { + return Err(document_too_large()); + } + let status = response.status(); + let headers = response.headers().clone(); + let mut body = Vec::new(); + while let Some(chunk) = response.chunk().await.map_err(upstream)? { + if body.len().saturating_add(chunk.len()) > MAX_DOCUMENT_BYTES { + return Err(document_too_large()); + } + body.extend_from_slice(&chunk); + } + let mut response = Response::new(body); + *response.status_mut() = status; + *response.headers_mut() = headers; + Ok(response) +} + +fn require_success(response: &reqwest::Response) -> Result<()> { + if response.status().is_success() { + return Ok(()); + } + Err(Error::Upstream(format!( + "outbound request returned {}", + response.status() + ))) +} + +fn upstream(error: impl std::fmt::Display) -> Error { + Error::Upstream(format!("outbound request failed: {error}")) +} + +fn document_too_large() -> Error { + Error::Upstream("remote document exceeds the byte limit".to_owned()) +} + +pub(super) fn validate_https_url(url: &Url) -> std::result::Result<(), &'static str> { + validate_url(url, DeploymentMode::Production) +} + +fn validate_url(url: &Url, mode: DeploymentMode) -> std::result::Result<(), &'static str> { + let transport_allowed = match (url.host(), mode, url.scheme()) { + (Some(Host::Domain(_)), _, "https") => true, + (Some(Host::Ipv4(ip)), DeploymentMode::Local, "http" | "https") => ip.is_loopback(), + (Some(Host::Ipv6(ip)), DeploymentMode::Local, "http" | "https") => ip.is_loopback(), + _ => false, + }; + if !transport_allowed + || !url.username().is_empty() + || url.password().is_some() + || url.fragment().is_some() + { + return Err( + "outbound URL requires credential-free public HTTPS; local mode also permits literal loopback HTTP(S)", + ); + } + Ok(()) +} + +pub(super) fn dns_resolver() -> Result { + let mut builder = + TokioResolver::builder_tokio().context(Error::Configuration, "system DNS configuration")?; + let options = builder.options_mut(); + options.timeout = Duration::from_secs(3); + options.attempts = 2; + options.cache_size = 256; + options.positive_max_ttl = Some(Duration::from_secs(300)); + options.negative_max_ttl = Some(Duration::from_secs(60)); + builder + .build() + .context(Error::Configuration, "DNS resolver") +} + +struct PublicDnsResolver { + dns: Arc>, + addresses: Arc, +} + +impl PublicDnsResolver { + fn new(dns: Option) -> Result { + let blocked = BLOCKED_ADDRESS_RANGES + .iter() + .map(|network| network.parse()) + .collect::, _>>() + .context(Error::Internal, "invalid blocked address range")?; + let addresses = AccessControlSetBuilder::new("outbound_http") + .deny(blocked.iter()) + .build() + .map_err(upstream)?; + Ok(Self { + dns: Arc::new(OnceCell::new_with(dns)), + addresses: Arc::new(addresses), + }) + } +} + +impl Resolve for PublicDnsResolver { + fn resolve(&self, name: Name) -> Resolving { + let dns = self.dns.clone(); + let addresses = self.addresses.clone(); + let name = format!("{}.", name.as_str().trim_end_matches('.')); + Box::pin(async move { + let resolver = dns.get_or_try_init(|| async { dns_resolver() }).await?; + let lookup = resolver.lookup_ip(name).await?; + // Filter at the socket boundary, including cached and hosts-file answers. + let addresses = lookup + .iter() + .filter(|ip| !addresses.denied(*ip)) + .map(|ip| SocketAddr::new(ip, 0)) + .collect::>(); + if addresses.is_empty() { + return Err(Error::Upstream("DNS returned no public addresses".to_owned()).into()); + } + Ok(Box::new(addresses.into_iter()) as Addrs) + }) + } +} + +#[cfg(test)] +mod tests; diff --git a/src/identity/http/tests.rs b/src/identity/http/tests.rs new file mode 100644 index 0000000..93db938 --- /dev/null +++ b/src/identity/http/tests.rs @@ -0,0 +1,398 @@ +use std::{ + io, + sync::atomic::{AtomicUsize, Ordering}, +}; + +use axum::{ + Router, + body::{Body, Bytes}, + http::StatusCode, + response::Redirect, + routing::{any, get}, +}; +use futures::{StreamExt as _, stream}; +use hickory_resolver::{Hosts, config::ResolverConfig, net::runtime::TokioRuntimeProvider}; +use serde_json::{Value, json}; + +use super::*; + +fn previous_url_policy(url: &Url, mode: DeploymentMode) -> std::result::Result<(), &'static str> { + let transport_allowed = match url.host() { + Some(Host::Domain(_)) => url.scheme() == "https", + Some(Host::Ipv4(ip)) => { + mode == DeploymentMode::Local + && ip.is_loopback() + && matches!(url.scheme(), "http" | "https") + } + Some(Host::Ipv6(ip)) => { + mode == DeploymentMode::Local + && ip.is_loopback() + && matches!(url.scheme(), "http" | "https") + } + None => false, + }; + if !transport_allowed + || !url.username().is_empty() + || url.password().is_some() + || url.fragment().is_some() + { + return Err( + "outbound URL requires credential-free public HTTPS; local mode also permits literal loopback HTTP(S)", + ); + } + Ok(()) +} + +#[test] +fn native_outbound_transport_match_preserves_complete_url_policy() { + for host in [ + "0.0.0.0", + "126.255.255.255", + "127.0.0.0", + "127.0.0.1", + "127.1.2.3", + "127.255.255.255", + "128.0.0.0", + "10.0.0.1", + "169.254.169.254", + "8.8.8.8", + "2130706433", + "0x7f000001", + "0177.0.0.1", + "127.1", + "[::]", + "[::1]", + "[::2]", + "[0:0:0:0:0:0:0:1]", + "[::ffff:127.0.0.1]", + "[::127.0.0.1]", + "[::ffff:7f00:1]", + "[2001:db8::1]", + "example.test", + "localhost", + "127.0.0.1.example.test", + "127.0.0.1.", + "bücher.example", + "xn--bcher-kva.example", + ] { + for scheme in ["https", "http", "ftp", "ws", "wss", "custom"] { + for userinfo in ["", "@", "user@", "user:password@", ":password@", ":@"] { + for suffix in [ + "doc", + "doc?query=yes", + "doc?", + "doc#fragment", + "doc#", + "doc?query=yes#fragment", + ] { + let url = Url::parse(&format!("{scheme}://{userinfo}{host}/{suffix}")).unwrap(); + for mode in [DeploymentMode::Local, DeploymentMode::Production] { + assert_eq!( + validate_url(&url, mode), + previous_url_policy(&url, mode), + "{url}, {mode:?}", + ); + } + } + } + } + } + for value in [ + "file:///tmp", + "mailto:alice@example.test", + "data:text/plain,hello", + "urn:example:test", + ] { + let url = Url::parse(value).unwrap(); + for mode in [DeploymentMode::Local, DeploymentMode::Production] { + assert_eq!( + validate_url(&url, mode), + previous_url_policy(&url, mode), + "{value}" + ); + } + } +} + +#[test] +fn production_and_local_urls_have_an_explicit_loopback_boundary() { + let production = OutboundHttp::new(None, DeploymentMode::Production).unwrap(); + let local = OutboundHttp::new(None, DeploymentMode::Local).unwrap(); + for value in ["http://127.0.0.1:3000/doc", "https://[::1]/doc"] { + let url = Url::parse(value).unwrap(); + assert!(production.validate_url(&url).is_err()); + assert!(local.validate_url(&url).is_ok()); + } + for value in [ + "http://example.com/doc", + "https://10.0.0.1/doc", + "https://169.254.169.254/doc", + "https://[::ffff:127.0.0.1]/doc", + "https://user@example.com/doc", + "https://example.com/doc#fragment", + "file:///etc/passwd", + ] { + let url = Url::parse(value).unwrap(); + assert!(production.validate_url(&url).is_err(), "{value}"); + assert!(local.validate_url(&url).is_err(), "{value}"); + } + assert!( + production + .validate_url(&Url::parse("https://example.com/doc").unwrap()) + .is_ok() + ); +} + +#[tokio::test] +async fn identity_transport_rejects_credentials_and_mutations_before_dispatch() { + let http = OutboundHttp::new(None, DeploymentMode::Production).unwrap(); + let uri = "https://example.com/did.json"; + for request in [ + Request::post(uri).body(Vec::new()).unwrap(), + Request::get(uri).body(vec![1]).unwrap(), + Request::get(uri) + .header(header::AUTHORIZATION, "Bearer secret") + .body(Vec::new()) + .unwrap(), + Request::get(uri) + .header(header::COOKIE, "session=secret") + .body(Vec::new()) + .unwrap(), + ] { + let error = http.send_http(request).await.unwrap_err(); + assert!(error.to_string().contains("unauthenticated empty GET")); + } +} + +#[tokio::test] +async fn identity_transport_preserves_guard_parse_and_policy_error_priority() { + for mode in [DeploymentMode::Production, DeploymentMode::Local] { + let http = OutboundHttp::new(None, mode).unwrap(); + for request in [ + Request::post("/did.json").body(Vec::new()).unwrap(), + Request::get("/did.json").body(vec![1]).unwrap(), + Request::get("/did.json") + .header(header::AUTHORIZATION, "Bearer secret") + .body(Vec::new()) + .unwrap(), + Request::get("/did.json") + .header(header::COOKIE, "session=secret") + .body(Vec::new()) + .unwrap(), + ] { + let error = http.send_http(request).await.unwrap_err(); + assert!(matches!( + error.downcast_ref::(), + Some(Error::Upstream(_)) + )); + assert_eq!( + error.to_string(), + "upstream service error: identity transport accepts only unauthenticated empty GET requests" + ); + } + + let error = http + .send_http(Request::get("/did.json").body(Vec::new()).unwrap()) + .await + .unwrap_err(); + assert_eq!( + error.downcast_ref::(), + Some(&url::ParseError::RelativeUrlWithoutBase) + ); + assert_eq!(error.to_string(), "relative URL without a base"); + + let error = http + .send_http( + Request::get("http://127.0.0.1/did.json") + .body(Vec::new()) + .unwrap(), + ) + .await + .unwrap_err(); + assert!(matches!( + error.downcast_ref::(), + Some(Error::Upstream(_)) + )); + assert_eq!( + error.to_string(), + "upstream service error: outbound URL requires credential-free public HTTPS; local mode also permits literal loopback HTTP(S)" + ); + } +} + +#[tokio::test] +async fn dns_filters_synthesized_and_hosts_file_addresses_before_connecting() { + let mut dns = TokioResolver::builder_with_config( + ResolverConfig::default(), + TokioRuntimeProvider::default(), + ) + .build() + .unwrap(); + let mut hosts = Hosts::default(); + hosts + .read_hosts_conf( + b"10.0.0.1 private.example\n169.254.169.254 metadata.example\n8.8.8.8 public.example\n" + .as_slice(), + ) + .unwrap(); + dns.set_hosts(Arc::new(hosts)); + assert!( + dns.lookup_ip("private.example") + .await + .unwrap() + .iter() + .any(|ip| ip == "10.0.0.1".parse::().unwrap()) + ); + let resolver = PublicDnsResolver::new(Some(dns)).unwrap(); + for name in ["localhost", "private.example", "metadata.example"] { + assert!( + resolver.resolve(name.parse().unwrap()).await.is_err(), + "{name}" + ); + } + let addresses = resolver + .resolve("public.example".parse().unwrap()) + .await + .unwrap() + .collect::>(); + assert_eq!(addresses, vec!["8.8.8.8:0".parse::().unwrap()]); +} + +#[test] +fn non_public_and_ip_translation_ranges_are_denied() { + let resolver = PublicDnsResolver::new(None).unwrap(); + for ip in [ + "0.0.0.1", + "10.1.2.3", + "100.64.0.1", + "127.0.0.1", + "169.254.169.254", + "172.16.1.1", + "192.168.1.1", + "224.0.0.1", + "::1", + "::127.0.0.1", + "::ffff:127.0.0.1", + "64:ff9b::7f00:1", + "64:ff9b:1::1", + "2001::1", + "2002:7f00:1::1", + "3fff::1", + "5f00::1", + "fd00::1", + "fe80::1", + "fec0::1", + ] { + assert!(resolver.addresses.denied(ip.parse().unwrap()), "{ip}"); + } + assert!(!resolver.addresses.denied("8.8.8.8".parse().unwrap())); + assert!( + !resolver + .addresses + .denied("2606:4700:4700::1111".parse().unwrap()) + ); +} + +#[tokio::test] +async fn documents_are_bounded_before_the_remote_stream_finishes() { + let router = Router::new() + .route( + "/exact", + get(|| async { + let document = format!("\"{}\"", "x".repeat(MAX_DOCUMENT_BYTES - 2)); + assert_eq!(document.len(), MAX_DOCUMENT_BYTES); + Body::from_stream(stream::once(async { Ok::<_, io::Error>(document) })) + }), + ) + .route( + "/large", + get(|| async { + let first_chunk = stream::once(async { + Ok::<_, io::Error>(Bytes::from(vec![b' '; MAX_DOCUMENT_BYTES + 1])) + }); + Body::from_stream(first_chunk.chain(stream::pending())) + }), + ); + let (url, server) = serve(router).await; + let http = OutboundHttp::new(None, DeploymentMode::Local).unwrap(); + let exact: String = http + .get_json(url.join("exact").unwrap(), None) + .await + .unwrap(); + assert_eq!(exact.len(), MAX_DOCUMENT_BYTES - 2); + let result = tokio::time::timeout( + Duration::from_secs(2), + http.get_json::(url.join("large").unwrap(), None), + ) + .await; + server.abort(); + let error = result + .expect("must stop without waiting for EOF") + .unwrap_err(); + assert!(error.to_string().contains("byte limit"), "{error}"); +} + +#[tokio::test] +async fn declared_oversize_documents_are_rejected_without_waiting_for_body_bytes() { + let router = Router::new().route( + "/large", + get(|| async { + ( + [(header::CONTENT_LENGTH, (MAX_DOCUMENT_BYTES + 1).to_string())], + Body::from_stream(stream::pending::>()), + ) + }), + ); + let (url, server) = serve(router).await; + let http = OutboundHttp::new(None, DeploymentMode::Local).unwrap(); + let result = tokio::time::timeout( + Duration::from_secs(2), + http.get_json::(url.join("large").unwrap(), None), + ) + .await; + server.abort(); + let error = result + .expect("must stop after reading response headers") + .unwrap_err(); + assert!(error.to_string().contains("byte limit"), "{error}"); +} + +#[tokio::test] +async fn credentials_and_notifications_never_follow_redirects() { + let destination_calls = Arc::new(AtomicUsize::new(0)); + let calls = destination_calls.clone(); + let router = Router::new() + .route( + "/redirect", + get(|| async { Redirect::temporary("/destination") }) + .post(|| async { Redirect::temporary("/destination") }), + ) + .route( + "/destination", + any(move || { + calls.fetch_add(1, Ordering::SeqCst); + async { StatusCode::NO_CONTENT } + }), + ); + let (url, server) = serve(router).await; + let http = OutboundHttp::new(None, DeploymentMode::Local).unwrap(); + assert!( + http.get_json::(url.join("redirect").unwrap(), Some("secret")) + .await + .is_err() + ); + assert!( + http.post_json(url.join("redirect").unwrap(), "secret", &json!({})) + .await + .is_err() + ); + assert_eq!(destination_calls.load(Ordering::SeqCst), 0); + server.abort(); +} + +async fn serve(router: Router) -> (Url, tokio::task::JoinHandle<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = Url::parse(&format!("http://{}/", listener.local_addr().unwrap())).unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + (url, server) +} diff --git a/src/identity/tests.rs b/src/identity/tests.rs new file mode 100644 index 0000000..e533e7a --- /dev/null +++ b/src/identity/tests.rs @@ -0,0 +1,628 @@ +use std::{net::SocketAddr, sync::atomic::AtomicUsize, time::Duration}; + +use crate::{ + AppState, IdentityRevalidation, + blob_store::MemoryBlobStore, + config::{AccountConfig, Config, HttpOrigin}, + database::Database, + events::NoopEventSink, +}; +use axum::{ + body::{Body, to_bytes}, + extract::ConnectInfo, + http::Request as AxumRequest, +}; +use tower::ServiceExt as _; + +use super::*; + +#[derive(Clone, Copy)] +enum DnsAnswer { + Found, + Missing, + Failure, + Unavailable, + Conflict, +} + +struct FakeNetwork { + dns: DnsAnswer, + did: Did, + document: DidDocument, + https_calls: AtomicUsize, +} + +#[async_trait::async_trait] +impl IdentityNetwork for FakeNetwork { + async fn resolve_dns_handle(&self, _handle: &Handle) -> Result> { + match self.dns { + DnsAnswer::Found => Ok(Some(self.did.clone())), + DnsAnswer::Missing => Ok(None), + DnsAnswer::Failure | DnsAnswer::Unavailable => { + Err(Error::Upstream("DNS unavailable".to_owned())) + } + DnsAnswer::Conflict => Err(handle_not_found("conflicting DNS records")), + } + } + + async fn resolve_https_handle(&self, _handle: &Handle) -> Result { + self.https_calls.fetch_add(1, Ordering::SeqCst); + if matches!(self.dns, DnsAnswer::Unavailable) { + return Err(Error::Upstream("HTTPS unavailable".to_owned())); + } + Ok(self.did.clone()) + } + + async fn resolve_did(&self, _did: &Did) -> Result { + Ok(self.document.clone()) + } +} + +#[tokio::test] +async fn concrete_network_dispatch_keeps_web_validation_and_unsupported_method_errors() { + let network = NetworkIdentity::new().unwrap(); + let resolver: &dyn IdentityNetwork = &network; + for (value, expected) in [ + ( + "did:web:example.com:%FF", + "invalid percent encoding in did:web", + ), + ( + "did:web:example.com:%2F", + "did:web contains an invalid path segment", + ), + ( + "did:web:127.0.0.1", + "outbound URL requires credential-free public HTTPS; local mode also permits literal loopback HTTP(S)", + ), + ] { + let did = value.parse().unwrap(); + assert!( + matches!( + resolver.resolve_did(&did).await, + Err(Error::Upstream(message)) if message == expected + ), + "{value}" + ); + } + let did: Did = "did:example:alice".parse().unwrap(); + let expected = format!( + "DID resolution failed: {}", + atrium_identity::Error::UnsupportedDidMethod(did.clone()) + ); + assert!(matches!( + resolver.resolve_did(&did).await, + Err(Error::Upstream(message)) if message == expected + )); +} + +#[tokio::test] +async fn configured_mode_keeps_local_handle_and_outbound_policy_in_sync() { + let network = fake_network(DnsAnswer::Found, "did:web:bob.example.com", None); + let mut service = service_with_network(network); + let handle = "alice.example.com".parse().unwrap(); + let loopback = Url::parse("http://127.0.0.1:3000/doc").unwrap(); + for (mode, expected_did, allows_loopback) in [ + (DeploymentMode::Local, "did:web:alice.example.com", true), + (DeploymentMode::Production, "did:web:bob.example.com", false), + ] { + service.mode = mode; + assert_eq!( + service.resolve_handle(&handle).await.unwrap().as_str(), + expected_did + ); + assert_eq!( + service + .outbound_http() + .unwrap() + .validate_url(&loopback) + .is_ok(), + allows_loopback + ); + } +} + +#[test] +fn dns_handle_selection_ignores_noise_and_rejects_conflicts() { + let selected = select_dns_handle([ + b"unrelated=value".to_vec(), + b"did=not-a-did".to_vec(), + b"did=did:plc:ewvi7nxzyoun6zhxrhs64oiz".to_vec(), + b"did=did:plc:ewvi7nxzyoun6zhxrhs64oiz".to_vec(), + ]) + .unwrap() + .unwrap(); + assert_eq!(selected.as_str(), "did:plc:ewvi7nxzyoun6zhxrhs64oiz"); + + assert!( + select_dns_handle([ + b"did=did:plc:ewvi7nxzyoun6zhxrhs64oiz".to_vec(), + b"did=did:web:other.example".to_vec(), + ]) + .is_err() + ); +} + +#[test] +fn web_dids_map_to_their_authoritative_documents() { + assert_eq!( + did_web_url(&"did:web:example.com".parse().unwrap()) + .unwrap() + .as_str(), + "https://example.com/.well-known/did.json" + ); + assert_eq!( + did_web_url(&"did:web:localhost%3A8443:users:alice".parse().unwrap()) + .unwrap() + .as_str(), + "https://localhost:8443/users/alice/did.json" + ); +} + +#[test] +fn native_txt_concatenation_preserves_binary_parts_and_did_selection() { + use hickory_resolver::proto::rr::rdata::TXT; + + let values: &[&[u8]] = &[ + b"", + b"unrelated=value", + b"did=not-a-did", + b"did=did:web:alice.example.com", + b"did=did:plc:ewvi7nxzyoun6zhxrhs64oiz", + b"did=did:web:caf\xc3\xa9.example", + b"did=did:web:invalid\xff.example", + b"\x00did=did:web:alice.example.com\x00", + ]; + for value in values { + for split in 0..=value.len() { + let txt = TXT::from_bytes(vec![&[], &value[..split], &[], &value[split..], &[]]); + let previous: Vec<_> = txt + .txt_data + .iter() + .flat_map(|part| part.iter().copied()) + .collect(); + let current = txt.txt_data.concat(); + assert_eq!(current, *value); + assert_eq!(current, previous); + let select = |bytes| { + select_dns_handle([bytes]) + .map(|did| did.map(|did| did.to_string())) + .map_err(|error| format!("{error:?}")) + }; + assert_eq!(select(current), select(previous)); + } + } + assert!(TXT::from_bytes(Vec::new()).txt_data.concat().is_empty()); +} + +#[test] +fn borrowed_web_segments_preserve_urls_and_error_precedence() { + fn previous_owned_url(did: &Did) -> Result { + let method_id = did + .as_str() + .strip_prefix("did:web:") + .ok_or_else(|| Error::Upstream("DID is not did:web".to_owned()))?; + let segments = method_id + .split(':') + .map(|segment| { + percent_decode_str(segment) + .decode_utf8() + .map(String::from) + .map_err(|_| Error::Upstream("invalid percent encoding in did:web".to_owned())) + }) + .collect::>>()?; + let (host, path) = segments + .split_first() + .ok_or_else(|| Error::Upstream("did:web has no host".to_owned()))?; + if segments + .iter() + .any(|segment| segment.is_empty() || segment.contains(['/', '?', '#'])) + { + return Err(Error::Upstream( + "did:web contains an invalid path segment".to_owned(), + )); + } + let suffix = if path.is_empty() { + "/.well-known/did.json".to_owned() + } else { + format!("/{}/did.json", path.join("/")) + }; + let url = Url::parse(&format!("https://{host}{suffix}")) + .context(Error::Upstream, "invalid did:web URL")?; + validate_https_url(&url).map_err(|error| Error::Upstream(error.to_owned()))?; + Ok(url) + } + + let mut inputs = [ + "did:example:alice", + "did:web::example.com", + "did:web:::example.com", + "did:web::%FF", + "did:web:example.com::alice", + "did:web:example.com:%00:alice", + ] + .map(str::to_owned) + .to_vec(); + for host in [ + "example.com", + "localhost%3A8443", + "127.0.0.1", + "user%40example.com", + "%5B%3A%3A1%5D", + "%FF", + ] { + for suffix in [ + "", + ":users:alice", + ":caf%C3%A9", + ":%2F", + ":%3F", + ":%23", + ":%00", + ":%20", + ":%25", + ":%FF", + ":%2F:%FF", + ":%FF:%2F", + ] { + inputs.push(format!("did:web:{host}{suffix}")); + } + } + for input in inputs { + let did = input.parse().unwrap(); + let shape = |result: Result| { + result + .map(String::from) + .map_err(|error| format!("{error:?}")) + }; + assert_eq!( + shape(did_web_url(&did)), + shape(previous_owned_url(&did)), + "{input}" + ); + } +} + +#[test] +fn bidirectional_validation_requires_an_explicit_handle_claim() { + let handle = "alice.example.com".parse::().unwrap(); + let mut document = DidDocument { + context: None, + id: "did:web:alice.example.com".to_owned(), + also_known_as: None, + verification_method: None, + service: None, + }; + assert!(!document_claims_handle(&document, &handle)); + document.also_known_as = Some(vec!["at://other.example.com".to_owned()]); + assert!(!document_claims_handle(&document, &handle)); + document.also_known_as = Some(vec!["at://alice.example.com".to_owned()]); + assert!(document_claims_handle(&document, &handle)); +} + +#[tokio::test] +async fn resolver_prefers_dns_and_falls_back_only_for_absence_or_failure() { + for (dns, expected_https_calls) in [ + (DnsAnswer::Found, 0), + (DnsAnswer::Missing, 1), + (DnsAnswer::Failure, 1), + ] { + let network = fake_network(dns, "did:web:bob.example.com", None); + let service = service_with_network(network.clone()); + let resolved = service + .resolve_handle(&"bob.example.com".parse().unwrap()) + .await + .unwrap(); + assert_eq!(resolved.as_str(), "did:web:bob.example.com"); + assert_eq!( + network.https_calls.load(Ordering::SeqCst), + expected_https_calls + ); + } + + let network = fake_network(DnsAnswer::Conflict, "did:web:bob.example.com", None); + let service = service_with_network(network.clone()); + assert!( + service + .resolve_handle(&"bob.example.com".parse().unwrap()) + .await + .is_err() + ); + assert_eq!(network.https_calls.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn bidirectional_resolution_rejects_missing_backlinks_and_mismatched_documents() { + let handle = "bob.example.com".parse::().unwrap(); + let missing = service_with_network(fake_network( + DnsAnswer::Found, + "did:web:bob.example.com", + None, + )); + assert!(missing.verify_handle(&handle).await.is_err()); + + let valid = service_with_network(fake_network( + DnsAnswer::Found, + "did:web:bob.example.com", + Some("at://bob.example.com"), + )); + assert_eq!( + valid.verify_handle(&handle).await.unwrap().as_str(), + "did:web:bob.example.com" + ); + + let mismatched = service_with_network(fake_network( + DnsAnswer::Found, + "did:web:bob.example.com", + Some("at://bob.example.com"), + )); + *mismatched.network.lock().unwrap() = Some(fake_network( + DnsAnswer::Found, + "did:web:mallory.example.com", + Some("at://bob.example.com"), + )); + assert!( + mismatched + .resolve_did(&"did:web:bob.example.com".parse().unwrap()) + .await + .is_err() + ); +} + +#[tokio::test] +async fn production_gate_marks_only_the_configured_bidirectional_identity_verified() { + let valid = production_service_with_network(fake_network( + DnsAnswer::Found, + "did:web:alice.example.com", + Some("at://alice.example.com"), + )); + assert!(!valid.local_handle_verified()); + valid.verify_local_handle().await.unwrap(); + assert!(valid.local_handle_verified()); + + let wrong_did = production_service_with_network(fake_network( + DnsAnswer::Found, + "did:web:mallory.example.com", + Some("at://alice.example.com"), + )); + assert!(wrong_did.verify_local_handle().await.is_err()); + assert!(!wrong_did.local_handle_verified()); +} + +#[tokio::test] +async fn revalidation_distinguishes_confirmed_invalidity_from_transient_failure() { + let service = production_service_with_network(fake_network( + DnsAnswer::Found, + "did:web:alice.example.com", + Some("at://alice.example.com"), + )); + assert!(service.revalidate_local_handle().await.unwrap()); + assert!(service.local_handle_verified()); + + *service.network.lock().unwrap() = Some(fake_network( + DnsAnswer::Unavailable, + "did:web:alice.example.com", + Some("at://alice.example.com"), + )); + assert!(service.revalidate_local_handle().await.is_err()); + assert!(service.local_handle_verified()); + + *service.network.lock().unwrap() = Some(fake_network( + DnsAnswer::Found, + "did:web:mallory.example.com", + Some("at://alice.example.com"), + )); + assert!(!service.revalidate_local_handle().await.unwrap()); + assert!(!service.local_handle_verified()); + + *service.network.lock().unwrap() = Some(fake_network( + DnsAnswer::Found, + "did:web:alice.example.com", + Some("at://alice.example.com"), + )); + assert!(service.revalidate_local_handle().await.unwrap()); + assert!(service.local_handle_verified()); +} + +#[tokio::test] +async fn hosted_identity_revalidation_emits_only_confirmed_state_changes() { + let mut state = AppState::assemble( + production_config(), + Database::open_in_memory().unwrap(), + Arc::new(MemoryBlobStore::default()), + Arc::new(NoopEventSink), + ) + .unwrap(); + Arc::get_mut(&mut state.config).unwrap().deployment_mode = DeploymentMode::Production; + state.identity = Arc::new(production_service_with_network(fake_network( + DnsAnswer::Found, + "did:web:alice.example.com", + Some("at://alice.example.com"), + ))); + + assert_eq!( + state.revalidate_network_identity().await.unwrap(), + IdentityRevalidation::Unchanged + ); + *state.identity.network.lock().unwrap() = Some(fake_network( + DnsAnswer::Found, + "did:web:mallory.example.com", + Some("at://alice.example.com"), + )); + assert_eq!( + state.revalidate_network_identity().await.unwrap(), + IdentityRevalidation::Invalidated + ); + assert_eq!( + state + .repository + .account_status("did:web:alice.example.com") + .unwrap(), + crate::domain::account::AccountStatus::Active + ); + let mut request = + AxumRequest::get("/xrpc/com.atproto.repo.describeRepo?repo=did%3Aweb%3Aalice.example.com") + .body(Body::empty()) + .unwrap(); + request.extensions_mut().insert(ConnectInfo( + "192.0.2.1:12345".parse::().unwrap(), + )); + let response = crate::http::router(state.clone()) + .oneshot(request) + .await + .unwrap(); + assert!(response.status().is_success()); + let body: serde_json::Value = + serde_json::from_slice(&to_bytes(response.into_body(), usize::MAX).await.unwrap()).unwrap(); + assert_eq!(body["handle"], "handle.invalid"); + assert_eq!(body["didDoc"]["alsoKnownAs"][0], "at://alice.example.com"); + assert_eq!(body["handleIsCorrect"], false); + + *state.identity.network.lock().unwrap() = Some(fake_network( + DnsAnswer::Unavailable, + "did:web:alice.example.com", + Some("at://alice.example.com"), + )); + assert!(state.revalidate_network_identity().await.is_err()); + assert_eq!( + state + .repository + .repo_messages_after(0, 10) + .await + .unwrap() + .len(), + 1 + ); + + *state.identity.network.lock().unwrap() = Some(fake_network( + DnsAnswer::Found, + "did:web:alice.example.com", + Some("at://alice.example.com"), + )); + assert_eq!( + state.revalidate_network_identity().await.unwrap(), + IdentityRevalidation::Recovered + ); + let messages = state.repository.repo_messages_after(0, 10).await.unwrap(); + assert!(matches!( + messages.as_slice(), + [ + atrium_api::com::atproto::sync::subscribe_repos::Message::Identity(invalid), + atrium_api::com::atproto::sync::subscribe_repos::Message::Identity(recovered), + ] if invalid.handle.as_ref().map(Handle::as_str) == Some("handle.invalid") + && recovered.handle.as_ref().map(Handle::as_str) == Some("alice.example.com") + )); +} + +#[tokio::test] +async fn space_notifications_use_the_identity_deployment_boundary() { + let calls = Arc::new(AtomicUsize::new(0)); + let handler_calls = calls.clone(); + let router = axum::Router::new().route( + "/xrpc/com.example.notify", + axum::routing::post(move || { + handler_calls.fetch_add(1, Ordering::SeqCst); + async { axum::http::StatusCode::NO_CONTENT } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = Url::parse(&format!("http://{}/", listener.local_addr().unwrap())).unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + for local in [true, false] { + let network = fake_network(DnsAnswer::Found, "did:web:alice.example.com", None); + let identity = if local { + service_with_network(network) + } else { + production_service_with_network(network) + }; + let auth = crate::space::auth::SpaceAuthService::new( + Database::open_in_memory().unwrap(), + "did:web:alice.example.com".to_owned(), + &[7; 32], + endpoint.clone(), + Arc::new(identity), + ) + .unwrap(); + let result = auth + .deliver_notification( + endpoint.as_str(), + "did:web:alice.example.com", + "com.example.notify", + &serde_json::json!({}), + ) + .await; + assert_eq!( + result.is_ok(), + local, + "only local mode can deliver to loopback" + ); + } + assert_eq!(calls.load(Ordering::SeqCst), 1); + server.abort(); +} + +fn service_with_network(network: Arc) -> IdentityService { + let mut service = IdentityService::new( + document("did:web:alice.example.com", Some("at://alice.example.com")), + DeploymentMode::Local, + ) + .unwrap(); + service.network = Mutex::new(Some(network)); + service +} + +fn production_service_with_network(network: Arc) -> IdentityService { + let mut service = service_with_network(network); + service.mode = DeploymentMode::Production; + service + .local_handle_verified + .store(false, Ordering::Release); + service +} + +fn fake_network(dns: DnsAnswer, did: &str, alias: Option<&str>) -> Arc { + let did = did.parse::().unwrap(); + Arc::new(FakeNetwork { + dns, + document: document(did.as_str(), alias), + did, + https_calls: AtomicUsize::new(0), + }) +} + +fn production_config() -> Config { + Config { + deployment_mode: DeploymentMode::Local, + listen_address: "127.0.0.1:3000".parse().unwrap(), + public_origin: HttpOrigin::parse_public( + "https://alice.example.com", + DeploymentMode::Production, + ) + .unwrap(), + database_path: ":memory:".into(), + blob_directory: "unused".into(), + blob_orphan_ttl: Duration::from_secs(86_400), + blob_gc_interval: Duration::from_secs(3_600), + identity_revalidation_interval: Duration::from_secs(3_600), + repo_event_retention: 100_000, + repo_stream_send_timeout: Duration::from_secs(10), + relay_origins: Vec::new(), + account: AccountConfig { + did: "did:web:alice.example.com".to_owned(), + handle: "alice.example.com".to_owned(), + password: "correct horse battery staple".to_owned(), + signing_key: [7; 32], + }, + oauth_clients: Vec::new(), + } +} + +fn document(did: &str, alias: Option<&str>) -> DidDocument { + DidDocument { + context: None, + id: did.to_owned(), + also_known_as: alias.map(|alias| vec![alias.to_owned()]), + verification_method: None, + service: None, + } +} diff --git a/src/lib.rs b/src/lib.rs index 710a263..74ce86f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -30,7 +30,7 @@ use database::Database; use domain::identity::{local_did_document, validate_local_did_document}; use error::Result; use events::EventSink; -use identity::{IdentityService, LocalHandleStatus}; +use identity::IdentityService; use oauth::{OAuthService, StaticClientRegistry}; use rate_limits::{PrincipalRateLimits, SubscriptionLimits}; use repository::RepositoryService; @@ -93,19 +93,19 @@ impl AppState { config.account.did.clone(), &config.account.signing_key, )?; - let public_key = spaces.public_key_multibase(); + let public_key = repository.public_key_multibase(); let document = local_did_document( &config.account.did, &config.account.handle, &config.public_origin.to_string(), - &public_key, + public_key, ); validate_local_did_document( &document, &config.account.did, &config.account.handle, &config.public_origin.to_string(), - &public_key, + public_key, )?; let identity = Arc::new(IdentityService::new(document, config.deployment_mode)?); let space_auth = SpaceAuthService::new( @@ -134,14 +134,6 @@ impl AppState { }) } - pub(crate) fn stream_shutdown(&self) -> watch::Receiver { - self.stream_shutdown.subscribe() - } - - pub(crate) fn stop_streams(&self) { - self.stream_shutdown.send_replace(true); - } - /// Verify the configured handle through public DNS or HTTPS before serving. /// /// Local mode keeps its derived identity offline. Production fails closed @@ -162,18 +154,19 @@ impl AppState { if self.config.deployment_mode == config::DeploymentMode::Local { return Ok(IdentityRevalidation::Unchanged); } - let status = self.identity.revalidate_local_handle().await?; - let handle = match status { - LocalHandleStatus::Valid => &self.config.account.handle, - LocalHandleStatus::Invalid => "handle.invalid", + let valid = self.identity.revalidate_local_handle().await?; + let handle = if valid { + &self.config.account.handle + } else { + "handle.invalid" }; let changed = self .repository .set_account_handle(&self.config.account.did, handle)?; - Ok(match (status, changed) { + Ok(match (valid, changed) { (_, false) => IdentityRevalidation::Unchanged, - (LocalHandleStatus::Valid, true) => IdentityRevalidation::Recovered, - (LocalHandleStatus::Invalid, true) => IdentityRevalidation::Invalidated, + (true, true) => IdentityRevalidation::Recovered, + (false, true) => IdentityRevalidation::Invalidated, }) } } diff --git a/src/main.rs b/src/main.rs index 2cd7182..fb25abd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -13,14 +13,17 @@ use tracing_subscriber::EnvFilter; #[tokio::main] async fn main() -> cpds::error::Result<()> { tracing_subscriber::fmt() - .with_env_filter(EnvFilter::from_default_env()) + .with_env_filter( + EnvFilter::builder() + .with_default_directive(tracing::Level::INFO.into()) + .from_env_lossy(), + ) .init(); - let operation = operation()?; + let operation = operation_from(std::env::args_os().skip(1))?; let config = Config::from_environment()?; if matches!(operation, Operation::RequestCrawl) { - relay::request_crawl(&config).await?; - return Ok(()); + return relay::request_crawl(&config).await; } let existing_only = matches!( operation, @@ -108,8 +111,7 @@ async fn main() -> cpds::error::Result<()> { let listener = tokio::net::TcpListener::bind(config.listen_address).await?; tracing::info!(address = %config.listen_address, "cpds is listening"); - http::serve(listener, state, http::shutdown_signal()).await?; - Ok(()) + http::serve(listener, state, http::shutdown_signal()).await } #[derive(Debug, Eq, PartialEq)] @@ -121,10 +123,6 @@ enum Operation { SetAccountStatus(AccountStatus), } -fn operation() -> cpds::error::Result { - operation_from(std::env::args_os().skip(1)) -} - fn operation_from(mut arguments: impl Iterator) -> cpds::error::Result { match (arguments.next(), arguments.next(), arguments.next()) { (None, None, None) => Ok(Operation::Serve), @@ -138,9 +136,8 @@ fn operation_from(mut arguments: impl Iterator) -> cpds::error: (Some(argument), Some(status), None) if argument == OsStr::new("set-account-status") => { let status = status .to_str() - .ok_or_else(invalid_account_status)? - .parse() - .map_err(|_| invalid_account_status())?; + .and_then(|status| status.parse().ok()) + .ok_or_else(invalid_account_status)?; Ok(Operation::SetAccountStatus(status)) } _ => Err(cpds::error::Error::Configuration( @@ -170,6 +167,49 @@ fn require_existing_storage(path: &Path, name: &str) -> cpds::error::Result<()> mod tests { use super::*; + #[test] + fn account_status_pipeline_keeps_values_and_exact_command_errors() { + #[cfg_attr(not(unix), allow(unused_mut))] + let mut values = [ + "active", + "deactivated", + "suspended", + "takendown", + "deleted", + "takedown", + "", + "Active", + " active", + "active ", + "active\0", + "é", + "active\ndeleted", + ] + .map(OsString::from) + .to_vec(); + #[cfg(unix)] + { + use std::os::unix::ffi::OsStringExt as _; + values.push(OsString::from_vec(vec![0xff])); + values.push(OsString::from_vec(b"active\xff".to_vec())); + } + for value in values { + let previous: cpds::error::Result = (|| { + let status = value + .to_str() + .ok_or_else(invalid_account_status)? + .parse() + .map_err(|_| invalid_account_status())?; + Ok(Operation::SetAccountStatus(status)) + })(); + let actual = operation_from([OsString::from("set-account-status"), value].into_iter()); + assert_eq!(format!("{actual:?}"), format!("{previous:?}")); + if let Err(error) = actual { + assert_eq!(error.to_string(), invalid_account_status().to_string()); + } + } + } + #[test] fn operator_account_status_command_accepts_only_durable_states() { for (value, expected) in [ diff --git a/src/oauth/mod.rs b/src/oauth/mod.rs index 38bc9f1..61abbbb 100644 --- a/src/oauth/mod.rs +++ b/src/oauth/mod.rs @@ -19,10 +19,7 @@ use url::Url; use crate::{ database::Database, - domain::{ - account::Account, - oauth::{AuthenticatedToken, AuthorizationGrant, AuthorizationRequest, TokenGrant}, - }, + domain::oauth::{AuthenticatedToken, AuthorizationRequest, TokenForm, TokenGrant}, error::{Error, Result}, }; @@ -64,11 +61,7 @@ impl OAuthService { .clients .find(&request.client_id)? .ok_or_else(|| Error::InvalidRequest("OAuth client is not registered".to_owned()))?; - if !client - .redirect_uris - .iter() - .any(|registered| registered == &request.redirect_uri) - { + if !client.redirect_uris.contains(&request.redirect_uri) { return Err(Error::InvalidRequest( "redirect_uri is not registered for this client".to_owned(), )); @@ -82,8 +75,7 @@ impl OAuthService { "scope must include `atproto`".to_owned(), )); } - pkce::validate_challenge(&request.code_challenge)?; - Ok(()) + pkce::validate_challenge(&request.code_challenge) } /// Authenticate the resource owner and issue a one-time authorization code. @@ -97,7 +89,7 @@ impl OAuthService { request: &AuthorizationRequest, identifier: &str, password: &str, - ) -> Result { + ) -> Result<(String, Url)> { self.validate_authorization_request(request)?; let account = self.database.authenticate_password(identifier, password)?; let code = random_credential(); @@ -106,10 +98,7 @@ impl OAuthService { self.database.store_authorization_code( &credential_hash(&code), &account.did, - request.client_id.as_str(), - request.redirect_uri.as_str(), - &request.code_challenge, - &request.scope, + request, expires_at, )?; @@ -119,7 +108,7 @@ impl OAuthService { .append_pair("code", &code) .append_pair("state", &request.state) .finish(); - Ok(AuthorizationGrant { code, redirect_to }) + Ok((code, redirect_to)) } /// Redeem a code using exact client, redirect, and PKCE binding. @@ -128,49 +117,27 @@ impl OAuthService { /// /// Returns an error when PKCE validation fails, the code is invalid or /// already used, or token persistence fails. - pub fn exchange_authorization_code( - &self, - code: &str, - client_id: &Url, - redirect_uri: &Url, - code_verifier: &str, - ) -> Result { - let presented_challenge = challenge_for_verifier(code_verifier)?; - let stored = self.database.redeem_authorization_code( - &credential_hash(code), - client_id.as_str(), - redirect_uri.as_str(), - &presented_challenge, - )?; - + pub fn exchange_authorization_code(&self, request: &TokenForm) -> Result { + let presented_challenge = challenge_for_verifier(&request.code_verifier)?; let access_token = random_credential(); let expires_at = (OffsetDateTime::now_utc() + ACCESS_TOKEN_LIFETIME).unix_timestamp(); - self.database.store_access_token( + let (did, scope) = self.database.redeem_authorization_code( + &credential_hash(&request.code), + request.client_id.as_str(), + request.redirect_uri.as_str(), + &presented_challenge, &credential_hash(&access_token), - &stored.did, - &stored.client_id, - &stored.scope, expires_at, )?; Ok(TokenGrant { access_token, + token_type: "Bearer", expires_in: u64::try_from(ACCESS_TOKEN_LIFETIME.whole_seconds()).unwrap_or_default(), - scope: stored.scope, - did: stored.did, + scope, + did, }) } - /// Resolve an opaque bearer credential to the hosted account. - /// - /// # Errors - /// - /// Returns `AuthenticationRequired` for unknown or expired credentials, or - /// a database error when lookup fails. - pub fn authenticate_access_token(&self, access_token: &str) -> Result { - self.database - .account_for_access_token(&credential_hash(access_token)) - } - /// Resolve an access token along with the exact OAuth grant it carries. pub(crate) fn authenticate_token(&self, access_token: &str) -> Result { self.database.access_token(&credential_hash(access_token)) @@ -186,3 +153,214 @@ fn random_credential() -> String { fn credential_hash(value: &str) -> String { URL_SAFE_NO_PAD.encode(Sha256::digest(value.as_bytes())) } + +#[cfg(test)] +mod tests { + use crate::domain::oauth::OAuthClient; + + use super::*; + + #[test] + fn native_redirect_membership_preserves_url_equality_and_validation_priority() { + let redirects = [ + "https://CLIENT.test:443/callback", + "https://client.test/callback", + "https://client.test/callback?first=1&second=2", + "https://client.test/callback?second=2&first=1", + "https://client.test/callback#fragment", + "https://client.test/caf%C3%A9", + "https://client.test/café", + "https://client.test/%2f", + ] + .map(|value| Url::parse(value).unwrap()); + let client_id = Url::parse("https://client.test/metadata.json").unwrap(); + let missing_id = Url::parse("https://missing.test/metadata.json").unwrap(); + let database = Database::open_in_memory().unwrap(); + for registered in [ + Vec::new(), + vec![redirects[0].clone()], + redirects.to_vec(), + redirects.iter().rev().cloned().collect(), + ] { + let service = OAuthService::new( + database.clone(), + Arc::new(StaticClientRegistry::new(vec![OAuthClient { + client_id: client_id.clone(), + redirect_uris: registered, + name: "test".to_owned(), + }])), + Url::parse("https://pds.test").unwrap(), + ); + for redirect in &redirects { + for id in [&client_id, &missing_id] { + for scope in ["atproto", "invalid"] { + for challenge in ["a".repeat(43), "invalid".to_owned()] { + let request = AuthorizationRequest { + client_id: id.clone(), + redirect_uri: redirect.clone(), + scope: scope.to_owned(), + state: String::new(), + code_challenge: challenge, + }; + let previous = (|| { + let client = + service.clients.find(&request.client_id)?.ok_or_else(|| { + Error::InvalidRequest( + "OAuth client is not registered".to_owned(), + ) + })?; + if !client + .redirect_uris + .iter() + .any(|registered| registered == &request.redirect_uri) + { + return Err(Error::InvalidRequest( + "redirect_uri is not registered for this client".to_owned(), + )); + } + if !request + .scope + .split_ascii_whitespace() + .any(|scope| scope == "atproto") + { + return Err(Error::InvalidRequest( + "scope must include `atproto`".to_owned(), + )); + } + pkce::validate_challenge(&request.code_challenge) + })(); + let actual = service.validate_authorization_request(&request); + assert_eq!(format!("{actual:?}"), format!("{previous:?}")); + } + } + } + } + } + } + + #[test] + fn authorization_pair_keeps_exact_redirect_parameters_and_one_time_code() { + let database = Database::open_in_memory().unwrap(); + let did = "did:web:alice.test"; + database + .ensure_account(did, "alice.test", "password") + .unwrap(); + let client_id: Url = "https://client.test/metadata.json".parse().unwrap(); + let verifier = "a".repeat(43); + for redirect in [ + "https://client.test/callback", + "https://client.test/callback?existing=1&code=old&state=old#fragment", + "https://client.test/café?x=%00+%2f#fragment", + ] { + let redirect_uri: Url = redirect.parse().unwrap(); + let service = OAuthService::new( + database.clone(), + Arc::new(StaticClientRegistry::new(vec![OAuthClient { + client_id: client_id.clone(), + redirect_uris: vec![redirect_uri.clone()], + name: "test client".to_owned(), + }])), + "https://pds.test".parse().unwrap(), + ); + for state in ["", "quoted '\" & = café\n"] { + let request = AuthorizationRequest { + client_id: client_id.clone(), + redirect_uri: redirect_uri.clone(), + scope: "atproto space:com.example.one".to_owned(), + state: state.to_owned(), + code_challenge: challenge_for_verifier(&verifier).unwrap(), + }; + let (code, redirect_to) = service + .authorize(&request, "alice.test", "password") + .unwrap(); + assert_eq!(code.len(), 43); + assert_eq!(URL_SAFE_NO_PAD.decode(&code).unwrap().len(), 32); + let mut expected = redirect_uri.query_pairs().into_owned().collect::>(); + expected.push(("code".to_owned(), code.clone())); + expected.push(("state".to_owned(), state.to_owned())); + assert_eq!( + redirect_to.query_pairs().into_owned().collect::>(), + expected + ); + let mut original = redirect_to.clone(); + original.set_query(redirect_uri.query()); + assert_eq!(original, redirect_uri); + let form = TokenForm { + grant_type: "authorization_code".to_owned(), + code, + client_id: client_id.clone(), + redirect_uri: redirect_uri.clone(), + code_verifier: verifier.clone(), + }; + let token = service.exchange_authorization_code(&form).unwrap(); + assert_eq!(token.did, did); + assert_eq!(token.scope, request.scope); + assert!(matches!( + service.exchange_authorization_code(&form), + Err(Error::Authorization(_)) + )); + } + } + } + + #[test] + fn failed_token_storage_does_not_consume_the_authorization_code() { + let database = Database::open_in_memory().unwrap(); + database + .ensure_account("did:web:alice.test", "alice.test", "password") + .unwrap(); + let client_id = Url::parse("https://client.test/metadata.json").unwrap(); + let redirect_uri = Url::parse("https://client.test/callback").unwrap(); + let verifier = "a".repeat(43); + let request = AuthorizationRequest { + client_id: client_id.clone(), + redirect_uri: redirect_uri.clone(), + scope: "atproto".to_owned(), + state: "state".to_owned(), + code_challenge: challenge_for_verifier(&verifier).unwrap(), + }; + let service = OAuthService::new( + database.clone(), + Arc::new(StaticClientRegistry::new(vec![OAuthClient { + client_id: client_id.clone(), + redirect_uris: vec![redirect_uri.clone()], + name: "test client".to_owned(), + }])), + Url::parse("https://pds.test").unwrap(), + ); + let (code, _redirect_to) = service + .authorize(&request, "alice.test", "password") + .unwrap(); + let form = TokenForm { + grant_type: "authorization_code".to_owned(), + code, + client_id, + redirect_uri, + code_verifier: verifier, + }; + database + .connection() + .unwrap() + .execute_batch( + "CREATE TEMP TRIGGER fail_token_storage BEFORE INSERT ON access_tokens + BEGIN SELECT RAISE(FAIL, 'injected token storage failure'); END;", + ) + .unwrap(); + + assert!(service.exchange_authorization_code(&form).is_err()); + let used: Option = database + .connection() + .unwrap() + .query_row("SELECT used_at FROM oauth_codes", [], |row| row.get(0)) + .unwrap(); + assert_eq!(used, None, "failed exchange must leave its code redeemable"); + database + .connection() + .unwrap() + .execute_batch("DROP TRIGGER fail_token_storage") + .unwrap(); + + service.exchange_authorization_code(&form).unwrap(); + assert!(service.exchange_authorization_code(&form).is_err()); + } +} diff --git a/src/oauth/pkce.rs b/src/oauth/pkce.rs index d4a1b72..0c7d7f7 100644 --- a/src/oauth/pkce.rs +++ b/src/oauth/pkce.rs @@ -1,8 +1,5 @@ //! Proof Key for Code Exchange (RFC 7636), S256 profile. -use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; -use sha2::{Digest, Sha256}; - use crate::error::{Error, Result}; /// Derive an S256 challenge from an RFC 7636 verifier. @@ -12,23 +9,18 @@ use crate::error::{Error, Result}; /// Returns an error when the verifier is outside the permitted length or /// contains a character outside the RFC 7636 unreserved set. pub fn challenge_for_verifier(verifier: &str) -> Result { - validate_verifier(verifier)?; - Ok(URL_SAFE_NO_PAD.encode(Sha256::digest(verifier.as_bytes()))) -} - -pub(super) fn validate_challenge(challenge: &str) -> Result<()> { - if challenge.len() != 43 || !is_unreserved(challenge) { + if !(43..=128).contains(&verifier.len()) || !is_unreserved(verifier) { return Err(Error::InvalidRequest( - "code_challenge must be a 43-character base64url SHA-256 digest".to_owned(), + "code_verifier must contain 43-128 RFC 7636 unreserved characters".to_owned(), )); } - Ok(()) + Ok(super::credential_hash(verifier)) } -fn validate_verifier(verifier: &str) -> Result<()> { - if !(43..=128).contains(&verifier.len()) || !is_unreserved(verifier) { +pub(super) fn validate_challenge(challenge: &str) -> Result<()> { + if challenge.len() != 43 || !is_unreserved(challenge) { return Err(Error::InvalidRequest( - "code_verifier must contain 43-128 RFC 7636 unreserved characters".to_owned(), + "code_challenge must be a 43-character base64url SHA-256 digest".to_owned(), )); } Ok(()) @@ -42,7 +34,47 @@ fn is_unreserved(value: &str) -> bool { #[cfg(test)] mod tests { - use super::challenge_for_verifier; + use super::{challenge_for_verifier, validate_challenge}; + use crate::error::Error; + use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; + use sha2::{Digest, Sha256}; + + #[test] + fn pkce_profiles_preserve_length_character_and_digest_behavior() { + for prefix_length in [0, 41, 42, 43, 126, 127, 128] { + for byte in u8::MIN..=u8::MAX { + let value = format!("{}{}", "a".repeat(prefix_length), char::from(byte)); + let unreserved = value.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') + }); + let verifier = challenge_for_verifier(&value); + if (43..=128).contains(&value.len()) && unreserved { + assert_eq!( + verifier.unwrap(), + URL_SAFE_NO_PAD.encode(Sha256::digest(value.as_bytes())), + ); + } else { + assert!(matches!(verifier, Err(Error::InvalidRequest(message)) + if message == "code_verifier must contain 43-128 RFC 7636 unreserved characters")); + } + let challenge = validate_challenge(&value); + if value.len() == 43 && unreserved { + challenge.unwrap(); + } else { + assert!(matches!(challenge, Err(Error::InvalidRequest(message)) + if message == "code_challenge must be a 43-character base64url SHA-256 digest")); + } + } + } + assert!(matches!( + challenge_for_verifier(""), + Err(Error::InvalidRequest(_)) + )); + assert!(matches!( + validate_challenge(""), + Err(Error::InvalidRequest(_)) + )); + } #[test] fn matches_the_rfc_7636_example() { diff --git a/src/private_fs.rs b/src/private_fs.rs index 9811ee2..52ab9c2 100644 --- a/src/private_fs.rs +++ b/src/private_fs.rs @@ -1,68 +1,155 @@ //! Synchronous durable writes for private storage snapshots. +#[cfg(unix)] +use std::os::unix::fs::{DirBuilderExt as _, OpenOptionsExt as _}; use std::{io::Write as _, path::Path}; use crate::error::Result; +pub(crate) fn parent_or_current(path: &Path) -> &Path { + path.parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")) +} + pub(crate) fn create_directory(path: &Path) -> Result<()> { + #[cfg_attr(not(unix), allow(unused_mut))] let mut builder = std::fs::DirBuilder::new(); - set_directory_mode(&mut builder); - builder.create(path)?; - Ok(()) + #[cfg(unix)] + builder.mode(0o700); + builder.create(path).map_err(Into::into) } pub(crate) fn write_file(path: &Path, bytes: &[u8]) -> Result<()> { let mut options = std::fs::OpenOptions::new(); options.write(true).create_new(true); - set_file_mode(&mut options); + #[cfg(unix)] + options.mode(0o600); let mut file = options.open(path)?; file.write_all(bytes)?; - file.sync_all()?; - Ok(()) + file.sync_all().map_err(Into::into) } -#[cfg(unix)] pub(crate) fn secure_file(path: &Path) -> Result<()> { - use std::os::unix::fs::PermissionsExt as _; - - std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600))?; - std::fs::File::open(path)?.sync_all()?; - Ok(()) + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600))?; + } + std::fs::File::open(path)?.sync_all().map_err(Into::into) } -#[cfg(not(unix))] -pub(crate) fn secure_file(path: &Path) -> Result<()> { - std::fs::File::open(path)?.sync_all()?; - Ok(()) -} - -#[cfg(unix)] pub(crate) fn sync_directory(path: &Path) -> Result<()> { - std::fs::File::open(path)?.sync_all()?; + if cfg!(unix) { + std::fs::File::open(path)?.sync_all()?; + } Ok(()) } -#[cfg(not(unix))] -pub(crate) fn sync_directory(_path: &Path) -> Result<()> { - Ok(()) -} +#[cfg(test)] +mod tests { + use super::*; -#[cfg(unix)] -fn set_directory_mode(builder: &mut std::fs::DirBuilder) { - use std::os::unix::fs::DirBuilderExt as _; + #[test] + fn directory_sync_preserves_platform_io_and_errors() { + let directory = tempfile::tempdir().unwrap(); + let file = directory.path().join("file"); + std::fs::write(&file, b"contents").unwrap(); + for path in [ + directory.path().to_owned(), + file.clone(), + directory.path().join("missing"), + file.join("not-a-directory"), + ] { + let expected: Result<()> = (|| { + #[cfg(unix)] + std::fs::File::open(&path)?.sync_all()?; + Ok(()) + })(); + assert_eq!( + format!("{:?}", sync_directory(&path)), + format!("{expected:?}") + ); + } + assert_eq!(std::fs::read(file).unwrap(), b"contents"); + } - builder.mode(0o700); -} + #[test] + fn private_snapshot_writes_never_replace_existing_paths() { + let directory = tempfile::tempdir().unwrap(); + let snapshot = directory.path().join("snapshot"); + create_directory(&snapshot).unwrap(); + let file = snapshot.join("data"); + write_file(&file, b"original").unwrap(); -#[cfg(not(unix))] -fn set_directory_mode(_builder: &mut std::fs::DirBuilder) {} + for result in [ + create_directory(&snapshot), + write_file(&file, b"replacement"), + ] { + let crate::error::Error::Io(error) = result.unwrap_err() else { + panic!("existing snapshot paths must produce an I/O error"); + }; + assert_eq!(error.kind(), std::io::ErrorKind::AlreadyExists); + } + assert_eq!(std::fs::read(file).unwrap(), b"original"); + } -#[cfg(unix)] -fn set_file_mode(options: &mut std::fs::OpenOptions) { - use std::os::unix::fs::OpenOptionsExt as _; + #[cfg(unix)] + #[test] + fn private_snapshot_modes_and_symlink_collisions_are_preserved() { + use std::os::unix::fs::{PermissionsExt as _, symlink}; - options.mode(0o600); -} + let directory = tempfile::tempdir().unwrap(); + let snapshot = directory.path().join("snapshot"); + create_directory(&snapshot).unwrap(); + let file = snapshot.join("data"); + write_file(&file, b"original").unwrap(); + for (path, mode) in [(&snapshot, 0o700), (&file, 0o600)] { + assert_eq!( + std::fs::metadata(path).unwrap().permissions().mode() & 0o777, + mode + ); + } + + let linked_file = directory.path().join("linked-file"); + let linked_directory = directory.path().join("linked-directory"); + symlink(&file, &linked_file).unwrap(); + symlink(&snapshot, &linked_directory).unwrap(); + for result in [ + write_file(&linked_file, b"replacement"), + create_directory(&linked_directory), + ] { + let crate::error::Error::Io(error) = result.unwrap_err() else { + panic!("symlink collisions must produce an I/O error"); + }; + assert_eq!(error.kind(), std::io::ErrorKind::AlreadyExists); + } + assert_eq!(std::fs::read(file).unwrap(), b"original"); + assert!( + std::fs::symlink_metadata(linked_file) + .unwrap() + .file_type() + .is_symlink() + ); + assert!( + std::fs::symlink_metadata(linked_directory) + .unwrap() + .file_type() + .is_symlink() + ); + } -#[cfg(not(unix))] -fn set_file_mode(_options: &mut std::fs::OpenOptions) {} + #[test] + fn storage_parent_preserves_relative_and_root_path_behavior() { + for (path, expected) in [ + ("", "."), + ("backup", "."), + ("./backup", "."), + ("storage/backup", "storage"), + ("/", "."), + ("/backup", "/"), + ] { + assert_eq!(parent_or_current(Path::new(path)), Path::new(expected)); + } + } +} diff --git a/src/relay.rs b/src/relay.rs index e2c1943..8012876 100644 --- a/src/relay.rs +++ b/src/relay.rs @@ -7,8 +7,8 @@ use futures::future::join_all; use reqwest::{Client, StatusCode, header::RETRY_AFTER, redirect::Policy}; use crate::{ - config::{Config, RelayOrigin}, - error::{Error, Result}, + config::{Config, HttpOrigin}, + error::{Error, ErrorContext as _, Result}, }; const REQUEST_CRAWL_PATH: &str = "/xrpc/com.atproto.sync.requestCrawl"; @@ -39,14 +39,13 @@ pub async fn request_crawl(config: &Config) -> Result<()> { .redirect(Policy::none()) .user_agent(concat!("cpds/", env!("CARGO_PKG_VERSION"))) .build() - .map_err(|error| Error::Internal(format!("could not build relay client: {error}")))?; + .context(Error::Internal, "could not build relay client")?; let hostname = config.public_origin.hostname(); let results = join_all( config .relay_origins .iter() - .cloned() - .map(|relay| request_one(client.clone(), relay, hostname.clone())), + .map(|relay| request_one(&client, relay, &hostname)), ) .await; let failures = results @@ -64,12 +63,12 @@ pub async fn request_crawl(config: &Config) -> Result<()> { } async fn request_one( - client: Client, - relay: RelayOrigin, - hostname: String, + client: &Client, + relay: &HttpOrigin, + hostname: &str, ) -> std::result::Result<(), String> { for attempt in 1..=MAX_ATTEMPTS { - match send_once(&client, &relay, &hostname).await { + match send_once(client, relay, hostname).await { Ok(()) => { tracing::info!(%relay, %hostname, attempt, "relay crawl request accepted"); return Ok(()); @@ -93,7 +92,7 @@ async fn request_one( async fn send_once( client: &Client, - relay: &RelayOrigin, + relay: &HttpOrigin, hostname: &str, ) -> std::result::Result<(), AttemptError> { let response = client diff --git a/src/repository.rs b/src/repository.rs deleted file mode 100644 index 9f7c31f..0000000 --- a/src/repository.rs +++ /dev/null @@ -1,1979 +0,0 @@ -//! Application service for records and blobs. -//! -//! This is the center of the PDS. It owns policy and sequencing; storage and -//! HTTP do not. Signed repository state and an outbox commit atomically before -//! the existing typed in-process event is emitted. - -use std::{ - collections::{BTreeMap, HashMap, HashSet}, - io::{self, Cursor}, - pin::Pin, - sync::Arc, - task::{Context, Poll}, - time::Duration, -}; - -use atrium_api::{ - com::atproto::sync::subscribe_repos, - types::{ - BlobRef as AtriumBlobRef, CidLink, DataModel as AtriumDataModel, TypedBlobRef, - string::{Datetime, Did, Handle, Tid}, - }, -}; -use atrium_crypto::{ - keypair::{Did as _, Secp256k1Keypair}, - verify::verify_signature, -}; -use atrium_repo::{ - Repository, - blockstore::{ - AsyncBlockStoreRead as _, AsyncBlockStoreWrite as _, CarStore, Error as AtriumBlockError, - SHA2_256, - }, - repo::Error as AtriumRepoError, -}; -use base64::{ - Engine as _, - engine::general_purpose::{STANDARD, STANDARD_NO_PAD}, -}; -use cid::Cid; -use futures::TryStreamExt as _; -use serde::{Serialize, Serializer}; -use serde_json::Value; -use sha2::{Digest, Sha256}; -use time::OffsetDateTime; -use tokio::io::{AsyncRead, AsyncSeek, AsyncWrite, ReadBuf}; - -use crate::{ - blob_store::BlobStore, - clock::TidClock, - database::{ - Database, ListedRepoHead, PendingRecordMutation, PendingRepoCommit, RepoBlockStore, - RepoHead, StoredAccountEvent, StoredCommitEvent, StoredIdentityEvent, StoredRepoEvent, - }, - domain::{ - account::{Account, AccountStatus}, - blob::Blob, - event::RepositoryEvent, - record::{CommitView, Record, RecordPage, RecordView, RecordWrite}, - }, - error::{Error, Result}, - events::EventSink, - syntax, -}; - -const DAG_CBOR_CODEC: u64 = 0x71; -const RAW_CODEC: u64 = 0x55; -pub(crate) const MAX_BLOB_BYTES: usize = 5 * 1024 * 1024; -const MAX_RECORD_BYTES: usize = 1_000_000; -const MAX_RECORD_DEPTH: usize = 32; -const MAX_RECORD_KEY_BYTES: usize = 8 * 1024; -const MAX_SAFE_JSON_INTEGER: f64 = 9_007_199_254_740_991.0; -const MAX_REPO_RETRIES: usize = 3; -const MAX_REPO_EXPORT_BYTES: usize = 64 * 1024 * 1024; -const MAX_COMMIT_EVENT_BYTES: usize = 2_000_000; -const MAX_BLOCKS_PER_REQUEST: usize = 1_000; -const MAX_REPO_EVENTS_PER_PAGE: usize = 1_000; -const MAX_ORPHAN_BLOBS_PER_RUN: usize = 1_000; -const MAX_REPO_BLOCKS_PER_GC_RUN: usize = 1_000; - -pub struct RepositoryService { - database: Database, - blobs: Arc, - events: Arc, - tids: TidClock, - signer: Secp256k1Keypair, - signing_did: String, - repo_event_retention: usize, -} - -pub struct RecordSwap<'a> { - pub record: Option<&'a Cid>, - pub commit: Option<&'a Cid>, -} - -pub struct RepoListEntry { - pub did: String, - pub head: Cid, - pub rev: String, - pub status: AccountStatus, -} - -pub struct MissingBlob { - pub cid: Cid, - pub record_uri: String, -} - -impl RepositoryService { - /// Compose the repository service with its persistent and signing adapters. - /// - /// # Errors - /// - /// Returns a configuration error when the signing key is invalid. - pub fn new( - database: Database, - blobs: Arc, - events: Arc, - signing_key: &[u8; 32], - repo_event_retention: usize, - ) -> Result { - if repo_event_retention == 0 { - return Err(Error::Configuration( - "repository event retention must be positive".to_owned(), - )); - } - let signer = Secp256k1Keypair::import(signing_key).map_err(|error| { - Error::Configuration(format!("invalid repository signing key: {error}")) - })?; - let signing_did = signer.did(); - Ok(Self { - database, - blobs, - events, - tids: TidClock::default(), - signer, - signing_did, - repo_event_retention, - }) - } - - /// Check the infrastructure adapters required to serve repository requests. - /// - /// # Errors - /// - /// Returns an error when the database, blob store, or event sink is not ready. - pub async fn check(&self) -> Result<()> { - self.database.check()?; - self.blobs.check().await?; - self.events.check().await - } - - /// Verify infrastructure and every initialized repository before serving traffic. - /// - /// This intentionally performs the expensive graph and record-index walk only at - /// startup. Routine readiness checks remain bounded to the infrastructure adapters. - /// - /// # Errors - /// - /// Returns an error when an adapter is unavailable or persisted repository state is - /// corrupt, incomplete, or inconsistent with the record index. - pub async fn verify(&self) -> Result<()> { - self.check().await?; - let heads = self.database.repo_heads_after(None, None, usize::MAX)?; - for head in &heads { - self.verify_repo(head).await?; - } - self.verify_repo_events(&heads).await?; - for (owner, blob) in self.database.all_blobs()? { - self.verify_blob(&owner, &blob).await?; - } - Ok(()) - } - - /// Delete one bounded batch of uploaded blobs outside their reference grace period. - /// - /// `SQLite` first claims each candidate so concurrent public or Space writes cannot - /// acquire it. A crash leaves a durable claim that the next run resumes. - /// - /// # Errors - /// - /// Returns an error when candidates cannot be claimed, object deletion fails, or a - /// completed deletion cannot remove its metadata claim. - pub async fn collect_orphan_blobs(&self, grace_period: Duration) -> Result { - let grace_seconds = i64::try_from(grace_period.as_secs()).unwrap_or(i64::MAX); - let created_before = OffsetDateTime::now_utc() - .unix_timestamp() - .saturating_sub(grace_seconds); - let candidates = self - .database - .claim_orphan_blobs(created_before, MAX_ORPHAN_BLOBS_PER_RUN)?; - for (owner, cid) in &candidates { - self.blobs.delete(owner, cid).await?; - self.database.finish_blob_deletion(owner, cid)?; - } - Ok(candidates.len()) - } - - /// Delete one bounded batch of blocks outside the current `ATrium` graph and - /// retained event window. - /// - /// # Errors - /// - /// Returns an error when a signed repository cannot be opened or traversed, or - /// when the transactional block deletion fails. - pub async fn collect_repository_blocks(&self) -> Result { - let heads = self.database.repo_heads_after(None, None, usize::MAX)?; - let mut reachable = HashSet::new(); - for head in heads { - let mut blocks = RepoBlockStore::new(self.database.clone()); - let mut repo = self - .open_repo( - &mut blocks, - &RepoHead { - root: head.root, - rev: head.rev, - }, - ) - .await?; - reachable.extend(repo.export().await.map_err(map_repo_error)?); - } - self.database - .prune_repo_blocks(&reachable, MAX_REPO_BLOCKS_PER_GC_RUN) - } - - /// Checkpoint durable repository metadata after request drain. - /// - /// # Errors - /// - /// Returns an error when `SQLite` cannot complete its WAL checkpoint. - pub fn checkpoint(&self) -> Result<()> { - self.database.checkpoint() - } - - /// Validate, address, persist, and announce one new record. - /// - /// # Errors - /// - /// Returns an error for unauthorized repositories, invalid record shape or - /// identifiers, duplicate paths, or failed persistence. - pub async fn create_record( - &self, - actor: &Account, - repo: &str, - collection: &str, - rkey: Option<&str>, - value: Value, - expected_commit: Option<&Cid>, - ) -> Result { - self.authorize_repo(actor, repo)?; - syntax::collection(collection)?; - validate_record_type(collection, &value)?; - self.database.validate_blob_refs(&actor.did, &value)?; - - let rkey = rkey.map_or_else(|| self.tids.next(), ToOwned::to_owned); - syntax::record_key(&rkey)?; - let cid = record_cid(&value)?; - let record = Record { - did: actor.did.clone(), - collection: collection.to_owned(), - rkey: rkey.clone(), - cid, - value, - }; - let commit = self - .commit_record_mutation( - actor, - expected_commit, - PendingRecordMutation::Create(record.clone()), - ) - .await?; - self.publish(RepositoryEvent::RecordCreated { - did: actor.did.clone(), - collection: collection.to_owned(), - rkey, - cid, - }) - .await; - Ok(RecordWrite { - record: record.into(), - commit, - }) - } - - /// Create or replace one record owned by the authenticated account. - /// - /// # Errors - /// - /// Returns an error for invalid data, unauthorized repositories, failed - /// commit or record swaps, or persistence failures. - pub async fn put_record( - &self, - actor: &Account, - repo: &str, - collection: &str, - rkey: &str, - value: Value, - swap: RecordSwap<'_>, - ) -> Result { - self.authorize_repo(actor, repo)?; - syntax::collection(collection)?; - syntax::record_key(rkey)?; - validate_record_type(collection, &value)?; - self.database.validate_blob_refs(&actor.did, &value)?; - let current = match self.database.get_record(&actor.did, collection, rkey) { - Ok(record) => Some(record), - Err(Error::NotFound) => None, - Err(error) => return Err(error), - }; - if swap.record.is_some_and(|expected| { - current - .as_ref() - .is_none_or(|record| expected != &record.cid) - }) { - return Err(invalid_swap("record CID does not match")); - } - let record = Record { - did: actor.did.clone(), - collection: collection.to_owned(), - rkey: rkey.to_owned(), - cid: record_cid(&value)?, - value, - }; - let mutation = current.clone().map_or_else( - || PendingRecordMutation::Create(record.clone()), - |current| PendingRecordMutation::Update { - current, - replacement: record.clone(), - }, - ); - let commit = self - .commit_record_mutation(actor, swap.commit, mutation) - .await?; - match current { - Some(current) => { - self.publish(RepositoryEvent::RecordUpdated { - did: actor.did.clone(), - collection: collection.to_owned(), - rkey: rkey.to_owned(), - cid: record.cid, - previous_cid: current.cid, - }) - .await; - } - None => { - self.publish(RepositoryEvent::RecordCreated { - did: actor.did.clone(), - collection: collection.to_owned(), - rkey: rkey.to_owned(), - cid: record.cid, - }) - .await; - } - } - Ok(RecordWrite { - record: record.into(), - commit, - }) - } - - /// Read one publicly addressable record. - /// - /// # Errors - /// - /// Returns `NotFound` for unknown repositories or paths, and validation or - /// persistence errors when the request cannot be fulfilled. - pub fn get_record( - &self, - repo: &str, - collection: &str, - rkey: &str, - cid: Option<&Cid>, - ) -> Result { - let account = self.resolve_repo(repo)?; - syntax::collection(collection)?; - syntax::record_key(rkey)?; - let record: RecordView = self - .database - .get_record(&account.did, collection, rkey) - .map(Into::into)?; - if cid.is_some_and(|cid| record.cid != cid.to_string()) { - return Err(Error::NotFound); - } - Ok(record) - } - - /// List a bounded page from one public collection. - /// - /// # Errors - /// - /// Returns an error for an unknown repository, invalid collection, or - /// failed database query. - pub fn list_records( - &self, - repo: &str, - collection: &str, - cursor: Option<&str>, - reverse: bool, - limit: usize, - ) -> Result { - let account = self.resolve_repo(repo)?; - syntax::collection(collection)?; - if let Some(cursor) = cursor { - syntax::record_key(cursor)?; - } - let limit = limit.clamp(1, 100); - let records = - self.database - .list_records(&account.did, collection, cursor, reverse, limit)?; - let cursor = records.last().map(|record| record.rkey.clone()); - let records = records.into_iter().map(Into::into).collect(); - Ok(RecordPage { records, cursor }) - } - - /// Return the current signed repository head. - /// - /// # Errors - /// - /// Returns `RepoNotFound` for an unhosted or not-yet-initialized repository. - pub fn latest_commit(&self, did: &str) -> Result { - let head = self.sync_head(did)?; - Ok(CommitView { - cid: head.root.to_string(), - rev: head.rev, - }) - } - - /// Return the current revision for a hosted repository, when initialized. - /// - /// # Errors - /// - /// Returns `RepoNotFound` when the DID is not hosted by this PDS. - pub fn repo_status(&self, did: &str) -> Result<(AccountStatus, Option)> { - let hosted = self - .database - .hosted_account_by_identifier(did)? - .ok_or_else(repo_not_found)?; - let rev = if hosted.status.is_active() { - self.database - .repo_head(&hosted.account.did)? - .map(|head| head.rev) - } else { - None - }; - Ok((hosted.status, rev)) - } - - /// Return the durable hosting state for an authenticated account. - /// - /// # Errors - /// - /// Returns `AuthenticationRequired` if the account no longer exists. - pub fn account_status(&self, did: &str) -> Result { - self.database - .hosted_account_by_identifier(did)? - .map(|hosted| hosted.status) - .ok_or(Error::AuthenticationRequired) - } - - /// Activate a user-deactivated account and emit its durable stream event. - /// - /// # Errors - /// - /// Host-enforced inactive states cannot be overridden by the account. - pub fn activate_account(&self, did: &str) -> Result<()> { - self.transition_user_account(did, AccountStatus::Active) - } - - /// Deactivate an active account and emit its durable stream event. - /// - /// # Errors - /// - /// Host-enforced inactive states cannot be replaced by user deactivation. - pub fn deactivate_account(&self, did: &str) -> Result<()> { - self.transition_user_account(did, AccountStatus::Deactivated) - } - - /// Apply an operator-authorized hosting state, including host-enforced states. - /// - /// # Errors - /// - /// Returns an error when the account is missing or the compare-and-set cannot - /// converge after bounded concurrent changes. - pub fn set_operator_account_status(&self, did: &str, next: AccountStatus) -> Result<()> { - self.transition_account(did, next, false) - } - - /// Persist the current handle and append its durable identity stream event. - /// - /// # Errors - /// - /// Returns an error when the handle is invalid, the account is missing, or - /// the compare-and-set cannot converge after bounded concurrent changes. - pub fn set_account_handle(&self, did: &str, next: &str) -> Result { - let next = Handle::new(next.to_owned()) - .map_err(|error| Error::InvalidRequest(format!("invalid account handle: {error}")))?; - for _ in 0..MAX_REPO_RETRIES { - let hosted = self - .database - .hosted_account_by_identifier(did)? - .ok_or(Error::AuthenticationRequired)?; - if hosted.account.handle == next.as_str() { - return Ok(false); - } - match self.database.transition_account_handle( - did, - &hosted.account.handle, - next.as_str(), - self.repo_event_retention, - ) { - Ok(changed) => return Ok(changed), - Err(Error::Conflict(message)) if message == "account handle changed" => {} - Err(error) => return Err(error), - } - } - Err(Error::Conflict( - "account handle changed too frequently; retry the request".to_owned(), - )) - } - - /// Describe a hosted account and the non-empty collections in its repository. - /// - /// # Errors - /// - /// Returns `RepoNotFound` when the identifier is not hosted and fails closed - /// when stored collection names are invalid. - pub fn describe_repo(&self, repo: &str) -> Result<(Account, Vec)> { - let account = self.sync_account(repo)?; - let collections = self.database.list_collections(&account.did)?; - for collection in &collections { - syntax::collection(collection).map_err(|error| { - Error::Internal(format!("stored repository collection is invalid: {error}")) - })?; - } - Ok((account, collections)) - } - - /// Enumerate initialized repositories using the reference PDS time/DID cursor. - /// - /// # Errors - /// - /// Returns `InvalidRequest` for malformed cursors and fails closed when a - /// stored repository head is invalid. - pub fn list_repos( - &self, - cursor: Option<&str>, - limit: usize, - ) -> Result<(Vec, Option)> { - let cursor = cursor.map(parse_repo_list_cursor).transpose()?; - let heads = self.database.repo_heads_after( - cursor.as_ref().map(|(time, did)| (*time, did.as_str())), - None, - limit, - )?; - let next = heads.last().map(repo_list_cursor); - let repos = heads - .into_iter() - .map(|head| RepoListEntry { - did: head.did, - head: head.root, - rev: head.rev, - status: head.status, - }) - .collect(); - Ok((repos, next)) - } - - /// Enumerate initialized repositories containing one public collection. - /// - /// # Errors - /// - /// Returns `InvalidRequest` for malformed collection names or cursors and - /// fails closed when a stored repository head is invalid. - pub fn list_repos_by_collection( - &self, - collection: &str, - cursor: Option<&str>, - limit: usize, - ) -> Result<(Vec, Option)> { - syntax::collection(collection)?; - let cursor = cursor.map(parse_repo_list_cursor).transpose()?; - let heads = self.database.repo_heads_after( - cursor.as_ref().map(|(time, did)| (*time, did.as_str())), - Some(collection), - limit, - )?; - let next = heads.last().map(repo_list_cursor); - Ok((heads.into_iter().map(|head| head.did).collect(), next)) - } - - /// Export the current signed repository as a bounded CAR v1 archive. - /// - /// # Errors - /// - /// Returns `RepoNotFound` for an unhosted or not-yet-initialized repository, - /// fails closed if stored history or the signed graph is invalid, and - /// bounds both full and incremental archives. - pub async fn export_repo(&self, did: &str, since: Option<&str>) -> Result> { - let head = self.sync_head(did)?; - let mut blocks = RepoBlockStore::new(self.database.clone()); - let mut repo = self.open_repo(&mut blocks, &head).await?; - if let Some(since) = since { - syntax::tid(since)?; - drop(repo); - let changed = self.database.repo_blocks_after_revision(did, since)?; - return repository_blocks_car(changed, head.root).await; - } - let mut bytes = Vec::new(); - let mut car = car_writer(&mut bytes, [head.root]).await?; - repo.export_into(&mut car).await.map_err(map_repo_error)?; - drop(car); - Ok(bytes) - } - - /// Export the signed MST proof and value blocks for one current record. - /// - /// # Errors - /// - /// Returns the canonical repository or record not-found error, or fails - /// closed if the stored graph or signature is invalid. - pub async fn export_record(&self, did: &str, collection: &str, rkey: &str) -> Result> { - syntax::collection(collection)?; - syntax::record_key(rkey)?; - let head = self.sync_head(did)?; - let mut blocks = RepoBlockStore::new(self.database.clone()); - let mut repo = self.open_repo(&mut blocks, &head).await?; - let path = format!("{collection}/{rkey}"); - if repo - .get_raw::(&path) - .await - .map_err(map_repo_error)? - .is_none() - { - return Err(Error::Xrpc { - name: "RecordNotFound", - message: "record does not exist".to_owned(), - }); - } - let mut bytes = Vec::new(); - let mut car = car_writer(&mut bytes, [head.root]).await?; - repo.extract_raw_into(&path, &mut car) - .await - .map_err(map_repo_error)?; - drop(car); - Ok(bytes) - } - - /// Export selected blocks from the current signed repository graph. - /// - /// # Errors - /// - /// Rejects empty or excessive requests, CIDs outside the repository's - /// current reachable graph, and invalid stored repository state. - pub async fn export_blocks(&self, did: &str, requested: &[Cid]) -> Result> { - if requested.is_empty() { - return Err(Error::InvalidRequest( - "at least one block CID is required".to_owned(), - )); - } - if requested.len() > MAX_BLOCKS_PER_REQUEST { - return Err(Error::InvalidRequest(format!( - "at most {MAX_BLOCKS_PER_REQUEST} block CIDs may be requested" - ))); - } - let head = self.sync_head(did)?; - let mut blocks = RepoBlockStore::new(self.database.clone()); - let mut repo = self.open_repo(&mut blocks, &head).await?; - let reachable = repo - .export() - .await - .map_err(map_repo_error)? - .collect::>(); - drop(repo); - if let Some(cid) = requested.iter().find(|cid| !reachable.contains(cid)) { - return Err(Error::Xrpc { - name: "BlockNotFound", - message: format!("block {cid} is not in the current repository"), - }); - } - - let mut bytes = Vec::new(); - let mut car = car_writer(&mut bytes, []).await?; - for cid in requested { - let block = blocks.read_block(*cid).await.map_err(map_block_error)?; - let written = car - .write_block(cid.codec(), SHA2_256, &block) - .await - .map_err(map_block_error)?; - if written != *cid { - return Err(Error::Internal(format!( - "ATrium CAR writer changed repository block CID {cid}" - ))); - } - } - drop(car); - Ok(bytes) - } - - /// Remove and announce a record owned by the authenticated account. - /// - /// # Errors - /// - /// Returns an error for unauthorized or invalid paths, missing records, or - /// failed persistence. - pub async fn delete_record( - &self, - actor: &Account, - repo: &str, - collection: &str, - rkey: &str, - expected_cid: Option<&Cid>, - expected_commit: Option<&Cid>, - ) -> Result { - self.authorize_repo(actor, repo)?; - syntax::collection(collection)?; - syntax::record_key(rkey)?; - let deleted = self.database.get_record(&actor.did, collection, rkey)?; - if expected_cid.is_some_and(|expected| expected != &deleted.cid) { - return Err(invalid_swap("record CID does not match")); - } - let commit = self - .commit_record_mutation( - actor, - expected_commit, - PendingRecordMutation::Delete(deleted.clone()), - ) - .await?; - self.publish(RepositoryEvent::RecordDeleted { - did: actor.did.clone(), - collection: collection.to_owned(), - rkey: rkey.to_owned(), - previous_cid: deleted.cid, - }) - .await; - Ok(commit) - } - - async fn commit_record_mutation( - &self, - actor: &Account, - expected_commit: Option<&Cid>, - mutation: PendingRecordMutation, - ) -> Result { - for _ in 0..MAX_REPO_RETRIES { - let previous = self.database.repo_head(&actor.did)?; - if expected_commit - .is_some_and(|expected| previous.as_ref().is_none_or(|head| expected != &head.root)) - { - return Err(invalid_swap("repository commit does not match")); - } - let rev = previous.as_ref().map_or_else( - || Ok(self.tids.next()), - |head| { - self.tids.next_after(&head.rev).ok_or_else(|| { - Error::Internal("stored repository revision is invalid".to_owned()) - }) - }, - )?; - let tid = Tid::new(rev.clone()).map_err(|error| { - Error::Internal(format!("generated repository revision is invalid: {error}")) - })?; - let mut blocks = RepoBlockStore::new(self.database.clone()); - let mut repo = self - .open_or_create_repo(&mut blocks, actor, previous.as_ref()) - .await?; - let previous_data = previous.as_ref().map(|_| repo.commit().data()); - let path = match &mutation { - PendingRecordMutation::Create(record) | PendingRecordMutation::Delete(record) => { - format!("{}/{}", record.collection, record.rkey) - } - PendingRecordMutation::Update { replacement, .. } => { - format!("{}/{}", replacement.collection, replacement.rkey) - } - }; - let root = self - .apply_record_mutation(&mut repo, &mutation, &path, &tid) - .await?; - let reachable = if previous.is_none() { - Some( - repo.export() - .await - .map_err(map_repo_error)? - .collect::>(), - ) - } else { - None - }; - drop(repo); - let staged = match reachable { - Some(reachable) => blocks.reachable_writes(reachable)?, - None => blocks.writes(), - }; - let commit = PendingRepoCommit { - did: actor.did.clone(), - expected_head: previous.as_ref().map(|head| head.root), - previous_rev: previous.as_ref().map(|head| head.rev.clone()), - previous_data, - head: RepoHead { - root, - rev: rev.clone(), - }, - blocks: staged, - mutation: mutation.clone(), - }; - match self - .database - .commit_repo(&commit, self.repo_event_retention) - { - Ok(()) => { - return Ok(CommitView { - cid: root.to_string(), - rev, - }); - } - Err(Error::Conflict(message)) if message == "repository head changed" => {} - Err(error) => return Err(error), - } - } - Err(Error::Conflict( - "repository changed too frequently; retry the request".to_owned(), - )) - } - - async fn apply_record_mutation( - &self, - repo: &mut Repository<&mut RepoBlockStore>, - mutation: &PendingRecordMutation, - path: &str, - tid: &Tid, - ) -> Result { - let (mut builder, expected_cid) = match mutation { - PendingRecordMutation::Create(record) => { - let data = atproto_record(&record.value)?; - let (builder, cid) = repo.add_raw(path, &data).await.map_err(map_repo_error)?; - (builder, Some((cid, record.cid))) - } - PendingRecordMutation::Update { replacement, .. } => { - let data = atproto_record(&replacement.value)?; - let (builder, cid) = repo.update_raw(path, &data).await.map_err(map_repo_error)?; - (builder, Some((cid, replacement.cid))) - } - PendingRecordMutation::Delete(_) => { - (repo.delete_raw(path).await.map_err(map_repo_error)?, None) - } - }; - if expected_cid.is_some_and(|(actual, expected)| actual != expected) { - return Err(Error::Internal( - "ATrium record CID differs from validated record CID".to_owned(), - )); - } - builder.rev(tid.clone()); - let signature = self.sign(&builder.bytes())?; - builder.finalize(signature).await.map_err(map_repo_error) - } - - async fn open_or_create_repo<'a>( - &self, - blocks: &'a mut RepoBlockStore, - actor: &Account, - head: Option<&RepoHead>, - ) -> Result> { - if let Some(head) = head { - return self.open_repo(blocks, head).await; - } - - let did = Did::new(actor.did.clone()) - .map_err(|error| Error::Internal(format!("hosted account DID is invalid: {error}")))?; - let builder = Repository::create(&mut *blocks, did) - .await - .map_err(map_repo_error)?; - let signature = self.sign(&builder.bytes())?; - let mut repo = builder.finalize(signature).await.map_err(map_repo_error)?; - - for record in self.database.all_records(&actor.did)? { - let path = format!("{}/{}", record.collection, record.rkey); - let data = atproto_record(&record.value).map_err(|error| { - Error::Internal(format!( - "stored record {} is invalid: {error}", - record.uri() - )) - })?; - let (builder, cid) = repo.add_raw(&path, &data).await.map_err(map_repo_error)?; - if cid != record.cid { - return Err(Error::Internal(format!( - "stored record {} has noncanonical CID", - record.uri() - ))); - } - let signature = self.sign(&builder.bytes())?; - builder.finalize(signature).await.map_err(map_repo_error)?; - } - Ok(repo) - } - - async fn open_repo<'a>( - &self, - blocks: &'a mut RepoBlockStore, - head: &RepoHead, - ) -> Result> { - let repo = Repository::open(&mut *blocks, head.root) - .await - .map_err(map_repo_error)?; - let commit = repo.commit(); - if commit.rev().to_string() != head.rev { - return Err(Error::Internal( - "stored repository head revision does not match its commit".to_owned(), - )); - } - verify_signature(&self.signing_did, &commit.bytes(), commit.sig()).map_err(|error| { - Error::Internal(format!("stored repository signature is invalid: {error}")) - })?; - Ok(repo) - } - - async fn verify_repo(&self, head: &ListedRepoHead) -> Result<()> { - let mut blocks = RepoBlockStore::new(self.database.clone()); - let mut repo = self - .open_repo( - &mut blocks, - &RepoHead { - root: head.root, - rev: head.rev.clone(), - }, - ) - .await?; - let entries = { - let mut tree = repo.tree(); - tree.entries() - .try_collect::>() - .await - .map_err(|error| map_repo_error(error.into()))? - }; - let records = self.database.all_records(&head.did)?; - if entries.len() != records.len() { - return Err(Error::Internal(format!( - "stored repository {} has {} MST entries but {} indexed records", - head.did, - entries.len(), - records.len() - ))); - } - - for ((path, cid), record) in entries.into_iter().zip(records) { - let expected_path = format!("{}/{}", record.collection, record.rkey); - if path != expected_path || cid != record.cid { - return Err(Error::Internal(format!( - "stored repository {} MST entry does not match indexed record {}", - head.did, - record.uri() - ))); - } - let expected_cid = record_cid(&record.value).map_err(|error| { - Error::Internal(format!( - "stored record {} is invalid: {error}", - record.uri() - )) - })?; - if expected_cid != record.cid { - return Err(Error::Internal(format!( - "stored record {} has a noncanonical CID", - record.uri() - ))); - } - let expected_value = atproto_record(&record.value).map_err(|error| { - Error::Internal(format!( - "stored record {} is invalid: {error}", - record.uri() - )) - })?; - let value = repo - .get_raw::(&path) - .await - .map_err(map_repo_error)?; - let actual_bytes = value - .as_ref() - .map(serde_ipld_dagcbor::to_vec) - .transpose() - .map_err(|error| { - Error::Internal(format!( - "stored record {} cannot be encoded: {error}", - record.uri() - )) - })?; - let expected_bytes = serde_ipld_dagcbor::to_vec(&expected_value).map_err(|error| { - Error::Internal(format!( - "stored record {} cannot be encoded: {error}", - record.uri() - )) - })?; - if actual_bytes.as_deref() != Some(expected_bytes.as_slice()) { - return Err(Error::Internal(format!( - "stored repository block does not match indexed record {}", - record.uri() - ))); - } - } - Ok(()) - } - - async fn verify_repo_events(&self, heads: &[ListedRepoHead]) -> Result<()> { - let Some(earliest) = self.database.earliest_repo_event_sequence()? else { - if heads.is_empty() { - return Ok(()); - } - return Err(Error::Internal( - "stored repository heads have no retained events".to_owned(), - )); - }; - let mut cursor = earliest - 1; - let mut expected = earliest; - let mut latest = HashMap::::new(); - loop { - let events = self - .database - .repo_events_after(cursor, MAX_REPO_EVENTS_PER_PAGE)?; - if events.is_empty() { - break; - } - for event in events { - let sequence = event.seq(); - if sequence != expected { - return Err(Error::Internal(format!( - "stored repository event sequence skips from {} to {}", - expected - 1, - sequence - ))); - } - match event { - StoredRepoEvent::Commit(event) => { - let event = *event; - let did = event.did.clone(); - let rev = event.rev.clone(); - let root = event.root; - let since = event.since.clone(); - let prev_data = event.prev_data; - let (_, data) = self.repo_commit_event(event).await?; - if let Some((previous_rev, _, previous_data)) = latest.get(&did) - && (since.as_deref() != Some(previous_rev) - || prev_data != Some(*previous_data)) - { - return Err(Error::Internal(format!( - "stored repository event {sequence} does not follow the prior {did} event" - ))); - } - latest.insert(did, (rev, root, data)); - } - StoredRepoEvent::Identity(event) => { - Self::repo_identity_event(event)?; - } - StoredRepoEvent::Account(event) => { - Self::repo_account_event(event)?; - } - } - cursor = sequence; - expected += 1; - } - } - for head in heads { - let Some((rev, root, _)) = latest.get(&head.did) else { - return Err(Error::Internal(format!( - "stored repository {} has no retained event", - head.did - ))); - }; - if rev != &head.rev || root != &head.root { - return Err(Error::Internal(format!( - "stored repository {} head does not match its latest retained event", - head.did - ))); - } - } - if let Some(did) = latest - .keys() - .find(|did| !heads.iter().any(|head| &head.did == *did)) - { - return Err(Error::Internal(format!( - "stored repository event exists without a head for {did}" - ))); - } - Ok(()) - } - - async fn verify_blob(&self, owner: &str, blob: &Blob) -> Result<()> { - let bytes = self.blobs.get(owner, &blob.cid).await?.ok_or_else(|| { - Error::Internal(format!( - "stored blob metadata has no bytes for {owner}/{}", - blob.cid - )) - })?; - if u64::try_from(bytes.len()).unwrap_or(u64::MAX) != blob.size { - return Err(Error::Internal(format!( - "stored blob size does not match metadata for {owner}/{}", - blob.cid - ))); - } - let cid = sha256_cid(RAW_CODEC, &bytes).map_err(|error| { - Error::Internal(format!( - "stored blob bytes are invalid for {owner}/{}: {error}", - blob.cid - )) - })?; - if cid != blob.cid { - return Err(Error::Internal(format!( - "stored blob bytes do not match metadata CID for {owner}/{}", - blob.cid - ))); - } - Ok(()) - } - - fn sign(&self, bytes: &[u8]) -> Result> { - self.signer - .sign(bytes) - .map_err(|error| Error::Internal(format!("repository commit signing failed: {error}"))) - } - - /// Validate and content-address opaque bytes, then store metadata and bytes. - /// - /// # Errors - /// - /// Returns an error for empty, oversized, or incorrectly typed blobs, or - /// when either the blob adapter or metadata store fails. - pub async fn upload_blob( - &self, - actor: &Account, - media_type: &str, - bytes: &[u8], - ) -> Result { - if bytes.is_empty() { - return Err(Error::InvalidRequest("blob must not be empty".to_owned())); - } - if bytes.len() > MAX_BLOB_BYTES { - return Err(Error::InvalidRequest(format!( - "blob exceeds the {MAX_BLOB_BYTES}-byte limit" - ))); - } - media_type - .parse::() - .map_err(|error| Error::InvalidRequest(format!("invalid content type: {error}")))?; - - let cid = sha256_cid(RAW_CODEC, bytes)?; - let blob = Blob { - cid, - media_type: media_type.to_owned(), - size: u64::try_from(bytes.len()).unwrap_or(u64::MAX), - }; - - self.database.begin_blob_upload(&actor.did, &blob)?; - self.blobs.put(&actor.did, &cid, bytes).await?; - self.database.finish_blob_upload(&actor.did, &cid)?; - self.publish(RepositoryEvent::BlobStored { - did: actor.did.clone(), - cid, - media_type: media_type.to_owned(), - size: blob.size, - }) - .await; - Ok(blob) - } - - /// Fetch public blob metadata and bytes by owner and CID. - /// - /// # Errors - /// - /// Returns the standard `RepoNotFound` or `BlobNotFound` XRPC error when the - /// owner, metadata, or bytes are absent, and propagates adapter failures. - pub async fn get_blob(&self, repo: &str, cid: &Cid) -> Result<(Blob, Vec)> { - let account = self.resolve_repo(repo).map_err(|error| match error { - Error::NotFound => Error::Xrpc { - name: "RepoNotFound", - message: "repository does not exist".to_owned(), - }, - error => error, - })?; - let blob = self - .database - .blob(&account.did, cid) - .map_err(|error| match error { - Error::NotFound => Error::Xrpc { - name: "BlobNotFound", - message: "blob does not exist".to_owned(), - }, - error => error, - })?; - let bytes = self - .blobs - .get(&account.did, cid) - .await? - .ok_or_else(|| Error::Xrpc { - name: "BlobNotFound", - message: "blob does not exist".to_owned(), - })?; - Ok((blob, bytes)) - } - - /// List blob CIDs referenced by the current public repository. - /// - /// # Errors - /// - /// Returns `RepoNotFound` for an unhosted DID and rejects malformed cursors - /// or revisions. Unattached uploads are intentionally not disclosed. - pub fn list_blobs( - &self, - did: &str, - since: Option<&str>, - cursor: Option<&str>, - limit: usize, - ) -> Result<(Vec, Option)> { - let account = self.sync_account(did)?; - if let Some(since) = since { - syntax::tid(since)?; - } - let cursor = cursor - .map(|cursor| { - cursor - .parse::() - .map(|cid| cid.to_string()) - .map_err(|error| { - Error::InvalidRequest(format!("invalid blob cursor CID: {error}")) - }) - }) - .transpose()?; - let cids = - self.database - .referenced_blob_cids(&account.did, since, cursor.as_deref(), limit)?; - let cursor = (cids.len() == limit) - .then(|| cids.last().map(ToString::to_string)) - .flatten(); - Ok((cids, cursor)) - } - - /// List referenced blob CIDs whose owner metadata is absent. - /// - /// # Errors - /// - /// Returns `InvalidRequest` for a malformed cursor and propagates storage - /// corruption or query failures. - pub fn list_missing_blobs( - &self, - actor: &Account, - cursor: Option<&str>, - limit: usize, - ) -> Result<(Vec, Option)> { - let cursor = cursor - .map(|cursor| { - cursor - .parse::() - .map(|cid| cid.to_string()) - .map_err(|error| { - Error::InvalidRequest(format!("invalid missing-blob cursor CID: {error}")) - }) - }) - .transpose()?; - let missing = self - .database - .missing_blob_references(&actor.did, cursor.as_deref(), limit.clamp(1, 1_000))? - .into_iter() - .map(|(cid, record_uri)| MissingBlob { cid, record_uri }) - .collect::>(); - let cursor = missing.last().map(|blob| blob.cid.to_string()); - Ok((missing, cursor)) - } - - /// Rebuild durable repository events as current generated `ATrium` wire models. - /// - /// Legacy events without inductive metadata and oversized commit diffs become - /// `#sync` messages containing the signed commit block. This gives consumers a - /// safe reset point instead of emitting a malformed `#commit` message. - /// - /// # Errors - /// - /// Returns `FutureCursor` for a cursor ahead of durable state and fails closed - /// when persisted event metadata or blocks are invalid. - pub async fn repo_messages_after( - &self, - cursor: i64, - limit: usize, - ) -> Result> { - let latest = self.database.latest_repo_event_sequence()?; - if cursor > latest { - return Err(Error::Xrpc { - name: "FutureCursor", - message: format!("cursor {cursor} is ahead of latest sequence {latest}"), - }); - } - if self - .database - .earliest_repo_event_sequence()? - .is_some_and(|earliest| cursor < earliest - 1) - { - return Ok(vec![subscribe_repos::Message::Info(Box::new( - subscribe_repos::InfoData { - message: Some( - "requested cursor is no longer retained; resynchronize repositories" - .to_owned(), - ), - name: "OutdatedCursor".to_owned(), - } - .into(), - ))]); - } - let events = self - .database - .repo_events_after(cursor, limit.clamp(1, MAX_REPO_EVENTS_PER_PAGE))?; - let mut messages = Vec::with_capacity(events.len()); - for event in events { - messages.push(match event { - StoredRepoEvent::Commit(event) => self.repo_commit_event(*event).await?.0, - StoredRepoEvent::Identity(event) => Self::repo_identity_event(event)?, - StoredRepoEvent::Account(event) => Self::repo_account_event(event)?, - }); - } - Ok(messages) - } - - /// Return the latest durable repository stream cursor. - /// - /// # Errors - /// - /// Returns an error when the outbox sequence cannot be read. - pub fn latest_repo_event_cursor(&self) -> Result { - self.database.latest_repo_event_sequence() - } - - fn repo_identity_event(event: StoredIdentityEvent) -> Result { - Ok(subscribe_repos::Message::Identity(Box::new( - subscribe_repos::IdentityData { - did: parse_event_did(&event.did)?, - handle: event.handle.map(Handle::new).transpose().map_err(|error| { - Error::Internal(format!( - "stored repository identity handle is invalid: {error}" - )) - })?, - seq: event.seq, - time: event_datetime(event.created_at)?, - } - .into(), - ))) - } - - fn repo_account_event(event: StoredAccountEvent) -> Result { - Ok(subscribe_repos::Message::Account(Box::new( - subscribe_repos::AccountData { - active: event.active, - did: parse_event_did(&event.did)?, - seq: event.seq, - status: event.status, - time: event_datetime(event.created_at)?, - } - .into(), - ))) - } - - async fn repo_commit_event( - &self, - event: StoredCommitEvent, - ) -> Result<(subscribe_repos::Message, Cid)> { - let mut store = RepoBlockStore::new(self.database.clone()); - let repo = self - .open_repo( - &mut store, - &RepoHead { - root: event.root, - rev: event.rev.clone(), - }, - ) - .await?; - let data = repo.commit().data(); - drop(repo); - let time = event_datetime(event.created_at)?; - validate_repo_event_metadata(&event)?; - let blocks = if event.incremental { - repository_event_car(&event.blocks, event.root).await? - } else { - Vec::new() - }; - if !event.incremental || blocks.len() > MAX_COMMIT_EVENT_BYTES { - let root_block = - if let Some(block) = event.blocks.first().filter(|(cid, _)| cid == &event.root) { - block.clone() - } else { - let mut store = RepoBlockStore::new(self.database.clone()); - let bytes = store - .read_block(event.root) - .await - .map_err(map_block_error)?; - (event.root, bytes) - }; - let blocks = repository_event_car(&[root_block], event.root).await?; - return Ok(( - subscribe_repos::Message::Sync(Box::new( - subscribe_repos::SyncData { - blocks, - did: parse_event_did(&event.did)?, - rev: event.rev, - seq: event.seq, - time, - } - .into(), - )), - data, - )); - } - let action = event.action; - Ok(( - subscribe_repos::Message::Commit(Box::new( - subscribe_repos::CommitData { - blobs: Vec::new(), - blocks, - commit: CidLink(event.root), - ops: vec![ - subscribe_repos::RepoOpData { - action, - cid: event.cid.map(CidLink), - path: event.path, - prev: event.prev.map(CidLink), - } - .into(), - ], - prev_data: event.prev_data.map(CidLink), - rebase: false, - repo: parse_event_did(&event.did)?, - rev: parse_event_tid(&event.rev)?, - seq: event.seq, - since: event.since.as_deref().map(parse_event_tid).transpose()?, - time, - too_big: false, - } - .into(), - )), - data, - )) - } - - fn authorize_repo(&self, actor: &Account, repo: &str) -> Result<()> { - let requested = self.resolve_repo(repo)?; - if requested.did != actor.did { - return Err(Error::Authorization( - "an access token may only mutate its own repository".to_owned(), - )); - } - Ok(()) - } - - fn resolve_repo(&self, repo: &str) -> Result { - let hosted = self - .database - .hosted_account_by_identifier(repo)? - .ok_or(Error::NotFound)?; - if let Some(name) = hosted.status.repo_error_name() { - return Err(Error::Xrpc { - name, - message: format!("repository is {}", hosted.status.as_str()), - }); - } - Ok(hosted.account) - } - - fn sync_account(&self, did: &str) -> Result { - self.resolve_repo(did).map_err(|error| match error { - Error::NotFound => repo_not_found(), - error => error, - }) - } - - fn transition_user_account(&self, did: &str, next: AccountStatus) -> Result<()> { - self.transition_account(did, next, true) - } - - fn transition_account( - &self, - did: &str, - next: AccountStatus, - user_initiated: bool, - ) -> Result<()> { - for _ in 0..MAX_REPO_RETRIES { - let hosted = self - .database - .hosted_account_by_identifier(did)? - .ok_or(Error::AuthenticationRequired)?; - let current = hosted.status; - if current == next { - return Ok(()); - } - if user_initiated - && !matches!( - (current, next), - (AccountStatus::Active, AccountStatus::Deactivated) - | (AccountStatus::Deactivated, AccountStatus::Active) - ) - { - return Err(account_inactive_error(current)); - } - match self.database.transition_account_status( - did, - current, - next, - self.repo_event_retention, - ) { - Ok(_) => return Ok(()), - Err(Error::Conflict(message)) if message == "account status changed" => {} - Err(error) => return Err(error), - } - } - Err(Error::Conflict( - "account status changed too frequently; retry the request".to_owned(), - )) - } - - fn sync_head(&self, did: &str) -> Result { - let account = self.sync_account(did)?; - self.database - .repo_head(&account.did)? - .ok_or_else(|| Error::Xrpc { - name: "RepoNotFound", - message: "repository has no commits".to_owned(), - }) - } - - async fn publish(&self, event: RepositoryEvent) { - if let Err(error) = self.events.publish(event).await { - // The database mutation is already committed. Returning an error - // would invite a client retry and duplicate the mutation, so event - // delivery failures are visible but do not change the write result. - tracing::error!(%error, "repository event delivery failed"); - } - } -} - -fn invalid_swap(message: &str) -> Error { - Error::Xrpc { - name: "InvalidSwap", - message: message.to_owned(), - } -} - -fn repo_not_found() -> Error { - Error::Xrpc { - name: "RepoNotFound", - message: "repository is not hosted by this PDS".to_owned(), - } -} - -fn account_inactive_error(status: AccountStatus) -> Error { - Error::Xrpc { - name: status.account_error_name().unwrap_or("AccountNotActive"), - message: format!("account is {}", status.as_str()), - } -} - -fn parse_event_did(value: &str) -> Result { - value.parse().map_err(|error| { - Error::Internal(format!("stored repository event DID is invalid: {error}")) - }) -} - -fn parse_repo_list_cursor(cursor: &str) -> Result<(i64, String)> { - let Some((time, did)) = cursor.split_once("::") else { - return Err(Error::InvalidRequest( - "malformed repository cursor".to_owned(), - )); - }; - if time.is_empty() || did.is_empty() || did.contains("::") { - return Err(Error::InvalidRequest( - "malformed repository cursor".to_owned(), - )); - } - let time = time - .parse() - .map_err(|_| Error::InvalidRequest("malformed repository cursor".to_owned()))?; - let did = Did::new(did.to_owned()) - .map_err(|_| Error::InvalidRequest("malformed repository cursor".to_owned()))?; - Ok((time, did.as_str().to_owned())) -} - -fn repo_list_cursor(head: &ListedRepoHead) -> String { - format!("{}::{}", head.created_at_millis, head.did) -} - -fn parse_event_tid(value: &str) -> Result { - value.parse().map_err(|error| { - Error::Internal(format!( - "stored repository event revision is invalid: {error}" - )) - }) -} - -fn event_datetime(timestamp: i64) -> Result { - let value = OffsetDateTime::from_unix_timestamp(timestamp).map_err(|error| { - Error::Internal(format!( - "stored repository event timestamp is invalid: {error}" - )) - })?; - let timestamp = format!( - "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}Z", - value.year(), - u8::from(value.month()), - value.day(), - value.hour(), - value.minute(), - value.second() - ); - timestamp.parse().map_err(|error| { - Error::Internal(format!( - "stored repository event timestamp cannot become a Lexicon datetime: {error}" - )) - }) -} - -fn validate_repo_event_metadata(event: &StoredCommitEvent) -> Result<()> { - let valid_operation = match event.action.as_str() { - "create" => event.cid.is_some() && event.prev.is_none(), - "update" => event.cid.is_some() && event.prev.is_some(), - "delete" => event.cid.is_none() && event.prev.is_some(), - action => { - return Err(Error::Internal(format!( - "stored repository event action is invalid: {action}" - ))); - } - }; - if !valid_operation { - return Err(Error::Internal(format!( - "stored {} repository event has invalid record links", - event.action - ))); - } - if event.since.is_some() != event.prev_data.is_some() { - return Err(Error::Internal( - "stored repository event has incomplete inductive metadata".to_owned(), - )); - } - Ok(()) -} - -async fn repository_event_car(blocks: &[(Cid, Vec)], root: Cid) -> Result> { - if blocks.first().is_none_or(|(cid, _)| cid != &root) { - return Err(Error::Internal( - "stored repository event does not begin with its commit block".to_owned(), - )); - } - repository_blocks_car(blocks.iter().cloned(), root).await -} - -async fn repository_blocks_car( - blocks: impl IntoIterator)>, - root: Cid, -) -> Result> { - let mut bytes = Vec::new(); - let mut car = car_writer(&mut bytes, [root]).await?; - for (cid, block) in blocks { - let written = car - .write_block(cid.codec(), SHA2_256, &block) - .await - .map_err(map_block_error)?; - if written != cid { - return Err(Error::Internal(format!( - "ATrium CAR writer changed repository block CID {cid}" - ))); - } - } - drop(car); - Ok(bytes) -} - -fn map_repo_error(error: AtriumRepoError) -> Error { - match error { - AtriumRepoError::BlockStore(error) => map_block_error(error), - AtriumRepoError::Mst(atrium_repo::mst::Error::KeyAlreadyExists) => { - Error::Conflict("record already exists".to_owned()) - } - AtriumRepoError::Mst(atrium_repo::mst::Error::KeyNotFound) => Error::NotFound, - error => Error::Internal(format!("ATrium repository operation failed: {error}")), - } -} - -fn map_block_error(error: AtriumBlockError) -> Error { - match error { - AtriumBlockError::Other(error) - if error - .downcast_ref::() - .is_some_and(|error| error.kind() == io::ErrorKind::FileTooLarge) => - { - Error::Xrpc { - name: "RepoTooBig", - message: format!( - "repository export exceeds the {MAX_REPO_EXPORT_BYTES}-byte limit" - ), - } - } - error => Error::Internal(format!("ATrium block operation failed: {error}")), - } -} - -async fn car_writer( - bytes: &mut Vec, - roots: impl IntoIterator, -) -> Result>> { - CarStore::create_with_roots(BoundedBuffer::new(bytes, MAX_REPO_EXPORT_BYTES), roots) - .await - .map_err(|error| Error::Internal(format!("could not create repository CAR: {error}"))) -} - -struct BoundedBuffer<'a> { - inner: Cursor<&'a mut Vec>, - max_bytes: usize, -} - -impl<'a> BoundedBuffer<'a> { - fn new(bytes: &'a mut Vec, max_bytes: usize) -> Self { - Self { - inner: Cursor::new(bytes), - max_bytes, - } - } -} - -impl AsyncRead for BoundedBuffer<'_> { - fn poll_read( - mut self: Pin<&mut Self>, - context: &mut Context<'_>, - buffer: &mut ReadBuf<'_>, - ) -> Poll> { - Pin::new(&mut self.inner).poll_read(context, buffer) - } -} - -impl AsyncSeek for BoundedBuffer<'_> { - fn start_seek(mut self: Pin<&mut Self>, position: io::SeekFrom) -> io::Result<()> { - Pin::new(&mut self.inner).start_seek(position) - } - - fn poll_complete(mut self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll> { - Pin::new(&mut self.inner).poll_complete(context) - } -} - -impl AsyncWrite for BoundedBuffer<'_> { - fn poll_write( - mut self: Pin<&mut Self>, - context: &mut Context<'_>, - buffer: &[u8], - ) -> Poll> { - let position = usize::try_from(self.inner.position()).unwrap_or(usize::MAX); - let remaining = self.max_bytes.saturating_sub(position); - if buffer.len() > remaining { - return Poll::Ready(Err(io::Error::new( - io::ErrorKind::FileTooLarge, - "repository export exceeded its byte limit", - ))); - } - Pin::new(&mut self.inner).poll_write(context, buffer) - } - - fn poll_flush(mut self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll> { - Pin::new(&mut self.inner).poll_flush(context) - } - - fn poll_shutdown(mut self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll> { - Pin::new(&mut self.inner).poll_shutdown(context) - } -} - -pub(crate) fn validate_record_type(collection: &str, value: &Value) -> Result<()> { - let record_type = value - .as_object() - .and_then(|object| object.get("$type")) - .and_then(Value::as_str) - .ok_or_else(|| Error::InvalidRequest("record requires a string `$type`".to_owned()))?; - if record_type != collection { - return Err(Error::InvalidRequest(format!( - "record `$type` must equal collection `{collection}`" - ))); - } - Ok(()) -} - -pub(crate) fn record_cid(value: &Value) -> Result { - let bytes = serde_ipld_dagcbor::to_vec(&atproto_record(value)?) - .map_err(|error| Error::InvalidRequest(format!("record is not valid DAG-CBOR: {error}")))?; - if bytes.len() > MAX_RECORD_BYTES { - return Err(Error::InvalidRequest(format!( - "record exceeds the {MAX_RECORD_BYTES}-byte DAG-CBOR limit" - ))); - } - dag_cbor_cid(&bytes) -} - -enum AtprotoData { - Null, - Boolean(bool), - Integer(i64), - Bytes(serde_bytes::ByteBuf), - String(String), - Link(CidLink), - List(Vec), - Map(BTreeMap), -} - -impl Serialize for AtprotoData { - fn serialize(&self, serializer: S) -> std::result::Result - where - S: Serializer, - { - match self { - Self::Null => serializer.serialize_none(), - Self::Boolean(value) => serializer.serialize_bool(*value), - Self::Integer(value) => serializer.serialize_i64(*value), - Self::String(value) => serializer.serialize_str(value), - Self::Bytes(value) => value.serialize(serializer), - Self::Link(value) => value.serialize(serializer), - Self::List(value) => value.serialize(serializer), - Self::Map(value) => value.serialize(serializer), - } - } -} - -fn atproto_record(value: &Value) -> Result { - if !value.is_object() { - return Err(Error::InvalidRequest( - "record must be an AT Protocol object".to_owned(), - )); - } - atproto_data(value, 0) -} - -fn atproto_data(value: &Value, depth: usize) -> Result { - if depth > MAX_RECORD_DEPTH { - return Err(Error::InvalidRequest(format!( - "record exceeds the {MAX_RECORD_DEPTH}-level nesting limit" - ))); - } - match value { - Value::Null => Ok(AtprotoData::Null), - Value::Bool(value) => Ok(AtprotoData::Boolean(*value)), - Value::Number(value) => { - let integer = value.as_i64().or_else(|| { - value.as_f64().and_then(|value| { - (value.fract() == 0.0 && value.abs() <= MAX_SAFE_JSON_INTEGER) - .then(|| format!("{value:.0}").parse().ok()) - .flatten() - }) - }); - integer.map(AtprotoData::Integer).ok_or_else(|| { - Error::InvalidRequest("record numbers must be signed 64-bit integers".to_owned()) - }) - } - Value::String(value) => Ok(AtprotoData::String(value.clone())), - Value::Array(values) => values - .iter() - .map(|value| atproto_data(value, depth + 1)) - .collect::>>() - .map(AtprotoData::List), - Value::Object(values) => atproto_object(values, depth), - } -} - -fn atproto_object(values: &serde_json::Map, depth: usize) -> Result { - if values - .get("$type") - .is_some_and(|value| value.as_str().is_none_or(str::is_empty)) - { - return Err(Error::InvalidRequest( - "record `$type` fields must be non-empty strings".to_owned(), - )); - } - validate_blob_ref(values)?; - if let Some(link) = atproto_link(values)? { - return Ok(link); - } - if let Some(bytes) = atproto_bytes(values)? { - return Ok(bytes); - } - values - .iter() - .map(|(key, value)| { - if key.is_empty() || key.len() > MAX_RECORD_KEY_BYTES { - return Err(Error::InvalidRequest(format!( - "record object keys must contain 1 to {MAX_RECORD_KEY_BYTES} UTF-8 bytes" - ))); - } - Ok((key.clone(), atproto_data(value, depth + 1)?)) - }) - .collect::>>() - .map(AtprotoData::Map) -} - -fn validate_blob_ref(values: &serde_json::Map) -> Result<()> { - if values.get("$type").and_then(Value::as_str) != Some("blob") { - return Ok(()); - } - let blob = serde_json::from_value::(Value::Object(values.clone())).map_err( - |error| { - Error::InvalidRequest(format!( - "record contains an invalid blob reference: {error}" - )) - }, - )?; - let AtriumBlobRef::Typed(TypedBlobRef::Blob(blob)) = blob else { - return Err(Error::InvalidRequest( - "record blob references must use the current typed format".to_owned(), - )); - }; - let hash = blob.r#ref.0.hash(); - if values.len() != 4 - || blob.mime_type.is_empty() - || blob.size == 0 - || blob.r#ref.0.version() != cid::Version::V1 - || blob.r#ref.0.codec() != RAW_CODEC - || hash.code() != SHA2_256 - || hash.digest().len() != 32 - { - return Err(Error::InvalidRequest( - "record contains a noncanonical blob reference".to_owned(), - )); - } - Ok(()) -} - -fn atproto_link(values: &serde_json::Map) -> Result> { - let Some(value) = values.get("$link") else { - return Ok(None); - }; - if values.len() != 1 { - return Err(Error::InvalidRequest( - "record CID links cannot contain extra fields".to_owned(), - )); - } - let value = value - .as_str() - .ok_or_else(|| Error::InvalidRequest("record CID link must be a string".to_owned()))?; - let link = CidLink::try_from(value) - .map_err(|error| Error::InvalidRequest(format!("record CID link is invalid: {error}")))?; - let hash = link.0.hash(); - if value != link.0.to_string() - || link.0.version() != cid::Version::V1 - || !matches!(link.0.codec(), DAG_CBOR_CODEC | RAW_CODEC) - || hash.code() != SHA2_256 - || hash.digest().len() != 32 - { - return Err(Error::InvalidRequest( - "record CID links must use canonical base32 CIDv1 with dag-cbor or raw codec and SHA-256".to_owned(), - )); - } - Ok(Some(AtprotoData::Link(link))) -} - -fn atproto_bytes(values: &serde_json::Map) -> Result> { - let Some(value) = values.get("$bytes") else { - return Ok(None); - }; - if values.len() != 1 { - return Err(Error::InvalidRequest( - "record bytes cannot contain extra fields".to_owned(), - )); - } - let value = value - .as_str() - .ok_or_else(|| Error::InvalidRequest("record bytes must be a base64 string".to_owned()))?; - let bytes = STANDARD - .decode(value) - .or_else(|_| STANDARD_NO_PAD.decode(value)) - .map_err(|error| { - Error::InvalidRequest(format!("record bytes are invalid base64: {error}")) - })?; - Ok(Some(AtprotoData::Bytes(bytes.into()))) -} - -pub(crate) fn dag_cbor_cid(bytes: &[u8]) -> Result { - sha256_cid(DAG_CBOR_CODEC, bytes) -} - -fn sha256_cid(codec: u64, bytes: &[u8]) -> Result { - let digest = Sha256::digest(bytes); - let multihash = cid::multihash::Multihash::<64>::wrap(0x12, &digest).map_err(|error| { - Error::Internal(format!("could not construct SHA-256 multihash: {error}")) - })?; - Ok(Cid::new_v1(codec, multihash)) -} - -#[cfg(test)] -mod tests; diff --git a/src/repository/accounts.rs b/src/repository/accounts.rs new file mode 100644 index 0000000..a030e0f --- /dev/null +++ b/src/repository/accounts.rs @@ -0,0 +1,248 @@ +//! Account authorization, hosting lifecycle, and repository discovery. +use super::{MAX_REPO_RETRIES, RepositoryService}; +use crate::error::ErrorContext; +use crate::{ + database::{LifecycleEvent, ListedRepoHead, RepoHead}, + domain::account::{Account, AccountStatus}, + error::{Error, Result}, + syntax, +}; +use atrium_api::types::string::{Did, Handle}; + +fn repo_not_found() -> Error { + Error::xrpc("RepoNotFound", "repository is not hosted by this PDS") +} + +fn parse_repo_list_cursor(cursor: &str) -> Result<(i64, String)> { + let invalid_cursor = || Error::InvalidRequest("malformed repository cursor".to_owned()); + let (time, did) = cursor + .split_once("::") + .filter(|(time, did)| !time.is_empty() && !did.is_empty() && !did.contains("::")) + .ok_or_else(invalid_cursor)?; + let time = time.parse().map_err(|_| invalid_cursor())?; + let did = Did::new(did.to_owned()).map_err(|_| invalid_cursor())?; + Ok((time, did.as_str().to_owned())) +} + +fn repo_list_cursor(head: &ListedRepoHead) -> String { + format!("{}::{}", head.created_at_millis, head.did) +} + +impl RepositoryService { + /// Return the current revision for a hosted repository, when initialized. + /// + /// # Errors + /// + /// Returns `RepoNotFound` when the DID is not hosted by this PDS. + pub fn repo_status(&self, did: &str) -> Result<(AccountStatus, Option)> { + let (account, current_status) = self + .database + .hosted_account_by_identifier(did)? + .ok_or_else(repo_not_found)?; + let rev = if current_status.is_active() { + self.database.repo_head(&account.did)?.map(|head| head.rev) + } else { + None + }; + Ok((current_status, rev)) + } + + /// Return the durable hosting state for an authenticated account. + /// + /// # Errors + /// + /// Returns `AuthenticationRequired` if the account no longer exists. + pub fn account_status(&self, did: &str) -> Result { + self.database + .hosted_account_by_identifier(did)? + .map(|(_account, status)| status) + .ok_or(Error::AuthenticationRequired) + } + + /// Activate a user-deactivated account and emit its durable stream event. + /// + /// # Errors + /// + /// Host-enforced inactive states cannot be overridden by the account. + pub fn activate_account(&self, did: &str) -> Result<()> { + self.transition_account(did, LifecycleEvent::Account(AccountStatus::Active), true) + .map(|_| ()) + } + + /// Deactivate an active account and emit its durable stream event. + /// + /// # Errors + /// + /// Host-enforced inactive states cannot be replaced by user deactivation. + pub fn deactivate_account(&self, did: &str) -> Result<()> { + self.transition_account( + did, + LifecycleEvent::Account(AccountStatus::Deactivated), + true, + ) + .map(|_| ()) + } + + /// Apply an operator-authorized hosting state, including host-enforced states. + /// + /// # Errors + /// + /// Returns an error when the account is missing or the compare-and-set cannot + /// converge after bounded concurrent changes. + pub fn set_operator_account_status(&self, did: &str, next: AccountStatus) -> Result<()> { + self.transition_account(did, LifecycleEvent::Account(next), false) + .map(|_| ()) + } + + /// Persist the current handle and append its durable identity stream event. + /// + /// # Errors + /// + /// Returns an error when the handle is invalid, the account is missing, or + /// the compare-and-set cannot converge after bounded concurrent changes. + pub fn set_account_handle(&self, did: &str, next: &str) -> Result { + let next = Handle::new(next.to_owned()) + .context(Error::InvalidRequest, "invalid account handle")?; + self.transition_account(did, LifecycleEvent::Identity(next.as_str()), false) + } + + /// Describe a hosted account and the non-empty collections in its repository. + /// + /// # Errors + /// + /// Returns `RepoNotFound` when the identifier is not hosted and fails closed + /// when stored collection names are invalid. + pub fn describe_repo(&self, repo: &str) -> Result<(Account, Vec)> { + let account = self.sync_account(repo)?; + let collections = self.database.list_collections(&account.did)?; + for collection in &collections { + syntax::collection(collection) + .context(Error::Internal, "stored repository collection is invalid")?; + } + Ok((account, collections)) + } + + /// Enumerate initialized repositories, optionally filtered by collection, + /// using the reference PDS time/DID cursor. + /// + /// # Errors + /// + /// Returns `InvalidRequest` for malformed cursors and fails closed when a + /// stored repository head is invalid. + pub(crate) fn list_repos( + &self, + cursor: Option<&str>, + collection: Option<&str>, + limit: usize, + ) -> Result<(Vec, Option)> { + let cursor = cursor.map(parse_repo_list_cursor).transpose()?; + let heads = self.database.repo_heads_after( + cursor.as_ref().map(|(time, did)| (*time, did.as_str())), + collection, + limit, + )?; + let next = heads.last().map(repo_list_cursor); + Ok((heads, next)) + } + + /// Enumerate initialized repositories containing one public collection. + /// + /// # Errors + /// + /// Returns `InvalidRequest` for malformed collection names or cursors and + /// fails closed when a stored repository head is invalid. + pub fn list_repos_by_collection( + &self, + collection: &str, + cursor: Option<&str>, + limit: usize, + ) -> Result<(Vec, Option)> { + syntax::collection(collection)?; + let (heads, next) = self.list_repos(cursor, Some(collection), limit)?; + Ok((heads.into_iter().map(|head| head.did).collect(), next)) + } + + pub(super) fn authorize_repo(&self, actor: &Account, repo: &str) -> Result<()> { + let requested = self.resolve_repo(repo)?; + if requested.did != actor.did { + return Err(Error::Authorization( + "an access token may only mutate its own repository".to_owned(), + )); + } + Ok(()) + } + + pub(super) fn resolve_repo(&self, repo: &str) -> Result { + let (account, current_status) = self + .database + .hosted_account_by_identifier(repo)? + .ok_or(Error::NotFound)?; + current_status.require_repo_active()?; + Ok(account) + } + + pub(super) fn sync_account(&self, did: &str) -> Result { + self.resolve_repo(did).map_err(|error| match error { + Error::NotFound => repo_not_found(), + error => error, + }) + } + + fn transition_account( + &self, + did: &str, + next: LifecycleEvent<'_>, + user_initiated: bool, + ) -> Result { + let changed = match next { + LifecycleEvent::Identity(_) => "account handle changed", + LifecycleEvent::Account(_) => "account status changed", + }; + for _ in 0..MAX_REPO_RETRIES { + let (account, current_status) = self + .database + .hosted_account_by_identifier(did)? + .ok_or(Error::AuthenticationRequired)?; + let (current, target) = match next { + LifecycleEvent::Identity(handle) => (account.handle.as_str(), handle), + LifecycleEvent::Account(status) => (current_status.as_str(), status.as_str()), + }; + if current == target { + return Ok(false); + } + if let LifecycleEvent::Account(status) = next + && user_initiated + && !matches!( + (current_status, status), + (AccountStatus::Active, AccountStatus::Deactivated) + | (AccountStatus::Deactivated, AccountStatus::Active) + ) + { + return Err(current_status.account_error()); + } + match self + .database + .transition_account(did, current, next, self.repo_event_retention) + { + Ok(changed) => return Ok(changed), + Err(Error::Conflict(message)) if message == changed => {} + Err(error) => return Err(error), + } + } + Err(Error::Conflict(format!( + "{changed} too frequently; retry the request" + ))) + } + + /// Return the current signed repository head. + /// + /// # Errors + /// + /// Returns `RepoNotFound` for an unhosted or not-yet-initialized repository. + pub(crate) fn sync_head(&self, did: &str) -> Result { + let account = self.sync_account(did)?; + self.database + .repo_head(&account.did)? + .ok_or_else(|| Error::xrpc("RepoNotFound", "repository has no commits")) + } +} diff --git a/src/repository/blobs.rs b/src/repository/blobs.rs new file mode 100644 index 0000000..35ed014 --- /dev/null +++ b/src/repository/blobs.rs @@ -0,0 +1,149 @@ +//! Blob publication, reference-gated reads, and inventories. +use super::RepositoryService; +use crate::error::ErrorContext; +use crate::{ + domain::{account::Account, blob::Blob, event::RepositoryEvent}, + error::{Error, Result}, + syntax, +}; +use cid::Cid; +use cpds_protocol::sha256_cid; +pub(super) const RAW_CODEC: u64 = 0x55; +pub(crate) const MAX_BLOB_BYTES: usize = 5 * 1024 * 1024; + +pub type MissingBlob = (Cid, String); + +impl RepositoryService { + /// Validate and content-address opaque bytes, then store metadata and bytes. + /// + /// # Errors + /// + /// Returns an error for empty, oversized, or incorrectly typed blobs, or + /// when either the blob adapter or metadata store fails. + pub async fn upload_blob( + &self, + actor: &Account, + media_type: &str, + bytes: &[u8], + ) -> Result { + if bytes.is_empty() { + return Err(Error::InvalidRequest("blob must not be empty".to_owned())); + } + if bytes.len() > MAX_BLOB_BYTES { + return Err(Error::InvalidRequest(format!( + "blob exceeds the {MAX_BLOB_BYTES}-byte limit" + ))); + } + media_type + .parse::() + .context(Error::InvalidRequest, "invalid content type")?; + + let cid = sha256_cid(RAW_CODEC, bytes); + let blob = Blob { + cid, + media_type: media_type.to_owned(), + size: u64::try_from(bytes.len()).unwrap_or(u64::MAX), + }; + + self.database.begin_blob_upload(&actor.did, &blob)?; + self.blobs.put(&actor.did, &cid, bytes).await?; + self.database.finish_blob_upload(&actor.did, &cid)?; + self.publish_events([RepositoryEvent::BlobStored { + did: actor.did.clone(), + cid, + media_type: media_type.to_owned(), + size: blob.size, + }]) + .await; + Ok(blob) + } + + /// Fetch public blob metadata and bytes by owner and CID. + /// + /// # Errors + /// + /// Returns the standard `RepoNotFound` or `BlobNotFound` XRPC error when the + /// owner, metadata, or bytes are absent, and propagates adapter failures. + pub async fn get_blob(&self, repo: &str, cid: &Cid) -> Result<(Blob, Vec)> { + let account = self.sync_account(repo)?; + if !self.database.is_public_blob(&account.did, cid)? { + return Err(Error::xrpc( + "BlobNotFound", + "blob is not referenced by the public repository", + )); + } + self.read_blob(&account.did, cid).await + } + + /// Read an available object after the caller has checked its public or Space reference. + pub(crate) async fn read_blob(&self, repo: &str, cid: &Cid) -> Result<(Blob, Vec)> { + let account = self.sync_account(repo)?; + let blob = self + .database + .find_blob(&account.did, cid)? + .ok_or_else(|| Error::xrpc("BlobNotFound", "blob does not exist"))?; + let bytes = self + .blobs + .get(&account.did, cid) + .await? + .ok_or_else(|| Error::xrpc("BlobNotFound", "blob does not exist"))?; + Ok((blob, bytes)) + } + + /// List blob CIDs referenced by the current public repository. + /// + /// # Errors + /// + /// Returns `RepoNotFound` for an unhosted DID and rejects malformed cursors + /// or revisions. Unattached uploads are intentionally not disclosed. + pub fn list_blobs( + &self, + did: &str, + since: Option<&str>, + cursor: Option<&str>, + limit: usize, + ) -> Result<(Vec, Option)> { + let account = self.sync_account(did)?; + if let Some(since) = since { + syntax::tid(since)?; + } + let cursor = cursor + .map(str::parse::) + .transpose() + .context(Error::InvalidRequest, "invalid blob cursor CID")? + .map(|cid| cid.to_string()); + let cids = + self.database + .referenced_blob_cids(&account.did, since, cursor.as_deref(), limit)?; + let cursor = (cids.len() == limit) + .then(|| cids.last().map(ToString::to_string)) + .flatten(); + Ok((cids, cursor)) + } + + /// List referenced blob CIDs whose owner metadata is absent. + /// + /// # Errors + /// + /// Returns `InvalidRequest` for a malformed cursor and propagates storage + /// corruption or query failures. + pub fn list_missing_blobs( + &self, + actor: &Account, + cursor: Option<&str>, + limit: usize, + ) -> Result<(Vec, Option)> { + let cursor = cursor + .map(str::parse::) + .transpose() + .context(Error::InvalidRequest, "invalid missing-blob cursor CID")? + .map(|cid| cid.to_string()); + let missing = self.database.missing_blob_references( + &actor.did, + cursor.as_deref(), + limit.clamp(1, 1_000), + )?; + let cursor = missing.last().map(|(cid, _)| cid.to_string()); + Ok((missing, cursor)) + } +} diff --git a/src/repository/car.rs b/src/repository/car.rs new file mode 100644 index 0000000..ce84812 --- /dev/null +++ b/src/repository/car.rs @@ -0,0 +1,120 @@ +//! A shared bounded buffer for `ATrium` CAR writers. +use crate::error::ErrorContext; +use crate::error::{Error, Result}; +use atrium_repo::blockstore::{ + AsyncBlockStoreWrite as _, CarStore, Error as AtriumBlockError, SHA2_256, +}; +use cid::Cid; +use std::{ + io::{self, Cursor}, + pin::Pin, + task::{Context, Poll}, +}; +use tokio::io::{AsyncRead, AsyncSeek, AsyncWrite, ReadBuf}; +const MAX_REPO_EXPORT_BYTES: usize = 64 * 1024 * 1024; + +pub(super) async fn repository_blocks_car( + blocks: impl IntoIterator)>, + root: Cid, +) -> Result> { + let mut bytes = Vec::new(); + let mut car = car_writer(&mut bytes, [root]).await?; + for (cid, block) in blocks { + let written = car + .write_block(cid.codec(), SHA2_256, &block) + .await + .map_err(map_block_error)?; + if written != cid { + return Err(Error::Internal(format!( + "ATrium CAR writer changed repository block CID {cid}" + ))); + } + } + drop(car); + Ok(bytes) +} + +pub(super) fn map_block_error(error: AtriumBlockError) -> Error { + match error { + AtriumBlockError::Other(error) + if error + .downcast_ref::() + .is_some_and(|error| error.kind() == io::ErrorKind::FileTooLarge) => + { + Error::xrpc( + "RepoTooBig", + format!("repository export exceeds the {MAX_REPO_EXPORT_BYTES}-byte limit"), + ) + } + error => Error::Internal(format!("ATrium block operation failed: {error}")), + } +} + +pub(super) async fn car_writer( + bytes: &mut Vec, + roots: impl IntoIterator, +) -> Result>> { + CarStore::create_with_roots(BoundedBuffer::new(bytes, MAX_REPO_EXPORT_BYTES), roots) + .await + .context(Error::Internal, "could not create repository CAR") +} + +pub(super) struct BoundedBuffer<'a> { + inner: Cursor<&'a mut Vec>, + max_bytes: usize, +} + +impl<'a> BoundedBuffer<'a> { + pub(super) fn new(bytes: &'a mut Vec, max_bytes: usize) -> Self { + Self { + inner: Cursor::new(bytes), + max_bytes, + } + } +} + +impl AsyncRead for BoundedBuffer<'_> { + fn poll_read( + mut self: Pin<&mut Self>, + context: &mut Context<'_>, + buffer: &mut ReadBuf<'_>, + ) -> Poll> { + Pin::new(&mut self.inner).poll_read(context, buffer) + } +} + +impl AsyncSeek for BoundedBuffer<'_> { + fn start_seek(mut self: Pin<&mut Self>, position: io::SeekFrom) -> io::Result<()> { + Pin::new(&mut self.inner).start_seek(position) + } + + fn poll_complete(mut self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.inner).poll_complete(context) + } +} + +impl AsyncWrite for BoundedBuffer<'_> { + fn poll_write( + mut self: Pin<&mut Self>, + context: &mut Context<'_>, + buffer: &[u8], + ) -> Poll> { + let position = usize::try_from(self.inner.position()).unwrap_or(usize::MAX); + let remaining = self.max_bytes.saturating_sub(position); + if buffer.len() > remaining { + return Poll::Ready(Err(io::Error::new( + io::ErrorKind::FileTooLarge, + "repository export exceeded its byte limit", + ))); + } + Pin::new(&mut self.inner).poll_write(context, buffer) + } + + fn poll_flush(mut self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.inner).poll_flush(context) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.inner).poll_shutdown(context) + } +} diff --git a/src/repository/commit.rs b/src/repository/commit.rs new file mode 100644 index 0000000..2adee8e --- /dev/null +++ b/src/repository/commit.rs @@ -0,0 +1,394 @@ +//! Stage `ATrium` blocks, sign the final root, and atomically publish the index, head, and outbox. +use super::{MAX_REPO_RETRIES, RepositoryService, car::map_block_error}; +use crate::error::ErrorContext; +use crate::{ + database::{PendingRepoCommit, RepoBlockStore, RepoHead}, + domain::{ + account::Account, + event::RepositoryEvent, + record::{Record, RecordMutation}, + }, + error::{Error, Result}, +}; +use atrium_api::com::atproto::repo::defs::CommitMetaData; +use atrium_api::types::string::{Cid as AtriumCid, Did, Tid}; +use atrium_crypto::verify::verify_signature; +use atrium_repo::{ + Repository, + blockstore::{AsyncBlockStoreWrite as _, DAG_CBOR as ATRIUM_DAG_CBOR, SHA2_256}, + repo::Error as AtriumRepoError, +}; +use cid::Cid; +use cpds_protocol::atproto_record; +use serde::Serialize; + +#[derive(Serialize)] +struct UnsignedCommit<'a> { + did: Did, + version: i64, + data: Cid, + rev: &'a Tid, + prev: Option, +} + +#[derive(Serialize)] +struct SignedCommit<'a> { + #[serde(flatten)] + content: UnsignedCommit<'a>, + #[serde(rename = "sig", with = "serde_bytes")] + signature: Vec, +} + +fn record_path(record: &Record) -> String { + format!("{}/{}", record.collection, record.rkey) +} + +async fn stage_record_blocks( + blocks: &mut RepoBlockStore, + mutations: &[RecordMutation], +) -> Result<()> { + for mutation in mutations { + let record = match mutation { + RecordMutation::Create(record) + | RecordMutation::Update { + replacement: record, + .. + } => record, + RecordMutation::Delete(_) => continue, + }; + let data = atproto_record(&record.value)?; + let bytes = serde_ipld_dagcbor::to_vec(&data) + .context(Error::Internal, "validated record cannot be encoded")?; + let cid = blocks + .write_block(ATRIUM_DAG_CBOR, SHA2_256, &bytes) + .await + .map_err(map_block_error)?; + if cid != record.cid { + return Err(Error::Internal( + "ATrium record CID differs from validated record CID".to_owned(), + )); + } + } + Ok(()) +} + +async fn apply_repo_mutations( + repo: &mut Repository<&mut RepoBlockStore>, + mutations: &[RecordMutation], +) -> Result { + let mut tree = repo.tree(); + for mutation in mutations { + match mutation { + RecordMutation::Create(record) => tree.add(&record_path(record), record.cid).await, + RecordMutation::Update { replacement, .. } => { + tree.update(&record_path(replacement), replacement.cid) + .await + } + RecordMutation::Delete(record) => tree.delete(&record_path(record)).await, + } + .map_err(|error| map_repo_error(error.into()))?; + } + Ok(tree.root()) +} + +async fn event_block_cids(blocks: &mut RepoBlockStore, root: Cid) -> Result> { + let reachable = { + let mut repo = Repository::open(&mut *blocks, root) + .await + .map_err(map_repo_error)?; + repo.export() + .await + .map_err(map_repo_error)? + .collect::>() + }; + Ok(blocks + .reachable_writes(reachable)? + .map(|(cid, _)| *cid) + .collect()) +} + +pub(super) fn invalid_swap(message: &str) -> Error { + Error::xrpc("InvalidSwap", message) +} + +pub(super) fn map_repo_error(error: AtriumRepoError) -> Error { + match error { + AtriumRepoError::BlockStore(error) => map_block_error(error), + AtriumRepoError::Mst(atrium_repo::mst::Error::KeyAlreadyExists) => { + Error::Conflict("record already exists".to_owned()) + } + AtriumRepoError::Mst(atrium_repo::mst::Error::KeyNotFound) => Error::NotFound, + error => Error::Internal(format!("ATrium repository operation failed: {error}")), + } +} + +impl RepositoryService { + fn record_event(mutation: &RecordMutation) -> RepositoryEvent { + match mutation { + RecordMutation::Create(record) => RepositoryEvent::RecordCreated { + did: record.did.clone(), + collection: record.collection.clone(), + rkey: record.rkey.clone(), + cid: record.cid, + }, + RecordMutation::Update { + current, + replacement, + } => RepositoryEvent::RecordUpdated { + did: replacement.did.clone(), + collection: replacement.collection.clone(), + rkey: replacement.rkey.clone(), + cid: replacement.cid, + previous_cid: current.cid, + }, + RecordMutation::Delete(record) => RepositoryEvent::RecordDeleted { + did: record.did.clone(), + collection: record.collection.clone(), + rkey: record.rkey.clone(), + previous_cid: record.cid, + }, + } + } + + pub(super) async fn commit_record_mutations( + &self, + actor: &Account, + expected_commit: Option<&Cid>, + mutations: &[RecordMutation], + ) -> Result { + for _ in 0..MAX_REPO_RETRIES { + let previous = self.database.repo_head(&actor.did)?; + if expected_commit + .is_some_and(|expected| previous.as_ref().is_none_or(|head| expected != &head.root)) + { + return Err(invalid_swap("repository commit does not match")); + } + let rev = previous.as_ref().map_or_else( + || Ok(self.tids.next()), + |head| { + self.tids.next_after(&head.rev).ok_or_else(|| { + Error::Internal("stored repository revision is invalid".to_owned()) + }) + }, + )?; + let tid = Tid::new(rev.clone()) + .context(Error::Internal, "generated repository revision is invalid")?; + let mut blocks = RepoBlockStore::new(self.database.clone()); + stage_record_blocks(&mut blocks, mutations).await?; + let mut repo = self + .open_or_create_repo(&mut blocks, actor, previous.as_ref()) + .await?; + let previous_data = previous.as_ref().map(|_| repo.commit().data()); + let data = apply_repo_mutations(&mut repo, mutations).await?; + drop(repo); + let root = self + .write_signed_commit(&mut blocks, actor, data, &tid) + .await?; + let event_blocks = event_block_cids(&mut blocks, root).await?; + let staged = blocks.writes(); + let commit = PendingRepoCommit { + did: actor.did.clone(), + expected_head: previous.as_ref().map(|head| head.root), + previous_rev: previous.as_ref().map(|head| head.rev.clone()), + previous_data, + head: RepoHead { + root, + rev: rev.clone(), + }, + blocks: staged, + event_blocks, + mutations: mutations.to_vec(), + }; + match self + .database + .commit_repo(&commit, self.repo_event_retention) + { + Ok(()) => { + self.publish_events(mutations.iter().map(Self::record_event)) + .await; + return Ok(CommitMetaData { + cid: AtriumCid::new(root), + rev: tid, + }); + } + Err(Error::Conflict(message)) if message == "repository head changed" => {} + Err(error) => return Err(error), + } + } + Err(Error::Conflict( + "repository changed too frequently; retry the request".to_owned(), + )) + } + + async fn write_signed_commit( + &self, + blocks: &mut RepoBlockStore, + actor: &Account, + data: Cid, + rev: &Tid, + ) -> Result { + let did = Did::new(actor.did.clone()) + .context(Error::Internal, "hosted account DID is invalid")?; + let commit = UnsignedCommit { + did, + version: 3, + data, + rev, + prev: None, + }; + let signature = self.sign( + &serde_ipld_dagcbor::to_vec(&commit) + .context(Error::Internal, "repository commit cannot be encoded")?, + )?; + let signed = SignedCommit { + content: commit, + signature, + }; + let bytes = serde_ipld_dagcbor::to_vec(&signed).context( + Error::Internal, + "signed repository commit cannot be encoded", + )?; + blocks + .write_block(ATRIUM_DAG_CBOR, SHA2_256, &bytes) + .await + .map_err(map_block_error) + } + + async fn open_or_create_repo<'a>( + &self, + blocks: &'a mut RepoBlockStore, + actor: &Account, + head: Option<&RepoHead>, + ) -> Result> { + if let Some(head) = head { + return self.open_repo(blocks, head.root, &head.rev).await; + } + + let did = Did::new(actor.did.clone()) + .context(Error::Internal, "hosted account DID is invalid")?; + let builder = Repository::create(&mut *blocks, did) + .await + .map_err(map_repo_error)?; + let signature = self.sign(&builder.bytes())?; + let mut repo = builder.finalize(signature).await.map_err(map_repo_error)?; + + for record in self.database.all_records(&actor.did)? { + let path = format!("{}/{}", record.collection, record.rkey); + let data = atproto_record(&record.value).context( + Error::Internal, + format_args!("stored record {record} is invalid"), + )?; + let (builder, cid) = repo.add_raw(&path, &data).await.map_err(map_repo_error)?; + if cid != record.cid { + return Err(Error::Internal(format!( + "stored record {record} has noncanonical CID" + ))); + } + let signature = self.sign(&builder.bytes())?; + builder.finalize(signature).await.map_err(map_repo_error)?; + } + Ok(repo) + } + + pub(super) async fn open_repo<'a>( + &self, + blocks: &'a mut RepoBlockStore, + root: Cid, + expected_rev: &str, + ) -> Result> { + let repo = Repository::open(&mut *blocks, root) + .await + .map_err(map_repo_error)?; + let commit = repo.commit(); + if commit.rev().to_string() != expected_rev { + return Err(Error::Internal( + "stored repository head revision does not match its commit".to_owned(), + )); + } + verify_signature(&self.signing_did, &commit.bytes(), commit.sig()) + .context(Error::Internal, "stored repository signature is invalid")?; + Ok(repo) + } + + fn sign(&self, bytes: &[u8]) -> Result> { + self.signer + .sign(bytes) + .context(Error::Internal, "repository commit signing failed") + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[derive(Serialize)] + struct PreviousUnsignedCommit { + did: Did, + version: i64, + data: Cid, + rev: Tid, + prev: Option, + } + + #[derive(Serialize)] + struct PreviousSignedCommit { + #[serde(flatten)] + content: PreviousUnsignedCommit, + #[serde(rename = "sig", with = "serde_bytes")] + signature: Vec, + } + + #[test] + fn borrowed_revision_keeps_unsigned_and_signed_commit_bytes() { + let data = cpds_protocol::dag_cbor_cid(b"data"); + let previous = cpds_protocol::dag_cbor_cid(b"previous"); + for rev in ["2222222222222", "3jqfcqzm3fo2j", "jzzzzzzzzzzzz"] { + let rev = Tid::new(rev.to_owned()).unwrap(); + for prev in [None, Some(previous)] { + for signature in [Vec::new(), vec![0, 255], (0..=255).collect()] { + let owned = PreviousUnsignedCommit { + did: Did::new("did:web:alice.test".to_owned()).unwrap(), + version: 3, + data, + rev: rev.clone(), + prev, + }; + let borrowed = UnsignedCommit { + did: owned.did.clone(), + version: owned.version, + data, + rev: &rev, + prev, + }; + assert_eq!( + serde_ipld_dagcbor::to_vec(&borrowed).unwrap(), + serde_ipld_dagcbor::to_vec(&owned).unwrap() + ); + assert_eq!( + serde_json::to_vec(&borrowed).unwrap(), + serde_json::to_vec(&owned).unwrap() + ); + let owned = PreviousSignedCommit { + content: owned, + signature: signature.clone(), + }; + let borrowed = SignedCommit { + content: borrowed, + signature, + }; + let expected = serde_ipld_dagcbor::to_vec(&owned).unwrap(); + let actual = serde_ipld_dagcbor::to_vec(&borrowed).unwrap(); + assert_eq!(actual, expected); + assert_eq!( + cpds_protocol::dag_cbor_cid(&actual), + cpds_protocol::dag_cbor_cid(&expected) + ); + assert_eq!( + serde_json::to_vec(&borrowed).unwrap(), + serde_json::to_vec(&owned).unwrap() + ); + } + } + } + } +} diff --git a/src/repository/export.rs b/src/repository/export.rs new file mode 100644 index 0000000..f212dc6 --- /dev/null +++ b/src/repository/export.rs @@ -0,0 +1,123 @@ +//! Bounded public repository CAR exports and proof extraction. +use super::{ + RepositoryService, + car::{car_writer, map_block_error, repository_blocks_car}, + commit::map_repo_error, +}; +use crate::{ + database::RepoBlockStore, + error::{Error, Result}, + syntax, +}; +use atrium_api::types::DataModel as AtriumDataModel; +use atrium_repo::blockstore::{AsyncBlockStoreRead as _, AsyncBlockStoreWrite as _, SHA2_256}; +use cid::Cid; +use std::collections::HashSet; +pub(super) const MAX_BLOCKS_PER_REQUEST: usize = 1_000; + +impl RepositoryService { + /// Export the current signed repository as a bounded CAR v1 archive. + /// + /// # Errors + /// + /// Returns `RepoNotFound` for an unhosted or not-yet-initialized repository, + /// fails closed if stored history or the signed graph is invalid, and + /// bounds both full and incremental archives. + pub async fn export_repo(&self, did: &str, since: Option<&str>) -> Result> { + let head = self.sync_head(did)?; + let mut blocks = RepoBlockStore::new(self.database.clone()); + let mut repo = self.open_repo(&mut blocks, head.root, &head.rev).await?; + if let Some(since) = since { + syntax::tid(since)?; + drop(repo); + let changed = self.database.repo_blocks_after_revision(did, since)?; + return repository_blocks_car(changed, head.root).await; + } + let mut bytes = Vec::new(); + let mut car = car_writer(&mut bytes, [head.root]).await?; + repo.export_into(&mut car).await.map_err(map_repo_error)?; + drop(car); + Ok(bytes) + } + + /// Export the signed MST proof and value blocks for one current record. + /// + /// # Errors + /// + /// Returns the canonical repository or record not-found error, or fails + /// closed if the stored graph or signature is invalid. + pub async fn export_record(&self, did: &str, collection: &str, rkey: &str) -> Result> { + syntax::collection(collection)?; + syntax::record_key(rkey)?; + let head = self.sync_head(did)?; + let mut blocks = RepoBlockStore::new(self.database.clone()); + let mut repo = self.open_repo(&mut blocks, head.root, &head.rev).await?; + let path = format!("{collection}/{rkey}"); + if repo + .get_raw::(&path) + .await + .map_err(map_repo_error)? + .is_none() + { + return Err(Error::xrpc("RecordNotFound", "record does not exist")); + } + let mut bytes = Vec::new(); + let mut car = car_writer(&mut bytes, [head.root]).await?; + repo.extract_raw_into(&path, &mut car) + .await + .map_err(map_repo_error)?; + drop(car); + Ok(bytes) + } + + /// Export selected blocks from the current signed repository graph. + /// + /// # Errors + /// + /// Rejects empty or excessive requests, CIDs outside the repository's + /// current reachable graph, and invalid stored repository state. + pub async fn export_blocks(&self, did: &str, requested: &[Cid]) -> Result> { + if requested.is_empty() { + return Err(Error::InvalidRequest( + "at least one block CID is required".to_owned(), + )); + } + if requested.len() > MAX_BLOCKS_PER_REQUEST { + return Err(Error::InvalidRequest(format!( + "at most {MAX_BLOCKS_PER_REQUEST} block CIDs may be requested" + ))); + } + let head = self.sync_head(did)?; + let mut blocks = RepoBlockStore::new(self.database.clone()); + let mut repo = self.open_repo(&mut blocks, head.root, &head.rev).await?; + let reachable = repo + .export() + .await + .map_err(map_repo_error)? + .collect::>(); + drop(repo); + if let Some(cid) = requested.iter().find(|cid| !reachable.contains(cid)) { + return Err(Error::xrpc( + "BlockNotFound", + format!("block {cid} is not in the current repository"), + )); + } + + let mut bytes = Vec::new(); + let mut car = car_writer(&mut bytes, []).await?; + for cid in requested { + let block = blocks.read_block(*cid).await.map_err(map_block_error)?; + let written = car + .write_block(cid.codec(), SHA2_256, &block) + .await + .map_err(map_block_error)?; + if written != *cid { + return Err(Error::Internal(format!( + "ATrium CAR writer changed repository block CID {cid}" + ))); + } + } + drop(car); + Ok(bytes) + } +} diff --git a/src/repository/maintenance.rs b/src/repository/maintenance.rs new file mode 100644 index 0000000..16eb0af --- /dev/null +++ b/src/repository/maintenance.rs @@ -0,0 +1,277 @@ +//! Readiness, durable-state verification, and bounded garbage collection. +use super::{ + RepositoryService, blobs::RAW_CODEC, commit::map_repo_error, stream::MAX_REPO_EVENTS_PER_PAGE, +}; +use crate::{ + clock::now, + database::{ListedRepoHead, RepoBlockStore, StoredRepoEvent}, + domain::blob::Blob, + error::{Error, ErrorContext as _, Result}, +}; +use atrium_api::types::DataModel as AtriumDataModel; +use cid::Cid; +use cpds_protocol::{atproto_record, record_cid, sha256_cid}; +use futures::TryStreamExt as _; +use std::{ + collections::{HashMap, HashSet}, + time::Duration, +}; +const MAX_ORPHAN_BLOBS_PER_RUN: usize = 1_000; +const MAX_REPO_BLOCKS_PER_GC_RUN: usize = 1_000; + +impl RepositoryService { + /// Check the infrastructure adapters required to serve repository requests. + /// + /// # Errors + /// + /// Returns an error when the database, blob store, or event sink is not ready. + pub async fn check(&self) -> Result<()> { + self.database.check()?; + self.blobs.check().await?; + self.events.check().await + } + + /// Verify infrastructure and every initialized repository before serving traffic. + /// + /// This intentionally performs the expensive graph and record-index walk only at + /// startup. Routine readiness checks remain bounded to the infrastructure adapters. + /// + /// # Errors + /// + /// Returns an error when an adapter is unavailable or persisted repository state is + /// corrupt, incomplete, or inconsistent with the record index. + pub async fn verify(&self) -> Result<()> { + self.check().await?; + let heads = self.database.repo_heads_after(None, None, usize::MAX)?; + for head in &heads { + self.verify_repo(head).await?; + } + self.verify_repo_events(&heads).await?; + for (owner, blob) in self.database.all_blobs()? { + self.verify_blob(&owner, &blob).await?; + } + Ok(()) + } + + /// Delete one bounded batch of uploaded blobs outside their reference grace period. + /// + /// `SQLite` first claims each candidate so concurrent public or Space writes cannot + /// acquire it. A crash leaves a durable claim that the next run resumes. + /// + /// # Errors + /// + /// Returns an error when candidates cannot be claimed, object deletion fails, or a + /// completed deletion cannot remove its metadata claim. + pub async fn collect_orphan_blobs(&self, grace_period: Duration) -> Result { + let grace_seconds = i64::try_from(grace_period.as_secs()).unwrap_or(i64::MAX); + let created_before = now().saturating_sub(grace_seconds); + let candidates = self + .database + .claim_orphan_blobs(created_before, MAX_ORPHAN_BLOBS_PER_RUN)?; + for (owner, cid) in &candidates { + self.blobs.delete(owner, cid).await?; + self.database.finish_blob_deletion(owner, cid)?; + } + Ok(candidates.len()) + } + + /// Delete one bounded batch of blocks outside the current `ATrium` graph and + /// retained event window. + /// + /// # Errors + /// + /// Returns an error when a signed repository cannot be opened or traversed, or + /// when the transactional block deletion fails. + pub async fn collect_repository_blocks(&self) -> Result { + let heads = self.database.repo_heads_after(None, None, usize::MAX)?; + let mut reachable = HashSet::new(); + for head in heads { + let mut blocks = RepoBlockStore::new(self.database.clone()); + let mut repo = self.open_repo(&mut blocks, head.root, &head.rev).await?; + reachable.extend(repo.export().await.map_err(map_repo_error)?); + } + self.database + .prune_repo_blocks(&reachable, MAX_REPO_BLOCKS_PER_GC_RUN) + } + + /// Checkpoint durable repository metadata after request drain. + /// + /// # Errors + /// + /// Returns an error when `SQLite` cannot complete its WAL checkpoint. + pub fn checkpoint(&self) -> Result<()> { + self.database.checkpoint() + } + + async fn verify_repo(&self, head: &ListedRepoHead) -> Result<()> { + let mut blocks = RepoBlockStore::new(self.database.clone()); + let mut repo = self.open_repo(&mut blocks, head.root, &head.rev).await?; + let entries = { + let mut tree = repo.tree(); + tree.entries() + .try_collect::>() + .await + .map_err(|error| map_repo_error(error.into()))? + }; + let records = self.database.all_records(&head.did)?; + if entries.len() != records.len() { + return Err(Error::Internal(format!( + "stored repository {} has {} MST entries but {} indexed records", + head.did, + entries.len(), + records.len() + ))); + } + + for ((path, cid), record) in entries.into_iter().zip(records) { + let expected_path = format!("{}/{}", record.collection, record.rkey); + if path != expected_path || cid != record.cid { + return Err(Error::Internal(format!( + "stored repository {} MST entry does not match indexed record {record}", + head.did + ))); + } + let expected_cid = record_cid(&record.value).context( + Error::Internal, + format_args!("stored record {record} is invalid"), + )?; + if expected_cid != record.cid { + return Err(Error::Internal(format!( + "stored record {record} has a noncanonical CID" + ))); + } + let expected_value = atproto_record(&record.value).context( + Error::Internal, + format_args!("stored record {record} is invalid"), + )?; + let value = repo + .get_raw::(&path) + .await + .map_err(map_repo_error)?; + let actual_bytes = value + .as_ref() + .map(serde_ipld_dagcbor::to_vec) + .transpose() + .context( + Error::Internal, + format_args!("stored record {record} cannot be encoded"), + )?; + let expected_bytes = serde_ipld_dagcbor::to_vec(&expected_value).context( + Error::Internal, + format_args!("stored record {record} cannot be encoded"), + )?; + if actual_bytes.as_deref() != Some(expected_bytes.as_slice()) { + return Err(Error::Internal(format!( + "stored repository block does not match indexed record {record}" + ))); + } + } + Ok(()) + } + + async fn verify_repo_events(&self, heads: &[ListedRepoHead]) -> Result<()> { + let Some(earliest) = self.database.earliest_repo_event_sequence()? else { + if heads.is_empty() { + return Ok(()); + } + return Err(Error::Internal( + "stored repository heads have no retained events".to_owned(), + )); + }; + let mut cursor = earliest - 1; + let mut expected = earliest; + let mut latest = HashMap::::new(); + loop { + let events = self + .database + .repo_events_after(cursor, MAX_REPO_EVENTS_PER_PAGE)?; + if events.is_empty() { + break; + } + for event in events { + let sequence = event.seq(); + if sequence != expected { + return Err(Error::Internal(format!( + "stored repository event sequence skips from {} to {}", + expected - 1, + sequence + ))); + } + match event { + StoredRepoEvent::Commit(event) => { + let event = *event; + let did = event.did.clone(); + let rev = event.rev.clone(); + let root = event.root; + let since = event.since.clone(); + let prev_data = event.prev_data; + let (_, data) = self.repo_commit_event(event).await?; + if let Some((previous_rev, _, previous_data)) = latest.get(&did) + && (since.as_deref() != Some(previous_rev) + || prev_data != Some(*previous_data)) + { + return Err(Error::Internal(format!( + "stored repository event {sequence} does not follow the prior {did} event" + ))); + } + latest.insert(did, (rev, root, data)); + } + StoredRepoEvent::Identity(event) => { + Self::repo_identity_event(event)?; + } + StoredRepoEvent::Account(event) => { + Self::repo_account_event(event)?; + } + } + cursor = sequence; + expected += 1; + } + } + for head in heads { + let Some((rev, root, _)) = latest.get(&head.did) else { + return Err(Error::Internal(format!( + "stored repository {} has no retained event", + head.did + ))); + }; + if rev != &head.rev || root != &head.root { + return Err(Error::Internal(format!( + "stored repository {} head does not match its latest retained event", + head.did + ))); + } + } + if let Some(did) = latest + .keys() + .find(|did| !heads.iter().any(|head| &head.did == *did)) + { + return Err(Error::Internal(format!( + "stored repository event exists without a head for {did}" + ))); + } + Ok(()) + } + + async fn verify_blob(&self, owner: &str, blob: &Blob) -> Result<()> { + let bytes = self.blobs.get(owner, &blob.cid).await?.ok_or_else(|| { + Error::Internal(format!( + "stored blob metadata has no bytes for {owner}/{}", + blob.cid + )) + })?; + if u64::try_from(bytes.len()).unwrap_or(u64::MAX) != blob.size { + return Err(Error::Internal(format!( + "stored blob size does not match metadata for {owner}/{}", + blob.cid + ))); + } + let cid = sha256_cid(RAW_CODEC, &bytes); + if cid != blob.cid { + return Err(Error::Internal(format!( + "stored blob bytes do not match metadata CID for {owner}/{}", + blob.cid + ))); + } + Ok(()) + } +} diff --git a/src/repository/mod.rs b/src/repository/mod.rs new file mode 100644 index 0000000..6136d28 --- /dev/null +++ b/src/repository/mod.rs @@ -0,0 +1,89 @@ +//! Public repository service. Each child module owns one cohesive part of its behavior. +mod accounts; +mod blobs; +mod car; +mod commit; +mod export; +mod maintenance; +mod records; +mod stream; + +use crate::error::ErrorContext; +use crate::{ + blob_store::BlobStore, + clock::TidClock, + database::Database, + domain::event::RepositoryEvent, + error::{Error, Result}, + events::EventSink, +}; +use atrium_crypto::keypair::{Did as _, Secp256k1Keypair}; +pub(crate) use blobs::MAX_BLOB_BYTES; +pub use blobs::MissingBlob; +pub use records::{RecordBatchWrite, RecordSwap}; +use std::sync::Arc; +const MAX_REPO_RETRIES: usize = 3; + +pub struct RepositoryService { + database: Database, + blobs: Arc, + events: Arc, + tids: TidClock, + signer: Secp256k1Keypair, + signing_did: String, + repo_event_retention: usize, +} + +impl RepositoryService { + /// Compose the repository service with its persistent and signing adapters. + /// + /// # Errors + /// + /// Returns a configuration error when the signing key is invalid. + pub fn new( + database: Database, + blobs: Arc, + events: Arc, + signing_key: &[u8; 32], + repo_event_retention: usize, + ) -> Result { + if repo_event_retention == 0 { + return Err(Error::Configuration( + "repository event retention must be positive".to_owned(), + )); + } + let signer = Secp256k1Keypair::import(signing_key) + .context(Error::Configuration, "invalid repository signing key")?; + let signing_did = signer.did(); + Ok(Self { + database, + blobs, + events, + tids: TidClock::default(), + signer, + signing_did, + repo_event_retention, + }) + } + + pub(crate) fn public_key_multibase(&self) -> &str { + self.signing_did + .strip_prefix("did:key:") + .expect("ATrium produces a did:key") + } + + async fn publish_events(&self, events: impl IntoIterator) { + crate::events::best_effort_notice("repository events", async { + for event in events { + if let Err(error) = self.events.publish(event).await { + tracing::warn!(%error, "repository event delivery failed"); + } + } + Ok(()) + }) + .await; + } +} + +#[cfg(test)] +mod tests; diff --git a/src/repository/records.rs b/src/repository/records.rs new file mode 100644 index 0000000..d28b077 --- /dev/null +++ b/src/repository/records.rs @@ -0,0 +1,274 @@ +//! Record validation and ordered mutation planning. Persistence is delegated to one commit path. +use super::{RepositoryService, commit::invalid_swap}; +pub use crate::domain::record::WriteRequest as RecordBatchWrite; +use crate::{ + domain::{ + account::Account, + record::{Record, RecordMutation, RecordView}, + }, + error::{Error, Result}, + syntax, +}; +use atrium_api::com::atproto::repo::defs::CommitMetaData; +use cid::Cid; +use cpds_protocol::{record_cid, validate_record_type}; +use serde_json::Value; +use std::collections::HashMap; +const MAX_WRITES_PER_COMMIT: usize = 200; + +pub struct RecordSwap<'a> { + pub record: Option<&'a Cid>, + pub commit: Option<&'a Cid>, +} + +impl RepositoryService { + /// Validate, address, persist, and announce one new record. + /// + /// # Errors + /// + /// Returns an error for unauthorized repositories, invalid record shape or + /// identifiers, duplicate paths, or failed persistence. + pub async fn create_record( + &self, + actor: &Account, + repo: &str, + collection: &str, + rkey: Option<&str>, + value: Value, + expected_commit: Option<&Cid>, + ) -> Result<(RecordView, CommitMetaData)> { + self.authorize_repo(actor, repo)?; + syntax::collection(collection)?; + let rkey = rkey.map_or_else(|| self.tids.next(), ToOwned::to_owned); + syntax::record_key(&rkey)?; + let record = self.prepare_record(actor, collection.to_owned(), rkey, value)?; + let commit = self + .commit_record_mutations( + actor, + expected_commit, + &[RecordMutation::Create(record.clone())], + ) + .await?; + Ok((record.into(), commit)) + } + + /// Create or replace one record owned by the authenticated account. + /// + /// # Errors + /// + /// Returns an error for invalid data, unauthorized repositories, failed + /// commit or record swaps, or persistence failures. + pub async fn put_record( + &self, + actor: &Account, + repo: &str, + collection: &str, + rkey: &str, + value: Value, + swap: RecordSwap<'_>, + ) -> Result<(RecordView, CommitMetaData)> { + self.authorize_repo(actor, repo)?; + syntax::collection(collection)?; + syntax::record_key(rkey)?; + let current = self.database.find_record(&actor.did, collection, rkey)?; + if swap.record.is_some_and(|expected| { + current + .as_ref() + .is_none_or(|record| expected != &record.cid) + }) { + return Err(invalid_swap("record CID does not match")); + } + let record = self.prepare_record(actor, collection.to_owned(), rkey.to_owned(), value)?; + let mutation = current.map_or_else( + || RecordMutation::Create(record.clone()), + |current| RecordMutation::Update { + current, + replacement: record.clone(), + }, + ); + let commit = self + .commit_record_mutations(actor, swap.commit, &[mutation]) + .await?; + Ok((record.into(), commit)) + } + + /// Read one publicly addressable record. + /// + /// # Errors + /// + /// Returns `NotFound` for unknown repositories or paths, and validation or + /// persistence errors when the request cannot be fulfilled. + pub fn get_record( + &self, + repo: &str, + collection: &str, + rkey: &str, + cid: Option<&Cid>, + ) -> Result { + let account = self.resolve_repo(repo)?; + syntax::collection(collection)?; + syntax::record_key(rkey)?; + let record = self + .database + .find_record(&account.did, collection, rkey)? + .ok_or(Error::NotFound)?; + if cid.is_some_and(|cid| record.cid != *cid) { + return Err(Error::NotFound); + } + Ok(record.into()) + } + + /// List a bounded page from one public collection. + /// + /// # Errors + /// + /// Returns an error for an unknown repository, invalid collection, or + /// failed database query. + pub fn list_records( + &self, + repo: &str, + collection: &str, + cursor: Option<&str>, + reverse: bool, + limit: usize, + ) -> Result<(Vec, Option)> { + let account = self.resolve_repo(repo)?; + syntax::collection(collection)?; + if let Some(cursor) = cursor { + syntax::record_key(cursor)?; + } + let limit = limit.clamp(1, 100); + let records = + self.database + .list_records(&account.did, collection, cursor, reverse, limit)?; + let cursor = records.last().map(|record| record.rkey.clone()); + let records = records.into_iter().map(Into::into).collect(); + Ok((records, cursor)) + } + + /// Remove and announce a record owned by the authenticated account. + /// + /// # Errors + /// + /// Returns an error for unauthorized or invalid paths, missing records, or + /// failed persistence. + pub async fn delete_record( + &self, + actor: &Account, + repo: &str, + collection: &str, + rkey: &str, + expected_cid: Option<&Cid>, + expected_commit: Option<&Cid>, + ) -> Result { + self.authorize_repo(actor, repo)?; + syntax::collection(collection)?; + syntax::record_key(rkey)?; + let deleted = self + .database + .find_record(&actor.did, collection, rkey)? + .ok_or(Error::NotFound)?; + if expected_cid.is_some_and(|expected| expected != &deleted.cid) { + return Err(invalid_swap("record CID does not match")); + } + self.commit_record_mutations(actor, expected_commit, &[RecordMutation::Delete(deleted)]) + .await + } + + /// Atomically apply an ordered batch as one signed repository commit. + /// + /// # Errors + /// + /// Returns an error when the batch is empty or oversized, a write is + /// invalid for the state produced by prior writes, the commit swap fails, + /// or persistence cannot publish the complete batch atomically. + pub async fn apply_writes( + &self, + actor: &Account, + repo: &str, + writes: Vec, + expected_commit: Option<&Cid>, + ) -> Result<(CommitMetaData, Vec)> { + self.authorize_repo(actor, repo)?; + if writes.is_empty() || writes.len() > MAX_WRITES_PER_COMMIT { + return Err(Error::InvalidRequest(format!( + "applyWrites requires 1 to {MAX_WRITES_PER_COMMIT} writes" + ))); + } + + let mut state = HashMap::<(String, String), Option>::new(); + let mut mutations = Vec::with_capacity(writes.len()); + for write in writes { + let (collection, rkey) = match &write { + RecordBatchWrite::Create { + collection, rkey, .. + } => ( + collection.clone(), + rkey.clone().unwrap_or_else(|| self.tids.next()), + ), + RecordBatchWrite::Update { + collection, rkey, .. + } + | RecordBatchWrite::Delete { collection, rkey } => { + (collection.clone(), rkey.clone()) + } + }; + syntax::collection(&collection)?; + syntax::record_key(&rkey)?; + let key = (collection.clone(), rkey.clone()); + let current = match state.remove(&key) { + Some(current) => current, + None => self.database.find_record(&actor.did, &collection, &rkey)?, + }; + + match (write, current) { + (RecordBatchWrite::Create { value, .. }, None) => { + let record = self.prepare_record(actor, collection, rkey, value)?; + state.insert(key, Some(record.clone())); + mutations.push(RecordMutation::Create(record)); + } + (RecordBatchWrite::Update { value, .. }, Some(current)) => { + let replacement = self.prepare_record(actor, collection, rkey, value)?; + state.insert(key, Some(replacement.clone())); + mutations.push(RecordMutation::Update { + current, + replacement, + }); + } + (RecordBatchWrite::Delete { .. }, Some(current)) => { + state.insert(key, None); + mutations.push(RecordMutation::Delete(current)); + } + (RecordBatchWrite::Create { .. }, Some(_)) => { + return Err(Error::Conflict("record already exists".to_owned())); + } + (RecordBatchWrite::Update { .. } | RecordBatchWrite::Delete { .. }, None) => { + return Err(Error::NotFound); + } + } + } + + let commit = self + .commit_record_mutations(actor, expected_commit, &mutations) + .await?; + Ok((commit, mutations)) + } + + fn prepare_record( + &self, + actor: &Account, + collection: String, + rkey: String, + value: Value, + ) -> Result { + validate_record_type(&collection, &value)?; + self.database.validate_blob_refs(&actor.did, &value)?; + Ok(Record { + did: actor.did.clone(), + collection, + rkey, + cid: record_cid(&value)?, + value, + }) + } +} diff --git a/src/repository/stream.rs b/src/repository/stream.rs new file mode 100644 index 0000000..fab4a51 --- /dev/null +++ b/src/repository/stream.rs @@ -0,0 +1,249 @@ +//! Reconstruct generated `ATrium` subscription messages from the durable outbox. +use super::{ + RepositoryService, + car::{map_block_error, repository_blocks_car}, +}; +use crate::error::ErrorContext; +use crate::{ + clock::format_utc_datetime, + database::{ + RepoBlockStore, StoredAccountEvent, StoredCommitEvent, StoredIdentityEvent, StoredRepoEvent, + }, + error::{Error, Result}, +}; +use atrium_api::{ + com::atproto::sync::subscribe_repos, + types::{ + CidLink, + string::{Datetime, Did, Handle, Tid}, + }, +}; +use atrium_repo::blockstore::AsyncBlockStoreRead as _; +use cid::Cid; +const MAX_COMMIT_EVENT_BYTES: usize = 2_000_000; +pub(super) const MAX_REPO_EVENTS_PER_PAGE: usize = 1_000; + +fn parse_event_did(value: &str) -> Result { + value + .parse() + .context(Error::Internal, "stored repository event DID is invalid") +} + +fn parse_event_tid(value: &str) -> Result { + value.parse().context( + Error::Internal, + "stored repository event revision is invalid", + ) +} + +fn event_datetime(timestamp: i64) -> Result { + let timestamp = format_utc_datetime(timestamp).context( + Error::Internal, + "stored repository event timestamp is invalid", + )?; + timestamp.parse().context( + Error::Internal, + "stored repository event timestamp cannot become a Lexicon datetime", + ) +} + +fn validate_repo_event_metadata(event: &StoredCommitEvent) -> Result<()> { + if event.operations.is_empty() { + return Err(Error::Internal( + "stored repository event has no operations".to_owned(), + )); + } + for operation in &event.operations { + let valid = match operation.action.as_str() { + "create" => operation.cid.is_some() && operation.prev.is_none(), + "update" => operation.cid.is_some() && operation.prev.is_some(), + "delete" => operation.cid.is_none() && operation.prev.is_some(), + action => { + return Err(Error::Internal(format!( + "stored repository event action is invalid: {action}" + ))); + } + }; + if !valid { + return Err(Error::Internal(format!( + "stored {} repository event has invalid record links", + operation.action + ))); + } + } + if event.since.is_some() != event.prev_data.is_some() { + return Err(Error::Internal( + "stored repository event has incomplete inductive metadata".to_owned(), + )); + } + Ok(()) +} + +async fn repository_event_car(blocks: &[(Cid, Vec)], root: Cid) -> Result> { + if blocks.first().is_none_or(|(cid, _)| cid != &root) { + return Err(Error::Internal( + "stored repository event does not begin with its commit block".to_owned(), + )); + } + repository_blocks_car(blocks.iter().cloned(), root).await +} + +impl RepositoryService { + /// Rebuild durable repository events as current generated `ATrium` wire models. + /// + /// Legacy events without inductive metadata and oversized commit diffs become + /// `#sync` messages containing the signed commit block. This gives consumers a + /// safe reset point instead of emitting a malformed `#commit` message. + /// + /// # Errors + /// + /// Returns `FutureCursor` for a cursor ahead of durable state and fails closed + /// when persisted event metadata or blocks are invalid. + pub async fn repo_messages_after( + &self, + cursor: i64, + limit: usize, + ) -> Result> { + let latest = self.database.latest_repo_event_sequence()?; + if cursor > latest { + return Err(Error::xrpc( + "FutureCursor", + format!("cursor {cursor} is ahead of latest sequence {latest}"), + )); + } + if self + .database + .earliest_repo_event_sequence()? + .is_some_and(|earliest| cursor < earliest - 1) + { + return Ok(vec![subscribe_repos::Message::Info(Box::new( + subscribe_repos::InfoData { + message: Some( + "requested cursor is no longer retained; resynchronize repositories" + .to_owned(), + ), + name: "OutdatedCursor".to_owned(), + } + .into(), + ))]); + } + let events = self + .database + .repo_events_after(cursor, limit.clamp(1, MAX_REPO_EVENTS_PER_PAGE))?; + let mut messages = Vec::with_capacity(events.len()); + for event in events { + messages.push(match event { + StoredRepoEvent::Commit(event) => self.repo_commit_event(*event).await?.0, + StoredRepoEvent::Identity(event) => Self::repo_identity_event(event)?, + StoredRepoEvent::Account(event) => Self::repo_account_event(event)?, + }); + } + Ok(messages) + } + + /// Return the latest durable repository stream cursor. + /// + /// # Errors + /// + /// Returns an error when the outbox sequence cannot be read. + pub fn latest_repo_event_cursor(&self) -> Result { + self.database.latest_repo_event_sequence() + } + + pub(super) fn repo_identity_event( + event: StoredIdentityEvent, + ) -> Result { + Ok(subscribe_repos::Message::Identity(Box::new( + subscribe_repos::IdentityData { + did: parse_event_did(&event.did)?, + handle: event.handle.map(Handle::new).transpose().context( + Error::Internal, + "stored repository identity handle is invalid", + )?, + seq: event.seq, + time: event_datetime(event.created_at)?, + } + .into(), + ))) + } + + pub(super) fn repo_account_event( + event: StoredAccountEvent, + ) -> Result { + Ok(subscribe_repos::Message::Account(Box::new( + subscribe_repos::AccountData { + active: event.active, + did: parse_event_did(&event.did)?, + seq: event.seq, + status: event.status, + time: event_datetime(event.created_at)?, + } + .into(), + ))) + } + + pub(super) async fn repo_commit_event( + &self, + event: StoredCommitEvent, + ) -> Result<(subscribe_repos::Message, Cid)> { + let mut store = RepoBlockStore::new(self.database.clone()); + let repo = self.open_repo(&mut store, event.root, &event.rev).await?; + let data = repo.commit().data(); + drop(repo); + let time = event_datetime(event.created_at)?; + validate_repo_event_metadata(&event)?; + let blocks = if event.incremental { + repository_event_car(&event.blocks, event.root).await? + } else { + Vec::new() + }; + if !event.incremental || blocks.len() > MAX_COMMIT_EVENT_BYTES { + let root_block = + if let Some(block) = event.blocks.first().filter(|(cid, _)| cid == &event.root) { + block.clone() + } else { + let mut store = RepoBlockStore::new(self.database.clone()); + let bytes = store + .read_block(event.root) + .await + .map_err(map_block_error)?; + (event.root, bytes) + }; + let blocks = repository_event_car(&[root_block], event.root).await?; + return Ok(( + subscribe_repos::Message::Sync(Box::new( + subscribe_repos::SyncData { + blocks, + did: parse_event_did(&event.did)?, + rev: event.rev, + seq: event.seq, + time, + } + .into(), + )), + data, + )); + } + let ops = event.operations.into_iter().map(Into::into).collect(); + Ok(( + subscribe_repos::Message::Commit(Box::new( + subscribe_repos::CommitData { + blobs: Vec::new(), + blocks, + commit: CidLink(event.root), + ops, + prev_data: event.prev_data.map(CidLink), + rebase: false, + repo: parse_event_did(&event.did)?, + rev: parse_event_tid(&event.rev)?, + seq: event.seq, + since: event.since.as_deref().map(parse_event_tid).transpose()?, + time, + too_big: false, + } + .into(), + )), + data, + )) + } +} diff --git a/src/repository/tests.rs b/src/repository/tests.rs deleted file mode 100644 index 759473f..0000000 --- a/src/repository/tests.rs +++ /dev/null @@ -1,1174 +0,0 @@ -use atrium_crypto::{ - did::parse_did_key, - keypair::{Did as _, Secp256k1Keypair}, - verify::Verifier, -}; -use base64::{Engine as _, engine::general_purpose::STANDARD_NO_PAD}; -use serde::Deserialize; -use serde_json::json; -use std::time::Duration; -use tokio::io::AsyncWriteExt as _; - -use crate::{blob_store::MemoryBlobStore, events::RecordingEventSink, space::domain::SpaceRecord}; - -use super::*; - -const DID: &str = "did:web:alice.test"; -const HANDLE: &str = "alice.test"; -const SECRET: [u8; 32] = [7; 32]; - -#[derive(Deserialize)] -struct DataModelFixture { - json: Value, - cbor_base64: String, - cid: String, -} - -#[derive(Deserialize)] -struct DataModelValidationFixture { - note: String, - json: Value, -} - -#[tokio::test] -async fn repository_export_buffer_enforces_its_byte_limit() { - let mut bytes = Vec::new(); - { - let mut buffer = BoundedBuffer::new(&mut bytes, 3); - buffer.write_all(b"abc").await.unwrap(); - let error = buffer.write_all(b"d").await.unwrap_err(); - assert_eq!(error.kind(), io::ErrorKind::FileTooLarge); - } - assert_eq!(bytes, b"abc"); -} - -#[test] -fn record_addressing_rejects_oversized_dag_cbor_before_repository_work() { - let oversized = json!({ - "$type": "app.bsky.feed.post", - "payload": "x".repeat(MAX_RECORD_BYTES) - }); - - let error = record_cid(&oversized).unwrap_err().to_string(); - - assert!(error.contains("1000000-byte DAG-CBOR limit")); -} - -#[test] -fn record_addressing_matches_official_atproto_data_model_fixtures() { - let fixtures: Vec = serde_json::from_str(include_str!( - "../../tests/fixtures/atproto-data-model-fixtures.json" - )) - .unwrap(); - - for fixture in fixtures { - let record = atproto_record(&fixture.json).unwrap(); - let bytes = serde_ipld_dagcbor::to_vec(&record).unwrap(); - assert_eq!(bytes, STANDARD_NO_PAD.decode(fixture.cbor_base64).unwrap()); - assert_eq!(record_cid(&fixture.json).unwrap().to_string(), fixture.cid); - } -} - -#[test] -fn generic_record_validation_matches_official_atproto_data_model_fixtures() { - let valid: Vec = serde_json::from_str(include_str!( - "../../tests/fixtures/atproto-data-model-valid.json" - )) - .unwrap(); - for fixture in valid { - assert!( - atproto_record(&fixture.json).is_ok(), - "rejected valid fixture: {}", - fixture.note - ); - } - - let invalid: Vec = serde_json::from_str(include_str!( - "../../tests/fixtures/atproto-data-model-invalid.json" - )) - .unwrap(); - for fixture in invalid { - assert!( - atproto_record(&fixture.json).is_err(), - "accepted invalid fixture: {}", - fixture.note - ); - } -} - -#[test] -fn record_addressing_rejects_values_outside_the_atproto_data_model() { - for value in [ - json!({"value": 1.5}), - json!({"value": u64::MAX}), - json!({"value": {"$bytes": true}}), - json!({"value": {"$bytes": "", "extra": true}}), - json!({"value": {"$link": "not-a-cid"}}), - json!({ - "value": { - "$link": "bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi" - } - }), - json!({ - "value": { - "$link": "bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a", - "extra": true - } - }), - ] { - assert!(record_cid(&value).is_err(), "accepted {value}"); - } - - let integer_like: Value = serde_json::from_str(r#"{"value": 123.0}"#).unwrap(); - assert_eq!( - record_cid(&integer_like).unwrap(), - record_cid(&json!({"value": 123})).unwrap() - ); - - let mut nested = Value::Null; - for _ in 0..=MAX_RECORD_DEPTH { - nested = json!({"nested": nested}); - } - assert!(record_cid(&nested).is_err()); -} - -#[tokio::test] -async fn repository_writes_native_atproto_bytes_without_a_parallel_encoder() { - let database = Database::open_in_memory().unwrap(); - let actor = account(&database); - let service = service(database.clone()); - let value = json!({ - "$type": "app.bsky.feed.post", - "bytes": {"$bytes": "nFERjvLLiw9qm45JrqH9QTzyC2Lu1Xb4ne6+sBrCzI0="} - }); - let expected = record_cid(&value).unwrap(); - - let created = service - .create_record( - &actor, - DID, - "app.bsky.feed.post", - Some("native"), - value, - None, - ) - .await - .unwrap(); - - assert_eq!(created.record.cid, expected.to_string()); - assert!( - RepoBlockStore::new(database) - .read_block(expected) - .await - .is_ok() - ); - service.verify().await.unwrap(); -} - -#[tokio::test] -async fn block_export_bounds_apply_before_repository_traversal() { - let service = service(Database::open_in_memory().unwrap()); - assert!(matches!( - service.export_blocks(DID, &[]).await, - Err(Error::InvalidRequest(_)) - )); - let cid = record_cid(&post("bounded")).unwrap(); - let excessive = vec![cid; MAX_BLOCKS_PER_REQUEST + 1]; - assert!(matches!( - service.export_blocks(DID, &excessive).await, - Err(Error::InvalidRequest(_)) - )); -} - -#[tokio::test] -async fn signed_repository_survives_restart_with_monotonic_revision() { - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("cpds.sqlite3"); - let first_database = Database::open(&path).unwrap(); - let actor = account(&first_database); - let first_service = service(first_database.clone()); - let first = first_service - .create_record( - &actor, - DID, - "app.bsky.feed.post", - Some("first"), - post("first"), - None, - ) - .await - .unwrap(); - let first_rev = first.commit.rev; - drop(first_service); - drop(first_database); - - let reopened = Database::open(&path).unwrap(); - let second_service = service(reopened.clone()); - let second = second_service - .create_record( - &actor, - DID, - "app.bsky.feed.post", - Some("second"), - post("second"), - None, - ) - .await - .unwrap(); - - assert!(second.commit.rev > first_rev); - let head = reopened.repo_head(DID).unwrap().unwrap(); - let mut blocks = RepoBlockStore::new(reopened); - let mut repo = Repository::open(&mut blocks, head.root).await.unwrap(); - assert!( - repo.get_raw::("app.bsky.feed.post/first") - .await - .unwrap() - .is_some() - ); - assert!( - repo.get_raw::("app.bsky.feed.post/second") - .await - .unwrap() - .is_some() - ); - - let key = Secp256k1Keypair::import(&SECRET).unwrap(); - let (algorithm, public_key) = parse_did_key(&key.did()).unwrap(); - Verifier::default() - .verify( - algorithm, - &public_key, - &repo.commit().bytes(), - repo.commit().sig(), - ) - .unwrap(); -} - -#[tokio::test] -async fn startup_verification_walks_reachable_atrium_blocks() { - let database = Database::open_in_memory().unwrap(); - let actor = account(&database); - let service = service(database.clone()); - let created = service - .create_record( - &actor, - DID, - "app.bsky.feed.post", - Some("corrupt"), - post("corrupt"), - None, - ) - .await - .unwrap(); - service.verify().await.unwrap(); - - database - .connection() - .unwrap() - .execute( - "UPDATE repo_blocks SET bytes = X'00' WHERE cid = ?1", - [created.record.cid], - ) - .unwrap(); - - service.check().await.unwrap(); - assert!(service.verify().await.is_err()); -} - -#[tokio::test] -async fn startup_verification_rejects_repository_record_index_drift() { - let database = Database::open_in_memory().unwrap(); - let actor = account(&database); - let service = service(database.clone()); - service - .create_record( - &actor, - DID, - "app.bsky.feed.post", - Some("missing"), - post("missing"), - None, - ) - .await - .unwrap(); - database - .connection() - .unwrap() - .execute("DELETE FROM records WHERE did = ?1", [DID]) - .unwrap(); - - let error = service.verify().await.unwrap_err().to_string(); - - assert!(error.contains("1 MST entries but 0 indexed records")); -} - -#[tokio::test] -async fn startup_verification_rejects_outbox_column_payload_drift() { - let database = Database::open_in_memory().unwrap(); - let actor = account(&database); - let service = service(database.clone()); - service - .create_record( - &actor, - DID, - "app.bsky.feed.post", - Some("event"), - post("event"), - None, - ) - .await - .unwrap(); - database - .connection() - .unwrap() - .execute( - "UPDATE repo_outbox SET rev = '2222222222222' - WHERE event_type = 'repo.commit'", - [], - ) - .unwrap(); - - let error = service.verify().await.unwrap_err().to_string(); - - assert!(error.contains("metadata does not match its payload")); -} - -#[tokio::test] -async fn startup_verification_rejects_broken_event_sequence() { - let database = Database::open_in_memory().unwrap(); - let actor = account(&database); - let service = service(database.clone()); - for rkey in ["first", "second", "third"] { - service - .create_record( - &actor, - DID, - "app.bsky.feed.post", - Some(rkey), - post(rkey), - None, - ) - .await - .unwrap(); - } - database - .connection() - .unwrap() - .execute("DELETE FROM repo_outbox WHERE seq = 2", []) - .unwrap(); - - let error = service.verify().await.unwrap_err().to_string(); - - assert!(error.contains("event sequence skips from 1 to 3")); -} - -#[tokio::test] -async fn startup_verification_rejects_broken_event_chain() { - let database = Database::open_in_memory().unwrap(); - let actor = account(&database); - let service = service(database.clone()); - service - .create_record( - &actor, - DID, - "app.bsky.feed.post", - Some("first"), - post("first"), - None, - ) - .await - .unwrap(); - let second = service - .create_record( - &actor, - DID, - "app.bsky.feed.post", - Some("second"), - post("second"), - None, - ) - .await - .unwrap(); - let payload: Vec = database - .connection() - .unwrap() - .query_row("SELECT payload FROM repo_outbox WHERE seq = 4", [], |row| { - row.get(0) - }) - .unwrap(); - let mut payload: Value = serde_json::from_slice(&payload).unwrap(); - payload["since"] = Value::String(second.commit.rev); - database - .connection() - .unwrap() - .execute( - "UPDATE repo_outbox SET payload = ?1 WHERE seq = 4", - [serde_json::to_vec(&payload).unwrap()], - ) - .unwrap(); - - let error = service.verify().await.unwrap_err().to_string(); - - assert!(error.contains("event 4 does not follow the prior")); -} - -#[tokio::test] -async fn startup_verification_rejects_incomplete_event_blocks() { - let database = Database::open_in_memory().unwrap(); - let actor = account(&database); - let service = service(database.clone()); - service - .create_record( - &actor, - DID, - "app.bsky.feed.post", - Some("event"), - post("event"), - None, - ) - .await - .unwrap(); - database - .connection() - .unwrap() - .execute("DELETE FROM repo_commit_blocks", []) - .unwrap(); - - let error = service.verify().await.unwrap_err().to_string(); - - assert!(error.contains("does not begin with its commit block")); -} - -#[tokio::test] -async fn startup_verification_rejects_blob_metadata_without_bytes() { - let database = Database::open_in_memory().unwrap(); - let actor = account(&database); - let blob = Blob { - cid: sha256_cid(RAW_CODEC, b"missing").unwrap(), - media_type: "image/png".to_owned(), - size: 7, - }; - database.begin_blob_upload(&actor.did, &blob).unwrap(); - database.finish_blob_upload(&actor.did, &blob.cid).unwrap(); - - let error = service(database).verify().await.unwrap_err().to_string(); - - assert!(error.contains("stored blob metadata has no bytes")); -} - -#[tokio::test] -async fn startup_verification_rejects_blob_size_drift() { - let database = Database::open_in_memory().unwrap(); - let actor = account(&database); - let service = service(database.clone()); - let blob = service - .upload_blob(&actor, "image/png", b"present") - .await - .unwrap(); - service.verify().await.unwrap(); - database - .connection() - .unwrap() - .execute( - "UPDATE blobs SET size = size + 1 WHERE did = ?1 AND cid = ?2", - rusqlite::params![DID, blob.cid.to_string()], - ) - .unwrap(); - - let error = service.verify().await.unwrap_err().to_string(); - - assert!(error.contains("stored blob size does not match metadata")); -} - -#[tokio::test] -async fn first_repository_commit_backfills_legacy_record_rows() { - let database = Database::open_in_memory().unwrap(); - let actor = account(&database); - let legacy_value = json!({ - "$type": "app.bsky.feed.post", - "text": "legacy", - "bytes": {"$bytes": "AQ=="} - }); - let legacy_cid = record_cid(&legacy_value).unwrap(); - database - .connection() - .unwrap() - .execute( - "INSERT INTO records - (did, collection, rkey, cid, record_json, created_at, updated_at) - VALUES (?1, 'app.bsky.feed.post', 'legacy', ?2, ?3, 1, 1)", - rusqlite::params![DID, legacy_cid.to_string(), legacy_value.to_string()], - ) - .unwrap(); - - service(database.clone()) - .create_record( - &actor, - DID, - "app.bsky.feed.post", - Some("new"), - post("new"), - None, - ) - .await - .unwrap(); - - let head = database.repo_head(DID).unwrap().unwrap(); - let mut blocks = RepoBlockStore::new(database); - let mut repo = Repository::open(&mut blocks, head.root).await.unwrap(); - let legacy = repo - .get_raw::("app.bsky.feed.post/legacy") - .await - .unwrap() - .unwrap(); - assert_eq!( - serde_ipld_dagcbor::to_vec(&legacy).unwrap(), - serde_ipld_dagcbor::to_vec(&atproto_record(&legacy_value).unwrap()).unwrap() - ); - assert!( - repo.get_raw::("app.bsky.feed.post/new") - .await - .unwrap() - .is_some() - ); -} - -#[test] -fn missing_blob_inventory_reuses_atrium_reference_scanning_and_cid_cursors() { - const BLOB_CID: &str = "bafkreibme22gw2h7y2h7tg2fhqotaqjucnbc24deqo72b6mkl2egezxhvy"; - let database = Database::open_in_memory().unwrap(); - let actor = account(&database); - let value = json!({ - "$type": "app.bsky.feed.post", - "image": { - "$type": "blob", - "ref": {"$link": BLOB_CID}, - "mimeType": "image/png", - "size": 3 - } - }); - let cid = record_cid(&value).unwrap(); - let connection = database.connection().unwrap(); - connection - .execute( - "INSERT INTO records - (did, collection, rkey, cid, record_json, created_at, updated_at) - VALUES (?1, 'app.bsky.feed.post', 'missing', ?2, ?3, 1, 1)", - rusqlite::params![DID, cid.to_string(), value.to_string()], - ) - .unwrap(); - connection - .execute( - "INSERT INTO record_blobs (did, collection, rkey, cid) - VALUES (?1, 'app.bsky.feed.post', 'missing', ?2)", - rusqlite::params![DID, BLOB_CID], - ) - .unwrap(); - drop(connection); - let service = service(database.clone()); - - let (missing, cursor) = service.list_missing_blobs(&actor, None, 500).unwrap(); - assert_eq!(missing.len(), 1); - assert_eq!(missing[0].cid.to_string(), BLOB_CID); - assert_eq!( - missing[0].record_uri, - "at://did:web:alice.test/app.bsky.feed.post/missing" - ); - assert_eq!(cursor.as_deref(), Some(BLOB_CID)); - assert!( - service - .list_missing_blobs(&actor, cursor.as_deref(), 500) - .unwrap() - .0 - .is_empty() - ); -} - -#[tokio::test] -async fn repository_mutations_atomically_replace_and_remove_blob_references() { - let database = Database::open_in_memory().unwrap(); - let actor = account(&database); - let service = service(database); - let first = service - .upload_blob(&actor, "image/png", b"first") - .await - .unwrap(); - let second = service - .upload_blob(&actor, "image/png", b"second") - .await - .unwrap(); - let record = |blob: &Blob| { - json!({ - "$type": "app.bsky.feed.post", - "image": { - "$type": "blob", - "ref": {"$link": blob.cid.to_string()}, - "mimeType": blob.media_type, - "size": blob.size - } - }) - }; - - service - .create_record( - &actor, - DID, - "app.bsky.feed.post", - Some("image"), - record(&first), - None, - ) - .await - .unwrap(); - assert_eq!( - service.list_blobs(DID, None, None, 100).unwrap().0, - vec![first.cid] - ); - - service - .put_record( - &actor, - DID, - "app.bsky.feed.post", - "image", - record(&second), - RecordSwap { - record: None, - commit: None, - }, - ) - .await - .unwrap(); - assert_eq!( - service.list_blobs(DID, None, None, 100).unwrap().0, - vec![second.cid] - ); - - service - .delete_record(&actor, DID, "app.bsky.feed.post", "image", None, None) - .await - .unwrap(); - assert!( - service - .list_blobs(DID, None, None, 100) - .unwrap() - .0 - .is_empty() - ); -} - -#[tokio::test] -async fn orphan_collection_preserves_public_space_and_fresh_blob_references() { - let database = Database::open_in_memory().unwrap(); - let actor = account(&database); - let service = service(database.clone()); - let public = service - .upload_blob(&actor, "image/png", b"public") - .await - .unwrap(); - let space = service - .upload_blob(&actor, "image/png", b"space") - .await - .unwrap(); - let orphan = service - .upload_blob(&actor, "image/png", b"orphan") - .await - .unwrap(); - service - .create_record( - &actor, - DID, - "app.bsky.feed.post", - Some("public-blob"), - blob_record("app.bsky.feed.post", &public), - None, - ) - .await - .unwrap(); - let space_value = blob_record("com.example.post", &space); - database - .create_space_record( - &SpaceRecord { - space: "at://did:web:alice.test/com.example.space/one".to_owned(), - did: DID.to_owned(), - collection: "com.example.post".to_owned(), - rkey: "space-blob".to_owned(), - cid: record_cid(&space_value).unwrap(), - value: space_value, - }, - "3jqfcqzm3fo2j", - ) - .unwrap(); - database - .connection() - .unwrap() - .execute("UPDATE blobs SET created_at = 0", []) - .unwrap(); - let fresh = service - .upload_blob(&actor, "image/png", b"fresh") - .await - .unwrap(); - - assert_eq!( - service - .collect_orphan_blobs(Duration::from_secs(60)) - .await - .unwrap(), - 1 - ); - for blob in [&public, &space, &fresh] { - service.get_blob(DID, &blob.cid).await.unwrap(); - } - assert!(service.get_blob(DID, &orphan.cid).await.is_err()); - service.verify().await.unwrap(); -} - -#[tokio::test] -async fn claimed_blob_deletion_survives_an_identical_reupload() { - let database = Database::open_in_memory().unwrap(); - let actor = account(&database); - let service = service(database.clone()); - let blob = service - .upload_blob(&actor, "image/png", b"claimed") - .await - .unwrap(); - database - .connection() - .unwrap() - .execute("UPDATE blobs SET created_at = 0", []) - .unwrap(); - assert_eq!(database.claim_orphan_blobs(i64::MAX, 1).unwrap().len(), 1); - - assert!( - service - .upload_blob(&actor, "image/png", b"claimed") - .await - .is_err() - ); - assert_eq!( - service - .collect_orphan_blobs(Duration::from_secs(60)) - .await - .unwrap(), - 1 - ); - assert!(matches!( - database.blob(DID, &blob.cid), - Err(Error::NotFound) - )); -} - -#[tokio::test] -async fn orphan_collection_recovers_an_interrupted_blob_publication() { - let database = Database::open_in_memory().unwrap(); - let actor = account(&database); - let blobs = Arc::new(MemoryBlobStore::new()); - let service = RepositoryService::new( - database.clone(), - blobs.clone(), - Arc::new(RecordingEventSink::default()), - &SECRET, - 100_000, - ) - .unwrap(); - let bytes = b"published-before-crash"; - let blob = Blob { - cid: sha256_cid(RAW_CODEC, bytes).unwrap(), - media_type: "image/png".to_owned(), - size: u64::try_from(bytes.len()).unwrap(), - }; - database.begin_blob_upload(&actor.did, &blob).unwrap(); - blobs.put(&actor.did, &blob.cid, bytes).await.unwrap(); - - assert!(matches!( - database.blob(DID, &blob.cid), - Err(Error::NotFound) - )); - assert!(blobs.get(DID, &blob.cid).await.unwrap().is_some()); - database - .connection() - .unwrap() - .execute("UPDATE blobs SET created_at = 0", []) - .unwrap(); - - assert_eq!( - service - .collect_orphan_blobs(Duration::from_secs(60)) - .await - .unwrap(), - 1 - ); - assert!(blobs.get(DID, &blob.cid).await.unwrap().is_none()); - assert_eq!( - database - .connection() - .unwrap() - .query_row("SELECT COUNT(*) FROM blobs", [], |row| row.get::<_, i64>(0)) - .unwrap(), - 0 - ); -} - -#[tokio::test] -async fn repository_block_collection_preserves_atrium_head_and_retained_events() { - let database = Database::open_in_memory().unwrap(); - let actor = account(&database); - let service = service_with_retention(database.clone(), 1); - let first = service - .create_record( - &actor, - DID, - "app.bsky.feed.post", - Some("replace"), - post("first"), - None, - ) - .await - .unwrap(); - let first_cid = first.record.cid.parse::().unwrap(); - service - .put_record( - &actor, - DID, - "app.bsky.feed.post", - "replace", - post("second"), - RecordSwap { - record: None, - commit: None, - }, - ) - .await - .unwrap(); - let mut blocks = RepoBlockStore::new(database.clone()); - assert!(blocks.read_block(first_cid).await.is_ok()); - - assert!(service.collect_repository_blocks().await.unwrap() > 0); - - let mut blocks = RepoBlockStore::new(database); - assert!(blocks.read_block(first_cid).await.is_err()); - service.export_repo(DID, None).await.unwrap(); - assert_eq!(service.repo_messages_after(1, 10).await.unwrap().len(), 1); - assert_eq!(service.collect_repository_blocks().await.unwrap(), 0); -} - -#[tokio::test] -async fn concurrent_record_creates_preserve_both_repository_paths() { - let database = Database::open_in_memory().unwrap(); - let actor = account(&database); - let service = service(database.clone()); - let (first, second) = tokio::join!( - service.create_record( - &actor, - DID, - "app.bsky.feed.post", - Some("first"), - post("first"), - None, - ), - service.create_record( - &actor, - DID, - "app.bsky.feed.post", - Some("second"), - post("second"), - None, - ) - ); - first.unwrap(); - second.unwrap(); - - let head = database.repo_head(DID).unwrap().unwrap(); - let mut blocks = RepoBlockStore::new(database); - let mut repo = Repository::open(&mut blocks, head.root).await.unwrap(); - assert!( - repo.get_raw::("app.bsky.feed.post/first") - .await - .unwrap() - .is_some() - ); - assert!( - repo.get_raw::("app.bsky.feed.post/second") - .await - .unwrap() - .is_some() - ); -} - -#[tokio::test] -async fn durable_repo_events_preserve_inductive_atrium_commit_metadata() { - let database = Database::open_in_memory().unwrap(); - let actor = account(&database); - let service = service(database.clone()); - let first = service - .create_record( - &actor, - DID, - "app.bsky.feed.post", - Some("first"), - post("first"), - None, - ) - .await - .unwrap(); - let first_root = first.commit.cid.parse::().unwrap(); - let mut store = RepoBlockStore::new(database); - let first_repo = Repository::open(&mut store, first_root).await.unwrap(); - let first_data = first_repo.commit().data(); - drop(first_repo); - - let second = service - .create_record( - &actor, - DID, - "app.bsky.feed.post", - Some("second"), - post("second"), - None, - ) - .await - .unwrap(); - let messages = service.repo_messages_after(0, 10).await.unwrap(); - assert_eq!(messages.len(), 4); - let subscribe_repos::Message::Identity(identity) = &messages[0] else { - panic!("first repository event should identify the repository"); - }; - assert_eq!(identity.did.as_str(), DID); - assert_eq!(identity.handle.as_ref().map(Handle::as_str), Some(HANDLE)); - assert_eq!(identity.seq, 1); - let subscribe_repos::Message::Account(account) = &messages[1] else { - panic!("second repository event should activate the account"); - }; - assert_eq!(account.did.as_str(), DID); - assert!(account.active); - assert_eq!(account.status, None); - assert_eq!(account.seq, 2); - let subscribe_repos::Message::Commit(first_message) = &messages[2] else { - panic!("third repository event should be a commit"); - }; - assert_eq!(first_message.commit.0, first_root); - assert_eq!(first_message.since, None); - assert_eq!(first_message.prev_data, None); - assert_eq!(first_message.ops[0].action, "create"); - assert_eq!(first_message.ops[0].path, "app.bsky.feed.post/first"); - let mut car = CarStore::open(Cursor::new(first_message.blocks.clone())) - .await - .unwrap(); - assert_eq!(car.roots().collect::>(), vec![first_root]); - assert!(car.read_block(first_root).await.is_ok()); - - let subscribe_repos::Message::Commit(second_message) = &messages[3] else { - panic!("fourth repository event should be a commit"); - }; - assert_eq!(second_message.commit.0, second.commit.cid.parse().unwrap()); - assert_eq!( - second_message - .since - .as_ref() - .map(|rev| rev.as_str().to_owned()), - Some(first.commit.rev) - ); - assert_eq!( - second_message.prev_data.as_ref().map(|cid| cid.0), - Some(first_data) - ); - assert_eq!( - service.repo_messages_after(4, 10).await.unwrap(), - Vec::new() - ); - assert!(matches!( - service.repo_messages_after(5, 10).await, - Err(Error::Xrpc { - name: "FutureCursor", - .. - }) - )); -} - -#[tokio::test] -async fn legacy_repo_events_replay_as_atrium_sync_reset_points() { - let database = Database::open_in_memory().unwrap(); - let actor = account(&database); - let service = service(database.clone()); - let created = service - .create_record( - &actor, - DID, - "app.bsky.feed.post", - Some("legacy"), - post("legacy"), - None, - ) - .await - .unwrap(); - let root = created.commit.cid.parse::().unwrap(); - let payload: Vec = database - .connection() - .unwrap() - .query_row("SELECT payload FROM repo_outbox WHERE seq = 3", [], |row| { - row.get(0) - }) - .unwrap(); - let mut payload: Value = serde_json::from_slice(&payload).unwrap(); - payload.as_object_mut().unwrap().remove("version"); - let payload = serde_json::to_vec(&payload).unwrap(); - { - let connection = database.connection().unwrap(); - connection - .execute( - "UPDATE repo_outbox SET payload = ?1 WHERE seq = 3", - [payload], - ) - .unwrap(); - connection - .execute("DELETE FROM repo_commit_blocks WHERE seq = 3", []) - .unwrap(); - } - - let messages = service.repo_messages_after(2, 10).await.unwrap(); - let [subscribe_repos::Message::Sync(message)] = messages.as_slice() else { - panic!("legacy repository event should be a sync reset point"); - }; - assert_eq!(message.did.as_str(), DID); - assert_eq!(message.rev, created.commit.rev); - let mut car = CarStore::open(Cursor::new(message.blocks.clone())) - .await - .unwrap(); - assert_eq!(car.roots().collect::>(), vec![root]); - assert!(car.read_block(root).await.is_ok()); -} - -fn service(database: Database) -> RepositoryService { - service_with_retention(database, 100_000) -} - -fn service_with_retention(database: Database, retention: usize) -> RepositoryService { - RepositoryService::new( - database, - Arc::new(MemoryBlobStore::new()), - Arc::new(RecordingEventSink::default()), - &SECRET, - retention, - ) - .unwrap() -} - -fn account(database: &Database) -> Account { - database.ensure_account(DID, HANDLE, "password").unwrap(); - Account { - did: DID.to_owned(), - handle: HANDLE.to_owned(), - } -} - -fn post(text: &str) -> Value { - json!({ - "$type": "app.bsky.feed.post", - "text": text, - "createdAt": "2026-08-22T00:00:00Z" - }) -} - -#[test] -fn account_cannot_override_a_host_enforced_status() { - let database = Database::open_in_memory().unwrap(); - account(&database); - database - .connection() - .unwrap() - .execute( - "UPDATE accounts SET status = 'suspended' WHERE did = ?1", - [DID], - ) - .unwrap(); - let service = service(database.clone()); - - assert!(matches!( - service.activate_account(DID), - Err(Error::Xrpc { - name: "AccountSuspended", - .. - }) - )); - assert_eq!( - database - .hosted_account_by_identifier(DID) - .unwrap() - .unwrap() - .status, - AccountStatus::Suspended - ); - assert!(database.repo_events_after(0, 10).unwrap().is_empty()); - - service - .set_operator_account_status(DID, AccountStatus::TakenDown) - .unwrap(); - service - .set_operator_account_status(DID, AccountStatus::Active) - .unwrap(); - let events = database.repo_events_after(0, 10).unwrap(); - assert!(matches!( - events.as_slice(), - [ - StoredRepoEvent::Account(StoredAccountEvent { - active: false, - status: Some(status), - .. - }), - StoredRepoEvent::Account(StoredAccountEvent { - active: true, - status: None, - .. - }) - ] if status == "takendown" - )); -} - -#[test] -fn identity_handle_transitions_are_typed_idempotent_and_durable() { - let database = Database::open_in_memory().unwrap(); - account(&database); - let service = service(database.clone()); - - service.set_account_handle(DID, "handle.invalid").unwrap(); - service.set_account_handle(DID, "handle.invalid").unwrap(); - service.set_account_handle(DID, HANDLE).unwrap(); - - assert_eq!( - database - .hosted_account_by_identifier(DID) - .unwrap() - .unwrap() - .account - .handle, - HANDLE - ); - let events = database.repo_events_after(0, 10).unwrap(); - assert!(matches!( - events.as_slice(), - [ - StoredRepoEvent::Identity(StoredIdentityEvent { - handle: Some(invalid), - .. - }), - StoredRepoEvent::Identity(StoredIdentityEvent { - handle: Some(recovered), - .. - }) - ] if invalid == "handle.invalid" && recovered == HANDLE - )); - assert!(service.set_account_handle(DID, "not a handle").is_err()); -} - -fn blob_record(collection: &str, blob: &Blob) -> Value { - json!({ - "$type": collection, - "image": { - "$type": "blob", - "ref": {"$link": blob.cid.to_string()}, - "mimeType": blob.media_type, - "size": blob.size - } - }) -} diff --git a/src/repository/tests/accounts.rs b/src/repository/tests/accounts.rs new file mode 100644 index 0000000..f331fa5 --- /dev/null +++ b/src/repository/tests/accounts.rs @@ -0,0 +1,626 @@ +use super::*; + +fn account_statuses() -> [AccountStatus; 5] { + [ + AccountStatus::Active, + AccountStatus::Deactivated, + AccountStatus::Suspended, + AccountStatus::TakenDown, + AccountStatus::Deleted, + ] +} + +#[derive(Debug)] +struct PreviousCommitView { + cid: String, + rev: String, +} + +fn compare_sync_head_projection( + service: &RepositoryService, + did: &str, +) -> Result { + let previous: Result = (|| { + let account = service.sync_account(did)?; + let head = service + .database + .repo_head(&account.did)? + .ok_or_else(|| Error::xrpc("RepoNotFound", "repository has no commits"))?; + Ok(PreviousCommitView { + cid: head.root.to_string(), + rev: head.rev, + }) + })(); + let current = service.sync_head(did).map(|head| PreviousCommitView { + cid: head.root.to_string(), + rev: head.rev, + }); + assert_eq!(format!("{current:?}"), format!("{previous:?}")); + current +} + +fn sync_head_cases(root: &str) -> [(rusqlite::types::Value, rusqlite::types::Value, bool); 7] { + use rusqlite::types::Value as SqlValue; + + let rev = "3jqfcqzm3fo2j"; + [ + ( + SqlValue::Text(root.to_owned()), + SqlValue::Text(rev.to_owned()), + true, + ), + ( + SqlValue::Text(root.to_owned()), + SqlValue::Text("invalid revision".to_owned()), + true, + ), + ( + SqlValue::Text("invalid CID".to_owned()), + SqlValue::Text(rev.to_owned()), + false, + ), + (SqlValue::Null, SqlValue::Text(rev.to_owned()), false), + ( + SqlValue::Blob(vec![1]), + SqlValue::Text(rev.to_owned()), + false, + ), + ( + SqlValue::Text("invalid CID".to_owned()), + SqlValue::Blob(vec![1]), + false, + ), + (SqlValue::Text(root.to_owned()), SqlValue::Null, false), + ] +} + +#[test] +fn sync_head_projection_preserves_missing_inactive_and_malformed_state() { + use rusqlite::{params, types::Value as SqlValue}; + + let database = Database::open_in_memory().unwrap(); + let service = service(database.clone()); + for message in [ + "repository is not hosted by this PDS", + "repository has no commits", + ] { + assert!(matches!( + compare_sync_head_projection(&service, DID), + Err(Error::Xrpc { name: "RepoNotFound", message: actual }) if actual == message + )); + account(&database); + } + database + .connection() + .unwrap() + .execute_batch("DROP TABLE repo_heads; CREATE TABLE repo_heads (did, root_cid, rev)") + .unwrap(); + let root = record_cid(&post("head")).unwrap().to_string(); + let cases = sync_head_cases(&root); + for status in account_statuses() { + database + .connection() + .unwrap() + .execute("UPDATE accounts SET status = ?1", [status.as_str()]) + .unwrap(); + for (cid, rev, valid_head) in &cases { + let connection = database.connection().unwrap(); + connection.execute("DELETE FROM repo_heads", []).unwrap(); + connection + .execute( + "INSERT INTO repo_heads VALUES (?1, ?2, ?3)", + params![DID, cid, rev], + ) + .unwrap(); + drop(connection); + for identifier in [DID, HANDLE] { + let actual = compare_sync_head_projection(&service, identifier); + assert_eq!(actual.is_ok(), status.is_active() && *valid_head); + if let Ok(commit) = actual { + assert_eq!(commit.cid, root); + assert_eq!(SqlValue::Text(commit.rev), *rev); + } + } + } + } + database + .connection() + .unwrap() + .execute("DROP TABLE repo_heads", []) + .unwrap(); + for status in account_statuses() { + database + .connection() + .unwrap() + .execute("UPDATE accounts SET status = ?1", [status.as_str()]) + .unwrap(); + let actual = compare_sync_head_projection(&service, DID); + assert_eq!( + matches!(actual, Err(Error::Database(_))), + status.is_active() + ); + } +} + +#[test] +fn lifecycle_status_matrix_preserves_policy_idempotence_and_events() { + let database = Database::open_in_memory().unwrap(); + account(&database); + let service = service(database.clone()); + let mut changes = account_statuses().map(|next| (next, false)).to_vec(); + changes.extend([ + (AccountStatus::Active, true), + (AccountStatus::Deactivated, true), + ]); + for current in account_statuses() { + for &(next, user_initiated) in &changes { + database + .connection() + .unwrap() + .execute( + "UPDATE accounts SET status = ?1 WHERE did = ?2", + [current.as_str(), DID], + ) + .unwrap(); + let cursor = database.latest_repo_event_sequence().unwrap(); + let result = if !user_initiated { + service.set_operator_account_status(DID, next) + } else if next == AccountStatus::Active { + service.activate_account(DID) + } else { + service.deactivate_account(DID) + }; + let allowed = !user_initiated + || current == next + || matches!( + (current, next), + (AccountStatus::Active, AccountStatus::Deactivated) + | (AccountStatus::Deactivated, AccountStatus::Active) + ); + if allowed { + result.unwrap(); + } else { + assert_eq!( + format!("{:?}", result.unwrap_err()), + format!("{:?}", current.account_error()), + ); + } + let (account, status) = database.hosted_account_by_identifier(DID).unwrap().unwrap(); + assert_eq!(status, if allowed { next } else { current }); + assert_eq!(account.handle, HANDLE); + let events = database.repo_events_after(cursor, 10).unwrap(); + if allowed && current != next { + let [StoredRepoEvent::Account(event)] = events.as_slice() else { + panic!("expected one account event: {events:?}"); + }; + assert_eq!(event.did, DID); + assert_eq!(event.active, next.is_active()); + assert_eq!(event.status.as_deref(), next.inactive_reason()); + } else { + assert!(events.is_empty(), "{current:?} -> {next:?}"); + } + } + } +} + +#[test] +fn lifecycle_handle_matrix_preserves_validation_idempotence_and_events() { + let database = Database::open_in_memory().unwrap(); + account(&database); + let service = service(database.clone()); + for status in account_statuses() { + for next in [HANDLE, "next.test", "handle.invalid", "not a handle", ""] { + database + .connection() + .unwrap() + .execute( + "UPDATE accounts SET status = ?1, handle = ?2 WHERE did = ?3", + [status.as_str(), HANDLE, DID], + ) + .unwrap(); + let cursor = database.latest_repo_event_sequence().unwrap(); + let expected = Handle::new(next.to_owned()).map(|handle| handle.as_str() != HANDLE); + let actual = service.set_account_handle(DID, next); + let changed = match expected { + Ok(changed) => { + assert_eq!(actual.unwrap(), changed); + changed + } + Err(error) => { + assert!(matches!(actual, Err(Error::InvalidRequest(message)) + if message == format!("invalid account handle: {error}"))); + false + } + }; + let (account, actual_status) = + database.hosted_account_by_identifier(DID).unwrap().unwrap(); + assert_eq!(actual_status, status); + assert_eq!(account.handle, if changed { next } else { HANDLE }); + let events = database.repo_events_after(cursor, 10).unwrap(); + if changed { + let [StoredRepoEvent::Identity(event)] = events.as_slice() else { + panic!("expected one identity event: {events:?}"); + }; + assert_eq!(event.did, DID); + assert_eq!(event.handle.as_deref(), Some(next)); + } else { + assert!(events.is_empty(), "{status:?}: {next:?}"); + } + } + } +} + +#[test] +fn lifecycle_lookup_errors_and_handle_validation_keep_their_priority() { + let database = Database::open_in_memory().unwrap(); + let service = service(database.clone()); + assert_lifecycle_errors(&service, &Error::AuthenticationRequired); + assert_invalid_handle_precedes_lookup(&service); + account(&database); + database + .connection() + .unwrap() + .execute_batch( + "PRAGMA ignore_check_constraints = ON; + UPDATE accounts SET status = 'unknown';", + ) + .unwrap(); + assert_lifecycle_errors( + &service, + &Error::Internal("stored unknown account status unknown".to_owned()), + ); + assert_invalid_handle_precedes_lookup(&service); + assert!(matches!( + service.set_account_handle(DID, HANDLE), + Err(Error::Internal(_)) + )); + database + .connection() + .unwrap() + .execute("UPDATE accounts SET handle = x'00' WHERE did = ?1", [DID]) + .unwrap(); + let expected = database.hosted_account_by_identifier(DID).unwrap_err(); + assert!(matches!( + &expected, + Error::Database(rusqlite::Error::InvalidColumnType(1, _, _)) + )); + assert_lifecycle_errors(&service, &expected); + assert_invalid_handle_precedes_lookup(&service); + database + .connection() + .unwrap() + .execute_batch("DROP TABLE accounts") + .unwrap(); + let expected = database.hosted_account_by_identifier(DID).unwrap_err(); + assert!(matches!(&expected, Error::Database(_))); + assert_lifecycle_errors(&service, &expected); + assert_invalid_handle_precedes_lookup(&service); +} + +fn assert_lifecycle_errors(service: &RepositoryService, expected: &Error) { + for result in [ + service.activate_account(DID), + service.deactivate_account(DID), + service.set_operator_account_status(DID, AccountStatus::Suspended), + service.set_account_handle(DID, "next.test").map(|_| ()), + ] { + assert_eq!( + format!("{:?}", result.unwrap_err()), + format!("{expected:?}") + ); + } +} + +fn assert_invalid_handle_precedes_lookup(service: &RepositoryService) { + let invalid = "not a handle"; + let expected = Handle::new(invalid.to_owned()).unwrap_err(); + assert!(matches!(service.set_account_handle(DID, invalid), + Err(Error::InvalidRequest(message)) if message == format!("invalid account handle: {expected}"))); +} + +#[test] +fn lifecycle_retry_exhaustion_preserves_exact_messages_and_rollback() { + let database = Database::open_in_memory().unwrap(); + account(&database); + let service = service(database.clone()); + database + .connection() + .unwrap() + .execute_batch( + "CREATE TRIGGER ignore_lifecycle_update BEFORE UPDATE ON accounts BEGIN + INSERT INTO repo_outbox + (did, rev, root_cid, event_type, payload, next_attempt_at, created_at) + VALUES (OLD.did, NULL, NULL, 'identity', x'00', 0, 0); + SELECT RAISE(IGNORE); + END;", + ) + .unwrap(); + for (result, field) in [ + ( + service.set_account_handle(DID, "next.test").map(|_| ()), + "handle", + ), + (service.deactivate_account(DID), "status"), + ( + service.set_operator_account_status(DID, AccountStatus::Suspended), + "status", + ), + ] { + assert!(matches!(result, Err(Error::Conflict(message)) + if message == format!("account {field} changed too frequently; retry the request"))); + } + let (account, status) = database.hosted_account_by_identifier(DID).unwrap().unwrap(); + assert_eq!(account.handle, HANDLE); + assert_eq!(status, AccountStatus::Active); + assert!(database.repo_events_after(0, 10).unwrap().is_empty()); + assert!(!service.set_account_handle(DID, HANDLE).unwrap()); + service.activate_account(DID).unwrap(); + service + .set_operator_account_status(DID, AccountStatus::Active) + .unwrap(); + assert!(database.repo_events_after(0, 10).unwrap().is_empty()); +} + +#[test] +fn repository_cursor_validation_preserves_boundaries_and_failure_priority() { + let service = service(Database::open_in_memory().unwrap()); + for cursor in [ + "0::did:example:alice", + "+1::did:web:alice.test", + "0001::did:example:alice", + "-9223372036854775808::did:example:alice", + "9223372036854775807::did:example:alice", + ] { + let (repos, next) = service.list_repos(Some(cursor), None, 1).unwrap(); + assert!(repos.is_empty()); + assert_eq!(next, None); + let (repos, next) = service + .list_repos_by_collection("app.bsky.feed.post", Some(cursor), 1) + .unwrap(); + assert!(repos.is_empty()); + assert_eq!(next, None); + } + for cursor in [ + "", + "123", + "::", + "::did:example:alice", + "0::", + "0::did:example:alice::bob", + "0::::did:example:alice", + " 0::did:example:alice", + "0 ::did:example:alice", + "+::did:example:alice", + "1.5::did:example:alice", + "9223372036854775808::did:example:alice", + "-9223372036854775809::did:example:alice", + "0::not-a-did", + ] { + for result in [ + service.list_repos(Some(cursor), None, 1).map(|_| ()), + service + .list_repos_by_collection("app.bsky.feed.post", Some(cursor), 1) + .map(|_| ()), + ] { + assert!( + matches!(result, Err(Error::InvalidRequest(message)) if message == "malformed repository cursor"), + "{cursor}", + ); + } + } + assert!(matches!( + service.list_repos_by_collection("invalid", Some("invalid"), 1), + Err(Error::InvalidRequest(message)) if message == "`invalid` is not a valid collection NSID", + )); +} + +fn previous_repository_listing( + database: &Database, + cursor: Option<&str>, + collection: Option<&str>, + limit: usize, +) -> Result<(Vec, Option)> { + use atrium_api::types::string::Did; + + let cursor = cursor + .map(|cursor| -> Result<_> { + let invalid = || Error::InvalidRequest("malformed repository cursor".to_owned()); + let (time, did) = cursor + .split_once("::") + .filter(|(time, did)| !time.is_empty() && !did.is_empty() && !did.contains("::")) + .ok_or_else(invalid)?; + let time = time.parse::().map_err(|_| invalid())?; + let did = Did::new(did.to_owned()).map_err(|_| invalid())?; + Ok((time, did.as_str().to_owned())) + }) + .transpose()?; + let heads = database.repo_heads_after( + cursor.as_ref().map(|(time, did)| (*time, did.as_str())), + collection, + limit, + )?; + let next = heads + .last() + .map(|head| format!("{}::{}", head.created_at_millis, head.did)); + Ok((heads, next)) +} + +fn repository_listing_database() -> Database { + let database = Database::open_in_memory().unwrap(); + let root = record_cid(&post("root")).unwrap().to_string(); + { + let connection = database.connection().unwrap(); + connection + .execute("INSERT INTO repo_blocks VALUES (?1, x'00', 0)", [&root]) + .unwrap(); + for (name, status, created_at, collection) in [ + ("a", "active", 1, "app.bsky.feed.post"), + ("b", "active", 1, "app.bsky.feed.post"), + ("c", "suspended", 2, "app.bsky.feed.post"), + ("d", "active", 3, "app.test.other"), + ] { + let did = format!("did:example:{name}"); + connection.execute( + "INSERT INTO accounts (did, handle, password_hash, created_at, status) VALUES (?1, ?2, '', ?3, ?4)", + rusqlite::params![did, format!("{name}.test"), created_at, status], + ).unwrap(); + connection + .execute( + "INSERT INTO repo_heads VALUES (?1, ?2, '2222222222222', 0)", + rusqlite::params![did, root], + ) + .unwrap(); + connection + .execute( + "INSERT INTO records VALUES (?1, ?2, 'one', ?3, '{}', 0, 0)", + rusqlite::params![did, collection, root], + ) + .unwrap(); + } + } + database +} + +#[test] +fn shared_repository_listing_preserves_filters_cursors_and_decode_errors() { + let database = repository_listing_database(); + let service = service(database.clone()); + for corrupt in [false, true] { + if corrupt { + database.connection().unwrap().execute_batch( + "PRAGMA ignore_check_constraints = ON; UPDATE accounts SET status = 'unknown' WHERE did = 'did:example:d';", + ).unwrap(); + } + for collection in [ + None, + Some("app.bsky.feed.post"), + Some("app.test.other"), + Some("app.test.missing"), + ] { + for cursor in [ + None, + Some("0::did:example:a"), + Some("1000::did:example:a"), + Some("3000::did:example:z"), + Some("invalid"), + ] { + for limit in [0, 1, 2, usize::MAX] { + let expected = + previous_repository_listing(&database, cursor, collection, limit); + let actual = service.list_repos(cursor, collection, limit); + assert_eq!( + format!("{actual:?}"), + format!("{expected:?}"), + "{collection:?} {cursor:?} {limit} {corrupt}" + ); + if let Some(collection) = collection { + let expected = expected.map(|(heads, next)| { + ( + heads.into_iter().map(|head| head.did).collect::>(), + next, + ) + }); + let actual = service.list_repos_by_collection(collection, cursor, limit); + assert_eq!(format!("{actual:?}"), format!("{expected:?}")); + } + } + } + } + } + database + .connection() + .unwrap() + .execute_batch("DROP TABLE records") + .unwrap(); + for (collection, cursor) in [ + ("invalid", "invalid"), + ("app.bsky.feed.post", "invalid"), + ("app.bsky.feed.post", "0::did:example:a"), + ] { + let expected = crate::syntax::collection(collection) + .and_then(|_| previous_repository_listing(&database, Some(cursor), Some(collection), 1)) + .map(|_| ()); + let actual = service + .list_repos_by_collection(collection, Some(cursor), 1) + .map(|_| ()); + assert_eq!(format!("{actual:?}"), format!("{expected:?}")); + } +} + +#[test] +fn account_cannot_override_a_host_enforced_status() { + let database = Database::open_in_memory().unwrap(); + account(&database); + database + .connection() + .unwrap() + .execute( + "UPDATE accounts SET status = 'suspended' WHERE did = ?1", + [DID], + ) + .unwrap(); + let service = service(database.clone()); + + assert!(matches!( + service.activate_account(DID), + Err(Error::Xrpc { + name: "AccountSuspended", + .. + }) + )); + let (_account, status) = database.hosted_account_by_identifier(DID).unwrap().unwrap(); + assert_eq!(status, AccountStatus::Suspended); + assert!(database.repo_events_after(0, 10).unwrap().is_empty()); + + service + .set_operator_account_status(DID, AccountStatus::TakenDown) + .unwrap(); + service + .set_operator_account_status(DID, AccountStatus::Active) + .unwrap(); + let events = database.repo_events_after(0, 10).unwrap(); + assert!(matches!( + events.as_slice(), + [ + StoredRepoEvent::Account(StoredAccountEvent { + active: false, + status: Some(status), + .. + }), + StoredRepoEvent::Account(StoredAccountEvent { + active: true, + status: None, + .. + }) + ] if status == "takendown" + )); +} + +#[test] +fn identity_handle_transitions_are_typed_idempotent_and_durable() { + let database = Database::open_in_memory().unwrap(); + account(&database); + let service = service(database.clone()); + + service.set_account_handle(DID, "handle.invalid").unwrap(); + service.set_account_handle(DID, "handle.invalid").unwrap(); + service.set_account_handle(DID, HANDLE).unwrap(); + + let (account, _status) = database.hosted_account_by_identifier(DID).unwrap().unwrap(); + assert_eq!(account.handle, HANDLE); + let events = database.repo_events_after(0, 10).unwrap(); + assert!(matches!( + events.as_slice(), + [ + StoredRepoEvent::Identity(StoredIdentityEvent { + handle: Some(invalid), + .. + }), + StoredRepoEvent::Identity(StoredIdentityEvent { + handle: Some(recovered), + .. + }) + ] if invalid == "handle.invalid" && recovered == HANDLE + )); + assert!(service.set_account_handle(DID, "not a handle").is_err()); +} diff --git a/src/repository/tests/blobs.rs b/src/repository/tests/blobs.rs new file mode 100644 index 0000000..bd21414 --- /dev/null +++ b/src/repository/tests/blobs.rs @@ -0,0 +1,406 @@ +use super::*; + +#[test] +fn blob_inventory_cursors_preserve_error_context_and_validation_priority() { + let database = Database::open_in_memory().unwrap(); + let actor = account(&database); + let service = service(database.clone()); + assert!(matches!( + service.list_blobs("did:example:missing", Some("invalid"), Some(""), 100), + Err(Error::Xrpc { + name: "RepoNotFound", + .. + }) + )); + assert!(matches!( + service.list_blobs(DID, Some("invalid"), Some(""), 100), + Err(Error::InvalidRequest(message)) if message == "`invalid` is not a valid TID" + )); + database + .connection() + .unwrap() + .execute_batch("DROP TABLE record_blobs") + .unwrap(); + for cursor in ["", "x"] { + assert!(matches!( + service.list_blobs(DID, None, Some(cursor), 100), + Err(Error::InvalidRequest(message)) if message == "invalid blob cursor CID: Input too short" + )); + assert!(matches!( + service.list_missing_blobs(&actor, Some(cursor), 100), + Err(Error::InvalidRequest(message)) if message == "invalid missing-blob cursor CID: Input too short" + )); + } + assert!(matches!( + service.list_blobs(DID, None, None, 100), + Err(Error::Database(_)) + )); + assert!(matches!( + service.list_missing_blobs(&actor, None, 100), + Err(Error::Database(_)) + )); +} + +#[test] +fn blob_inventory_cursors_keep_legacy_and_multibase_canonicalization() { + let database = Database::open_in_memory().unwrap(); + let actor = account(&database); + let first = sha256_cid(RAW_CODEC, b"first"); + let legacy = Cid::new_v0(*first.hash()).unwrap(); + let mut cids = vec![first, sha256_cid(RAW_CODEC, b"second"), legacy]; + cids.sort_by_key(ToString::to_string); + let value = post("inventory"); + let connection = database.connection().unwrap(); + connection + .execute( + "INSERT INTO records (did, collection, rkey, cid, record_json, created_at, updated_at) + VALUES (?1, 'app.bsky.feed.post', 'inventory', ?2, ?3, 1, 1)", + rusqlite::params![ + DID, + record_cid(&value).unwrap().to_string(), + value.to_string() + ], + ) + .unwrap(); + for cid in &cids { + connection + .execute( + "INSERT INTO record_blobs (did, collection, rkey, cid) + VALUES (?1, 'app.bsky.feed.post', 'inventory', ?2)", + rusqlite::params![DID, cid.to_string()], + ) + .unwrap(); + } + drop(connection); + let service = service(database); + assert_eq!( + service.list_blobs(DID, None, None, 100).unwrap(), + (cids.clone(), None) + ); + let (missing, cursor) = service.list_missing_blobs(&actor, None, 100).unwrap(); + assert_eq!( + missing.iter().map(|(cid, _)| *cid).collect::>(), + cids + ); + assert_eq!(cursor, cids.last().map(ToString::to_string)); + for cid in &cids { + let canonical = cid.to_string(); + let mut cursors = vec![ + canonical.clone(), + format!("/ipfs/{canonical}"), + format!("https://gateway.test/ipfs/{canonical}"), + ]; + if cid.version() == cid::Version::V1 { + cursors.push( + cid.to_string_of_base(cid::multibase::Base::Base58Btc) + .unwrap(), + ); + } + let expected: Vec<_> = cids + .iter() + .copied() + .filter(|cid| cid.to_string() > canonical) + .collect(); + for cursor in cursors { + assert_eq!( + service.list_blobs(DID, None, Some(&cursor), 100).unwrap(), + (expected.clone(), None), + "public cursor {cursor}" + ); + let (missing, next) = service + .list_missing_blobs(&actor, Some(&cursor), 100) + .unwrap(); + assert_eq!( + missing.iter().map(|(cid, _)| *cid).collect::>(), + expected, + "missing cursor {cursor}" + ); + assert!( + missing + .iter() + .all(|(_, uri)| uri == "at://did:web:alice.test/app.bsky.feed.post/inventory") + ); + assert_eq!(next, expected.last().map(ToString::to_string)); + } + } +} + +#[test] +fn missing_blob_inventory_reuses_atrium_reference_scanning_and_cid_cursors() { + const BLOB_CID: &str = "bafkreibme22gw2h7y2h7tg2fhqotaqjucnbc24deqo72b6mkl2egezxhvy"; + let database = Database::open_in_memory().unwrap(); + let actor = account(&database); + let value = json!({ + "$type": "app.bsky.feed.post", + "image": { + "$type": "blob", + "ref": {"$link": BLOB_CID}, + "mimeType": "image/png", + "size": 3 + } + }); + let cid = record_cid(&value).unwrap(); + let connection = database.connection().unwrap(); + connection + .execute( + "INSERT INTO records + (did, collection, rkey, cid, record_json, created_at, updated_at) + VALUES (?1, 'app.bsky.feed.post', 'missing', ?2, ?3, 1, 1)", + rusqlite::params![DID, cid.to_string(), value.to_string()], + ) + .unwrap(); + connection + .execute( + "INSERT INTO record_blobs (did, collection, rkey, cid) + VALUES (?1, 'app.bsky.feed.post', 'missing', ?2)", + rusqlite::params![DID, BLOB_CID], + ) + .unwrap(); + drop(connection); + let service = service(database.clone()); + + let (missing, cursor) = service.list_missing_blobs(&actor, None, 500).unwrap(); + assert_eq!(missing.len(), 1); + assert_eq!(missing[0].0.to_string(), BLOB_CID); + assert_eq!( + missing[0].1, + "at://did:web:alice.test/app.bsky.feed.post/missing" + ); + assert_eq!(cursor.as_deref(), Some(BLOB_CID)); + assert!( + service + .list_missing_blobs(&actor, cursor.as_deref(), 500) + .unwrap() + .0 + .is_empty() + ); +} + +#[tokio::test] +async fn repository_mutations_atomically_replace_and_remove_blob_references() { + let database = Database::open_in_memory().unwrap(); + let actor = account(&database); + let service = service(database); + let first = service + .upload_blob(&actor, "image/png", b"first") + .await + .unwrap(); + let second = service + .upload_blob(&actor, "image/png", b"second") + .await + .unwrap(); + let record = |blob: &Blob| { + json!({ + "$type": "app.bsky.feed.post", + "image": { + "$type": "blob", + "ref": {"$link": blob.cid.to_string()}, + "mimeType": blob.media_type, + "size": blob.size + } + }) + }; + + service + .create_record( + &actor, + DID, + "app.bsky.feed.post", + Some("image"), + record(&first), + None, + ) + .await + .unwrap(); + assert_eq!( + service.list_blobs(DID, None, None, 100).unwrap().0, + vec![first.cid] + ); + + service + .put_record( + &actor, + DID, + "app.bsky.feed.post", + "image", + record(&second), + RecordSwap { + record: None, + commit: None, + }, + ) + .await + .unwrap(); + assert_eq!( + service.list_blobs(DID, None, None, 100).unwrap().0, + vec![second.cid] + ); + + service + .delete_record(&actor, DID, "app.bsky.feed.post", "image", None, None) + .await + .unwrap(); + assert!( + service + .list_blobs(DID, None, None, 100) + .unwrap() + .0 + .is_empty() + ); +} + +#[tokio::test] +async fn orphan_collection_preserves_public_space_and_fresh_blob_references() { + let database = Database::open_in_memory().unwrap(); + let actor = account(&database); + let service = service(database.clone()); + let public = service + .upload_blob(&actor, "image/png", b"public") + .await + .unwrap(); + let space = service + .upload_blob(&actor, "image/png", b"space") + .await + .unwrap(); + let orphan = service + .upload_blob(&actor, "image/png", b"orphan") + .await + .unwrap(); + service + .create_record( + &actor, + DID, + "app.bsky.feed.post", + Some("public-blob"), + blob_record("app.bsky.feed.post", &public), + None, + ) + .await + .unwrap(); + let space_value = blob_record("com.example.post", &space); + database + .create_space_record( + &SpaceRecord { + space: "at://did:web:alice.test/com.example.space/one".to_owned(), + did: DID.to_owned(), + collection: "com.example.post".to_owned(), + rkey: "space-blob".to_owned(), + cid: record_cid(&space_value).unwrap(), + value: space_value, + }, + "3jqfcqzm3fo2j", + ) + .unwrap(); + database + .connection() + .unwrap() + .execute("UPDATE blobs SET created_at = 0", []) + .unwrap(); + let fresh = service + .upload_blob(&actor, "image/png", b"fresh") + .await + .unwrap(); + + assert_eq!( + service + .collect_orphan_blobs(Duration::from_secs(60)) + .await + .unwrap(), + 1 + ); + for blob in [&public, &space, &fresh] { + service.read_blob(DID, &blob.cid).await.unwrap(); + } + assert!(service.get_blob(DID, &orphan.cid).await.is_err()); + service.verify().await.unwrap(); +} + +#[tokio::test] +async fn claimed_blob_deletion_survives_an_identical_reupload() { + let database = Database::open_in_memory().unwrap(); + let actor = account(&database); + let service = service(database.clone()); + let blob = service + .upload_blob(&actor, "image/png", b"claimed") + .await + .unwrap(); + database + .connection() + .unwrap() + .execute("UPDATE blobs SET created_at = 0", []) + .unwrap(); + assert_eq!(database.claim_orphan_blobs(i64::MAX, 1).unwrap().len(), 1); + + assert!( + service + .upload_blob(&actor, "image/png", b"claimed") + .await + .is_err() + ); + assert_eq!( + service + .collect_orphan_blobs(Duration::from_secs(60)) + .await + .unwrap(), + 1 + ); + assert!(matches!( + database + .find_blob(DID, &blob.cid) + .and_then(|blob| blob.ok_or(Error::NotFound)), + Err(Error::NotFound) + )); +} + +#[tokio::test] +async fn orphan_collection_recovers_an_interrupted_blob_publication() { + let database = Database::open_in_memory().unwrap(); + let actor = account(&database); + let blobs = Arc::new(MemoryBlobStore::default()); + let service = RepositoryService::new( + database.clone(), + blobs.clone(), + Arc::new(RecordingEventSink::default()), + &SECRET, + 100_000, + ) + .unwrap(); + let bytes = b"published-before-crash"; + let blob = Blob { + cid: sha256_cid(RAW_CODEC, bytes), + media_type: "image/png".to_owned(), + size: u64::try_from(bytes.len()).unwrap(), + }; + database.begin_blob_upload(&actor.did, &blob).unwrap(); + blobs.put(&actor.did, &blob.cid, bytes).await.unwrap(); + + assert!(matches!( + database + .find_blob(DID, &blob.cid) + .and_then(|blob| blob.ok_or(Error::NotFound)), + Err(Error::NotFound) + )); + assert!(blobs.get(DID, &blob.cid).await.unwrap().is_some()); + database + .connection() + .unwrap() + .execute("UPDATE blobs SET created_at = 0", []) + .unwrap(); + + assert_eq!( + service + .collect_orphan_blobs(Duration::from_secs(60)) + .await + .unwrap(), + 1 + ); + assert!(blobs.get(DID, &blob.cid).await.unwrap().is_none()); + assert_eq!( + database + .connection() + .unwrap() + .query_row("SELECT COUNT(*) FROM blobs", [], |row| row.get::<_, i64>(0)) + .unwrap(), + 0 + ); +} diff --git a/src/repository/tests/export.rs b/src/repository/tests/export.rs new file mode 100644 index 0000000..28eece9 --- /dev/null +++ b/src/repository/tests/export.rs @@ -0,0 +1,28 @@ +use super::*; + +#[tokio::test] +async fn repository_export_buffer_enforces_its_byte_limit() { + let mut bytes = Vec::new(); + { + let mut buffer = BoundedBuffer::new(&mut bytes, 3); + buffer.write_all(b"abc").await.unwrap(); + let error = buffer.write_all(b"d").await.unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::FileTooLarge); + } + assert_eq!(bytes, b"abc"); +} + +#[tokio::test] +async fn block_export_bounds_apply_before_repository_traversal() { + let service = service(Database::open_in_memory().unwrap()); + assert!(matches!( + service.export_blocks(DID, &[]).await, + Err(Error::InvalidRequest(_)) + )); + let cid = record_cid(&post("bounded")).unwrap(); + let excessive = vec![cid; MAX_BLOCKS_PER_REQUEST + 1]; + assert!(matches!( + service.export_blocks(DID, &excessive).await, + Err(Error::InvalidRequest(_)) + )); +} diff --git a/src/repository/tests/maintenance.rs b/src/repository/tests/maintenance.rs new file mode 100644 index 0000000..9b32cc9 --- /dev/null +++ b/src/repository/tests/maintenance.rs @@ -0,0 +1,446 @@ +use super::*; + +#[tokio::test] +async fn opening_repository_checks_revision_before_signature() { + let database = Database::open_in_memory().unwrap(); + let actor = account(&database); + let service = service(database.clone()); + service + .create_record( + &actor, + DID, + "app.bsky.feed.post", + Some("one"), + post("signed"), + None, + ) + .await + .unwrap(); + let head = database.repo_head(DID).unwrap().unwrap(); + let other_signer = RepositoryService::new( + database.clone(), + Arc::new(MemoryBlobStore::default()), + Arc::new(RecordingEventSink::default()), + &[8; 32], + 100_000, + ) + .unwrap(); + for verifier in [&service, &other_signer] { + for expected_rev in ["", "invalid", "2222222222222"] { + let mut blocks = RepoBlockStore::new(database.clone()); + let Err(error) = verifier + .open_repo(&mut blocks, head.root, expected_rev) + .await + else { + panic!("a mismatched revision must fail before signature verification"); + }; + assert!(matches!( + error, + Error::Internal(message) + if message == "stored repository head revision does not match its commit" + )); + } + } + let mut blocks = RepoBlockStore::new(database.clone()); + let repo = service + .open_repo(&mut blocks, head.root, &head.rev) + .await + .unwrap(); + assert_eq!(repo.commit().rev().to_string(), head.rev); + drop(repo); + let Err(error) = other_signer + .open_repo(&mut blocks, head.root, &head.rev) + .await + else { + panic!("a matching revision must still verify the signature"); + }; + assert!(matches!( + error, + Error::Internal(message) if message.starts_with("stored repository signature is invalid:") + )); + assert_eq!(database.repo_head(DID).unwrap(), Some(head)); +} + +#[tokio::test] +async fn stored_record_diagnostics_preserve_exact_text_and_failure_priority() { + let database = Database::open_in_memory().unwrap(); + let actor = account(&database); + let service = service(database.clone()); + let (created, _) = service + .create_record( + &actor, + DID, + "app.bsky.feed.post", + Some("diagnostic"), + post("original"), + None, + ) + .await + .unwrap(); + let uri = &created.uri; + let invalid_value = json!("private record value"); + let other_value = post("different private record value"); + for (cid, value, expected) in [ + ( + created.cid.clone(), + invalid_value.clone(), + format!("stored record {uri} is invalid: record must be an AT Protocol object"), + ), + ( + created.cid.clone(), + other_value.clone(), + format!("stored record {uri} has a noncanonical CID"), + ), + ( + record_cid(&other_value).unwrap().to_string(), + invalid_value.clone(), + format!("stored repository {DID} MST entry does not match indexed record {uri}"), + ), + ( + "invalid-cid".to_owned(), + invalid_value, + format!( + "stored CID is invalid: {}", + "invalid-cid".parse::().unwrap_err() + ), + ), + ] { + database + .connection() + .unwrap() + .execute( + "UPDATE records SET cid = ?1, record_json = ?2 WHERE did = ?3", + rusqlite::params![cid, value.to_string(), DID], + ) + .unwrap(); + let error = service.verify().await.unwrap_err(); + assert!(matches!(&error, Error::Internal(message) if message == &expected)); + assert_eq!(error.to_string(), format!("internal error: {expected}")); + assert!(!error.to_string().contains("private record value")); + } +} + +#[tokio::test] +async fn legacy_record_backfill_keeps_invalid_and_noncanonical_diagnostics() { + let uri = format!("at://{DID}/app.bsky.feed.post/legacy"); + for (value, expected) in [ + ( + json!("private record value"), + format!("stored record {uri} is invalid: record must be an AT Protocol object"), + ), + ( + post("different private record value"), + format!("stored record {uri} has noncanonical CID"), + ), + ] { + let database = Database::open_in_memory().unwrap(); + let actor = account(&database); + database + .connection() + .unwrap() + .execute( + "INSERT INTO records + (did, collection, rkey, cid, record_json, created_at, updated_at) + VALUES (?1, 'app.bsky.feed.post', 'legacy', ?2, ?3, 1, 1)", + rusqlite::params![ + DID, + record_cid(&post("original")).unwrap().to_string(), + value.to_string() + ], + ) + .unwrap(); + let error = service(database) + .create_record( + &actor, + DID, + "app.bsky.feed.post", + Some("new"), + post("new"), + None, + ) + .await + .unwrap_err(); + assert!(matches!(&error, Error::Internal(message) if message == &expected)); + assert_eq!(error.to_string(), format!("internal error: {expected}")); + assert!(!error.to_string().contains("private record value")); + } +} + +#[tokio::test] +async fn startup_verification_walks_reachable_atrium_blocks() { + let database = Database::open_in_memory().unwrap(); + let actor = account(&database); + let service = service(database.clone()); + let (created, _) = service + .create_record( + &actor, + DID, + "app.bsky.feed.post", + Some("corrupt"), + post("corrupt"), + None, + ) + .await + .unwrap(); + service.verify().await.unwrap(); + + database + .connection() + .unwrap() + .execute( + "UPDATE repo_blocks SET bytes = X'00' WHERE cid = ?1", + [created.cid], + ) + .unwrap(); + + service.check().await.unwrap(); + assert!(service.verify().await.is_err()); +} + +#[tokio::test] +async fn startup_verification_rejects_repository_record_index_drift() { + let database = Database::open_in_memory().unwrap(); + let actor = account(&database); + let service = service(database.clone()); + service + .create_record( + &actor, + DID, + "app.bsky.feed.post", + Some("missing"), + post("missing"), + None, + ) + .await + .unwrap(); + database + .connection() + .unwrap() + .execute("DELETE FROM records WHERE did = ?1", [DID]) + .unwrap(); + + let error = service.verify().await.unwrap_err().to_string(); + + assert!(error.contains("1 MST entries but 0 indexed records")); +} + +#[tokio::test] +async fn startup_verification_rejects_outbox_column_payload_drift() { + let database = Database::open_in_memory().unwrap(); + let actor = account(&database); + let service = service(database.clone()); + service + .create_record( + &actor, + DID, + "app.bsky.feed.post", + Some("event"), + post("event"), + None, + ) + .await + .unwrap(); + database + .connection() + .unwrap() + .execute( + "UPDATE repo_outbox SET rev = '2222222222222' + WHERE event_type = 'repo.commit'", + [], + ) + .unwrap(); + + let error = service.verify().await.unwrap_err().to_string(); + + assert!(error.contains("metadata does not match its payload")); +} + +#[tokio::test] +async fn startup_verification_rejects_broken_event_sequence() { + let database = Database::open_in_memory().unwrap(); + let actor = account(&database); + let service = service(database.clone()); + for rkey in ["first", "second", "third"] { + service + .create_record( + &actor, + DID, + "app.bsky.feed.post", + Some(rkey), + post(rkey), + None, + ) + .await + .unwrap(); + } + database + .connection() + .unwrap() + .execute("DELETE FROM repo_outbox WHERE seq = 2", []) + .unwrap(); + + let error = service.verify().await.unwrap_err().to_string(); + + assert!(error.contains("event sequence skips from 1 to 3")); +} + +#[tokio::test] +async fn startup_verification_rejects_broken_event_chain() { + let database = Database::open_in_memory().unwrap(); + let actor = account(&database); + let service = service(database.clone()); + service + .create_record( + &actor, + DID, + "app.bsky.feed.post", + Some("first"), + post("first"), + None, + ) + .await + .unwrap(); + let (_, second_commit) = service + .create_record( + &actor, + DID, + "app.bsky.feed.post", + Some("second"), + post("second"), + None, + ) + .await + .unwrap(); + let payload: Vec = database + .connection() + .unwrap() + .query_row("SELECT payload FROM repo_outbox WHERE seq = 4", [], |row| { + row.get(0) + }) + .unwrap(); + let mut payload: Value = serde_json::from_slice(&payload).unwrap(); + payload["since"] = Value::String(second_commit.rev.into()); + database + .connection() + .unwrap() + .execute( + "UPDATE repo_outbox SET payload = ?1 WHERE seq = 4", + [serde_json::to_vec(&payload).unwrap()], + ) + .unwrap(); + + let error = service.verify().await.unwrap_err().to_string(); + + assert!(error.contains("event 4 does not follow the prior")); +} + +#[tokio::test] +async fn startup_verification_rejects_incomplete_event_blocks() { + let database = Database::open_in_memory().unwrap(); + let actor = account(&database); + let service = service(database.clone()); + service + .create_record( + &actor, + DID, + "app.bsky.feed.post", + Some("event"), + post("event"), + None, + ) + .await + .unwrap(); + database + .connection() + .unwrap() + .execute("DELETE FROM repo_commit_blocks", []) + .unwrap(); + + let error = service.verify().await.unwrap_err().to_string(); + + assert!(error.contains("does not begin with its commit block")); +} + +#[tokio::test] +async fn startup_verification_rejects_blob_metadata_without_bytes() { + let database = Database::open_in_memory().unwrap(); + let actor = account(&database); + let blob = Blob { + cid: sha256_cid(RAW_CODEC, b"missing"), + media_type: "image/png".to_owned(), + size: 7, + }; + database.begin_blob_upload(&actor.did, &blob).unwrap(); + database.finish_blob_upload(&actor.did, &blob.cid).unwrap(); + + let error = service(database).verify().await.unwrap_err().to_string(); + + assert!(error.contains("stored blob metadata has no bytes")); +} + +#[tokio::test] +async fn startup_verification_rejects_blob_size_drift() { + let database = Database::open_in_memory().unwrap(); + let actor = account(&database); + let service = service(database.clone()); + let blob = service + .upload_blob(&actor, "image/png", b"present") + .await + .unwrap(); + service.verify().await.unwrap(); + database + .connection() + .unwrap() + .execute( + "UPDATE blobs SET size = size + 1 WHERE did = ?1 AND cid = ?2", + rusqlite::params![DID, blob.cid.to_string()], + ) + .unwrap(); + + let error = service.verify().await.unwrap_err().to_string(); + + assert!(error.contains("stored blob size does not match metadata")); +} + +#[tokio::test] +async fn repository_block_collection_preserves_atrium_head_and_retained_events() { + let database = Database::open_in_memory().unwrap(); + let actor = account(&database); + let service = service_with_retention(database.clone(), 1); + let (first_record, _) = service + .create_record( + &actor, + DID, + "app.bsky.feed.post", + Some("replace"), + post("first"), + None, + ) + .await + .unwrap(); + let first_cid = first_record.cid.parse::().unwrap(); + service + .put_record( + &actor, + DID, + "app.bsky.feed.post", + "replace", + post("second"), + RecordSwap { + record: None, + commit: None, + }, + ) + .await + .unwrap(); + let mut blocks = RepoBlockStore::new(database.clone()); + assert!(blocks.read_block(first_cid).await.is_ok()); + + assert!(service.collect_repository_blocks().await.unwrap() > 0); + + let mut blocks = RepoBlockStore::new(database); + assert!(blocks.read_block(first_cid).await.is_err()); + service.export_repo(DID, None).await.unwrap(); + assert_eq!(service.repo_messages_after(1, 10).await.unwrap().len(), 1); + assert_eq!(service.collect_repository_blocks().await.unwrap(), 0); +} diff --git a/src/repository/tests/mod.rs b/src/repository/tests/mod.rs new file mode 100644 index 0000000..13ad74d --- /dev/null +++ b/src/repository/tests/mod.rs @@ -0,0 +1,86 @@ +use atrium_crypto::{ + did::parse_did_key, + keypair::{Did as _, Secp256k1Keypair}, + verify::Verifier, +}; +use serde_json::json; +use std::time::Duration; +use tokio::io::AsyncWriteExt as _; + +use crate::{blob_store::MemoryBlobStore, events::RecordingEventSink, space::domain::SpaceRecord}; + +use super::*; +use super::{blobs::RAW_CODEC, car::BoundedBuffer, export::MAX_BLOCKS_PER_REQUEST}; +use crate::{ + database::{RepoBlockStore, StoredAccountEvent, StoredIdentityEvent, StoredRepoEvent}, + domain::{ + account::{Account, AccountStatus}, + blob::Blob, + }, +}; +use atrium_api::{ + com::atproto::sync::subscribe_repos, + types::{DataModel as AtriumDataModel, string::Handle}, +}; +use atrium_repo::{ + Repository, + blockstore::{AsyncBlockStoreRead as _, CarStore}, +}; +use cid::Cid; +use cpds_protocol::{atproto_record, record_cid, sha256_cid}; +use serde_json::Value; +use std::io::{self, Cursor}; + +const DID: &str = "did:web:alice.test"; +const HANDLE: &str = "alice.test"; +const SECRET: [u8; 32] = [7; 32]; + +fn service(database: Database) -> RepositoryService { + service_with_retention(database, 100_000) +} + +fn service_with_retention(database: Database, retention: usize) -> RepositoryService { + RepositoryService::new( + database, + Arc::new(MemoryBlobStore::default()), + Arc::new(RecordingEventSink::default()), + &SECRET, + retention, + ) + .unwrap() +} + +fn account(database: &Database) -> Account { + database.ensure_account(DID, HANDLE, "password").unwrap(); + Account { + did: DID.to_owned(), + handle: HANDLE.to_owned(), + } +} + +fn post(text: &str) -> Value { + json!({ + "$type": "app.bsky.feed.post", + "text": text, + "createdAt": "2026-08-22T00:00:00Z" + }) +} + +fn blob_record(collection: &str, blob: &Blob) -> Value { + json!({ + "$type": collection, + "image": { + "$type": "blob", + "ref": {"$link": blob.cid.to_string()}, + "mimeType": blob.media_type, + "size": blob.size + } + }) +} + +mod accounts; +mod blobs; +mod export; +mod maintenance; +mod records; +mod stream; diff --git a/src/repository/tests/records.rs b/src/repository/tests/records.rs new file mode 100644 index 0000000..f2baf92 --- /dev/null +++ b/src/repository/tests/records.rs @@ -0,0 +1,785 @@ +use super::*; +use crate::domain::record::{Record, RecordMutation, RecordView}; +use crate::syntax; + +#[tokio::test] +async fn single_write_pairs_preserve_create_update_and_upsert_results() { + let database = Database::open_in_memory().unwrap(); + let actor = account(&database); + let service = service(database.clone()); + let collection = "app.bsky.feed.post"; + let mut previous_revision = String::new(); + for (method, rkey, text) in [ + ("create", "first", "created"), + ("put", "first", "updated"), + ("put", "second", "upserted"), + ] { + let value = post(text); + let (record, commit) = if method == "create" { + service + .create_record(&actor, DID, collection, Some(rkey), value.clone(), None) + .await + .unwrap() + } else { + service + .put_record( + &actor, + DID, + collection, + rkey, + value.clone(), + RecordSwap { + record: None, + commit: None, + }, + ) + .await + .unwrap() + }; + assert_eq!(record.uri, format!("at://{DID}/{collection}/{rkey}")); + assert_eq!(record.cid, record_cid(&value).unwrap().to_string()); + assert_eq!(record.value, value); + assert_eq!( + serde_json::to_value(service.get_record(DID, collection, rkey, None).unwrap()).unwrap(), + serde_json::to_value(&record).unwrap() + ); + let head = database.repo_head(DID).unwrap().unwrap(); + assert_eq!(commit.cid.as_ref(), &head.root); + assert_eq!(commit.rev.as_str(), head.rev); + assert!(commit.rev.as_str() > previous_revision.as_str()); + previous_revision = commit.rev.into(); + } + service.verify().await.unwrap(); +} + +#[tokio::test] +async fn listed_record_tuple_preserves_records_and_cursor_for_every_page_direction() { + let database = Database::open_in_memory().unwrap(); + let actor = account(&database); + let service = service(database); + let collection = "app.bsky.feed.post"; + for reverse in [false, true] { + let (records, cursor) = service + .list_records(DID, collection, None, reverse, 2) + .unwrap(); + assert!(records.is_empty()); + assert!(cursor.is_none()); + } + for rkey in ["b", "a", "c"] { + service + .create_record(&actor, DID, collection, Some(rkey), post(rkey), None) + .await + .unwrap(); + } + for (cursor, reverse, limit, keys, next) in [ + (None, false, 2, ["c", "b"].as_slice(), Some("b")), + (Some("b"), false, 2, ["a"].as_slice(), Some("a")), + (Some("a"), false, 2, [].as_slice(), None), + (None, true, 2, ["a", "b"].as_slice(), Some("b")), + (Some("b"), true, 2, ["c"].as_slice(), Some("c")), + (Some("c"), true, 2, [].as_slice(), None), + (None, false, 0, ["c"].as_slice(), Some("c")), + ( + None, + false, + usize::MAX, + ["c", "b", "a"].as_slice(), + Some("a"), + ), + ] { + for repo in [DID, HANDLE] { + let (records, returned_cursor) = service + .list_records(repo, collection, cursor, reverse, limit) + .unwrap(); + assert_eq!(returned_cursor.as_deref(), next); + assert_eq!(records.len(), keys.len()); + for (record, key) in records.iter().zip(keys) { + assert_eq!(record.uri, format!("at://{DID}/{collection}/{key}")); + assert_eq!(record.cid, record_cid(&post(key)).unwrap().to_string()); + assert_eq!(record.value, post(key)); + } + } + } +} + +fn assert_record_getter_parity( + service: &RepositoryService, + repo: &str, + collection: &str, + rkey: &str, + cid: Option<&Cid>, +) -> Result { + let previous = (|| { + let account = service.resolve_repo(repo)?; + syntax::collection(collection)?; + syntax::record_key(rkey)?; + let record: RecordView = service + .database + .find_record(&account.did, collection, rkey)? + .ok_or(Error::NotFound) + .map(Into::into)?; + if cid.is_some_and(|cid| record.cid != cid.to_string()) { + return Err(Error::NotFound); + } + Ok(record) + })(); + let actual = service.get_record(repo, collection, rkey, cid); + assert_eq!(format!("{actual:?}"), format!("{previous:?}")); + if let (Ok(actual), Ok(previous)) = (&actual, &previous) { + assert_eq!( + serde_json::to_vec(actual).unwrap(), + serde_json::to_vec(previous).unwrap(), + ); + } + actual +} + +fn store_getter_record( + database: &Database, + cid: &rusqlite::types::Value, + json: &rusqlite::types::Value, +) { + database + .connection() + .unwrap() + .execute( + "INSERT OR REPLACE INTO records + (did, collection, rkey, cid, record_json, created_at, updated_at) + VALUES (?1, 'app.bsky.feed.post', 'one', ?2, ?3, 0, 0)", + rusqlite::params![DID, cid, json], + ) + .unwrap(); +} + +#[test] +fn typed_record_cid_comparison_preserves_getter_values_and_encodings() { + use cid::{Version, multibase::Base, multihash::Multihash}; + use rusqlite::types::Value as SqlValue; + + let database = Database::open_in_memory().unwrap(); + account(&database); + let service = service(database.clone()); + let original = record_cid(&post("original")).unwrap(); + let candidates = [ + original, + Cid::new_v0(*original.hash()).unwrap(), + Cid::new_v1(0x70, *original.hash()), + Cid::new_v1(0x55, *original.hash()), + Cid::new_v1( + 0x71, + Multihash::wrap(0x13, original.hash().digest()).unwrap(), + ), + Cid::new_v1( + 0x71, + Multihash::wrap(0x12, &original.hash().digest()[..31]).unwrap(), + ), + record_cid(&post("different digest")).unwrap(), + ]; + for stored in candidates { + let mut encodings = vec![stored.to_string(), format!("/ipfs/{stored}")]; + if stored.version() == Version::V1 { + encodings.extend( + [Base::Base32Upper, Base::Base58Btc, Base::Base64] + .map(|base| stored.to_string_of_base(base).unwrap()), + ); + } + for encoding in encodings { + assert_eq!(encoding.parse::().unwrap(), stored); + store_getter_record( + &database, + &SqlValue::Text(encoding), + &SqlValue::Text(post("stored value").to_string()), + ); + for repo in [DID, HANDLE] { + for requested in std::iter::once(None).chain(candidates.iter().map(Some)) { + let result = assert_record_getter_parity( + &service, + repo, + "app.bsky.feed.post", + "one", + requested, + ); + if requested.is_none_or(|requested| *requested == stored) { + let record = result.unwrap(); + assert_eq!(record.uri, format!("at://{DID}/app.bsky.feed.post/one")); + assert_eq!(record.cid, stored.to_string()); + assert_eq!(record.value, post("stored value")); + } else { + assert!(matches!(result, Err(Error::NotFound))); + } + } + } + } + } +} + +#[test] +fn typed_record_cid_comparison_keeps_lookup_and_decode_error_priority() { + use rusqlite::types::Value as SqlValue; + + let database = Database::open_in_memory().unwrap(); + account(&database); + let service = service(database.clone()); + let stored = record_cid(&post("stored")).unwrap(); + let mismatch = record_cid(&post("mismatch")).unwrap(); + for (cid, json, kind) in [ + ( + SqlValue::Text("invalid CID".to_owned()), + SqlValue::Text("{".to_owned()), + "cid", + ), + ( + SqlValue::Text("invalid CID".to_owned()), + SqlValue::Blob(vec![0]), + "sql", + ), + ( + SqlValue::Blob(vec![0]), + SqlValue::Text("{".to_owned()), + "sql", + ), + ( + SqlValue::Text(stored.to_string()), + SqlValue::Text("{".to_owned()), + "json", + ), + ( + SqlValue::Text(stored.to_string()), + SqlValue::Blob(vec![0]), + "sql", + ), + ] { + store_getter_record(&database, &cid, &json); + for requested in [None, Some(&mismatch)] { + let error = + assert_record_getter_parity(&service, DID, "app.bsky.feed.post", "one", requested) + .unwrap_err(); + assert!(match kind { + "cid" => matches!(error, Error::Internal(_)), + "sql" => matches!(error, Error::Database(_)), + "json" => matches!(error, Error::Json(_)), + _ => unreachable!(), + }); + } + } + for (repo, collection, rkey, expected) in [ + ( + "did:example:missing", + "invalid", + "/", + "the requested object does not exist", + ), + ( + DID, + "invalid", + "/", + "the request is invalid: `invalid` is not a valid collection NSID", + ), + ( + DID, + "app.bsky.feed.post", + "/", + "the request is invalid: `/` is not a valid record key", + ), + ( + DID, + "app.bsky.feed.post", + "missing", + "the requested object does not exist", + ), + ] { + let error = assert_record_getter_parity(&service, repo, collection, rkey, Some(&mismatch)) + .unwrap_err(); + assert_eq!(error.to_string(), expected); + } +} + +#[tokio::test] +async fn tree_mutation_failures_preserve_errors_and_publish_nothing() { + let database = Database::open_in_memory().unwrap(); + let actor = account(&database); + let service = service(database.clone()); + let value = post("staged only"); + let record = Record { + did: DID.to_owned(), + collection: "app.bsky.feed.post".to_owned(), + rkey: "one".to_owned(), + cid: record_cid(&value).unwrap(), + value, + }; + let head = database.repo_head(DID).unwrap(); + let events = database.repo_events_after(0, 10).unwrap(); + for mutations in [ + vec![ + RecordMutation::Create(record.clone()), + RecordMutation::Create(record.clone()), + ], + vec![RecordMutation::Update { + current: record.clone(), + replacement: record.clone(), + }], + vec![RecordMutation::Delete(record.clone())], + ] { + let error = service + .commit_record_mutations(&actor, None, &mutations) + .await + .unwrap_err(); + match mutations[0] { + RecordMutation::Create(_) => assert!(matches!( + error, Error::Conflict(message) if message == "record already exists" + )), + _ => assert!(matches!(error, Error::NotFound)), + } + assert_eq!(database.repo_head(DID).unwrap(), head); + assert_eq!(database.repo_events_after(0, 10).unwrap(), events); + assert!(database.all_records(DID).unwrap().is_empty()); + let mut store = RepoBlockStore::new(database.clone()); + assert!(matches!( + store.read_block(record.cid).await, + Err(atrium_repo::blockstore::Error::CidNotFound) + )); + } +} + +#[test] +fn swap_error_constructor_retains_protocol_name_and_message() { + for message in [ + "repository commit does not match", + "record CID does not match", + "", + "quote '\" café", + ] { + let error = crate::repository::commit::invalid_swap(message); + assert_eq!(error.to_string(), message); + assert!(matches!( + error, Error::Xrpc { name: "InvalidSwap", message: actual } if actual == message + )); + } +} + +#[tokio::test] +async fn ordered_repeated_key_writes_use_the_previous_planned_record() { + let database = Database::open_in_memory().unwrap(); + let actor = account(&database); + let service = service(database.clone()); + let collection = "app.bsky.feed.post"; + let rkey = "repeated"; + let (written_commit, results) = service + .apply_writes( + &actor, + DID, + vec![ + RecordBatchWrite::Create { + collection: collection.to_owned(), + rkey: Some(rkey.to_owned()), + value: post("first"), + }, + RecordBatchWrite::Update { + collection: collection.to_owned(), + rkey: rkey.to_owned(), + value: post("second"), + }, + RecordBatchWrite::Delete { + collection: collection.to_owned(), + rkey: rkey.to_owned(), + }, + RecordBatchWrite::Create { + collection: collection.to_owned(), + rkey: Some(rkey.to_owned()), + value: post("final"), + }, + ], + None, + ) + .await + .unwrap(); + let [ + RecordMutation::Create(created), + RecordMutation::Update { + current, + replacement, + }, + RecordMutation::Delete(deleted), + RecordMutation::Create(recreated), + ] = results.as_slice() + else { + panic!("results must retain the requested mutation order"); + }; + for (record, text) in [ + (created, "first"), + (current, "first"), + (replacement, "second"), + (deleted, "second"), + (recreated, "final"), + ] { + assert_eq!(record.value, post(text)); + assert_eq!(record.cid, record_cid(&post(text)).unwrap()); + assert_eq!(record.did, DID); + assert_eq!(record.collection, collection); + assert_eq!(record.rkey, rkey); + } + assert_eq!( + service + .get_record(DID, collection, rkey, None) + .unwrap() + .value, + post("final") + ); + let messages = service.repo_messages_after(0, 10).await.unwrap(); + let commits = messages + .iter() + .filter_map(|message| match message { + subscribe_repos::Message::Commit(commit) => Some(commit), + _ => None, + }) + .collect::>(); + let [commit] = commits.as_slice() else { + panic!("the entire batch must emit one commit"); + }; + assert_eq!(&commit.commit.0, written_commit.cid.as_ref()); + assert_eq!( + commit + .ops + .iter() + .map(|op| op.action.as_str()) + .collect::>(), + ["create", "update", "delete", "create"] + ); + assert_final_record_car(commit, [created.cid, replacement.cid], recreated).await; + service.verify().await.unwrap(); +} + +async fn assert_final_record_car( + commit: &subscribe_repos::Commit, + transient_cids: [Cid; 2], + record: &Record, +) { + let mut car = CarStore::open(Cursor::new(commit.blocks.clone())) + .await + .unwrap(); + assert_eq!(car.roots().collect::>(), [commit.commit.0]); + for transient in transient_cids { + assert!(car.read_block(transient).await.is_err()); + } + assert!(car.read_block(record.cid).await.is_ok()); + let mut repo = Repository::open(&mut car, commit.commit.0).await.unwrap(); + assert_eq!( + repo.get_raw::(&format!("{}/{}", record.collection, record.rkey)) + .await + .unwrap() + .unwrap(), + record.value + ); +} + +#[tokio::test] +async fn cached_deletion_does_not_reload_the_persisted_record() { + let database = Database::open_in_memory().unwrap(); + let actor = account(&database); + let service = service(database.clone()); + let collection = "app.bsky.feed.post"; + let rkey = "persisted"; + service + .create_record(&actor, DID, collection, Some(rkey), post("stored"), None) + .await + .unwrap(); + let head = database.repo_head(DID).unwrap(); + let events = database.repo_events_after(0, 10).unwrap(); + let delete = || RecordBatchWrite::Delete { + collection: collection.to_owned(), + rkey: rkey.to_owned(), + }; + + for next in [ + RecordBatchWrite::Update { + collection: collection.to_owned(), + rkey: rkey.to_owned(), + value: json!({"$type": "wrong.record.type"}), + }, + delete(), + ] { + assert!(matches!( + service + .apply_writes(&actor, DID, vec![delete(), next], None) + .await, + Err(Error::NotFound) + )); + assert_eq!(database.repo_head(DID).unwrap(), head); + assert_eq!(database.repo_events_after(0, 10).unwrap(), events); + assert_eq!( + service + .get_record(DID, collection, rkey, None) + .unwrap() + .value, + post("stored") + ); + } + + service + .apply_writes( + &actor, + DID, + vec![ + delete(), + RecordBatchWrite::Create { + collection: collection.to_owned(), + rkey: Some(rkey.to_owned()), + value: post("recreated"), + }, + ], + None, + ) + .await + .unwrap(); + assert_eq!( + service + .get_record(DID, collection, rkey, None) + .unwrap() + .value, + post("recreated") + ); + service.verify().await.unwrap(); +} + +#[tokio::test] +async fn stalled_optional_sink_does_not_fail_an_atomic_record_batch() { + struct StalledSink; + #[async_trait::async_trait] + impl EventSink for StalledSink { + async fn check(&self) -> Result<()> { + Ok(()) + } + async fn publish(&self, _: crate::domain::event::RepositoryEvent) -> Result<()> { + std::future::pending().await + } + } + let database = Database::open_in_memory().unwrap(); + let actor = account(&database); + let service = RepositoryService::new( + database, + Arc::new(MemoryBlobStore::default()), + Arc::new(StalledSink), + &SECRET, + 100, + ) + .unwrap(); + let writes = ["first", "second"] + .into_iter() + .map(|rkey| RecordBatchWrite::Create { + collection: "app.bsky.feed.post".to_owned(), + rkey: Some(rkey.to_owned()), + value: post(rkey), + }) + .collect(); + let (_, results) = tokio::time::timeout( + Duration::from_secs(3), + service.apply_writes(&actor, DID, writes, None), + ) + .await + .expect("one delivery budget applies to the entire batch") + .unwrap(); + assert_eq!(results.len(), 2); + for key in ["first", "second"] { + service + .get_record(DID, "app.bsky.feed.post", key, None) + .unwrap(); + } + assert!(matches!( + service.repo_messages_after(0, 10).await.unwrap().last(), + Some(subscribe_repos::Message::Commit(_)) + )); +} + +#[tokio::test] +async fn repository_writes_native_atproto_bytes_without_a_parallel_encoder() { + let database = Database::open_in_memory().unwrap(); + let actor = account(&database); + let service = service(database.clone()); + let value = json!({ + "$type": "app.bsky.feed.post", + "bytes": {"$bytes": "nFERjvLLiw9qm45JrqH9QTzyC2Lu1Xb4ne6+sBrCzI0="} + }); + let expected = record_cid(&value).unwrap(); + + let (created, _) = service + .create_record( + &actor, + DID, + "app.bsky.feed.post", + Some("native"), + value, + None, + ) + .await + .unwrap(); + + assert_eq!(created.cid, expected.to_string()); + assert!( + RepoBlockStore::new(database) + .read_block(expected) + .await + .is_ok() + ); + service.verify().await.unwrap(); +} + +#[tokio::test] +async fn signed_repository_survives_restart_with_monotonic_revision() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("cpds.sqlite3"); + let first_database = Database::open(&path).unwrap(); + let actor = account(&first_database); + let first_service = service(first_database.clone()); + let (_, first_commit) = first_service + .create_record( + &actor, + DID, + "app.bsky.feed.post", + Some("first"), + post("first"), + None, + ) + .await + .unwrap(); + let first_rev = first_commit.rev; + drop(first_service); + drop(first_database); + + let reopened = Database::open(&path).unwrap(); + let second_service = service(reopened.clone()); + let (_, second_commit) = second_service + .create_record( + &actor, + DID, + "app.bsky.feed.post", + Some("second"), + post("second"), + None, + ) + .await + .unwrap(); + + assert!(second_commit.rev.as_str() > first_rev.as_str()); + let head = reopened.repo_head(DID).unwrap().unwrap(); + let mut blocks = RepoBlockStore::new(reopened); + let mut repo = Repository::open(&mut blocks, head.root).await.unwrap(); + assert!( + repo.get_raw::("app.bsky.feed.post/first") + .await + .unwrap() + .is_some() + ); + assert!( + repo.get_raw::("app.bsky.feed.post/second") + .await + .unwrap() + .is_some() + ); + + let key = Secp256k1Keypair::import(&SECRET).unwrap(); + let (algorithm, public_key) = parse_did_key(&key.did()).unwrap(); + Verifier::default() + .verify( + algorithm, + &public_key, + &repo.commit().bytes(), + repo.commit().sig(), + ) + .unwrap(); +} + +#[tokio::test] +async fn first_repository_commit_backfills_legacy_record_rows() { + let database = Database::open_in_memory().unwrap(); + let actor = account(&database); + let legacy_value = json!({ + "$type": "app.bsky.feed.post", + "text": "legacy", + "bytes": {"$bytes": "AQ=="} + }); + let legacy_cid = record_cid(&legacy_value).unwrap(); + database + .connection() + .unwrap() + .execute( + "INSERT INTO records + (did, collection, rkey, cid, record_json, created_at, updated_at) + VALUES (?1, 'app.bsky.feed.post', 'legacy', ?2, ?3, 1, 1)", + rusqlite::params![DID, legacy_cid.to_string(), legacy_value.to_string()], + ) + .unwrap(); + + service(database.clone()) + .create_record( + &actor, + DID, + "app.bsky.feed.post", + Some("new"), + post("new"), + None, + ) + .await + .unwrap(); + + let head = database.repo_head(DID).unwrap().unwrap(); + let mut blocks = RepoBlockStore::new(database); + let mut repo = Repository::open(&mut blocks, head.root).await.unwrap(); + let legacy = repo + .get_raw::("app.bsky.feed.post/legacy") + .await + .unwrap() + .unwrap(); + assert_eq!( + serde_ipld_dagcbor::to_vec(&legacy).unwrap(), + serde_ipld_dagcbor::to_vec(&atproto_record(&legacy_value).unwrap()).unwrap() + ); + assert!( + repo.get_raw::("app.bsky.feed.post/new") + .await + .unwrap() + .is_some() + ); +} + +#[tokio::test] +async fn concurrent_record_creates_preserve_both_repository_paths() { + let database = Database::open_in_memory().unwrap(); + let actor = account(&database); + let service = service(database.clone()); + let (first, second) = tokio::join!( + service.create_record( + &actor, + DID, + "app.bsky.feed.post", + Some("first"), + post("first"), + None, + ), + service.create_record( + &actor, + DID, + "app.bsky.feed.post", + Some("second"), + post("second"), + None, + ) + ); + first.unwrap(); + second.unwrap(); + + let head = database.repo_head(DID).unwrap().unwrap(); + let mut blocks = RepoBlockStore::new(database); + let mut repo = Repository::open(&mut blocks, head.root).await.unwrap(); + assert!( + repo.get_raw::("app.bsky.feed.post/first") + .await + .unwrap() + .is_some() + ); + assert!( + repo.get_raw::("app.bsky.feed.post/second") + .await + .unwrap() + .is_some() + ); +} diff --git a/src/repository/tests/stream.rs b/src/repository/tests/stream.rs new file mode 100644 index 0000000..aec53b6 --- /dev/null +++ b/src/repository/tests/stream.rs @@ -0,0 +1,157 @@ +use super::*; + +#[tokio::test] +async fn durable_repo_events_preserve_inductive_atrium_commit_metadata() { + let database = Database::open_in_memory().unwrap(); + let actor = account(&database); + let service = service(database.clone()); + let (first_record, first_commit) = service + .create_record( + &actor, + DID, + "app.bsky.feed.post", + Some("first"), + post("first"), + None, + ) + .await + .unwrap(); + let first_root = *first_commit.cid.as_ref(); + let mut store = RepoBlockStore::new(database); + let first_repo = Repository::open(&mut store, first_root).await.unwrap(); + let first_data = first_repo.commit().data(); + drop(first_repo); + + let (_, second_commit) = service + .create_record( + &actor, + DID, + "app.bsky.feed.post", + Some("second"), + post("second"), + None, + ) + .await + .unwrap(); + let messages = service.repo_messages_after(0, 10).await.unwrap(); + assert_eq!(messages.len(), 4); + let subscribe_repos::Message::Identity(identity) = &messages[0] else { + panic!("first repository event should identify the repository"); + }; + assert_eq!(identity.did.as_str(), DID); + assert_eq!(identity.handle.as_ref().map(Handle::as_str), Some(HANDLE)); + assert_eq!(identity.seq, 1); + let subscribe_repos::Message::Account(account) = &messages[1] else { + panic!("second repository event should activate the account"); + }; + assert_eq!(account.did.as_str(), DID); + assert!(account.active); + assert_eq!(account.status, None); + assert_eq!(account.seq, 2); + let subscribe_repos::Message::Commit(first_message) = &messages[2] else { + panic!("third repository event should be a commit"); + }; + assert_eq!(first_message.commit.0, first_root); + assert_eq!(first_message.since, None); + assert_eq!(first_message.prev_data, None); + assert_eq!(first_message.ops[0].action, "create"); + assert_eq!(first_message.ops[0].path, "app.bsky.feed.post/first"); + assert_eq!( + serde_json::to_value(&first_message.ops[0]).unwrap(), + json!({ + "action": "create", + "path": "app.bsky.feed.post/first", + "cid": {"$link": first_record.cid} + }) + ); + let mut car = CarStore::open(Cursor::new(first_message.blocks.clone())) + .await + .unwrap(); + assert_eq!(car.roots().collect::>(), vec![first_root]); + assert!(car.read_block(first_root).await.is_ok()); + + let subscribe_repos::Message::Commit(second_message) = &messages[3] else { + panic!("fourth repository event should be a commit"); + }; + assert_eq!(&second_message.commit.0, second_commit.cid.as_ref()); + assert_eq!( + second_message + .since + .as_ref() + .map(|rev| rev.as_str().to_owned()), + Some(first_commit.rev.into()) + ); + assert_eq!( + second_message.prev_data.as_ref().map(|cid| cid.0), + Some(first_data) + ); + assert_eq!( + service.repo_messages_after(4, 10).await.unwrap(), + Vec::new() + ); + assert!(matches!( + service.repo_messages_after(5, 10).await, + Err(Error::Xrpc { + name: "FutureCursor", + .. + }) + )); +} + +#[tokio::test] +async fn legacy_repo_events_replay_as_atrium_sync_reset_points() { + let database = Database::open_in_memory().unwrap(); + let actor = account(&database); + let service = service(database.clone()); + let (_, created_commit) = service + .create_record( + &actor, + DID, + "app.bsky.feed.post", + Some("legacy"), + post("legacy"), + None, + ) + .await + .unwrap(); + let root = *created_commit.cid.as_ref(); + let payload: Vec = database + .connection() + .unwrap() + .query_row("SELECT payload FROM repo_outbox WHERE seq = 3", [], |row| { + row.get(0) + }) + .unwrap(); + let mut payload: Value = serde_json::from_slice(&payload).unwrap(); + let payload = payload.as_object_mut().unwrap(); + payload.remove("version"); + let operation = payload.remove("operations").unwrap()[0].clone(); + for field in ["action", "path", "cid", "prev"] { + payload.insert(field.to_owned(), operation[field].clone()); + } + let payload = serde_json::to_vec(&payload).unwrap(); + { + let connection = database.connection().unwrap(); + connection + .execute( + "UPDATE repo_outbox SET payload = ?1 WHERE seq = 3", + [payload], + ) + .unwrap(); + connection + .execute("DELETE FROM repo_commit_blocks WHERE seq = 3", []) + .unwrap(); + } + + let messages = service.repo_messages_after(2, 10).await.unwrap(); + let [subscribe_repos::Message::Sync(message)] = messages.as_slice() else { + panic!("legacy repository event should be a sync reset point"); + }; + assert_eq!(message.did.as_str(), DID); + assert_eq!(message.rev, created_commit.rev.as_str()); + let mut car = CarStore::open(Cursor::new(message.blocks.clone())) + .await + .unwrap(); + assert_eq!(car.roots().collect::>(), vec![root]); + assert!(car.read_block(root).await.is_ok()); +} diff --git a/src/space/auth.rs b/src/space/auth.rs index e55f28d..58abf59 100644 --- a/src/space/auth.rs +++ b/src/space/auth.rs @@ -1,6 +1,9 @@ //! JWT, `DPoP`, attestation, and service-auth primitives for Spaces. -use std::{fmt::Write as _, sync::Arc, time::Duration}; +mod attestation; +mod service; + +use std::sync::Arc; use atrium_api::types::string::Did; use atrium_crypto::{ @@ -17,14 +20,14 @@ use p256::{ use rand::Rng as _; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; -use time::OffsetDateTime; use url::Url; use crate::{ + clock::now, database::Database, - error::{Error, Result}, - identity::IdentityService, - syntax, + error::{Error, ErrorContext as _, Result}, + identity::{IdentityService, http::OutboundHttp}, + syntax::SpaceRef, }; const DELEGATION_TYP: &str = "atproto-space-delegation+jwt"; @@ -37,27 +40,11 @@ pub(crate) struct SpaceAuthService { database: Database, did: String, signing_key: Secp256k1Keypair, - http: reqwest::Client, + http: OutboundHttp, local_endpoint: Url, identity: Arc, } -#[derive(Debug)] -pub(crate) struct Delegation { - pub(crate) user_did: String, - pub(crate) space: String, -} - -#[derive(Debug)] -pub(crate) struct Credential { - pub(crate) space: String, -} - -pub(crate) struct ServiceIdentity { - pub(crate) iss: String, - pub(crate) aud: String, -} - #[derive(Serialize, Deserialize)] struct JwtHeader { alg: String, @@ -69,9 +56,9 @@ struct JwtHeader { } #[derive(Serialize, Deserialize)] -struct SpaceClaims { - iss: String, - sub: String, +pub(crate) struct SpaceClaims { + pub(crate) iss: String, + pub(crate) sub: String, #[serde(default)] aud: Option, iat: i64, @@ -104,55 +91,6 @@ struct DpopClaims { ath: Option, } -#[derive(Deserialize, Serialize)] -struct ServiceClaims { - iss: String, - aud: String, - iat: i64, - exp: i64, - lxm: String, - #[serde(default)] - jti: Option, -} - -#[derive(Deserialize)] -struct AttestationClaims { - iss: String, - sub: String, - aud: String, - iat: i64, - exp: i64, - jti: String, -} - -#[derive(Deserialize)] -struct ClientMetadata { - client_id: String, - #[serde(default)] - jwks: Option, - #[serde(default)] - jwks_uri: Option, -} - -#[derive(Deserialize)] -struct Jwks { - keys: Vec, -} - -#[derive(Deserialize)] -struct PublicJwk { - kid: String, - kty: String, - crv: String, - x: String, - y: String, -} - -#[derive(Deserialize)] -struct AccessDecision { - authorized: bool, -} - impl SpaceAuthService { pub(crate) fn new( database: Database, @@ -161,26 +99,23 @@ impl SpaceAuthService { local_endpoint: Url, identity: Arc, ) -> Result { - let signing_key = Secp256k1Keypair::import(key) - .map_err(|error| Error::Configuration(format!("invalid signing key: {error}")))?; + let signing_key = + Secp256k1Keypair::import(key).context(Error::Configuration, "invalid signing key")?; Ok(Self { database, did, signing_key, - http: reqwest::Client::builder() - .timeout(Duration::from_secs(5)) - .redirect(reqwest::redirect::Policy::none()) - .build() - .map_err(|error| Error::Configuration(format!("HTTP client: {error}")))?, + http: identity.outbound_http()?, local_endpoint, identity, }) } pub(crate) fn mint_delegation(&self, space: &str, issuer: &str) -> Result { - let space_ref = syntax::space_ref(space)?; - self.sign_token( + let space_ref = space.parse::()?; + self.sign_jwt( DELEGATION_TYP, + Some("#atproto"), &SpaceClaims { iss: issuer.to_owned(), sub: space.to_owned(), @@ -199,9 +134,9 @@ impl SpaceAuthService { proof: &str, method: &str, htu: &Url, - ) -> Result<(Delegation, String)> { + ) -> Result<(SpaceClaims, String)> { let claims = self.verify_space_token(token, DELEGATION_TYP).await?; - let space_ref = syntax::space_ref(&claims.sub)?; + let space_ref = claims.sub.parse::()?; if claims.aud.as_deref() != Some(format!("{}#atproto_space_host", space_ref.authority).as_str()) { @@ -228,18 +163,13 @@ impl SpaceAuthService { "delegation token has already been used", )); } - Ok(( - Delegation { - user_did: claims.iss, - space: claims.sub, - }, - jkt, - )) + Ok((claims, jkt)) } pub(crate) fn mint_credential(&self, space: &str, jkt: String) -> Result { - self.sign_token( + self.sign_jwt( CREDENTIAL_TYP, + Some("#atproto"), &SpaceClaims { iss: self.did.clone(), sub: space.to_owned(), @@ -258,9 +188,9 @@ impl SpaceAuthService { proof: &str, method: &str, htu: &Url, - ) -> Result { + ) -> Result { let claims = self.verify_space_token(credential, CREDENTIAL_TYP).await?; - let space = syntax::space_ref(&claims.sub)?; + let space = claims.sub.parse::()?; if claims.iss != space.authority { return Err(auth_error( "BadJwtIss", @@ -273,270 +203,23 @@ impl SpaceAuthService { .map(|cnf| cnf.jkt.as_str()) .ok_or_else(|| auth_error("BadJwtCnf", "credential has no key binding"))?; self.verify_dpop(proof, method, htu, Some(credential), Some(jkt))?; - Ok(Credential { space: claims.sub }) + Ok(claims.sub) } - pub(crate) async fn verify_service_token( - &self, - token: &str, - expected_lxm: &str, - ) -> Result { - let (header, claims, input, signature) = decode_jwt::(token)?; - if header.alg != "ES256K" - || matches!(header.typ.as_str(), "at+jwt" | "refresh+jwt" | "dpop+jwt") - { - return Err(auth_error("BadJwtType", "unsupported service JWT type")); - } - if claims.lxm != expected_lxm { - return Err(auth_error( - "BadJwtLexiconMethod", - "service JWT method does not match this endpoint", - )); - } - if now() > claims.exp || claims.iat > now() + CLOCK_SKEW { - return Err(auth_error( - "JwtExpired", - "service JWT is expired or not yet valid", - )); - } - if claims.jti.as_deref().is_some_and(str::is_empty) { - return Err(auth_error("BadJwt", "service JWT has an empty identifier")); - } - let issuer_did = claims - .iss - .split_once('#') - .map_or(claims.iss.as_str(), |v| v.0); - syntax::did(issuer_did) - .map_err(|_| auth_error("BadJwtIss", "service JWT issuer is not a DID"))?; - verify_signature( - &self - .resolve_signing_key(issuer_did, Some("#atproto")) - .await?, - input.as_bytes(), - &signature, - ) - .map_err(|_| auth_error("BadJwtSignature", "invalid service JWT signature"))?; - Ok(ServiceIdentity { - iss: claims.iss, - aud: claims.aud, - }) - } - - pub(crate) async fn verify_client_attestation( - &self, - attestation: &str, - expected_audience: &str, - ) -> Result { - let (header, claims, input, signature) = - decode_jwt::(attestation).map_err(|_| invalid_attestation())?; - if header.typ != "atproto-client-attestation+jwt" || header.alg != "ES256" { - return Err(invalid_attestation()); - } - if claims.iss != claims.sub - || claims.aud != expected_audience - || claims.jti.is_empty() - || now() - CLOCK_SKEW >= claims.exp - || claims.iat > now() + CLOCK_SKEW - { - return Err(invalid_attestation()); - } - let client_url = safe_document_url(&claims.iss).map_err(|_| invalid_attestation())?; - let metadata: ClientMetadata = self - .fetch_json(client_url) - .await - .map_err(|_| invalid_attestation())?; - if metadata.client_id != claims.iss { - return Err(invalid_attestation()); - } - let keys = if let Some(keys) = metadata.jwks { - keys - } else { - let uri = metadata.jwks_uri.ok_or_else(invalid_attestation)?; - self.fetch_json(safe_document_url(&uri).map_err(|_| invalid_attestation())?) - .await - .map_err(|_| invalid_attestation())? - }; - let kid = header.kid.as_deref().ok_or_else(invalid_attestation)?; - let jwk = keys - .keys - .into_iter() - .find(|key| key.kid == kid && key.kty == "EC" && key.crv == "P-256") - .ok_or_else(invalid_attestation)?; - let x = decode_coordinate(&jwk.x).map_err(|_| invalid_attestation())?; - let y = decode_coordinate(&jwk.y).map_err(|_| invalid_attestation())?; - let point = EncodedPoint::from_affine_coordinates((&x).into(), (&y).into(), false); - let key = VerifyingKey::from_encoded_point(&point).map_err(|_| invalid_attestation())?; - let signature = P256Signature::from_slice(&signature).map_err(|_| invalid_attestation())?; - key.verify(input.as_bytes(), &signature) - .map_err(|_| invalid_attestation())?; - if !self.database.consume_space_auth_jti( - "attestation", - &claims.iss, - &claims.jti, - claims.exp, - )? { - return Err(invalid_attestation()); - } - Ok(claims.iss) - } - - pub(crate) async fn resolve_service_endpoint(&self, service: &str) -> Result { - let (did, fragment) = service - .split_once('#') - .map_or((service, None), |(did, fragment)| (did, Some(fragment))); - syntax::did(did)?; - if fragment.is_some_and(str::is_empty) { - return Err(Error::InvalidRequest( - "service identifier has an empty fragment".to_owned(), - )); - } - if did == self.did { - if fragment.is_none_or(|value| value == "atproto_pds") { - return Ok(self.local_endpoint.clone()); - } - return Err(named_request( - "ServiceNotResolvable", - "local DID has no matching service entry", - )); - } - let did = did.parse::().map_err(|_| { - named_request( - "ServiceNotResolvable", - "service identifier is not a valid DID", - ) - })?; - let document = self.identity.resolve_did(&did).await.map_err(|_| { - named_request( - "ServiceNotResolvable", - "could not resolve the service DID document", - ) - })?; - let fragment = fragment.unwrap_or("atproto_pds"); - let service_id = format!("{}#{fragment}", did.as_str()); - let endpoint = document - .service - .unwrap_or_default() - .into_iter() - .find(|entry| entry.id == service_id || entry.id == format!("#{fragment}")) - .map(|entry| entry.service_endpoint) - .ok_or_else(|| { - named_request( - "ServiceNotResolvable", - "DID document has no matching string service endpoint", - ) - })?; - let endpoint = Url::parse(&endpoint) - .map_err(|_| named_request("ServiceNotResolvable", "service endpoint is not a URL"))?; - if !matches!(endpoint.scheme(), "http" | "https") { - return Err(named_request( - "ServiceNotResolvable", - "service endpoint must use HTTP or HTTPS", - )); - } - Ok(endpoint) - } - - pub(crate) async fn deliver_notification( - &self, - endpoint: &str, - audience: &str, - lxm: &str, - body: &serde_json::Value, - ) -> Result<()> { - let target = format!("{}/xrpc/{lxm}", endpoint.trim_end_matches('/')); - let response = self - .http - .post(target) - .bearer_auth(self.mint_service_token(audience, lxm)?) - .json(body) - .send() - .await - .map_err(|error| Error::Internal(format!("notification delivery failed: {error}")))?; - if !response.status().is_success() { - return Err(Error::Internal(format!( - "notification delivery returned {}", - response.status() - ))); - } - Ok(()) - } - - pub(crate) async fn check_user_access( - &self, - service: &str, - space: &str, - user: &str, - client_id: Option<&str>, - ) -> bool { - let Ok(endpoint) = self.resolve_service_endpoint(service).await else { - return false; - }; - let lxm = "com.atproto.simplespace.checkUserAccess"; - let target = format!("{}/xrpc/{lxm}", endpoint.as_str().trim_end_matches('/')); - let Ok(jwt) = self.mint_service_token(service, lxm) else { - return false; - }; - let mut query = vec![("space", space), ("user", user)]; - if let Some(client_id) = client_id { - query.push(("clientId", client_id)); - } - let Ok(response) = self - .http - .get(target) - .bearer_auth(jwt) - .query(&query) - .send() - .await - else { - return false; - }; - if !response.status().is_success() { - return false; - } - response - .json::() - .await - .is_ok_and(|decision| decision.authorized) - } - - fn sign_token(&self, typ: &str, claims: &SpaceClaims) -> Result { + fn sign_jwt(&self, typ: &str, kid: Option<&str>, claims: &impl Serialize) -> Result { let header = JwtHeader { alg: "ES256K".to_owned(), typ: typ.to_owned(), - kid: Some("#atproto".to_owned()), + kid: kid.map(ToOwned::to_owned), jwk: None, }; - self.sign_jwt(&header, claims) - } - - fn mint_service_token(&self, audience: &str, lxm: &str) -> Result { - let issued_at = now(); - self.sign_jwt( - &JwtHeader { - alg: "ES256K".to_owned(), - typ: "JWT".to_owned(), - kid: None, - jwk: None, - }, - &ServiceClaims { - iss: self.did.clone(), - aud: audience.to_owned(), - iat: issued_at, - exp: issued_at + 60, - lxm: lxm.to_owned(), - jti: Some(random_jti()), - }, - ) - } - - fn sign_jwt(&self, header: &JwtHeader, claims: &impl Serialize) -> Result { - let head = URL_SAFE_NO_PAD.encode(serde_json::to_vec(header)?); + let head = URL_SAFE_NO_PAD.encode(serde_json::to_vec(&header)?); let body = URL_SAFE_NO_PAD.encode(serde_json::to_vec(claims)?); let input = format!("{head}.{body}"); let signature = self .signing_key .sign(input.as_bytes()) - .map_err(|error| Error::Internal(format!("JWT signing failed: {error}")))?; + .context(Error::Internal, "JWT signing failed")?; Ok(format!("{input}.{}", URL_SAFE_NO_PAD.encode(signature))) } @@ -598,30 +281,6 @@ impl SpaceAuthService { secp256k1_did_key(&multikey) } - async fn fetch_json Deserialize<'de>>(&self, url: Url) -> Result { - let response = self - .http - .get(url) - .header("accept", "application/json") - .send() - .await - .map_err(|error| Error::Internal(format!("document fetch failed: {error}")))?; - if !response.status().is_success() { - return Err(Error::Internal(format!( - "document fetch returned {}", - response.status() - ))); - } - let bytes = response - .bytes() - .await - .map_err(|error| Error::Internal(format!("document read failed: {error}")))?; - if bytes.len() > 256 * 1024 { - return Err(Error::InvalidRequest("document is too large".to_owned())); - } - serde_json::from_slice(&bytes).map_err(Into::into) - } - fn verify_dpop( &self, proof: &str, @@ -646,15 +305,7 @@ impl SpaceAuthService { "DPoP proof requires a P-256 EC key", )); } - let x = decode_coordinate(&jwk.x)?; - let y = decode_coordinate(&jwk.y)?; - let point = EncodedPoint::from_affine_coordinates((&x).into(), (&y).into(), false); - let key = VerifyingKey::from_encoded_point(&point) - .map_err(|_| auth_error("BadDpopProofSignature", "invalid DPoP public key"))?; - let signature = P256Signature::from_slice(&signature) - .map_err(|_| auth_error("BadDpopProofSignature", "invalid DPoP signature encoding"))?; - key.verify(input.as_bytes(), &signature) - .map_err(|_| auth_error("BadDpopProofSignature", "invalid DPoP signature"))?; + verify_p256(&jwk.x, &jwk.y, input, &signature)?; if claims.jti.is_empty() || claims.htm != method || claims.htu != normalize_htu(htu) { return Err(auth_error("BadDpopProof", "DPoP request binding mismatch")); @@ -691,24 +342,17 @@ impl SpaceAuthService { } } -fn decode_jwt Deserialize<'de>>(jwt: &str) -> Result<(JwtHeader, T, String, Vec)> { - let mut parts = jwt.split('.'); - let head = parts.next().ok_or_else(bad_jwt)?; - let body = parts.next().ok_or_else(bad_jwt)?; - let signature = parts.next().ok_or_else(bad_jwt)?; - if parts.next().is_some() { +fn decode_jwt Deserialize<'de>>(jwt: &str) -> Result<(JwtHeader, T, &str, Vec)> { + let (input, signature) = jwt.rsplit_once('.').ok_or_else(bad_jwt)?; + let (head, body) = input.split_once('.').ok_or_else(bad_jwt)?; + if body.contains('.') { return Err(bad_jwt()); } let header = serde_json::from_slice(&decode_part(head)?) .map_err(|_| auth_error("BadJwt", "invalid JWT header"))?; let claims = serde_json::from_slice(&decode_part(body)?) .map_err(|_| auth_error("BadJwt", "invalid JWT claims"))?; - Ok(( - header, - claims, - format!("{head}.{body}"), - decode_part(signature)?, - )) + Ok((header, claims, input, decode_part(signature)?)) } fn decode_part(value: &str) -> Result> { @@ -723,25 +367,16 @@ fn decode_coordinate(value: &str) -> Result<[u8; 32]> { .map_err(|_| auth_error("BadDpopProof", "invalid P-256 coordinate")) } -fn safe_document_url(value: &str) -> Result { - let url = Url::parse(value)?; - if !url.username().is_empty() || url.password().is_some() || url.fragment().is_some() { - return Err(Error::InvalidRequest("unsafe document URL".to_owned())); - } - let valid_scheme = match url.scheme() { - "https" => true, - "http" => { - matches!(url.host(), Some(url::Host::Ipv4(ip)) if ip.is_loopback()) - || matches!(url.host(), Some(url::Host::Ipv6(ip)) if ip.is_loopback()) - } - _ => false, - }; - if !valid_scheme { - return Err(Error::InvalidRequest( - "document URL must use HTTPS, except for loopback development clients".to_owned(), - )); - } - Ok(url) +fn verify_p256(x: &str, y: &str, input: &str, signature: &[u8]) -> Result<()> { + let x = decode_coordinate(x)?; + let y = decode_coordinate(y)?; + let point = EncodedPoint::from_affine_coordinates((&x).into(), (&y).into(), false); + let key = VerifyingKey::from_encoded_point(&point) + .map_err(|_| auth_error("BadDpopProofSignature", "invalid DPoP public key"))?; + let signature = P256Signature::from_slice(signature) + .map_err(|_| auth_error("BadDpopProofSignature", "invalid DPoP signature encoding"))?; + key.verify(input.as_bytes(), &signature) + .map_err(|_| auth_error("BadDpopProofSignature", "invalid DPoP signature")) } fn secp256k1_did_key(multikey: &str) -> Result { @@ -774,15 +409,7 @@ fn normalize_htu(url: &Url) -> String { fn random_jti() -> String { let mut value = [0_u8; 16]; rand::rng().fill(&mut value); - let mut output = String::with_capacity(32); - for byte in value { - let _ = write!(output, "{byte:02x}"); - } - output -} - -fn now() -> i64 { - OffsetDateTime::now_utc().unix_timestamp() + format!("{:032x}", u128::from_be_bytes(value)) } fn bad_jwt() -> Error { @@ -796,25 +423,565 @@ fn auth_error(name: &'static str, message: &str) -> Error { } } -fn named_request(name: &'static str, message: &str) -> Error { - Error::Xrpc { - name, - message: message.to_owned(), - } -} - -fn invalid_attestation() -> Error { - named_request( - "InvalidClientAttestation", - "client attestation could not be verified", - ) -} - #[cfg(test)] mod tests { use atrium_crypto::keypair::{Did as _, Secp256k1Keypair}; + use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; + use p256::ecdsa::{Signature, SigningKey, signature::Signer as _}; + use sha2::{Digest as _, Sha256}; + + use super::{Error, secp256k1_did_key, verify_p256}; + + pub(super) fn signing_service() -> super::SpaceAuthService { + let did = "did:web:alice.test"; + let endpoint = "http://alice.test/"; + let key = Secp256k1Keypair::import(&[7; 32]).unwrap(); + let document = crate::domain::identity::local_did_document( + did, + "alice.test", + endpoint, + key.did().strip_prefix("did:key:").unwrap(), + ); + let identity = + super::IdentityService::new(document, crate::config::DeploymentMode::Local).unwrap(); + super::SpaceAuthService::new( + super::Database::open_in_memory().unwrap(), + did.to_owned(), + &[7; 32], + endpoint.parse().unwrap(), + super::Arc::new(identity), + ) + .unwrap() + } + + pub(super) fn assert_signer_parity( + service: &super::SpaceAuthService, + typ: &str, + kid: Option<&str>, + claims: &impl serde::Serialize, + expected_header: &[u8], + ) { + let previous_header = super::JwtHeader { + alg: "ES256K".to_owned(), + typ: typ.to_owned(), + kid: kid.map(ToOwned::to_owned), + jwk: None, + }; + let header = serde_json::to_vec(&previous_header).unwrap(); + assert_eq!(header, expected_header); + let head = URL_SAFE_NO_PAD.encode(header); + let body = URL_SAFE_NO_PAD.encode(serde_json::to_vec(claims).unwrap()); + let input = format!("{head}.{body}"); + let signature = service.signing_key.sign(input.as_bytes()).unwrap(); + let previous = format!("{input}.{}", URL_SAFE_NO_PAD.encode(signature)); + let actual = service.sign_jwt(typ, kid, claims).unwrap(); + assert_eq!(actual, previous); + let (input, signature) = actual.rsplit_once('.').unwrap(); + atrium_crypto::verify::verify_signature( + &service.signing_key.did(), + input.as_bytes(), + &URL_SAFE_NO_PAD.decode(signature).unwrap(), + ) + .unwrap(); + } + + #[tokio::test] + async fn shared_signer_preserves_delegation_and_credential_jwts() { + let service = signing_service(); + let contexts: [(&str, &[u8]); 2] = [ + ( + super::DELEGATION_TYP, + br##"{"alg":"ES256K","typ":"atproto-space-delegation+jwt","kid":"#atproto","jwk":null}"##, + ), + ( + super::CREDENTIAL_TYP, + br##"{"alg":"ES256K","typ":"atproto-space-credential+jwt","kid":"#atproto","jwk":null}"##, + ), + ]; + for text in [ + "", + "did:web:alice.test", + "quotes \" slash \\ newline\n nul\0 café", + ] { + for audience in [None, Some(text)] { + for binding in [None, Some(text)] { + let claims = super::SpaceClaims { + iss: text.to_owned(), + sub: text.to_owned(), + aud: audience.map(ToOwned::to_owned), + iat: i64::MIN, + exp: i64::MAX, + jti: text.to_owned(), + cnf: binding.map(|jkt| super::Confirmation { + jkt: jkt.to_owned(), + }), + }; + for (typ, header) in contexts { + assert_signer_parity(&service, typ, Some("#atproto"), &claims, header); + } + } + } + } + } + + #[tokio::test] + async fn shared_signer_preserves_claim_serialization_error() { + let service = signing_service(); + let claims = std::collections::BTreeMap::from([(vec!["not", "a string"], 1)]); + let previous = Error::Json(serde_json::to_vec(&claims).unwrap_err()); + for (typ, kid) in [ + (super::DELEGATION_TYP, Some("#atproto")), + (super::CREDENTIAL_TYP, Some("#atproto")), + ("JWT", None), + ] { + let actual = service.sign_jwt(typ, kid, &claims).unwrap_err(); + assert!(matches!(actual, Error::Json(_))); + assert_eq!(actual.to_string(), previous.to_string()); + } + } + + fn delegation_claims() -> super::SpaceClaims { + super::SpaceClaims { + iss: "did:web:alice.test".to_owned(), + sub: "at://did:web:alice.test/space/com.example.forum/default".to_owned(), + aud: Some("did:web:alice.test#atproto_space_host".to_owned()), + iat: super::now(), + exp: super::now() + 600, + jti: "delegation-id".to_owned(), + cnf: Some(super::Confirmation { + jkt: "unused delegation binding".to_owned(), + }), + } + } + + fn dpop_proof(jti: &str, htu: &url::Url, credential: Option<&str>) -> (String, String) { + let key = SigningKey::from_slice(&[9; 32]).unwrap(); + let point = key.verifying_key().to_encoded_point(false); + let jwk = super::EcJwk { + kty: "EC".to_owned(), + crv: "P-256".to_owned(), + x: URL_SAFE_NO_PAD.encode(point.x().unwrap()), + y: URL_SAFE_NO_PAD.encode(point.y().unwrap()), + }; + let jkt = super::jwk_thumbprint(&jwk); + let header = super::JwtHeader { + alg: "ES256".to_owned(), + typ: super::DPOP_TYP.to_owned(), + kid: None, + jwk: Some(jwk), + }; + let mut claims = serde_json::json!({ + "jti": jti, + "htm": "POST", + "htu": super::normalize_htu(htu), + "iat": super::now(), + }); + if let Some(credential) = credential { + claims["ath"] = serde_json::json!(URL_SAFE_NO_PAD.encode(Sha256::digest(credential))); + } + let head = URL_SAFE_NO_PAD.encode(serde_json::to_vec(&header).unwrap()); + let body = URL_SAFE_NO_PAD.encode(serde_json::to_vec(&claims).unwrap()); + let input = format!("{head}.{body}"); + let signature: Signature = key.sign(input.as_bytes()); + ( + format!("{input}.{}", URL_SAFE_NO_PAD.encode(signature.to_bytes())), + jkt, + ) + } + + fn assert_authentication_error( + result: super::Result, + expected_name: &str, + expected_message: &str, + ) { + let Err(Error::Authentication { name, message }) = result else { + panic!("expected authentication failure") + }; + assert_eq!(name, expected_name); + assert_eq!(message, expected_message); + } + + #[tokio::test] + async fn delegation_exchange_preserves_claims_proof_binding_and_replay_priority() { + let service = signing_service(); + let claims = delegation_claims(); + let token = service + .sign_jwt(super::DELEGATION_TYP, Some("#atproto"), &claims) + .unwrap(); + let htu = "http://alice.test/exchange?query=ignored#fragment" + .parse() + .unwrap(); + let (proof, expected_jkt) = dpop_proof("first-proof", &htu, None); + assert_authentication_error( + service + .exchange_delegation(&token, &proof, "GET", &htu) + .await, + "BadDpopProof", + "DPoP request binding mismatch", + ); + let (actual_claims, actual_jkt) = service + .exchange_delegation(&token, &proof, "POST", &htu) + .await + .unwrap(); + assert_eq!( + serde_json::to_vec(&actual_claims).unwrap(), + serde_json::to_vec(&claims).unwrap() + ); + assert_eq!(actual_jkt, expected_jkt); + let (fresh_proof, _) = dpop_proof("second-proof", &htu, None); + for (proof, name, message) in [ + (&proof, "BadDpopProof", "DPoP proof has been replayed"), + ( + &fresh_proof, + "JwtReplayed", + "delegation token has already been used", + ), + (&fresh_proof, "BadDpopProof", "DPoP proof has been replayed"), + ] { + assert_authentication_error( + service + .exchange_delegation(&token, proof, "POST", &htu) + .await, + name, + message, + ); + } + } + + #[tokio::test] + async fn credential_verification_returns_the_exact_scope_after_bound_proof_checks() { + let service = signing_service(); + let htu = "http://alice.test/read?query=ignored#fragment" + .parse() + .unwrap(); + let (unbound_proof, jkt) = dpop_proof("credential-proof", &htu, None); + let space = "at://did:web:alice.test/space/com.example.forum/key:with:colons"; + let credential = service.mint_credential(space, jkt).unwrap(); + let (proof, _) = dpop_proof("credential-proof", &htu, Some(&credential)); + for (proof, method, message) in [ + (&proof, "GET", "DPoP request binding mismatch"), + ( + &unbound_proof, + "POST", + "DPoP credential hash binding mismatch", + ), + ] { + assert_authentication_error( + service + .verify_credential(&credential, proof, method, &htu) + .await, + "BadDpopProof", + message, + ); + } + let actual = service + .verify_credential(&credential, &proof, "POST", &htu) + .await + .unwrap(); + assert_eq!(actual, space); + assert_authentication_error( + service + .verify_credential(&credential, &proof, "POST", &htu) + .await, + "BadDpopProof", + "DPoP proof has been replayed", + ); + + for (sub, cnf, name, message) in [ + ( + "at://did:web:elsewhere.test/space/com.example.forum/default", + None, + "BadJwtIss", + "credential issuer is not the space authority", + ), + (space, None, "BadJwtCnf", "credential has no key binding"), + ] { + let claims = super::SpaceClaims { + sub: sub.to_owned(), + cnf, + ..delegation_claims() + }; + let credential = service + .sign_jwt(super::CREDENTIAL_TYP, Some("#atproto"), &claims) + .unwrap(); + assert_authentication_error( + service + .verify_credential(&credential, "malformed", "POST", &htu) + .await, + name, + message, + ); + } + } + + #[tokio::test] + async fn delegation_exchange_validates_claims_before_proof_and_replay_consumption() { + let htu = "http://alice.test/exchange".parse().unwrap(); + let claims = serde_json::to_value(delegation_claims()).unwrap(); + for (field, value, name, message) in [ + ( + "exp", + serde_json::json!(super::now() - 600), + "JwtExpired", + "JWT has expired", + ), + ( + "iat", + serde_json::json!(super::now() + 600), + "BadJwt", + "invalid JWT issue time or identifier", + ), + ( + "jti", + serde_json::json!(""), + "BadJwt", + "invalid JWT issue time or identifier", + ), + ( + "aud", + serde_json::Value::Null, + "BadJwtAudience", + "delegation token audience mismatch", + ), + ( + "aud", + serde_json::json!("did:web:elsewhere.test#atproto_space_host"), + "BadJwtAudience", + "delegation token audience mismatch", + ), + ] { + let service = signing_service(); + let mut invalid_claims = claims.clone(); + invalid_claims[field] = value; + let token = service + .sign_jwt(super::DELEGATION_TYP, Some("#atproto"), &invalid_claims) + .unwrap(); + let (proof, jkt) = dpop_proof("unused-proof", &htu, None); + for proof in [proof.as_str(), "not-a-proof"] { + assert_authentication_error( + service + .exchange_delegation(&token, proof, "POST", &htu) + .await, + name, + message, + ); + } + assert_eq!( + service + .verify_dpop(&proof, "POST", &htu, None, None) + .unwrap(), + jkt + ); + assert!( + service + .database + .consume_space_auth_jti( + "delegation", + invalid_claims["iss"].as_str().unwrap(), + invalid_claims["jti"].as_str().unwrap(), + super::now() + 600, + ) + .unwrap() + ); + } + } + + fn previous_decode_jwt( + jwt: &str, + ) -> super::Result<(super::JwtHeader, T, String, Vec)> { + let mut parts = jwt.split('.'); + let head = parts.next().ok_or_else(super::bad_jwt)?; + let body = parts.next().ok_or_else(super::bad_jwt)?; + let signature = parts.next().ok_or_else(super::bad_jwt)?; + if parts.next().is_some() { + return Err(super::bad_jwt()); + } + let header = serde_json::from_slice(&super::decode_part(head)?) + .map_err(|_| super::auth_error("BadJwt", "invalid JWT header"))?; + let claims = serde_json::from_slice(&super::decode_part(body)?) + .map_err(|_| super::auth_error("BadJwt", "invalid JWT claims"))?; + Ok(( + header, + claims, + format!("{head}.{body}"), + super::decode_part(signature)?, + )) + } + + fn assert_jwt_decoder_parity(jwt: &str) { + let previous = previous_decode_jwt::(jwt) + .map(|(header, claims, input, signature)| { + ( + serde_json::to_value(header).unwrap(), + serde_json::to_value(claims).unwrap(), + input, + signature, + ) + }) + .map_err(|error| format!("{error:?}")); + let actual = super::decode_jwt::(jwt) + .map(|(header, claims, input, signature)| { + assert_eq!(input.as_ptr(), jwt.as_ptr()); + ( + serde_json::to_value(header).unwrap(), + serde_json::to_value(claims).unwrap(), + input.to_owned(), + signature, + ) + }) + .map_err(|error| format!("{error:?}")); + assert_eq!(actual, previous, "JWT {jwt:?}"); + } + + #[test] + fn borrowed_jwt_input_preserves_shape_sections_and_error_priority() { + let encode = |value| URL_SAFE_NO_PAD.encode(value); + let header = encode(r##"{"alg":"ES256K","typ":"JWT","kid":"#atproto"}"##); + let claims = + encode(r#"{"iss":"did:example:alice","sub":"space","iat":1,"exp":2,"jti":"id"}"#); + let headers = [ + header.clone(), + encode( + r#"{"alg":"ES256","typ":"dpop+jwt","jwk":{"kty":"EC","crv":"P-256","x":"x","y":"y"}}"#, + ), + encode(r#"{"alg":"ES256K","alg":"duplicate","typ":"JWT"}"#), + encode("{}"), + encode("null"), + encode("[\"ES256K\",\"JWT\",null,null]"), + encode("{"), + String::new(), + "!".to_owned(), + "é".to_owned(), + ]; + let bodies = [ + claims.clone(), + encode("{}"), + encode("null"), + encode("[]"), + encode(r#"{"iss":"first","iss":"duplicate"}"#), + encode("{"), + String::new(), + "!".to_owned(), + ]; + for header in &headers { + for body in &bodies { + for signature in ["", "AA", "AQID", "_w", "AA==", "!", "é"] { + let jwt = format!("{header}.{body}.{signature}"); + assert_jwt_decoder_parity::(&jwt); + assert_jwt_decoder_parity::(&jwt); + } + } + } + for jwt in [ + String::new(), + ".".to_owned(), + "..".to_owned(), + "...".to_owned(), + "a".to_owned(), + "a.b".to_owned(), + "a.b.c.d".to_owned(), + format!("{header}.{claims}"), + format!("{header}.{claims}.AA.extra"), + format!(".{header}.{claims}.AA"), + ] { + assert_jwt_decoder_parity::(&jwt); + assert_jwt_decoder_parity::(&jwt); + } + } + + #[test] + fn fixed_width_jti_hex_matches_byte_order_and_leading_zeroes() { + use std::fmt::Write as _; + + for value in [ + 0, + u128::MAX, + 0x0001_0203_0405_0607_0809_0a0b_0c0d_0e0f, + 0xff00_0000_0000_0000_0000_0000_0000_0000, + ] + .into_iter() + .chain((0..128).map(|bit| 1_u128 << bit)) + { + let bytes = value.to_be_bytes(); + let mut previous = String::with_capacity(32); + for byte in bytes { + write!(previous, "{byte:02x}").unwrap(); + } + let current = format!("{:032x}", u128::from_be_bytes(bytes)); + assert_eq!(current, previous); + assert_eq!(current.len(), 32); + } + for _ in 0..32 { + let jti = super::random_jti(); + assert_eq!(jti.len(), 32); + assert!( + jti.bytes() + .all(|byte| matches!(byte, b'0'..=b'9' | b'a'..=b'f')) + ); + } + } - use super::secp256k1_did_key; + #[test] + fn shared_p256_verification_preserves_error_order_and_dpop_diagnostics() { + let key = SigningKey::from_slice(&[9; 32]).unwrap(); + let point = key.verifying_key().to_encoded_point(false); + let x = URL_SAFE_NO_PAD.encode(point.x().unwrap()); + let y = URL_SAFE_NO_PAD.encode(point.y().unwrap()); + let signature: Signature = key.sign(b"signed input"); + assert!(verify_p256(&x, &y, "signed input", &signature.to_bytes()).is_ok()); + for (x, y, input, signature, name, message) in [ + ( + "!".to_owned(), + "!".to_owned(), + "input", + vec![], + "BadJwt", + "invalid base64url in JWT", + ), + ( + URL_SAFE_NO_PAD.encode([0; 31]), + y.clone(), + "input", + vec![], + "BadDpopProof", + "invalid P-256 coordinate", + ), + ( + URL_SAFE_NO_PAD.encode([0; 32]), + URL_SAFE_NO_PAD.encode([0; 32]), + "input", + vec![], + "BadDpopProofSignature", + "invalid DPoP public key", + ), + ( + x.clone(), + y.clone(), + "input", + vec![], + "BadDpopProofSignature", + "invalid DPoP signature encoding", + ), + ( + x, + y, + "tampered input", + signature.to_bytes().to_vec(), + "BadDpopProofSignature", + "invalid DPoP signature", + ), + ] { + let Error::Authentication { + name: actual_name, + message: actual_message, + } = verify_p256(&x, &y, input, &signature).unwrap_err() + else { + panic!("expected authentication failure") + }; + assert_eq!(actual_name, name); + assert_eq!(actual_message, message); + } + } #[test] fn accepts_only_secp256k1_multikeys() { diff --git a/src/space/auth/attestation.rs b/src/space/auth/attestation.rs new file mode 100644 index 0000000..5e649ea --- /dev/null +++ b/src/space/auth/attestation.rs @@ -0,0 +1,101 @@ +//! Verify a client's signed identity against its published metadata and keys. + +use serde::Deserialize; +use url::Url; + +use super::{CLOCK_SKEW, SpaceAuthService, decode_jwt, now, verify_p256}; +use crate::error::{Error, Result}; + +#[derive(Deserialize)] +struct AttestationClaims { + iss: String, + sub: String, + aud: String, + iat: i64, + exp: i64, + jti: String, +} + +#[derive(Deserialize)] +struct ClientMetadata { + client_id: String, + #[serde(default)] + jwks: Option, + #[serde(default)] + jwks_uri: Option, +} + +#[derive(Deserialize)] +struct Jwks { + keys: Vec, +} + +#[derive(Deserialize)] +struct PublicJwk { + kid: String, + kty: String, + crv: String, + x: String, + y: String, +} + +impl SpaceAuthService { + pub(crate) async fn verify_client_attestation( + &self, + attestation: &str, + expected_audience: &str, + ) -> Result { + let claims = self + .client_attestation(attestation, expected_audience) + .await + .map_err(|_| invalid_attestation())?; + if !self.database.consume_space_auth_jti( + "attestation", + &claims.iss, + &claims.jti, + claims.exp, + )? { + return Err(invalid_attestation()); + } + Ok(claims.iss) + } + + async fn client_attestation(&self, token: &str, audience: &str) -> Result { + let (header, claims, input, signature) = decode_jwt::(token)?; + if header.typ != "atproto-client-attestation+jwt" + || header.alg != "ES256" + || claims.iss != claims.sub + || claims.aud != audience + || claims.jti.is_empty() + || now() - CLOCK_SKEW >= claims.exp + || claims.iat > now() + CLOCK_SKEW + { + return Err(invalid_attestation()); + } + let metadata: ClientMetadata = self.http.get_json(Url::parse(&claims.iss)?, None).await?; + if metadata.client_id != claims.iss { + return Err(invalid_attestation()); + } + let keys = if let Some(keys) = metadata.jwks { + keys + } else { + let uri = metadata.jwks_uri.ok_or_else(invalid_attestation)?; + self.http.get_json(Url::parse(&uri)?, None).await? + }; + let kid = header.kid.as_deref().ok_or_else(invalid_attestation)?; + let jwk = keys + .keys + .into_iter() + .find(|key| key.kid == kid && key.kty == "EC" && key.crv == "P-256") + .ok_or_else(invalid_attestation)?; + verify_p256(&jwk.x, &jwk.y, input, &signature)?; + Ok(claims) + } +} + +fn invalid_attestation() -> Error { + Error::xrpc( + "InvalidClientAttestation", + "client attestation could not be verified", + ) +} diff --git a/src/space/auth/service.rs b/src/space/auth/service.rs new file mode 100644 index 0000000..cbaee66 --- /dev/null +++ b/src/space/auth/service.rs @@ -0,0 +1,314 @@ +//! DID-resolved service authentication, callbacks, and notifications. + +use atrium_api::types::string::Did; +use atrium_crypto::verify::verify_signature; +use serde::{Deserialize, Serialize}; +use url::Url; + +use super::{CLOCK_SKEW, SpaceAuthService, auth_error, decode_jwt, now, random_jti}; +use crate::{ + error::{Error, Result}, + syntax, +}; + +#[derive(Deserialize, Serialize)] +struct ServiceClaims { + iss: String, + aud: String, + iat: i64, + exp: i64, + lxm: String, + #[serde(default)] + jti: Option, +} + +#[derive(Deserialize)] +struct AccessDecision { + authorized: bool, +} + +impl SpaceAuthService { + pub(crate) async fn verify_service_token( + &self, + token: &str, + expected_lxm: &str, + ) -> Result<(String, String)> { + let (header, claims, input, signature) = decode_jwt::(token)?; + if header.alg != "ES256K" + || matches!(header.typ.as_str(), "at+jwt" | "refresh+jwt" | "dpop+jwt") + { + return Err(auth_error("BadJwtType", "unsupported service JWT type")); + } + if claims.lxm != expected_lxm { + return Err(auth_error( + "BadJwtLexiconMethod", + "service JWT method does not match this endpoint", + )); + } + if now() > claims.exp || claims.iat > now() + CLOCK_SKEW { + return Err(auth_error( + "JwtExpired", + "service JWT is expired or not yet valid", + )); + } + if claims.jti.as_deref().is_some_and(str::is_empty) { + return Err(auth_error("BadJwt", "service JWT has an empty identifier")); + } + let issuer_did = claims + .iss + .split_once('#') + .map_or(claims.iss.as_str(), |v| v.0); + syntax::did(issuer_did) + .map_err(|_| auth_error("BadJwtIss", "service JWT issuer is not a DID"))?; + verify_signature( + &self + .resolve_signing_key(issuer_did, Some("#atproto")) + .await?, + input.as_bytes(), + &signature, + ) + .map_err(|_| auth_error("BadJwtSignature", "invalid service JWT signature"))?; + Ok((claims.iss, claims.aud)) + } + + pub(crate) async fn resolve_service_endpoint(&self, service: &str) -> Result { + let (did, fragment) = service + .split_once('#') + .map_or((service, None), |(did, fragment)| (did, Some(fragment))); + let did = syntax::parse::(did, "DID")?; + if fragment.is_some_and(str::is_empty) { + return Err(Error::InvalidRequest( + "service identifier has an empty fragment".to_owned(), + )); + } + if did.as_str() == self.did { + if fragment.is_none_or(|value| value == "atproto_pds") { + return Ok(self.local_endpoint.clone()); + } + return Err(Error::xrpc( + "ServiceNotResolvable", + "local DID has no matching service entry", + )); + } + let document = self.identity.resolve_did(&did).await.map_err(|_| { + Error::xrpc( + "ServiceNotResolvable", + "could not resolve the service DID document", + ) + })?; + let fragment = fragment.unwrap_or("atproto_pds"); + let service_id = format!("{}#{fragment}", did.as_str()); + let endpoint = document + .service + .unwrap_or_default() + .into_iter() + .find(|entry| entry.id == service_id || entry.id == format!("#{fragment}")) + .map(|entry| entry.service_endpoint) + .ok_or_else(|| { + Error::xrpc( + "ServiceNotResolvable", + "DID document has no matching string service endpoint", + ) + })?; + let endpoint = Url::parse(&endpoint) + .map_err(|_| Error::xrpc("ServiceNotResolvable", "service endpoint is not a URL"))?; + self.http.validate_url(&endpoint).map_err(|_| { + Error::xrpc( + "ServiceNotResolvable", + "service endpoint does not satisfy the outbound URL policy", + ) + })?; + Ok(endpoint) + } + + pub(crate) async fn deliver_notification( + &self, + endpoint: &str, + audience: &str, + lxm: &str, + body: &serde_json::Value, + ) -> Result<()> { + let target = service_request_url(Url::parse(endpoint)?, lxm); + self.http + .post_json(target, &self.mint_service_token(audience, lxm)?, body) + .await + } + + pub(crate) async fn check_user_access( + &self, + service: &str, + space: &str, + user: &str, + client_id: Option<&str>, + ) -> bool { + let Ok(endpoint) = self.resolve_service_endpoint(service).await else { + return false; + }; + let lxm = "com.atproto.simplespace.checkUserAccess"; + let mut target = service_request_url(endpoint, lxm); + let Ok(jwt) = self.mint_service_token(service, lxm) else { + return false; + }; + target + .query_pairs_mut() + .append_pair("space", space) + .append_pair("user", user); + if let Some(client_id) = client_id { + target.query_pairs_mut().append_pair("clientId", client_id); + } + self.http + .get_json::(target, Some(&jwt)) + .await + .is_ok_and(|decision| decision.authorized) + } + + fn mint_service_token(&self, audience: &str, lxm: &str) -> Result { + let issued_at = now(); + self.sign_jwt( + "JWT", + None, + &ServiceClaims { + iss: self.did.clone(), + aud: audience.to_owned(), + iat: issued_at, + exp: issued_at + 60, + lxm: lxm.to_owned(), + jti: Some(random_jti()), + }, + ) + } +} + +fn service_request_url(mut endpoint: Url, lxm: &str) -> Url { + endpoint.set_path(&format!( + "{}/xrpc/{lxm}", + endpoint.path().trim_end_matches('/') + )); + endpoint +} + +#[cfg(test)] +mod tests { + use super::super::tests::{assert_signer_parity, signing_service}; + use super::{Did, Error, ServiceClaims, syntax}; + + #[test] + fn reused_service_did_matches_both_previous_parses_without_normalization() { + let fixtures = concat!( + include_str!("../../../tests/fixtures/atproto-did-syntax-valid.txt"), + "\n", + include_str!("../../../tests/fixtures/atproto-did-syntax-invalid.txt"), + ); + for value in fixtures + .lines() + .filter(|line| !line.is_empty() && !line.starts_with('#')) + .chain([ + "", + "did:web:ALICE.test", + "did:web:localhost%3A1234", + "did:method:val%BB", + "did:method:val%bb", + ]) + { + let previous = syntax::did(value).and_then(|did| { + did.parse::().map_err(|_| { + Error::xrpc( + "ServiceNotResolvable", + "service identifier is not a valid DID", + ) + }) + }); + let actual = syntax::parse::(value, "DID"); + assert_eq!(format!("{actual:?}"), format!("{previous:?}"), "{value}"); + if let Ok(did) = actual { + assert_eq!(did.as_str(), value); + for fragment in ["atproto_pds", "custom", "MixedCase%3A", "x#y"] { + assert_eq!( + format!("{}#{fragment}", did.as_str()), + format!("{value}#{fragment}"), + ); + } + } + } + } + + #[tokio::test] + async fn service_endpoint_preserves_did_fragment_and_local_branch_priority() { + let mut service = signing_service(); + for value in [ + "", + "invalid", + "DID:web:alice.test", + "did:web:", + "did:example:🦀", + ] { + for fragment in ["", "#", "#atproto_pds", "#missing"] { + let error = service + .resolve_service_endpoint(&format!("{value}{fragment}")) + .await + .unwrap_err(); + assert!(matches!( + error, + Error::InvalidRequest(message) + if message == format!("`{value}` is not a valid DID") + )); + } + } + for did in [ + "did:web:alice.test", + "did:web:ALICE.test", + "did:web:localhost%3A1234", + "did:method:val%BB", + "did:method:val%bb", + ] { + service.did = did.to_owned(); + for fragment in ["", "#atproto_pds"] { + let endpoint = service + .resolve_service_endpoint(&format!("{did}{fragment}")) + .await + .unwrap(); + assert_eq!(endpoint, service.local_endpoint); + } + assert!(matches!( + service.resolve_service_endpoint(&format!("{did}#")).await, + Err(Error::InvalidRequest(message)) + if message == "service identifier has an empty fragment" + )); + for fragment in ["missing", "atproto_pds#extra", "#"] { + assert!(matches!( + service.resolve_service_endpoint(&format!("{did}#{fragment}")).await, + Err(Error::Xrpc { name: "ServiceNotResolvable", message }) + if message == "local DID has no matching service entry" + )); + } + } + } + + #[tokio::test] + async fn shared_signer_preserves_service_jwts() { + let service = signing_service(); + for text in [ + "", + "did:web:alice.test", + "quotes \" slash \\ newline\n nul\0 café", + ] { + for identifier in [None, Some(text)] { + let claims = ServiceClaims { + iss: text.to_owned(), + aud: text.to_owned(), + iat: i64::MIN, + exp: i64::MAX, + lxm: text.to_owned(), + jti: identifier.map(ToOwned::to_owned), + }; + assert_signer_parity( + &service, + "JWT", + None, + &claims, + br#"{"alg":"ES256K","typ":"JWT","kid":null,"jwk":null}"#, + ); + } + } + } +} diff --git a/src/space/authorization.rs b/src/space/authorization.rs new file mode 100644 index 0000000..d6ab755 --- /dev/null +++ b/src/space/authorization.rs @@ -0,0 +1,325 @@ +//! Space access policy and OAuth scope matching; JWT verification is in auth. +use super::{SpaceService, domain::Space, space_not_found}; +use crate::{ + domain::oauth::AuthenticatedToken, + error::{Error, Result}, + syntax::{self, SpaceRef}, +}; +use serde_json::Value; + +impl SpaceService { + pub(super) fn require_active_account(&self, did: &str) -> Result<()> { + let (_account, status) = self + .database + .hosted_account_by_identifier(did)? + .ok_or_else(|| Error::xrpc("RepoNotFound", "repository is not hosted by this PDS"))?; + status.require_repo_active() + } + + pub(crate) fn authorize_delegation( + &self, + auth: &AuthenticatedToken, + space: &str, + ) -> Result<()> { + if auth.account.did != self.hosted_did { + return Err(Error::Authorization( + "OAuth account is not hosted by this PDS".to_owned(), + )); + } + let space_ref = space.parse::()?; + require_scope(auth, &space_ref, Capability::Read) + } + + pub(crate) fn authorize_credential( + &self, + space: &str, + user_did: &str, + client_id: Option<&str>, + ) -> Result> { + self.require_active_account(&self.hosted_did)?; + let space_ref = space.parse::()?; + if space_ref.authority != self.hosted_did { + return Err(Error::xrpc( + "SpaceNotFound", + "this PDS is not the space host", + )); + } + match self.database.space_is_deleted(space)? { + Some(true) => return Err(Error::xrpc("SpaceDeleted", "space has been deleted")), + None => return Err(Error::xrpc("SpaceNotFound", "space does not exist")), + Some(false) => {} + } + let local = self.database.space(space)?; + + if union_type(&local.app_access) == Some("com.atproto.simplespace.defs#allowList") { + let allowed = local.app_access["allowed"] + .as_array() + .expect("validated allowList policy"); + if !client_id.is_some_and(|client| { + allowed + .iter() + .any(|allowed| allowed.as_str() == Some(client)) + }) { + return Err(Error::xrpc( + "AppNotAuthorized", + "application is not authorized for this space", + )); + } + } + if user_did == local.authority || self.user_can_read(&local, user_did)? { + return Ok(None); + } + match union_type(&local.policy) { + Some("com.atproto.simplespace.defs#managingAppPolicy") => local.policy["managingApp"] + .as_str() + .map(ToOwned::to_owned) + .map(Some) + .ok_or_else(|| Error::xrpc("UserNotAuthorized", "managing app is not configured")), + _ => Err(Error::xrpc( + "UserNotAuthorized", + "user is not authorized for this space", + )), + } + } + + pub(super) fn authorize_write( + &self, + auth: &AuthenticatedToken, + space: &str, + repo: &str, + collection: &str, + action: &'static str, + ) -> Result { + if repo != auth.account.did { + return Err(Error::Authorization( + "a permissioned write must target the authenticated account DID".to_owned(), + )); + } + self.require_active_account(repo)?; + syntax::collection(collection)?; + let space_ref = space.parse::()?; + require_scope(auth, &space_ref, Capability::Record(action, collection))?; + if space_ref.authority == self.hosted_did { + let local = self.database.space(space).map_err(space_not_found)?; + if !self.user_can_read(&local, &auth.account.did)? { + return Err(Error::Authorization( + "the account is not a member of this space".to_owned(), + )); + } + } + Ok(space_ref) + } + + pub(super) fn authorize_read( + &self, + auth: &ReadAuthorization, + space: &str, + repo: &str, + ) -> Result { + let space_ref = space.parse::()?; + match auth { + ReadAuthorization::OAuth(auth) => { + if repo != auth.account.did { + return Err(Error::Authorization( + "an OAuth credential can only read its account's own permissioned repo" + .to_owned(), + )); + } + require_scope(auth, &space_ref, Capability::ReadSelf)?; + if space_ref.authority == self.hosted_did { + let local = self.database.space(space).map_err(space_not_found)?; + if !self.user_can_read(&local, &auth.account.did)? { + return Err(Error::Authorization( + "the account is not a member of this space".to_owned(), + )); + } + } + } + ReadAuthorization::Credential(credential_space) => { + if credential_space != space { + return Err(Error::Authorization( + "space credential does not cover the requested space".to_owned(), + )); + } + } + } + self.require_active_account(repo)?; + Ok(space_ref) + } + + pub(super) fn require_owner(&self, auth: &AuthenticatedToken, space: &SpaceRef) -> Result<()> { + if space.authority != auth.account.did || space.authority != self.hosted_did { + return Err(Error::Authorization( + "the authenticated account is not the space owner".to_owned(), + )); + } + Ok(()) + } + + pub(super) fn assert_local_credential_space( + &self, + credential_space: &str, + space: &str, + ) -> Result<()> { + if credential_space != space { + return Err(Error::Authorization( + "space credential does not cover the requested space".to_owned(), + )); + } + let space_ref = space.parse::()?; + if space_ref.authority != self.hosted_did { + return Err(Error::xrpc( + "SpaceNotFound", + "this PDS is not the space host", + )); + } + self.database.space(space).map_err(space_not_found)?; + Ok(()) + } + + pub(super) fn user_can_read(&self, space: &Space, did: &str) -> Result { + match union_type(&space.policy) { + Some("com.atproto.simplespace.defs#publicPolicy") => Ok(true), + Some("com.atproto.simplespace.defs#memberListPolicy") => { + self.database.is_space_member(&space.uri, did) + } + // A managing-app callback is intentionally not guessed. Until a + // configured verifier exists, only the owner is admitted. + Some("com.atproto.simplespace.defs#managingAppPolicy") => Ok(space.authority == did), + _ => Ok(false), + } + } +} + +pub(crate) enum ReadAuthorization { + OAuth(AuthenticatedToken), + Credential(String), +} +#[derive(Clone, Copy)] +pub(super) enum Capability<'a> { + Read, + ReadSelf, + Record(&'static str, &'a str), + Manage(&'static str), +} + +pub(super) fn require_scope( + auth: &AuthenticatedToken, + target: &SpaceRef, + capability: Capability<'_>, +) -> Result<()> { + if auth.scope.split_ascii_whitespace().any(|scope| { + parse_space_scope(scope).is_some_and(|grant| { + grant.matches(&auth.account.did, target) && grant.permits(capability) + }) + }) { + return Ok(()); + } + Err(Error::Authorization(format!( + "OAuth token has no covering `space:` grant for {target}" + ))) +} + +struct SpaceGrant { + space_type: String, + authority: String, + skey: String, + collections: Vec, + actions: Vec, + manage: Vec, +} + +impl SpaceGrant { + fn matches(&self, actor: &str, target: &SpaceRef) -> bool { + (self.space_type == "*" || self.space_type == target.space_type) + && (self.authority == "*" + || self.authority == target.authority + || (self.authority == "self" && target.authority == actor)) + && (self.skey == "*" || self.skey == target.skey) + } + + fn permits(&self, capability: Capability<'_>) -> bool { + match capability { + Capability::Read => self.actions.iter().any(|value| value == "read"), + Capability::ReadSelf => { + self.actions.is_empty() + || self.actions.iter().any(|value| value == "read") + || self.actions.iter().any(|value| value == "read_self") + } + Capability::Record(action, collection) => { + let action_allowed = + self.actions.is_empty() || self.actions.iter().any(|value| value == action); + let collection_allowed = self + .collections + .iter() + .any(|value| value == "*" || value == collection); + action_allowed && collection_allowed + } + Capability::Manage(action) => self.manage.iter().any(|value| value == action), + } + } +} + +fn parse_space_scope(value: &str) -> Option { + let value = value.strip_prefix("space:")?; + let (space_type, query) = value.split_once('?').unwrap_or((value, "")); + if space_type != "*" && syntax::collection(space_type).is_err() { + return None; + } + let mut grant = SpaceGrant { + space_type: space_type.to_owned(), + authority: "self".to_owned(), + skey: "*".to_owned(), + collections: Vec::new(), + actions: Vec::new(), + manage: Vec::new(), + }; + for (name, value) in url::form_urlencoded::parse(query.as_bytes()) { + match name.as_ref() { + "authority" => grant.authority = value.into_owned(), + "skey" => grant.skey = value.into_owned(), + "collection" => grant.collections.push(value.into_owned()), + "action" => grant.actions.push(value.into_owned()), + "manage" => grant.manage.push(value.into_owned()), + _ => return None, + } + } + Some(grant) +} + +pub(super) fn validate_policy(value: &Value) -> Result<()> { + match union_type(value) { + Some( + "com.atproto.simplespace.defs#publicPolicy" + | "com.atproto.simplespace.defs#memberListPolicy", + ) => Ok(()), + Some("com.atproto.simplespace.defs#managingAppPolicy") + if value["managingApp"].as_str().is_some() => + { + Ok(()) + } + _ => Err(Error::InvalidRequest( + "unsupported simplespace policy".to_owned(), + )), + } +} + +pub(super) fn validate_app_access(value: &Value) -> Result<()> { + match union_type(value) { + Some("com.atproto.simplespace.defs#open") => Ok(()), + Some("com.atproto.simplespace.defs#allowList") + if value["allowed"] + .as_array() + .is_some_and(|values| values.iter().all(Value::is_string)) => + { + Ok(()) + } + _ => Err(Error::InvalidRequest( + "unsupported simplespace appAccess policy".to_owned(), + )), + } +} + +pub(super) fn union_type(value: &Value) -> Option<&str> { + value.as_object()?.get("$type")?.as_str() +} diff --git a/src/space/car.rs b/src/space/car.rs index 51e65e0..9dbb14f 100644 --- a/src/space/car.rs +++ b/src/space/car.rs @@ -1,16 +1,18 @@ //! CAR serialization for full Spaces repository recovery. -use std::{cmp::Ordering, collections::BTreeMap}; +use std::collections::BTreeMap; use cid::Cid; +use cpds_protocol::{atproto_record, dag_cbor_cid}; use serde::Serialize; use crate::{ - error::{Error, Result}, - repository::dag_cbor_cid, + error::{Error, ErrorContext as _, Result}, space::{commit::SignedCommit, domain::SpaceRecord}, }; +pub(super) const MAX_REPO_EXPORT_BYTES: usize = 64 * 1024 * 1024; + #[derive(Serialize)] struct CarHeader { roots: Vec, @@ -22,7 +24,10 @@ pub(crate) fn serialize_repo( mut records: Vec, exclude_values: bool, ) -> Result> { - records.sort_by(|left, right| canonical_path_cmp(&left.path(), &right.path())); + records.sort_by_key(|record| { + let path = record.path(); + (path.len(), path) + }); let index = records .iter() @@ -30,74 +35,269 @@ pub(crate) fn serialize_repo( .collect::>(); let commit_bytes = commit.dag_cbor()?; let index_bytes = serde_ipld_dagcbor::to_vec(&index) - .map_err(|error| Error::Internal(format!("could not encode repo index: {error}")))?; - let commit_cid = dag_cbor_cid(&commit_bytes)?; - let index_cid = dag_cbor_cid(&index_bytes)?; + .context(Error::Internal, "could not encode repo index")?; + let commit_cid = dag_cbor_cid(&commit_bytes); + let index_cid = dag_cbor_cid(&index_bytes); let header = serde_ipld_dagcbor::to_vec(&CarHeader { roots: vec![commit_cid, index_cid], version: 1, }) - .map_err(|error| Error::Internal(format!("could not encode CAR header: {error}")))?; + .context(Error::Internal, "could not encode CAR header")?; let mut output = Vec::new(); - append_section(&mut output, &header); - append_block(&mut output, &commit_cid, &commit_bytes); - append_block(&mut output, &index_cid, &index_bytes); + append_section(&mut output, &[&header])?; + append_section(&mut output, &[&commit_cid.to_bytes(), &commit_bytes])?; + append_section(&mut output, &[&index_cid.to_bytes(), &index_bytes])?; if !exclude_values { for record in records { - let bytes = serde_ipld_dagcbor::to_vec(&record.value).map_err(|error| { - Error::Internal(format!("could not encode space record: {error}")) - })?; - if dag_cbor_cid(&bytes)? != record.cid { + let bytes = serde_ipld_dagcbor::to_vec(&atproto_record(&record.value)?) + .context(Error::Internal, "could not encode space record")?; + if dag_cbor_cid(&bytes) != record.cid { return Err(Error::Internal(format!( "stored CID does not match record bytes at {}", record.path() ))); } - append_block(&mut output, &record.cid, &bytes); + append_section(&mut output, &[&record.cid.to_bytes(), &bytes])?; } } Ok(output) } -fn append_block(output: &mut Vec, cid: &Cid, bytes: &[u8]) { - let cid = cid.to_bytes(); - append_varint(output, cid.len() + bytes.len()); - output.extend_from_slice(&cid); - output.extend_from_slice(bytes); +fn append_section(output: &mut Vec, parts: &[&[u8]]) -> Result<()> { + let length = parts.iter().map(|part| part.len()).sum::(); + let mut buffer = unsigned_varint::encode::usize_buffer(); + let prefix = unsigned_varint::encode::usize(length, &mut buffer); + if output + .len() + .saturating_add(prefix.len()) + .saturating_add(length) + > MAX_REPO_EXPORT_BYTES + { + return Err(Error::xrpc( + "RepoTooBig", + format!("space repository export exceeds the {MAX_REPO_EXPORT_BYTES}-byte limit"), + )); + } + output.extend_from_slice(prefix); + for part in parts { + output.extend_from_slice(part); + } + Ok(()) } -fn append_section(output: &mut Vec, bytes: &[u8]) { - append_varint(output, bytes.len()); - output.extend_from_slice(bytes); -} +#[cfg(test)] +mod tests { + use super::*; -fn append_varint(output: &mut Vec, mut value: usize) { - loop { - let mut byte = u8::try_from(value & 0x7f).unwrap_or_default(); - value >>= 7; - if value != 0 { - byte |= 0x80; + fn previous_append_block(output: &mut Vec, cid: &Cid, bytes: &[u8]) -> Result<()> { + append_section(output, &[&cid.to_bytes(), bytes]) + } + + #[test] + fn native_block_parts_preserve_cid_versions_payloads_and_existing_prefixes() { + let cid = dag_cbor_cid(b"record"); + for cid in [ + cid, + Cid::new_v0(*cid.hash()).unwrap(), + Cid::new_v1(0x55, *cid.hash()), + ] { + for length in [0, 1, 91, 92, 93, 94, 127, 128, 16_384] { + let bytes = vec![0xa5; length]; + for prefix in [Vec::new(), b"existing CAR header".to_vec(), vec![0xff; 16]] { + let mut expected = prefix.clone(); + previous_append_block(&mut expected, &cid, &bytes).unwrap(); + let mut actual = prefix.clone(); + append_section(&mut actual, &[&cid.to_bytes(), &bytes]).unwrap(); + assert_eq!(actual, expected); + assert_eq!(&actual[..prefix.len()], prefix); + let (section_length, section) = + unsigned_varint::decode::usize(&actual[prefix.len()..]).unwrap(); + let cid_bytes = cid.to_bytes(); + assert_eq!(section_length, cid_bytes.len() + bytes.len()); + assert_eq!(§ion[..cid_bytes.len()], cid_bytes); + assert_eq!(§ion[cid_bytes.len()..], bytes); + } + } } - output.push(byte); - if value == 0 { - break; + } + + #[test] + fn native_block_parts_keep_exact_size_boundary_and_failed_buffer() { + let cid = dag_cbor_cid(b"record"); + let payload = b"payload"; + let mut block = Vec::new(); + previous_append_block(&mut block, &cid, payload).unwrap(); + let prefix_length = MAX_REPO_EXPORT_BYTES - block.len(); + let mut output = Vec::with_capacity(MAX_REPO_EXPORT_BYTES); + output.resize(prefix_length, 0x5a); + + previous_append_block(&mut output, &cid, payload).unwrap(); + assert_eq!(output.len(), MAX_REPO_EXPORT_BYTES); + assert_eq!(&output[prefix_length..], block); + assert!(output[..prefix_length].iter().all(|byte| *byte == 0x5a)); + output.truncate(prefix_length); + append_section(&mut output, &[&cid.to_bytes(), payload]).unwrap(); + assert_eq!(output.len(), MAX_REPO_EXPORT_BYTES); + assert_eq!(&output[prefix_length..], block); + assert!(output[..prefix_length].iter().all(|byte| *byte == 0x5a)); + + output.clear(); + output.resize(prefix_length + 1, 0x5a); + let previous = previous_append_block(&mut output, &cid, payload).unwrap_err(); + assert_eq!(output.len(), prefix_length + 1); + assert!(output.iter().all(|byte| *byte == 0x5a)); + let actual = append_section(&mut output, &[&cid.to_bytes(), payload]).unwrap_err(); + assert_eq!(format!("{actual:?}"), format!("{previous:?}")); + assert_eq!(actual.to_string(), previous.to_string()); + assert_eq!( + actual.to_string(), + format!("space repository export exceeds the {MAX_REPO_EXPORT_BYTES}-byte limit") + ); + assert!(matches!( + actual, + Error::Xrpc { + name: "RepoTooBig", + .. + } + )); + assert_eq!(output.len(), prefix_length + 1); + assert!(output.iter().all(|byte| *byte == 0x5a)); + } + + #[test] + fn car_record_order_matches_length_then_bytes_and_keeps_duplicate_paths_stable() { + use crate::space::commit::LexBytes; + + let commit = SignedCommit { + ver: 1, + hash: LexBytes::new(b"hash"), + ikm: LexBytes::new(b"ikm"), + sig: LexBytes::new(b"sig"), + mac: LexBytes::new(b"mac"), + rev: "3jqfcqzm3fo2j".to_owned(), + }; + let mut records = [ + ("x", "z", "short"), + ("x", "é", "unicode"), + ("aa", "a", "long collection"), + ("x", "a", "first duplicate"), + ("x", "a", "second duplicate"), + ("x", "b", "middle"), + ("x", "aa", "long key"), + ] + .map(|(collection, rkey, label)| { + let value = serde_json::json!({"label": label}); + SpaceRecord { + space: "space".to_owned(), + did: "did:example:alice".to_owned(), + collection: collection.to_owned(), + rkey: rkey.to_owned(), + cid: cpds_protocol::record_cid(&value).unwrap(), + value, + } + }) + .to_vec(); + for _ in 0..2 { + let mut expected = records.clone(); + expected.sort_by(|left, right| { + let left = left.path(); + let right = right.path(); + left.len() + .cmp(&right.len()) + .then_with(|| left.as_bytes().cmp(right.as_bytes())) + }); + assert_eq!( + expected.iter().map(SpaceRecord::path).collect::>(), + ["x/a", "x/a", "x/b", "x/z", "aa/a", "x/aa", "x/é"] + ); + let bytes = serialize_repo(&commit, records.clone(), false).unwrap(); + assert_eq!( + car_record_cids(&bytes), + expected.iter().map(|record| record.cid).collect::>() + ); + records.reverse(); } } -} -fn canonical_path_cmp(left: &str, right: &str) -> Ordering { - left.len() - .cmp(&right.len()) - .then_with(|| left.as_bytes().cmp(right.as_bytes())) -} + fn car_record_cids(mut bytes: &[u8]) -> Vec { + let mut sections = Vec::new(); + while !bytes.is_empty() { + let (length, body) = unsigned_varint::decode::usize(bytes).unwrap(); + let (section, remaining) = body.split_at(length); + sections.push(section); + bytes = remaining; + } + sections + .into_iter() + .skip(3) + .map(|section| Cid::read_bytes(std::io::Cursor::new(section)).unwrap()) + .collect() + } -trait RecordPath { - fn path(&self) -> String; -} + #[test] + fn native_varints_match_the_previous_encoder_at_every_prefix_boundary() { + let mut lengths = vec![0, 1, usize::MAX]; + for bit in (7..usize::BITS).step_by(7) { + let boundary = 1_usize << bit; + lengths.extend([boundary - 1, boundary, boundary + 1]); + } + for length in lengths { + let mut expected = Vec::new(); + previous_varint(&mut expected, length); + let mut buffer = unsigned_varint::encode::usize_buffer(); + let actual = unsigned_varint::encode::usize(length, &mut buffer); + assert_eq!(actual, expected, "varint length {length}"); + } + } + + #[test] + fn car_sections_preserve_varint_prefixes_and_all_parts() { + for length in [0, 1, 127, 128, 129, 16_383, 16_384, 16_385] { + let bytes = vec![b'a'; length]; + let mut expected = Vec::new(); + previous_varint(&mut expected, length); + expected.extend_from_slice(&bytes); + let mut actual = Vec::new(); + let (left, right) = bytes.split_at(length / 2); + append_section(&mut actual, &[left, right]).unwrap(); + assert_eq!(actual, expected, "CAR section length {length}"); + } + } + + #[test] + fn car_sections_are_bounded_before_mutating_the_buffer() { + let mut bytes = vec![0; MAX_REPO_EXPORT_BYTES - 2]; + append_section(&mut bytes, &[b"a"]).unwrap(); + assert_eq!(bytes.len(), MAX_REPO_EXPORT_BYTES); + let error = append_section(&mut bytes, &[b"b"]).unwrap_err(); + assert!(matches!( + error, + Error::Xrpc { + name: "RepoTooBig", + .. + } + )); + assert_eq!(bytes.len(), MAX_REPO_EXPORT_BYTES); + assert!( + bytes[..MAX_REPO_EXPORT_BYTES - 2] + .iter() + .all(|byte| *byte == 0) + ); + assert_eq!(&bytes[MAX_REPO_EXPORT_BYTES - 2..], b"\x01a"); + } -impl RecordPath for SpaceRecord { - fn path(&self) -> String { - format!("{}/{}", self.collection, self.rkey) + fn previous_varint(output: &mut Vec, mut value: usize) { + loop { + let mut byte = u8::try_from(value & 0x7f).unwrap_or_default(); + value >>= 7; + if value != 0 { + byte |= 0x80; + } + output.push(byte); + if value == 0 { + break; + } + } } } diff --git a/src/space/commit.rs b/src/space/commit.rs index 95dc89d..dd3fb80 100644 --- a/src/space/commit.rs +++ b/src/space/commit.rs @@ -1,6 +1,6 @@ //! Cryptographic commits for permissioned Spaces repositories. -use atrium_crypto::keypair::{Did as _, Secp256k1Keypair}; +use atrium_crypto::keypair::Secp256k1Keypair; use base64::{Engine as _, engine::general_purpose::STANDARD_NO_PAD}; use hkdf::Hkdf; use hmac::{Hmac, Mac}; @@ -10,7 +10,7 @@ use serde_bytes::ByteBuf; use sha2::{Digest, Sha256}; use crate::{ - error::{Error, Result}, + error::{Error, ErrorContext as _, Result}, space::domain::SpaceRecord, }; @@ -18,74 +18,46 @@ const LANES: usize = 1024; const STATE_BYTES: usize = LANES * 2; const DOMAIN: &[u8] = b"atproto-space-v1"; -pub(crate) struct CommitSigner { - key: Secp256k1Keypair, - public_key_multibase: String, -} - -impl CommitSigner { - pub(crate) fn new(secret: &[u8; 32]) -> Result { - let key = Secp256k1Keypair::import(secret).map_err(|error| { - Error::Configuration(format!("invalid account secp256k1 signing key: {error}")) - })?; - let did_key = key.did(); - let public_key_multibase = did_key - .strip_prefix("did:key:") - .ok_or_else(|| Error::Internal("ATrium returned an invalid did:key".to_owned()))? - .to_owned(); - Ok(Self { - key, - public_key_multibase, - }) - } +pub(crate) fn sign( + key: &Secp256k1Keypair, + space: &str, + author: &str, + rev: &str, + records: &[SpaceRecord], +) -> Result { + let hash = set_hash_digest(records); + let mut ikm = [0_u8; 32]; + rand::rng().fill(&mut ikm); + let context = encode_context(space, author, rev, &ikm)?; + let signature = key + .sign(&context) + .context(Error::Internal, "commit signing failed")?; - pub(crate) fn public_key_multibase(&self) -> &str { - &self.public_key_multibase - } - - pub(crate) fn sign( - &self, - space: &str, - author: &str, - rev: &str, - records: &[SpaceRecord], - ) -> Result { - let hash = set_hash_digest(records); - let mut ikm = [0_u8; 32]; - rand::rng().fill(&mut ikm); - let context = encode_context(space, author, rev, &ikm)?; - let signature = self - .key - .sign(&context) - .map_err(|error| Error::Internal(format!("commit signing failed: {error}")))?; - - let hkdf = Hkdf::::from_prk(&ikm) - .map_err(|error| Error::Internal(format!("commit HKDF failed: {error}")))?; - let mut mac_key = [0_u8; 32]; - hkdf.expand(&context, &mut mac_key) - .map_err(|error| Error::Internal(format!("commit HKDF failed: {error}")))?; - let mut mac = Hmac::::new_from_slice(&mac_key) - .map_err(|error| Error::Internal(format!("commit HMAC failed: {error}")))?; - mac.update(&hash); + let hkdf = Hkdf::::from_prk(&ikm).context(Error::Internal, "commit HKDF failed")?; + let mut mac_key = [0_u8; 32]; + hkdf.expand(&context, &mut mac_key) + .context(Error::Internal, "commit HKDF failed")?; + let mut mac = + Hmac::::new_from_slice(&mac_key).context(Error::Internal, "commit HMAC failed")?; + mac.update(&hash); - Ok(SignedCommit { - ver: 1, - hash: LexBytes::new(hash), - ikm: LexBytes::new(ikm), - sig: LexBytes::new(signature), - mac: LexBytes::new(mac.finalize().into_bytes()), - rev: rev.to_owned(), - }) - } + Ok(SignedCommit { + ver: 1, + hash: LexBytes::new(hash), + ikm: LexBytes::new(ikm), + sig: LexBytes::new(signature), + mac: LexBytes::new(mac.finalize().into_bytes()), + rev: rev.to_owned(), + }) } #[derive(Clone, Debug, Serialize)] -pub(crate) struct SignedCommit { +pub(crate) struct SignedCommit { pub(crate) ver: u8, - pub(crate) hash: LexBytes, - pub(crate) ikm: LexBytes, - pub(crate) sig: LexBytes, - pub(crate) mac: LexBytes, + pub(crate) hash: Bytes, + pub(crate) ikm: Bytes, + pub(crate) sig: Bytes, + pub(crate) mac: Bytes, pub(crate) rev: String, } @@ -103,33 +75,23 @@ impl LexBytes { } pub(crate) fn decode(&self) -> Result> { - STANDARD_NO_PAD.decode(&self.value).map_err(|error| { - Error::Internal(format!("invalid signed commit byte encoding: {error}")) - }) + STANDARD_NO_PAD + .decode(&self.value) + .context(Error::Internal, "invalid signed commit byte encoding") } } impl SignedCommit { pub(crate) fn dag_cbor(&self) -> Result> { - #[derive(Serialize)] - struct CborCommit<'a> { - ver: u8, - hash: ByteBuf, - ikm: ByteBuf, - sig: ByteBuf, - mac: ByteBuf, - rev: &'a str, - } - - serde_ipld_dagcbor::to_vec(&CborCommit { + serde_ipld_dagcbor::to_vec(&SignedCommit:: { ver: self.ver, hash: self.hash.decode()?.into(), ikm: self.ikm.decode()?.into(), sig: self.sig.decode()?.into(), mac: self.mac.decode()?.into(), - rev: &self.rev, + rev: self.rev.clone(), }) - .map_err(|error| Error::Internal(format!("could not encode signed commit: {error}"))) + .context(Error::Internal, "could not encode signed commit") } } @@ -145,11 +107,7 @@ pub(crate) fn set_hash_digest(records: &[SpaceRecord]) -> [u8; 32] { *lane = lane.wrapping_add(u16::from_le_bytes([bytes[0], bytes[1]])); } } - let mut state = [0_u8; STATE_BYTES]; - for (lane, bytes) in lanes.iter().zip(state.chunks_exact_mut(2)) { - bytes.copy_from_slice(&lane.to_le_bytes()); - } - Sha256::digest(state).into() + Sha256::digest(lanes.map(u16::to_le_bytes).as_flattened()).into() } fn encode_context(space: &str, author: &str, rev: &str, ikm: &[u8]) -> Result> { @@ -170,16 +128,44 @@ fn encode_context(space: &str, author: &str, rev: &str, ikm: &[u8]) -> Result SpaceRecord { SpaceRecord { @@ -212,25 +198,59 @@ mod tests { assert_ne!(set_hash_digest(&[a]), set_hash_digest(&[moved])); } + #[test] + fn set_hash_retains_little_endian_lanes_and_wrapping_addition() { + let record = record( + "com.example.post", + "é-one", + "bafyreidefdycgbfy3oglcb6ism3eqhyp5llsrpzxjsuac2gsy4mtrtx244", + ); + let element = format!("{}/{}/{}", record.collection, record.rkey, record.cid); + let mut expanded = [0_u8; super::STATE_BYTES]; + blake3::Hasher::new() + .update(element.as_bytes()) + .finalize_xof() + .fill(&mut expanded); + let lanes: Vec<_> = expanded + .chunks_exact(2) + .map(|bytes| u16::from_le_bytes([bytes[0], bytes[1]])) + .collect(); + assert!(lanes.iter().any(|&lane| lane > u16::MAX / 2)); + assert!(lanes.iter().any(|&lane| lane != lane.swap_bytes())); + + for copies in [0_u16, 1, 2, 3, 257] { + let mut state = [0_u8; super::STATE_BYTES]; + for (lane, bytes) in lanes.iter().zip(state.chunks_exact_mut(2)) { + bytes.copy_from_slice(&lane.wrapping_mul(copies).to_le_bytes()); + } + let expected: [u8; 32] = Sha256::digest(state).into(); + assert_eq!( + set_hash_digest(&vec![record.clone(); usize::from(copies)]), + expected, + "{copies} copies" + ); + } + } + #[test] fn each_signed_view_uses_fresh_binding_material() { - let signer = CommitSigner::new(&[7; 32]).unwrap(); - let first = signer - .sign( - "at://did:example:o/space/com.example.x/a", - "did:example:a", - "3kbcq3p7ad400", - &[], - ) - .unwrap(); - let second = signer - .sign( - "at://did:example:o/space/com.example.x/a", - "did:example:a", - "3kbcq3p7ad400", - &[], - ) - .unwrap(); + let signer = Secp256k1Keypair::import(&[7; 32]).unwrap(); + let first = sign( + &signer, + "at://did:example:o/space/com.example.x/a", + "did:example:a", + "3kbcq3p7ad400", + &[], + ) + .unwrap(); + let second = sign( + &signer, + "at://did:example:o/space/com.example.x/a", + "did:example:a", + "3kbcq3p7ad400", + &[], + ) + .unwrap(); assert_eq!(first.hash, second.hash); assert_ne!(first.ikm, second.ikm); assert_ne!(first.mac, second.mac); @@ -239,14 +259,14 @@ mod tests { #[test] fn commit_signature_and_mac_verify_against_the_published_key() { - let signer = CommitSigner::new(&[7; 32]).unwrap(); + let signer = Secp256k1Keypair::import(&[7; 32]).unwrap(); let space = "at://did:example:o/space/com.example.x/a"; let author = "did:example:a"; - let commit = signer.sign(space, author, "3kbcq3p7ad400", &[]).unwrap(); + let commit = sign(&signer, space, author, "3kbcq3p7ad400", &[]).unwrap(); let ikm = STANDARD_NO_PAD.decode(&commit.ikm.value).unwrap(); let context = encode_context(space, author, &commit.rev, &ikm).unwrap(); verify_signature( - &format!("did:key:{}", signer.public_key_multibase()), + &atrium_crypto::keypair::Did::did(&signer), &context, &STANDARD_NO_PAD.decode(&commit.sig.value).unwrap(), ) diff --git a/src/space/database/auth.rs b/src/space/database/auth.rs index 6d9b912..82ee327 100644 --- a/src/space/database/auth.rs +++ b/src/space/database/auth.rs @@ -14,18 +14,17 @@ impl Database { jti: &str, expires_at: i64, ) -> Result { - let mut connection = self.connection()?; - let transaction = connection.transaction()?; - transaction.execute( - "DELETE FROM space_auth_replays WHERE expires_at < ?1", - params![now()], - )?; - let inserted = transaction.execute( - "INSERT OR IGNORE INTO space_auth_replays (kind, issuer, jti, expires_at) + self.transaction(|transaction| { + transaction.execute( + "DELETE FROM space_auth_replays WHERE expires_at < ?1", + params![now()], + )?; + let inserted = transaction.execute( + "INSERT OR IGNORE INTO space_auth_replays (kind, issuer, jti, expires_at) VALUES (?1, ?2, ?3, ?4)", - params![kind, issuer, jti, expires_at], - )?; - transaction.commit()?; - Ok(inserted == 1) + params![kind, issuer, jti, expires_at], + )?; + Ok(inserted == 1) + }) } } diff --git a/src/space/database/metadata.rs b/src/space/database/metadata.rs new file mode 100644 index 0000000..e8c569e --- /dev/null +++ b/src/space/database/metadata.rs @@ -0,0 +1,220 @@ +//! Hosted Space metadata, membership, writer announcements, and notification subscriptions. + +use crate::space::{NotifiedWrite, NotifyRegistrationInput, SpacePageQuery, UpdateSpaceInput}; +use rusqlite::params; + +use crate::{ + database::{Database, constraint_conflict, now, sql_row}, + error::{Error, Result}, + space::domain::{Space, SpaceWriter}, +}; + +impl Database { + pub(crate) fn space_is_deleted(&self, uri: &str) -> Result> { + self.query_optional( + "SELECT deleted_at IS NOT NULL FROM spaces WHERE uri = ?1", + [uri], + |row| Ok(row.get(0)?), + ) + } + + pub(crate) fn create_space(&self, space: &Space) -> Result<()> { + let policy = serde_json::to_string(&space.policy)?; + let app_access = serde_json::to_string(&space.app_access)?; + self.transaction(|transaction| { + let changed = transaction + .execute( + "INSERT INTO spaces + (uri, authority, space_type, skey, policy_json, app_access_json, + created_at, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?7) + ON CONFLICT(uri) DO UPDATE SET + policy_json = excluded.policy_json, + app_access_json = excluded.app_access_json, + deleted_at = NULL, + created_at = excluded.created_at, + updated_at = excluded.updated_at + WHERE spaces.deleted_at IS NOT NULL", + params![ + space.uri, + space.authority, + space.kind, + space.skey, + policy, + app_access, + now(), + ], + ) + .map_err(|error| constraint_conflict(error, "space already exists"))?; + if changed == 0 { + return Err(Error::Conflict("space already exists".to_owned())); + } + transaction.execute( + "DELETE FROM space_members WHERE space_uri = ?1", + [&space.uri], + )?; + transaction.execute( + "INSERT INTO space_members (space_uri, did, created_at) VALUES (?1, ?2, ?3)", + params![space.uri, space.authority, now()], + )?; + Ok(()) + }) + } + + pub(crate) fn space(&self, uri: &str) -> Result { + self.query_optional( + "SELECT authority, space_type, skey, policy_json, app_access_json + FROM spaces WHERE uri = ?1 AND deleted_at IS NULL", + [uri], + |row| { + let (authority, kind, skey, policy, app_access) = + <(_, _, _, String, String)>::try_from(row)?; + Ok(Space { + uri: uri.to_owned(), + authority, + kind, + skey, + policy: serde_json::from_str(&policy)?, + app_access: serde_json::from_str(&app_access)?, + }) + }, + )? + .ok_or(Error::NotFound) + } + + pub(crate) fn update_space(&self, input: &UpdateSpaceInput) -> Result<()> { + let policy = input + .policy + .as_ref() + .map(serde_json::to_string) + .transpose()?; + let app_access = input + .app_access + .as_ref() + .map(serde_json::to_string) + .transpose()?; + let changed = self.connection()?.execute( + "UPDATE spaces + SET policy_json = COALESCE(?2, policy_json), + app_access_json = COALESCE(?3, app_access_json), updated_at = ?4 + WHERE uri = ?1 AND deleted_at IS NULL", + params![input.space, policy, app_access, now()], + )?; + if changed == 0 { + return Err(Error::NotFound); + } + Ok(()) + } + + pub(crate) fn delete_space(&self, uri: &str, authority: &str) -> Result<()> { + self.transaction(|transaction| { + transaction.execute( + "UPDATE spaces SET deleted_at = COALESCE(deleted_at, ?3), updated_at = ?3 + WHERE uri = ?1 AND authority = ?2", + params![uri, authority, now()], + )?; + transaction.execute( + "DELETE FROM space_records WHERE space_uri = ?1 AND did = ?2", + params![uri, authority], + )?; + transaction.execute( + "DELETE FROM space_repo_ops WHERE space_uri = ?1 AND did = ?2", + params![uri, authority], + )?; + Ok(()) + }) + } + + pub(crate) fn add_space_member(&self, uri: &str, did: &str) -> Result<()> { + self.connection()?.execute( + "INSERT OR IGNORE INTO space_members (space_uri, did, created_at) + VALUES (?1, ?2, ?3)", + params![uri, did, now()], + )?; + Ok(()) + } + + pub(crate) fn remove_space_member(&self, uri: &str, did: &str) -> Result<()> { + self.connection()?.execute( + "DELETE FROM space_members WHERE space_uri = ?1 AND did = ?2", + params![uri, did], + )?; + Ok(()) + } + + pub(crate) fn list_space_members(&self, query: &SpacePageQuery) -> Result> { + let limit = i64::try_from(query.limit.unwrap_or(100).clamp(1, 1000)).unwrap_or(i64::MAX); + self.query( + "SELECT did FROM space_members + WHERE space_uri = ?1 AND (?2 IS NULL OR did > ?2) + ORDER BY did LIMIT ?3", + params![query.space, query.cursor, limit], + |row| Ok(row.get(0)?), + ) + } + + pub(crate) fn is_space_member(&self, uri: &str, did: &str) -> Result { + self.connection()? + .prepare("SELECT 1 FROM space_members WHERE space_uri = ?1 AND did = ?2")? + .exists(params![uri, did]) + .map_err(Into::into) + } + + pub(crate) fn record_space_writer(&self, input: &NotifiedWrite<'_>) -> Result<()> { + self.connection()?.execute( + "INSERT INTO space_writers (space_uri, did, rev, hash, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5) + ON CONFLICT(space_uri, did) DO UPDATE SET + rev = excluded.rev, hash = excluded.hash, updated_at = excluded.updated_at", + params![input.space, input.repo, input.rev, input.hash, now()], + )?; + Ok(()) + } + + pub(crate) fn list_space_writers(&self, query: &SpacePageQuery) -> Result> { + let limit = i64::try_from(query.limit.unwrap_or(100).clamp(1, 1000)).unwrap_or(i64::MAX); + self.query( + "SELECT did, rev, hash FROM space_writers + WHERE space_uri = ?1 AND (?2 IS NULL OR did > ?2) + ORDER BY did LIMIT ?3", + params![query.space, query.cursor, limit], + |row| Ok(sql_row!(row => SpaceWriter { did: 0, rev: 1, hash: 2 })), + ) + } + + pub(crate) fn register_space_notify( + &self, + input: &NotifyRegistrationInput, + endpoint: &str, + expires_at: i64, + ) -> Result<()> { + self.connection()?.execute( + "INSERT INTO space_notify_registrations + (space_uri, service, endpoint, expires_at, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5) + ON CONFLICT(space_uri, service) DO UPDATE SET + endpoint = excluded.endpoint, + expires_at = excluded.expires_at, + updated_at = excluded.updated_at", + params![input.space, input.service, endpoint, expires_at, now()], + )?; + Ok(()) + } + + pub(crate) fn unregister_space_notify(&self, input: &NotifyRegistrationInput) -> Result<()> { + self.connection()?.execute( + "DELETE FROM space_notify_registrations WHERE space_uri = ?1 AND service = ?2", + params![input.space, input.service], + )?; + Ok(()) + } + + pub(crate) fn space_notify_registrations(&self, space: &str) -> Result> { + self.query( + "SELECT service, endpoint FROM space_notify_registrations + WHERE space_uri = ?1 AND expires_at > ?2 ORDER BY service", + params![space, now()], + |row| Ok(row.try_into()?), + ) + } +} diff --git a/src/space/database/mod.rs b/src/space/database/mod.rs index 9e2eb1b..ea47de7 100644 --- a/src/space/database/mod.rs +++ b/src/space/database/mod.rs @@ -1,4 +1,8 @@ //! `SQLite` implementations used by the Spaces feature. mod auth; +mod metadata; mod repository; + +#[cfg(test)] +mod tests; diff --git a/src/space/database/repository.rs b/src/space/database/repository.rs index 6e4edee..8a55d50 100644 --- a/src/space/database/repository.rs +++ b/src/space/database/repository.rs @@ -1,453 +1,129 @@ //! Persistence for spaces and their per-user repositories. -use std::str::FromStr; - use cid::Cid; -use rusqlite::{OptionalExtension, Transaction, params}; -use serde_json::Value; +use rusqlite::{Connection, OptionalExtension, Row, Transaction, params}; +use crate::space::{ListBlobsQuery, ListSpacesQuery, RecordQuery}; use crate::{ - database::{Database, active_blob_ref_cids, now}, - error::{Error, Result}, - space::domain::{NotifyRegistration, Space, SpaceOp, SpaceRecord, SpaceWrite, SpaceWriter}, + database::{ + Database, active_blob_ref_cids, constraint_conflict, now, query, require_account_active, + }, + error::{Error, ErrorContext as _, Result}, + space::domain::{SpaceOp, SpaceRecord, SpaceWrite}, }; impl Database { - pub(crate) fn space_is_deleted(&self, uri: &str) -> Result> { - self.connection()? - .query_row( - "SELECT deleted_at IS NOT NULL FROM spaces WHERE uri = ?1", - [uri], - |row| row.get(0), - ) - .optional() - .map_err(Into::into) - } - - pub(crate) fn create_space(&self, space: &Space) -> Result<()> { - let policy = serde_json::to_string(&space.policy)?; - let app_access = serde_json::to_string(&space.app_access)?; - let mut connection = self.connection()?; - let transaction = connection.transaction()?; - let changed = transaction - .execute( - "INSERT INTO spaces - (uri, authority, space_type, skey, policy_json, app_access_json, - created_at, updated_at) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?7) - ON CONFLICT(uri) DO UPDATE SET - policy_json = excluded.policy_json, - app_access_json = excluded.app_access_json, - deleted_at = NULL, - created_at = excluded.created_at, - updated_at = excluded.updated_at - WHERE spaces.deleted_at IS NOT NULL", - params![ - space.uri, - space.authority, - space.kind, - space.skey, - policy, - app_access, - now(), - ], - ) - .map_err(map_space_conflict)?; - if changed == 0 { - return Err(Error::Conflict("space already exists".to_owned())); - } - transaction.execute( - "DELETE FROM space_members WHERE space_uri = ?1", - [&space.uri], - )?; - transaction.execute( - "INSERT INTO space_members (space_uri, did, created_at) VALUES (?1, ?2, ?3)", - params![space.uri, space.authority, now()], - )?; - transaction.commit()?; - Ok(()) - } - - pub(crate) fn space(&self, uri: &str) -> Result { - self.connection()? - .query_row( - "SELECT authority, space_type, skey, policy_json, app_access_json - FROM spaces WHERE uri = ?1 AND deleted_at IS NULL", - [uri], - |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, String>(2)?, - row.get::<_, String>(3)?, - row.get::<_, String>(4)?, - )) - }, - ) - .optional()? - .map(|(authority, space_type, skey, policy, app_access)| { - Ok::(Space { - uri: uri.to_owned(), - authority, - kind: space_type, - skey, - policy: serde_json::from_str(&policy)?, - app_access: serde_json::from_str(&app_access)?, - }) - }) - .transpose()? - .ok_or(Error::NotFound) - } - - pub(crate) fn update_space( - &self, - uri: &str, - policy: Option<&Value>, - app_access: Option<&Value>, - ) -> Result<()> { - let policy = policy.map(serde_json::to_string).transpose()?; - let app_access = app_access.map(serde_json::to_string).transpose()?; - let changed = self.connection()?.execute( - "UPDATE spaces - SET policy_json = COALESCE(?2, policy_json), - app_access_json = COALESCE(?3, app_access_json), updated_at = ?4 - WHERE uri = ?1 AND deleted_at IS NULL", - params![uri, policy, app_access, now()], - )?; - if changed == 0 { - return Err(Error::NotFound); - } - Ok(()) - } - - pub(crate) fn delete_space(&self, uri: &str, authority: &str) -> Result<()> { - let mut connection = self.connection()?; - let transaction = connection.transaction()?; - transaction.execute( - "UPDATE spaces SET deleted_at = COALESCE(deleted_at, ?3), updated_at = ?3 - WHERE uri = ?1 AND authority = ?2", - params![uri, authority, now()], - )?; - transaction.execute( - "DELETE FROM space_records WHERE space_uri = ?1 AND did = ?2", - params![uri, authority], - )?; - transaction.execute( - "DELETE FROM space_repo_ops WHERE space_uri = ?1 AND did = ?2", - params![uri, authority], - )?; - transaction.commit()?; - Ok(()) - } - - pub(crate) fn add_space_member(&self, uri: &str, did: &str) -> Result<()> { - self.connection()?.execute( - "INSERT OR IGNORE INTO space_members (space_uri, did, created_at) - VALUES (?1, ?2, ?3)", - params![uri, did, now()], - )?; - Ok(()) - } - - pub(crate) fn remove_space_member(&self, uri: &str, did: &str) -> Result<()> { - self.connection()?.execute( - "DELETE FROM space_members WHERE space_uri = ?1 AND did = ?2", - params![uri, did], - )?; - Ok(()) - } - - pub(crate) fn list_space_members( - &self, - uri: &str, - cursor: Option<&str>, - limit: usize, - ) -> Result> { - let connection = self.connection()?; - let mut statement = connection.prepare( - "SELECT did FROM space_members - WHERE space_uri = ?1 AND (?2 IS NULL OR did > ?2) - ORDER BY did LIMIT ?3", - )?; - let rows = statement.query_map( - params![uri, cursor, i64::try_from(limit).unwrap_or(i64::MAX)], - |row| row.get(0), - )?; - rows.collect::>() - .map_err(Into::into) - } - - pub(crate) fn is_space_member(&self, uri: &str, did: &str) -> Result { - Ok(self - .connection()? - .query_row( - "SELECT 1 FROM space_members WHERE space_uri = ?1 AND did = ?2", - params![uri, did], - |_| Ok(()), - ) - .optional()? - .is_some()) - } - - pub(crate) fn record_space_writer( + pub(crate) fn space_snapshot( &self, space: &str, did: &str, - rev: &str, - hash: &[u8], - ) -> Result<()> { - self.connection()?.execute( - "INSERT INTO space_writers (space_uri, did, rev, hash, updated_at) - VALUES (?1, ?2, ?3, ?4, ?5) - ON CONFLICT(space_uri, did) DO UPDATE SET - rev = excluded.rev, hash = excluded.hash, updated_at = excluded.updated_at", - params![space, did, rev, hash, now()], - )?; - Ok(()) - } - - pub(crate) fn list_space_writers( - &self, - space: &str, - cursor: Option<&str>, - limit: usize, - ) -> Result> { - let connection = self.connection()?; - let mut statement = connection.prepare( - "SELECT did, rev, hash FROM space_writers - WHERE space_uri = ?1 AND (?2 IS NULL OR did > ?2) - ORDER BY did LIMIT ?3", - )?; - let rows = statement.query_map( - params![space, cursor, i64::try_from(limit).unwrap_or(i64::MAX)], - |row| { - Ok(SpaceWriter { - did: row.get(0)?, - rev: row.get(1)?, - hash: row.get(2)?, - }) - }, - )?; - rows.collect::>() - .map_err(Into::into) - } - - pub(crate) fn register_space_notify( - &self, - space: &str, - service: &str, - endpoint: &str, - expires_at: i64, - ) -> Result<()> { - self.connection()?.execute( - "INSERT INTO space_notify_registrations - (space_uri, service, endpoint, expires_at, updated_at) - VALUES (?1, ?2, ?3, ?4, ?5) - ON CONFLICT(space_uri, service) DO UPDATE SET - endpoint = excluded.endpoint, - expires_at = excluded.expires_at, - updated_at = excluded.updated_at", - params![space, service, endpoint, expires_at, now()], - )?; - Ok(()) + max_bytes: usize, + ) -> Result<(String, Vec)> { + self.transaction(|transaction| { + require_account_active(transaction, did)?; + let rev = load_latest_space_rev(transaction, space, did)?; + let size: i64 = transaction.query_row( + "SELECT COALESCE(SUM(length(CAST(record_json AS BLOB))), 0) + FROM space_records WHERE space_uri = ?1 AND did = ?2", + params![space, did], + |row| row.get(0), + )?; + if size > i64::try_from(max_bytes).unwrap_or(i64::MAX) { + return Err(Error::xrpc( + "RepoTooBig", + "Space repository exceeds the export size limit", + )); + } + let records = query( + transaction, + "SELECT space_uri, did, collection, rkey, cid, record_json FROM space_records + WHERE space_uri = ?1 AND did = ?2 ORDER BY collection, rkey", + params![space, did], + decode_record, + )?; + Ok((rev, records)) + }) } - pub(crate) fn unregister_space_notify(&self, space: &str, service: &str) -> Result<()> { - self.connection()?.execute( - "DELETE FROM space_notify_registrations WHERE space_uri = ?1 AND service = ?2", - params![space, service], - )?; - Ok(()) + pub(crate) fn space_blob_cids(&self, query: &ListBlobsQuery) -> Result> { + let limit = query.limit.unwrap_or(500).clamp(1, 1000); + self.query( + "SELECT DISTINCT b.cid FROM space_record_blobs b + WHERE b.space_uri = ?1 AND b.did = ?2 + AND (?3 IS NULL OR EXISTS ( + SELECT 1 FROM space_repo_ops o + WHERE o.space_uri = b.space_uri AND o.did = b.did + AND o.collection = b.collection AND o.rkey = b.rkey AND o.rev > ?3 + )) + AND (?4 IS NULL OR b.cid > ?4) + ORDER BY b.cid LIMIT ?5", + params![ + query.space, + query.repo, + query.since, + query.cursor, + i64::try_from(limit).unwrap_or(i64::MAX) + ], + |row| Ok(row.get(0)?), + ) } - pub(crate) fn space_notify_registrations( - &self, - space: &str, - ) -> Result> { - let connection = self.connection()?; - let mut statement = connection.prepare( - "SELECT service, endpoint FROM space_notify_registrations - WHERE space_uri = ?1 AND expires_at > ?2 ORDER BY service", - )?; - let rows = statement.query_map(params![space, now()], |row| { - Ok(NotifyRegistration { - service: row.get(0)?, - endpoint: row.get(1)?, - }) - })?; - rows.collect::>() - .map_err(Into::into) + pub(crate) fn space_references_blob(&self, space: &str, did: &str, cid: &Cid) -> Result { + self.query_one( + "SELECT EXISTS(SELECT 1 FROM space_record_blobs + WHERE space_uri = ?1 AND did = ?2 AND cid = ?3)", + params![space, did, cid.to_string()], + |row| Ok(row.get(0)?), + ) } pub(crate) fn create_space_record(&self, record: &SpaceRecord, rev: &str) -> Result<()> { - let mut connection = self.connection()?; - let transaction = connection.transaction()?; - insert_space_record(&transaction, record).map_err(map_record_conflict)?; - replace_space_record_blob_refs(&transaction, record)?; - insert_op(&transaction, record, rev, None, Some(&record.value))?; - transaction.commit()?; - Ok(()) + self.transaction(|transaction| write_record(transaction, record, rev, WriteMode::Create)) } pub(crate) fn put_space_record(&self, record: &SpaceRecord, rev: &str) -> Result<()> { - let mut connection = self.connection()?; - let transaction = connection.transaction()?; - let previous = record_cid(&transaction, record)?; - let json = serde_json::to_string(&record.value)?; - transaction.execute( - "INSERT INTO space_records - (space_uri, did, collection, rkey, cid, record_json, created_at, updated_at) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?7) - ON CONFLICT(space_uri, did, collection, rkey) DO UPDATE SET - cid = excluded.cid, record_json = excluded.record_json, - updated_at = excluded.updated_at", - params![ - record.space, - record.did, - record.collection, - record.rkey, - record.cid.to_string(), - json, - now(), - ], - )?; - replace_space_record_blob_refs(&transaction, record)?; - insert_op( - &transaction, - record, - rev, - previous.as_deref(), - Some(&record.value), - )?; - transaction.commit()?; - Ok(()) + self.transaction(|transaction| write_record(transaction, record, rev, WriteMode::Put)) } pub(crate) fn apply_space_writes(&self, writes: &[SpaceWrite], rev: &str) -> Result<()> { if writes.is_empty() { return Ok(()); } - let mut connection = self.connection()?; - let transaction = connection.transaction()?; - for write in writes { - match write { - SpaceWrite::Create(record) => { - insert_space_record(&transaction, record).map_err(map_record_conflict)?; - replace_space_record_blob_refs(&transaction, record)?; - insert_op(&transaction, record, rev, None, Some(&record.value))?; - } - SpaceWrite::Update(record) => { - let previous = record_cid(&transaction, record)?.ok_or(Error::NotFound)?; - transaction.execute( - "UPDATE space_records SET cid = ?5, record_json = ?6, updated_at = ?7 - WHERE space_uri = ?1 AND did = ?2 AND collection = ?3 AND rkey = ?4", - params![ - record.space, - record.did, - record.collection, - record.rkey, - record.cid.to_string(), - serde_json::to_string(&record.value)?, - now(), - ], - )?; - replace_space_record_blob_refs(&transaction, record)?; - insert_op( - &transaction, - record, - rev, - Some(&previous), - Some(&record.value), - )?; - } - SpaceWrite::Delete { - space, - did, - collection, - rkey, - } => { - let previous: String = transaction - .query_row( - "SELECT cid FROM space_records - WHERE space_uri = ?1 AND did = ?2 AND collection = ?3 AND rkey = ?4", - params![space, did, collection, rkey], - |row| row.get(0), - ) - .optional()? - .ok_or(Error::NotFound)?; - transaction.execute( - "DELETE FROM space_records - WHERE space_uri = ?1 AND did = ?2 AND collection = ?3 AND rkey = ?4", - params![space, did, collection, rkey], - )?; - transaction.execute( - "INSERT INTO space_repo_ops - (space_uri, did, rev, collection, rkey, cid, prev, created_at) - VALUES (?1, ?2, ?3, ?4, ?5, NULL, ?6, ?7)", - params![space, did, rev, collection, rkey, previous, now()], - )?; + self.transaction(|transaction| { + for write in writes { + match write { + SpaceWrite::Create(record) => { + write_record(transaction, record, rev, WriteMode::Create)?; + } + SpaceWrite::Update(record) => { + write_record(transaction, record, rev, WriteMode::Update)?; + } + SpaceWrite::Delete(query) => { + if !delete_record(transaction, query, rev)? { + return Err(Error::NotFound); + } + } } } - } - transaction.commit()?; - Ok(()) + Ok(()) + }) } - pub(crate) fn delete_space_record( - &self, - space: &str, - did: &str, - collection: &str, - rkey: &str, - rev: &str, - ) -> Result<()> { - let mut connection = self.connection()?; - let transaction = connection.transaction()?; - let previous: Option = transaction - .query_row( - "SELECT cid FROM space_records - WHERE space_uri = ?1 AND did = ?2 AND collection = ?3 AND rkey = ?4", - params![space, did, collection, rkey], - |row| row.get(0), - ) - .optional()?; - if previous.is_none() { - transaction.commit()?; - return Ok(()); - } - transaction.execute( - "DELETE FROM space_records - WHERE space_uri = ?1 AND did = ?2 AND collection = ?3 AND rkey = ?4", - params![space, did, collection, rkey], - )?; - transaction.execute( - "INSERT INTO space_repo_ops - (space_uri, did, rev, collection, rkey, cid, prev, created_at) - VALUES (?1, ?2, ?3, ?4, ?5, NULL, ?6, ?7)", - params![space, did, rev, collection, rkey, previous, now()], - )?; - transaction.commit()?; - Ok(()) + pub(crate) fn delete_space_record(&self, query: &RecordQuery, rev: &str) -> Result<()> { + self.transaction(|transaction| { + delete_record(transaction, query, rev)?; + Ok(()) + }) } - pub(crate) fn get_space_record( - &self, - space: &str, - did: &str, - collection: &str, - rkey: &str, - ) -> Result { - self.connection()? - .query_row( - "SELECT cid, record_json FROM space_records + pub(crate) fn get_space_record(&self, query: &RecordQuery) -> Result { + self.query_optional( + "SELECT space_uri, did, collection, rkey, cid, record_json FROM space_records WHERE space_uri = ?1 AND did = ?2 AND collection = ?3 AND rkey = ?4", - params![space, did, collection, rkey], - |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), - ) - .optional()? - .map(|(cid, json)| decode_record(space, did, collection, rkey, &cid, &json)) - .transpose()? - .ok_or(Error::NotFound) + params![query.space, query.repo, query.collection, query.rkey], + decode_record, + )? + .ok_or(Error::NotFound) } pub(crate) fn list_space_records( @@ -459,10 +135,9 @@ impl Database { limit: usize, reverse: bool, ) -> Result> { - let connection = self.connection()?; let order = if reverse { "ASC" } else { "DESC" }; let sql = format!( - "SELECT collection, rkey, cid, record_json FROM space_records + "SELECT space_uri, did, collection, rkey, cid, record_json FROM space_records WHERE space_uri = ?1 AND did = ?2 AND (?3 IS NULL OR collection = ?3) AND (?4 IS NULL OR @@ -470,8 +145,8 @@ impl Database { (NOT ?5 AND space_uri || '/' || did || '/' || collection || '/' || rkey < ?4)) ORDER BY collection {order}, rkey {order} LIMIT ?6" ); - let mut statement = connection.prepare(&sql)?; - let rows = statement.query_map( + self.query( + &sql, params![ space, did, @@ -480,51 +155,36 @@ impl Database { reverse, i64::try_from(limit).unwrap_or(i64::MAX) ], - |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, String>(2)?, - row.get::<_, String>(3)?, - )) - }, - )?; - rows.map(|row| { - let (collection, rkey, cid, json) = row?; - decode_record(space, did, &collection, &rkey, &cid, &json) - }) - .collect() + decode_record, + ) } pub(crate) fn list_spaces_for_repo( &self, did: &str, - space_type: Option<&str>, - authority: Option<&str>, - cursor: Option<&str>, - limit: usize, + query: &ListSpacesQuery, ) -> Result> { - let connection = self.connection()?; - let mut statement = connection.prepare( + let limit = query.limit.unwrap_or(50).clamp(1, 100); + // Both filters are validated protocol identifiers, so neither can contain + // GLOB metacharacters. Match canonical URIs even for remotely hosted spaces. + let pattern = format!( + "at://{}/space/{}/*", + query.did.as_deref().unwrap_or("*"), + query.space_type.as_deref().unwrap_or("*"), + ); + self.query( "SELECT DISTINCT r.space_uri FROM space_records r - LEFT JOIN spaces s ON s.uri = r.space_uri - WHERE r.did = ?1 AND (?2 IS NULL OR s.space_type = ?2) - AND (?3 IS NULL OR s.authority = ?3) - AND (?4 IS NULL OR r.space_uri > ?4) - ORDER BY r.space_uri LIMIT ?5", - )?; - let rows = statement.query_map( + WHERE r.did = ?1 AND r.space_uri GLOB ?2 + AND (?3 IS NULL OR r.space_uri > ?3) + ORDER BY r.space_uri LIMIT ?4", params![ did, - space_type, - authority, - cursor, + pattern, + query.cursor, i64::try_from(limit).unwrap_or(i64::MAX) ], - |row| row.get(0), - )?; - rows.collect::>() - .map_err(Into::into) + |row| Ok(row.get(0)?), + ) } pub(crate) fn list_space_ops( @@ -535,8 +195,7 @@ impl Database { cursor: Option, limit: usize, ) -> Result> { - let connection = self.connection()?; - let mut statement = connection.prepare( + self.query( "SELECT o.id, o.rev, o.collection, o.rkey, o.cid, o.prev, r.record_json FROM space_repo_ops o LEFT JOIN space_records r @@ -546,8 +205,6 @@ impl Database { AND ((?4 IS NOT NULL AND o.id > ?4) OR (?4 IS NULL AND (?3 IS NULL OR o.rev > ?3))) ORDER BY o.id LIMIT ?5", - )?; - let rows = statement.query_map( params![ space, did, @@ -556,61 +213,129 @@ impl Database { i64::try_from(limit).unwrap_or(i64::MAX) ], |row| { - Ok(( - row.get::<_, i64>(0)?, - row.get::<_, String>(1)?, - row.get::<_, String>(2)?, - row.get::<_, String>(3)?, - row.get::<_, Option>(4)?, - row.get::<_, Option>(5)?, - row.get::<_, Option>(6)?, - )) + Ok(SpaceOp { + id: row.get(0)?, + rev: row.get(1)?, + collection: row.get(2)?, + rkey: row.get(3)?, + cid: row.get(4)?, + prev: row.get(5)?, + value: row + .get::<_, Option>(6)? + .map(|json| serde_json::from_str(&json)) + .transpose()?, + }) }, - )?; - rows.map(|row| { - let (id, rev, collection, rkey, cid, prev, json) = row?; - Ok(SpaceOp { - id, - rev, - collection, - rkey, - cid, - prev, - value: json.map(|value| serde_json::from_str(&value)).transpose()?, - }) - }) - .collect() + ) } pub(crate) fn latest_space_rev(&self, space: &str, did: &str) -> Result { - self.connection()? - .query_row( - "SELECT rev FROM space_repo_ops - WHERE space_uri = ?1 AND did = ?2 ORDER BY rev DESC, id DESC LIMIT 1", - params![space, did], - |row| row.get(0), - ) - .optional()? - .ok_or(Error::NotFound) + load_latest_space_rev(&*self.connection()?, space, did) } } -fn insert_space_record(transaction: &Transaction<'_>, record: &SpaceRecord) -> Result<()> { +fn load_latest_space_rev(connection: &Connection, space: &str, did: &str) -> Result { + connection + .query_row( + "SELECT rev FROM space_repo_ops + WHERE space_uri = ?1 AND did = ?2 ORDER BY rev DESC, id DESC LIMIT 1", + params![space, did], + |row| row.get(0), + ) + .optional()? + .ok_or(Error::NotFound) +} + +fn delete_record(transaction: &Transaction<'_>, query: &RecordQuery, rev: &str) -> Result { + let RecordQuery { + space, + repo: did, + collection, + rkey, + } = query; + require_account_active(transaction, did)?; + let previous: Option = transaction + .query_row( + "SELECT cid FROM space_records + WHERE space_uri = ?1 AND did = ?2 AND collection = ?3 AND rkey = ?4", + params![space, did, collection, rkey], + |row| row.get(0), + ) + .optional()?; + let Some(previous) = previous else { + return Ok(false); + }; transaction.execute( - "INSERT INTO space_records - (space_uri, did, collection, rkey, cid, record_json, created_at, updated_at) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?7)", - params![ - record.space, - record.did, - record.collection, - record.rkey, - record.cid.to_string(), - serde_json::to_string(&record.value)?, - now(), - ], + "DELETE FROM space_records + WHERE space_uri = ?1 AND did = ?2 AND collection = ?3 AND rkey = ?4", + params![space, did, collection, rkey], )?; - Ok(()) + transaction.execute( + "INSERT INTO space_repo_ops + (space_uri, did, rev, collection, rkey, cid, prev, created_at) + VALUES (?1, ?2, ?3, ?4, ?5, NULL, ?6, ?7)", + params![space, did, rev, collection, rkey, previous, now()], + )?; + Ok(true) +} + +#[derive(Clone, Copy)] +enum WriteMode { + Create, + Update, + Put, +} + +fn write_record( + transaction: &Transaction<'_>, + record: &SpaceRecord, + rev: &str, + mode: WriteMode, +) -> Result<()> { + require_account_active(transaction, &record.did)?; + let previous = match mode { + WriteMode::Create => None, + WriteMode::Put => record_cid(transaction, record)?, + WriteMode::Update => Some(record_cid(transaction, record)?.ok_or(Error::NotFound)?), + }; + let sql = match mode { + WriteMode::Create => { + "INSERT INTO space_records + (space_uri, did, collection, rkey, cid, record_json, created_at, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?7)" + } + WriteMode::Put => { + "INSERT INTO space_records + (space_uri, did, collection, rkey, cid, record_json, created_at, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?7) + ON CONFLICT(space_uri, did, collection, rkey) DO UPDATE SET + cid = excluded.cid, record_json = excluded.record_json, + updated_at = excluded.updated_at" + } + WriteMode::Update => { + "UPDATE space_records SET cid = ?5, record_json = ?6, updated_at = ?7 + WHERE space_uri = ?1 AND did = ?2 AND collection = ?3 AND rkey = ?4" + } + }; + transaction + .execute( + sql, + params![ + record.space, + record.did, + record.collection, + record.rkey, + record.cid.to_string(), + serde_json::to_string(&record.value)?, + now(), + ], + ) + .map_err(|error| match mode { + WriteMode::Create => constraint_conflict(error, "record already exists"), + _ => Error::Database(error), + })?; + replace_space_record_blob_refs(transaction, record)?; + insert_op(transaction, record, rev, previous.as_deref()) } fn replace_space_record_blob_refs( @@ -655,12 +380,11 @@ fn insert_op( record: &SpaceRecord, rev: &str, prev: Option<&str>, - value: Option<&Value>, ) -> Result<()> { transaction.execute( "INSERT INTO space_repo_ops - (space_uri, did, rev, collection, rkey, cid, prev, record_json, created_at) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)", + (space_uri, did, rev, collection, rkey, cid, prev, created_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", params![ record.space, record.did, @@ -669,50 +393,633 @@ fn insert_op( record.rkey, record.cid.to_string(), prev, - value.map(serde_json::to_string).transpose()?, now(), ], )?; Ok(()) } -fn decode_record( - space: &str, - did: &str, - collection: &str, - rkey: &str, - cid: &str, - json: &str, -) -> Result { +fn decode_record(row: &Row<'_>) -> Result { + let (space, did, collection, rkey, cid, json) = <(_, _, _, _, String, String)>::try_from(row)?; Ok(SpaceRecord { - space: space.to_owned(), - did: did.to_owned(), - collection: collection.to_owned(), - rkey: rkey.to_owned(), - cid: Cid::from_str(cid) - .map_err(|error| Error::Internal(format!("stored CID is invalid: {error}")))?, - value: serde_json::from_str(json)?, + space, + did, + collection, + rkey, + cid: cid + .parse() + .context(Error::Internal, "stored CID is invalid")?, + value: serde_json::from_str(&json)?, }) } -fn map_space_conflict(error: rusqlite::Error) -> Error { - match error { - rusqlite::Error::SqliteFailure(ref failure, _) - if failure.code == rusqlite::ErrorCode::ConstraintViolation => - { - Error::Conflict("space already exists".to_owned()) +#[cfg(test)] +mod tests { + use super::*; + use rusqlite::types::Value as SqlValue; + + fn delete_query_fixture(query: &RecordQuery, cid: Option<&SqlValue>) -> Database { + let database = Database::open_in_memory().unwrap(); + let connection = database.connection().unwrap(); + connection.execute( + "INSERT INTO accounts (did, handle, password_hash, created_at) VALUES (?1, ?1, 'raw', 0)", + [&query.repo], + ).unwrap(); + if let Some(cid) = cid { + connection + .execute( + "INSERT INTO space_records + (space_uri, did, collection, rkey, cid, record_json, created_at, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5, '{', 0, 0)", + params![query.space, query.repo, query.collection, query.rkey, cid], + ) + .unwrap(); } - other => Error::Database(other), + drop(connection); + database + } + + fn deleted_repo_state(database: &Database) -> [Vec>; 2] { + [ + "SELECT space_uri, did, collection, rkey, cid, record_json FROM space_records", + "SELECT space_uri, did, rev, collection, rkey, cid, prev FROM space_repo_ops ORDER BY id", + ].map(|sql| database.query(sql, [], |row| { + Ok((0..row.as_ref().column_count()) + .map(|column| row.get(column)) + .collect::>>()?) + }).unwrap()) + } + + fn previous_delete_record( + transaction: &Transaction<'_>, + space: &str, + did: &str, + collection: &str, + rkey: &str, + rev: &str, + ) -> Result { + require_account_active(transaction, did)?; + let previous: Option = transaction + .query_row( + "SELECT cid FROM space_records + WHERE space_uri = ?1 AND did = ?2 AND collection = ?3 AND rkey = ?4", + params![space, did, collection, rkey], + |row| row.get(0), + ) + .optional()?; + let Some(previous) = previous else { + return Ok(false); + }; + transaction.execute( + "DELETE FROM space_records + WHERE space_uri = ?1 AND did = ?2 AND collection = ?3 AND rkey = ?4", + params![space, did, collection, rkey], + )?; + transaction.execute( + "INSERT INTO space_repo_ops + (space_uri, did, rev, collection, rkey, cid, prev, created_at) + VALUES (?1, ?2, ?3, ?4, ?5, NULL, ?6, ?7)", + params![space, did, rev, collection, rkey, previous, now()], + )?; + Ok(true) } -} -fn map_record_conflict(error: Error) -> Error { - match error { - Error::Database(rusqlite::Error::SqliteFailure(ref failure, _)) - if failure.code == rusqlite::ErrorCode::ConstraintViolation => + #[test] + fn shared_delete_query_preserves_raw_inputs_idempotence_and_sql_failures() { + for fields in [ + ["space", "repo", "collection", "key"], + ["", "", "", ""], + [ + "at://did:web:localhost%3A1234/space/com.example.fooBar/~key", + "did:web:localhost%3A1234", + "com.example.fooBar", + "literal:self", + ], + ["space\0🦀", "not a DID\0é", "invalid collection", "bad/key"], + ] { + let [space, repo, collection, rkey] = fields.map(str::to_owned); + let query = RecordQuery { + space, + repo, + collection, + rkey, + }; + for cid in [ + None, + Some(SqlValue::Text("invalid CID".to_owned())), + Some(SqlValue::Blob(vec![0])), + ] { + let previous = delete_query_fixture(&query, cid.as_ref()); + let actual = delete_query_fixture(&query, cid.as_ref()); + for _ in 0..2 { + let before = previous.transaction(|transaction| { + previous_delete_record( + transaction, + &query.space, + &query.repo, + &query.collection, + &query.rkey, + "rev\0é", + )?; + Ok(()) + }); + let after = actual.delete_space_record(&query, "rev\0é"); + assert_eq!(format!("{after:?}"), format!("{before:?}")); + let state = deleted_repo_state(&actual); + assert_eq!(state, deleted_repo_state(&previous)); + assert_eq!( + state[0].len(), + usize::from(matches!(&cid, Some(SqlValue::Blob(_)))) + ); + assert_eq!( + state[1].len(), + usize::from(matches!(&cid, Some(SqlValue::Text(_)))) + ); + assert_eq!(after.is_err(), matches!(&cid, Some(SqlValue::Blob(_)))); + } + } + } + } + + #[test] + fn batch_delete_queries_keep_strict_absence_and_atomic_rollback() { + for fields in [ + ["space", "repo", "collection", "key"], + ["", "", "", ""], + [ + "at://raw%2F", + "did:example:raw%2F", + "com.example.raw", + "literal:self", + ], + ["space\0🦀", "not a DID\0é", "invalid collection", "bad/key"], + ] { + let [space, repo, collection, rkey] = fields.map(str::to_owned); + let query = RecordQuery { + space, + repo, + collection, + rkey, + }; + for cid in [ + None, + Some(SqlValue::Text("invalid CID".to_owned())), + Some(SqlValue::Blob(vec![0])), + ] { + let database = delete_query_fixture(&query, cid.as_ref()); + let initial = deleted_repo_state(&database); + let result = + database.apply_space_writes(&[SpaceWrite::Delete(query.clone())], "rev\0é"); + match cid { + None => { + assert!(matches!(result, Err(Error::NotFound))); + assert_eq!(deleted_repo_state(&database), initial); + } + Some(SqlValue::Blob(_)) => { + assert!(matches!(result, Err(Error::Database(_)))); + assert_eq!(deleted_repo_state(&database), initial); + } + Some(SqlValue::Text(_)) => { + result.unwrap(); + let committed = deleted_repo_state(&database); + assert!(committed[0].is_empty()); + assert_eq!(committed[1].len(), 1); + assert!(matches!( + database + .apply_space_writes(&[SpaceWrite::Delete(query.clone())], "again",), + Err(Error::NotFound), + )); + assert_eq!(deleted_repo_state(&database), committed); + database.delete_space_record(&query, "again").unwrap(); + assert_eq!(deleted_repo_state(&database), committed); + } + _ => unreachable!(), + } + } + let database = + delete_query_fixture(&query, Some(&SqlValue::Text("raw CID".to_owned()))); + let initial = deleted_repo_state(&database); + assert!(matches!( + database.apply_space_writes( + &[SpaceWrite::Delete(query.clone()), SpaceWrite::Delete(query)], + "rev\0é", + ), + Err(Error::NotFound), + )); + assert_eq!(deleted_repo_state(&database), initial); + } + } + + fn inventory_query_fixture() -> Database { + let database = Database::open_in_memory().unwrap(); + database + .connection() + .unwrap() + .execute_batch( + "CREATE TEMP TABLE space_record_blobs (space_uri, did, collection, rkey, cid); + CREATE TEMP TABLE space_repo_ops (space_uri, did, collection, rkey, rev); + CREATE TEMP TABLE space_records (space_uri, did);", + ) + .unwrap(); + database + } + + fn previous_blob_page(database: &Database, query: &ListBlobsQuery) -> Result> { + let limit = query.limit.unwrap_or(500).clamp(1, 1000); + database.query( + "SELECT DISTINCT b.cid FROM space_record_blobs b + WHERE b.space_uri = ?1 AND b.did = ?2 + AND (?3 IS NULL OR EXISTS ( + SELECT 1 FROM space_repo_ops o + WHERE o.space_uri = b.space_uri AND o.did = b.did + AND o.collection = b.collection AND o.rkey = b.rkey AND o.rev > ?3 + )) + AND (?4 IS NULL OR b.cid > ?4) + ORDER BY b.cid LIMIT ?5", + params![ + query.space.as_str(), + query.repo.as_str(), + query.since.as_deref(), + query.cursor.as_deref(), + i64::try_from(limit).unwrap_or(i64::MAX), + ], + |row| Ok(row.get(0)?), + ) + } + + fn compare_blob_page(database: &Database, query: &ListBlobsQuery) -> Result> { + let previous = previous_blob_page(database, query); + let actual = database.space_blob_cids(query); + assert_eq!(format!("{actual:?}"), format!("{previous:?}")); + actual + } + + fn seed_blob_inventory(database: &Database, space: &str, repo: &str) { + for (index, cid) in ["", "alpha", "alpha", "omega", "🦀", "z\0end"] + .into_iter() + .enumerate() { - Error::Conflict("record already exists".to_owned()) + let connection = database.connection().unwrap(); + connection + .execute( + "INSERT INTO space_record_blobs VALUES (?1, ?2, 'collection', ?3, ?4)", + params![space, repo, index.to_string(), cid], + ) + .unwrap(); + connection + .execute( + "INSERT INTO space_repo_ops VALUES (?1, ?2, 'collection', ?3, ?4)", + params![ + space, + repo, + index.to_string(), + if index % 2 == 0 { "1" } else { "3" } + ], + ) + .unwrap(); + } + } + + #[test] + fn shared_blob_page_query_preserves_raw_bindings_limits_and_sql_errors() { + let database = inventory_query_fixture(); + for (space, repo) in [ + ("space", "repo"), + ("", ""), + ( + "at://did:web:localhost%3A1234/space/com.example.fooBar/~key", + "did:web:localhost%3A1234", + ), + ("space\0🦀", "not a DID\0é"), + ] { + seed_blob_inventory(&database, space, repo); + let mut query = ListBlobsQuery { + space: space.to_owned(), + repo: repo.to_owned(), + since: None, + cursor: None, + limit: None, + }; + for limit in [ + None, + Some(0), + Some(1), + Some(500), + Some(1001), + Some(usize::MAX), + ] { + for since in [None, Some(""), Some("2"), Some("zz")] { + for cursor in [None, Some(""), Some("alpha"), Some("zz")] { + query.limit = limit; + query.since = since.map(str::to_owned); + query.cursor = cursor.map(str::to_owned); + compare_blob_page(&database, &query).unwrap(); + } + } + } + query.since = None; + query.cursor = None; + for cid in [SqlValue::Blob(vec![0]), SqlValue::Null] { + database + .connection() + .unwrap() + .execute( + "INSERT INTO space_record_blobs VALUES (?1, ?2, 'invalid', 'invalid', ?3)", + params![space, repo, cid], + ) + .unwrap(); + assert!(matches!( + compare_blob_page(&database, &query), + Err(Error::Database(_)) + )); + } } - other => other, + database + .connection() + .unwrap() + .execute_batch( + "DROP TABLE temp.space_record_blobs; DROP TABLE main.space_record_blobs;", + ) + .unwrap(); + let query = ListBlobsQuery { + space: "space".to_owned(), + repo: "repo".to_owned(), + since: None, + cursor: None, + limit: None, + }; + assert!(matches!( + compare_blob_page(&database, &query), + Err(Error::Database(_)) + )); + } + + fn compare_space_page( + database: &Database, + repo: &str, + query: &ListSpacesQuery, + ) -> Result> { + let limit = query.limit.unwrap_or(50).clamp(1, 100); + let pattern = format!( + "at://{}/space/{}/*", + query.did.as_deref().unwrap_or("*"), + query.space_type.as_deref().unwrap_or("*"), + ); + let previous = database.query( + "SELECT DISTINCT r.space_uri FROM space_records r + WHERE r.did = ?1 AND r.space_uri GLOB ?2 + AND (?3 IS NULL OR r.space_uri > ?3) + ORDER BY r.space_uri LIMIT ?4", + params![ + repo, + pattern, + query.cursor.as_deref(), + i64::try_from(limit).unwrap_or(i64::MAX) + ], + |row| Ok(row.get::<_, String>(0)?), + ); + let actual = database.list_spaces_for_repo(repo, query); + assert_eq!(format!("{actual:?}"), format!("{previous:?}")); + actual + } + + fn seed_space_inventory(database: &Database, repo: &str) { + for space in [ + "at://did:web:localhost%3A1234/space/com.example.fooBar/a", + "at://did:web:localhost%3A1234/space/com.example.fooBar/a", + "at://did:web:localhost%3A1234/space/com.example.fooBar/b", + "at://did:example:other/space/com.example.other/c", + "not a URI", + "at://🦀/space/invalid collection/\0", + ] { + database + .connection() + .unwrap() + .execute( + "INSERT INTO space_records VALUES (?1, ?2)", + params![space, repo], + ) + .unwrap(); + } + } + + #[test] + fn shared_space_page_query_preserves_raw_filters_limits_and_sql_errors() { + let database = inventory_query_fixture(); + for repo in ["repo", "", "did:web:localhost%3A1234", "not a DID\0é"] { + seed_space_inventory(&database, repo); + for limit in [ + None, + Some(0), + Some(1), + Some(100), + Some(101), + Some(usize::MAX), + ] { + for authority in [None, Some(""), Some("did:web:localhost%3A1234"), Some("🦀")] { + for kind in [ + None, + Some(""), + Some("com.example.fooBar"), + Some("invalid collection"), + ] { + for cursor in [ + None, + Some(""), + Some("at://did:web:localhost%3A1234/space/com.example.fooBar/a"), + ] { + let query = ListSpacesQuery { + did: authority.map(str::to_owned), + space_type: kind.map(str::to_owned), + cursor: cursor.map(str::to_owned), + limit, + }; + compare_space_page(&database, repo, &query).unwrap(); + } + } + } + } + } + database + .connection() + .unwrap() + .execute_batch("DROP TABLE temp.space_records; DROP TABLE main.space_records;") + .unwrap(); + let query = ListSpacesQuery { + did: None, + space_type: None, + cursor: None, + limit: None, + }; + assert!(matches!( + compare_space_page(&database, "repo", &query), + Err(Error::Database(_)) + )); + } + + fn compare_space_record_lookup( + database: &Database, + query: &RecordQuery, + ) -> Result { + let previous = database + .query_optional( + "SELECT space_uri, did, collection, rkey, cid, record_json FROM space_records + WHERE space_uri = ?1 AND did = ?2 AND collection = ?3 AND rkey = ?4", + params![query.space, query.repo, query.collection, query.rkey], + decode_record, + ) + .and_then(|record| record.ok_or(Error::NotFound)); + let actual = database.get_space_record(query); + assert_eq!(format!("{actual:?}"), format!("{previous:?}")); + actual + } + + #[test] + fn shared_record_query_preserves_raw_sql_inputs_and_decoding_errors() { + let database = Database::open_in_memory().unwrap(); + let cid = cpds_protocol::sha256_cid(0x71, b"record"); + let legacy = Cid::new_v0(*cid.hash()).unwrap().to_string(); + for fields in [ + ["space", "repo", "collection", "key"], + ["", "", "", ""], + [ + "at://did:web:localhost%3A1234/space/com.example.fooBar/~1.2-3_", + "did:web:localhost%3A1234", + "com.example.fooBar", + "literal:self", + ], + [ + "sp ace%2F\0🦀", + "not a DID\0é", + "invalid collection", + "bad/key", + ], + ] { + let [space, repo, collection, rkey] = fields.map(str::to_owned); + let query = RecordQuery { + space, + repo, + collection, + rkey, + }; + assert!(matches!( + compare_space_record_lookup(&database, &query), + Err(Error::NotFound) + )); + database.connection().unwrap().execute( + "INSERT INTO accounts (did, handle, password_hash, created_at) VALUES (?1, ?1, 'raw', 0)", + [&query.repo], + ).unwrap(); + for stored_cid in [cid.to_string(), legacy.clone(), "invalid".to_owned()] { + for (kind, json) in [ + ("null", SqlValue::Text("null".to_owned())), + ("object", SqlValue::Text(r#"{"raw":true}"#.to_owned())), + ("json", SqlValue::Text("{".to_owned())), + ("sql", SqlValue::Blob(vec![0])), + ] { + database.connection().unwrap().execute( + "INSERT OR REPLACE INTO space_records + (space_uri, did, collection, rkey, cid, record_json, created_at, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, 0, 0)", + params![query.space, query.repo, query.collection, query.rkey, stored_cid, json], + ).unwrap(); + let actual = compare_space_record_lookup(&database, &query); + if kind == "sql" { + assert!(matches!(actual, Err(Error::Database(_)))); + } else if stored_cid == "invalid" { + assert!(matches!(actual, Err(Error::Internal(_)))); + } else if kind == "json" { + assert!(matches!(actual, Err(Error::Json(_)))); + } else { + let record = actual.unwrap(); + assert_eq!( + (&record.space, &record.did, &record.collection, &record.rkey), + (&query.space, &query.repo, &query.collection, &query.rkey), + ); + assert_eq!(record.cid.to_string(), stored_cid); + assert_eq!( + record.value, + if kind == "null" { + serde_json::Value::Null + } else { + serde_json::json!({"raw": true}) + } + ); + } + } + } + } + } + + #[test] + fn stored_space_record_cid_context_preserves_decoding_results_and_error_order() { + let database = Database::open_in_memory().unwrap(); + let cid = cpds_protocol::sha256_cid(0x71, b"record"); + let v0 = Cid::new_v0(*cid.hash()).unwrap().to_string(); + let valid = cid.to_string(); + for cid in [ + valid.as_str(), + v0.as_str(), + "", + "invalid-cid", + "b", + "Qm", + "é", + " bafyreia", + ] { + for json in [r#"{"text":"value"}"#, "null", "{"] { + assert_record_decoding( + &database, + &[ + "space", + "did:web:test", + "app.test.record", + "rkey", + cid, + json, + ] + .map(|value| SqlValue::Text(value.to_owned())), + ); + } + } + for index in 0..6 { + for invalid in [ + SqlValue::Null, + SqlValue::Integer(7), + SqlValue::Real(7.5), + SqlValue::Blob(vec![0]), + ] { + let mut row = [ + "space", + "did:web:test", + "app.test.record", + "rkey", + "invalid-cid", + "{", + ] + .map(|value| SqlValue::Text(value.to_owned())); + row[index] = invalid; + assert_record_decoding(&database, &row); + } + } + } + + fn assert_record_decoding(database: &Database, values: &[SqlValue; 6]) { + let sql = "SELECT ?1, ?2, ?3, ?4, ?5, ?6"; + let old = database.query_one(sql, rusqlite::params_from_iter(values), |row| { + let (space, did, collection, rkey, cid, json) = + <(_, _, _, _, String, String)>::try_from(row)?; + Ok(SpaceRecord { + space, + did, + collection, + rkey, + cid: ::from_str(&cid) + .map_err(|error| Error::Internal(format!("stored CID is invalid: {error}")))?, + value: serde_json::from_str(&json)?, + }) + }); + let current = database.query_one(sql, rusqlite::params_from_iter(values), decode_record); + assert_eq!(format!("{current:?}"), format!("{old:?}"), "{values:?}"); } } diff --git a/src/space/database/tests.rs b/src/space/database/tests.rs new file mode 100644 index 0000000..a7ae494 --- /dev/null +++ b/src/space/database/tests.rs @@ -0,0 +1,611 @@ +use rusqlite::params; +use serde_json::json; + +use crate::{ + database::Database, + domain::account::AccountStatus, + domain::blob::Blob, + error::Error, + space::{ + ListBlobsQuery, NotifiedWrite, NotifyRegistrationInput, RecordQuery, SpacePageQuery, + domain::{Space, SpaceRecord, SpaceWrite}, + }, +}; + +const DID: &str = "did:web:alice.test"; +const SPACE: &str = "at://did:web:authority.test/space/com.example.space/one"; +const REV: &str = "3jqfcqzm3fo2j"; + +fn previous_notification_rows( + database: &Database, + sql: &str, + params: impl rusqlite::Params, +) -> crate::error::Result> { + struct PreviousRegistration { + service: String, + endpoint: String, + } + + database + .query(sql, params, |row| { + Ok(PreviousRegistration { + service: row.get(0)?, + endpoint: row.get(1)?, + }) + }) + .map(|rows| { + rows.into_iter() + .map(|row| (row.service, row.endpoint)) + .collect() + }) +} + +#[test] +fn notification_pairs_preserve_order_expiry_and_raw_values() { + let database = Database::open_in_memory().unwrap(); + for (index, space) in [SPACE, "raw\0space/é"].into_iter().enumerate() { + database + .create_space(&Space { + uri: space.to_owned(), + authority: DID.to_owned(), + kind: "com.example.space".to_owned(), + skey: index.to_string(), + policy: json!(null), + app_access: json!(null), + }) + .unwrap(); + for (service, endpoint, expires_at) in [ + ("z", "https://example.test/path", i64::MAX), + ("", "", i64::MAX), + ("a", "raw\0/雪", i64::MAX), + ("é\0", "not a URL", i64::MAX), + ("expired", "must not be returned", i64::MIN), + ] { + database + .register_space_notify( + &NotifyRegistrationInput { + space: space.to_owned(), + service: service.to_owned(), + }, + endpoint, + expires_at, + ) + .unwrap(); + } + } + let sql = "SELECT service, endpoint FROM space_notify_registrations + WHERE space_uri = ?1 AND expires_at > ?2 ORDER BY service"; + for space in [SPACE, "raw\0space/é", "missing"] { + let expected = previous_notification_rows(&database, sql, params![space, 0]).unwrap(); + let actual = database.space_notify_registrations(space).unwrap(); + assert_eq!(actual, expected); + if space == "missing" { + assert!(actual.is_empty()); + } else { + assert_eq!( + actual, + [ + ("", ""), + ("a", "raw\0/雪"), + ("z", "https://example.test/path"), + ("é\0", "not a URL"), + ] + .map(|(service, endpoint)| (service.to_owned(), endpoint.to_owned())) + ); + } + let at_expiry = + previous_notification_rows(&database, sql, params![space, i64::MAX]).unwrap(); + let native_at_expiry: Vec<(String, String)> = database + .query(sql, params![space, i64::MAX], |row| Ok(row.try_into()?)) + .unwrap(); + assert_eq!(native_at_expiry, at_expiry); + assert!(native_at_expiry.is_empty()); + } +} + +#[test] +fn notification_pair_decoding_preserves_first_column_type_failure() { + use rusqlite::types::Value as SqlValue; + + let database = Database::open_in_memory().unwrap(); + let values = [ + SqlValue::Null, + SqlValue::Integer(7), + SqlValue::Real(1.5), + SqlValue::Text(String::new()), + SqlValue::Text("raw\0/é".to_owned()), + SqlValue::Blob(vec![0, 255]), + ]; + for service in &values { + for endpoint in &values { + let sql = "SELECT ?1 AS service, ?2 AS endpoint"; + let previous = previous_notification_rows(&database, sql, params![service, endpoint]); + let actual: crate::error::Result> = + database.query(sql, params![service, endpoint], |row| Ok(row.try_into()?)); + assert_eq!(format!("{actual:?}"), format!("{previous:?}")); + let first_invalid = [service, endpoint] + .into_iter() + .position(|value| !matches!(value, SqlValue::Text(_))); + match (actual, previous, first_invalid) { + (Ok(actual), Ok(previous), None) => assert_eq!(actual, previous), + (Err(actual), Err(previous), Some(index)) => { + assert_eq!(actual.to_string(), previous.to_string()); + assert!(matches!( + actual, + Error::Database(rusqlite::Error::InvalidColumnType(column, _, _)) + if column == index + )); + } + other => panic!("unexpected notification decoding result: {other:?}"), + } + } + } +} + +#[test] +fn shared_notification_input_preserves_raw_values_updates_and_idempotent_deletes() { + for (space, service) in [ + (SPACE, "did:example:subscriber"), + ("", ""), + ("at://DID/space/type/%2f", "did:example:caf%C3%A9"), + ("raw\0space/é", "not a DID\0é"), + ] { + let database = Database::open_in_memory().unwrap(); + database + .create_space(&Space { + uri: space.to_owned(), + authority: DID.to_owned(), + kind: "com.example.space".to_owned(), + skey: "one".to_owned(), + policy: json!(null), + app_access: json!(null), + }) + .unwrap(); + let input = NotifyRegistrationInput { + space: space.to_owned(), + service: service.to_owned(), + }; + for (endpoint, expires_at) in [("before", i64::MIN), ("", 0), ("raw\0/é", i64::MAX)] { + let before = crate::database::now(); + database + .register_space_notify(&input, endpoint, expires_at) + .unwrap(); + let after = crate::database::now(); + let stored: (String, String, String, i64, i64) = database + .query_one( + "SELECT space_uri, service, endpoint, expires_at, updated_at + FROM space_notify_registrations WHERE space_uri = ?1 AND service = ?2", + params![space, service], + |row| Ok(row.try_into()?), + ) + .unwrap(); + assert_eq!( + ( + stored.0.as_str(), + stored.1.as_str(), + stored.2.as_str(), + stored.3 + ), + (space, service, endpoint, expires_at) + ); + assert!((before..=after).contains(&stored.4)); + } + for _ in 0..2 { + database.unregister_space_notify(&input).unwrap(); + let remaining: i64 = database + .query_one( + "SELECT COUNT(*) FROM space_notify_registrations + WHERE space_uri = ?1 AND service = ?2", + params![space, service], + |row| Ok(row.get(0)?), + ) + .unwrap(); + assert_eq!(remaining, 0); + } + } +} + +fn space_page_fixture() -> Database { + let database = Database::open_in_memory().unwrap(); + database + .create_space(&Space { + uri: SPACE.to_owned(), + authority: DID.to_owned(), + kind: "com.example.space".to_owned(), + skey: "one".to_owned(), + policy: json!(null), + app_access: json!(null), + }) + .unwrap(); + for index in 0_u16..1002 { + let did = format!("did:example:writer{index:04}"); + database.add_space_member(SPACE, &did).unwrap(); + database + .record_space_writer(&NotifiedWrite { + issuer: "", + audience: "", + space: SPACE, + repo: &did, + rev: &format!("revision{index:04}"), + hash: &index.to_le_bytes(), + managing_app_authorized: false, + }) + .unwrap(); + } + database +} + +#[test] +fn shared_space_page_query_preserves_database_limits_cursors_and_bindings() { + let database = space_page_fixture(); + for limit in [ + None, + Some(0), + Some(1), + Some(100), + Some(1001), + Some(usize::MAX), + ] { + for cursor in [ + None, + Some(""), + Some("did:example:writer0500a"), + Some("zzzz"), + ] { + let query = SpacePageQuery { + space: SPACE.to_owned(), + limit, + cursor: cursor.map(str::to_owned), + }; + let limit = query.limit.unwrap_or(100).clamp(1, 1000); + let old_limit = i64::try_from(limit).unwrap_or(i64::MAX); + let previous_members = database + .query( + "SELECT did FROM space_members + WHERE space_uri = ?1 AND (?2 IS NULL OR did > ?2) + ORDER BY did LIMIT ?3", + params![query.space.as_str(), query.cursor.as_deref(), old_limit], + |row| Ok(row.get::<_, String>(0)?), + ) + .unwrap(); + assert_eq!( + database.list_space_members(&query).unwrap(), + previous_members + ); + let previous_writers: Vec<(String, String, Vec)> = database + .query( + "SELECT did, rev, hash FROM space_writers + WHERE space_uri = ?1 AND (?2 IS NULL OR did > ?2) + ORDER BY did LIMIT ?3", + params![query.space.as_str(), query.cursor.as_deref(), old_limit], + |row| Ok(<(_, _, _)>::try_from(row)?), + ) + .unwrap(); + let writers = database + .list_space_writers(&query) + .unwrap() + .into_iter() + .map(|writer| (writer.did, writer.rev, writer.hash)) + .collect::>(); + assert_eq!(writers, previous_writers); + let decode = |row: &rusqlite::Row<'_>| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, Option>(1)?, + row.get::<_, i64>(2)?, + )) + }; + assert_eq!( + database + .query_one( + "SELECT ?1, ?2, ?3", + params![query.space, query.cursor, old_limit], + decode + ) + .unwrap(), + database + .query_one( + "SELECT ?1, ?2, ?3", + params![query.space.as_str(), query.cursor.as_deref(), old_limit], + decode, + ) + .unwrap() + ); + } + } +} + +#[test] +fn snapshot_bounds_utf8_bytes_and_keeps_empty_repositories() { + let database = Database::open_in_memory().unwrap(); + database + .ensure_account(DID, "alice.test", "password") + .unwrap(); + assert!(matches!( + database.space_snapshot(SPACE, DID, 1_000), + Err(Error::NotFound) + )); + let record = record("one", "🌊"); + database.create_space_record(&record, REV).unwrap(); + let size = serde_json::to_vec(&record.value).unwrap().len(); + assert!(matches!( + database.space_snapshot(SPACE, DID, size - 1), + Err(Error::Xrpc { + name: "RepoTooBig", + .. + }) + )); + let (rev, records) = database.space_snapshot(SPACE, DID, size).unwrap(); + assert_eq!(rev, REV); + assert_eq!(records.len(), 1); + assert_eq!(records[0].value, record.value); + + let later = "3jqfcqzm3fo2k"; + database + .delete_space_record( + &record_query(SPACE, DID, &record.collection, &record.rkey), + later, + ) + .unwrap(); + let (rev, records) = database.space_snapshot(SPACE, DID, 0).unwrap(); + assert_eq!(rev, later); + assert!(records.is_empty()); +} + +#[test] +fn blob_pages_use_current_references_and_filter_changes_before_pagination() { + let database = Database::open_in_memory().unwrap(); + database + .ensure_account(DID, "alice.test", "password") + .unwrap(); + let mut blobs: Vec<_> = [b"one".as_slice(), b"two", b"three"] + .into_iter() + .map(|bytes| Blob { + cid: cpds_protocol::sha256_cid(0x55, bytes), + media_type: "image/png".to_owned(), + size: bytes.len() as u64, + }) + .collect(); + blobs.sort_by_key(|blob| blob.cid.to_string()); + for (index, blob) in blobs.iter().enumerate() { + database.begin_blob_upload(DID, blob).unwrap(); + database.finish_blob_upload(DID, &blob.cid).unwrap(); + database + .create_space_record(&record_with_blob(&format!("record{index}"), blob), REV) + .unwrap(); + } + let cids: Vec<_> = blobs.iter().map(|blob| blob.cid.to_string()).collect(); + assert_eq!( + database + .space_blob_cids(&blob_page_query(SPACE, DID, None, None, 1)) + .unwrap(), + cids[..1] + ); + assert_eq!( + database + .space_blob_cids(&blob_page_query(SPACE, DID, None, Some(&cids[0]), 1)) + .unwrap(), + cids[1..2] + ); + assert_eq!( + database + .space_blob_cids(&blob_page_query(SPACE, DID, None, Some(&cids[1]), 1)) + .unwrap(), + cids[2..] + ); + assert!( + database + .space_blob_cids(&blob_page_query(SPACE, DID, Some(REV), None, 10)) + .unwrap() + .is_empty() + ); + + let later = "3jqfcqzm3fo2k"; + database + .put_space_record(&record_with_blob("record0", &blobs[2]), later) + .unwrap(); + database + .create_space_record(&record_with_blob("duplicate", &blobs[2]), later) + .unwrap(); + database + .delete_space_record( + &record_query(SPACE, DID, "com.example.record", "record1"), + later, + ) + .unwrap(); + assert_eq!( + database + .space_blob_cids(&blob_page_query(SPACE, DID, None, None, 10)) + .unwrap(), + cids[2..] + ); + assert_eq!( + database + .space_blob_cids(&blob_page_query(SPACE, DID, Some(REV), None, 1)) + .unwrap(), + cids[2..] + ); + assert!( + database + .space_blob_cids(&blob_page_query(SPACE, DID, Some(REV), Some(&cids[2]), 1)) + .unwrap() + .is_empty() + ); + assert!( + database + .space_blob_cids(&blob_page_query( + "at://did:web:other.test/space/com.example.space/one", + DID, + None, + None, + 10 + )) + .unwrap() + .is_empty() + ); +} + +fn blob_page_query( + space: &str, + repo: &str, + since: Option<&str>, + cursor: Option<&str>, + limit: usize, +) -> ListBlobsQuery { + ListBlobsQuery { + space: space.to_owned(), + repo: repo.to_owned(), + since: since.map(str::to_owned), + cursor: cursor.map(str::to_owned), + limit: Some(limit), + } +} + +fn record_query(space: &str, repo: &str, collection: &str, rkey: &str) -> RecordQuery { + RecordQuery { + space: space.to_owned(), + repo: repo.to_owned(), + collection: collection.to_owned(), + rkey: rkey.to_owned(), + } +} + +fn record_with_blob(rkey: &str, blob: &Blob) -> SpaceRecord { + let value = json!({ + "$type": "com.example.record", + "image": {"$type": "blob", "ref": {"$link": blob.cid.to_string()}, + "mimeType": blob.media_type, "size": blob.size}, + }); + SpaceRecord { + value: value.clone(), + cid: cpds_protocol::record_cid(&value).unwrap(), + ..record(rkey, "") + } +} + +#[test] +fn deactivation_prevents_every_space_record_mutation_at_commit_time() { + let database = Database::open_in_memory().unwrap(); + database + .ensure_account(DID, "alice.test", "password") + .unwrap(); + let original = record("existing", "original"); + database.create_space_record(&original, REV).unwrap(); + database + .transition_account( + DID, + AccountStatus::Active.as_str(), + crate::database::LifecycleEvent::Account(AccountStatus::Deactivated), + 100, + ) + .unwrap(); + + let created = record("new", "new"); + let updated = record("existing", "updated"); + let deleted = SpaceWrite::Delete(RecordQuery { + space: SPACE.to_owned(), + repo: DID.to_owned(), + collection: original.collection.clone(), + rkey: original.rkey.clone(), + }); + for result in [ + database.create_space_record(&created, REV), + database.put_space_record(&created, REV), + database.put_space_record(&updated, REV), + database.delete_space_record( + &record_query(SPACE, DID, &original.collection, &original.rkey), + REV, + ), + database.apply_space_writes(&[SpaceWrite::Create(created)], REV), + database.apply_space_writes(&[SpaceWrite::Update(updated)], REV), + database.apply_space_writes(&[deleted], REV), + ] { + assert!(matches!( + result, + Err(Error::Xrpc { + name: "AccountDeactivated", + .. + }) + )); + } + let records = database + .list_space_records(SPACE, DID, None, None, 10, false) + .unwrap(); + assert_eq!(records.len(), 1); + assert_eq!(records[0].value, original.value); + assert_eq!( + database + .list_space_ops(SPACE, DID, None, None, 10) + .unwrap() + .len(), + 1 + ); +} + +#[test] +fn account_deactivation_between_batch_items_rolls_back_every_mutation() { + let database = Database::open_in_memory().unwrap(); + database + .ensure_account(DID, "alice.test", "password") + .unwrap(); + let original = record("existing", "original"); + database.create_space_record(&original, REV).unwrap(); + database + .connection() + .unwrap() + .execute_batch( + "CREATE TEMP TRIGGER deactivate_after_first_space_write + AFTER INSERT ON space_records WHEN NEW.rkey = 'first' + BEGIN + UPDATE accounts SET status = 'deactivated' WHERE did = NEW.did; + END;", + ) + .unwrap(); + + for second in [ + SpaceWrite::Create(record("second", "new")), + SpaceWrite::Update(record("existing", "updated")), + SpaceWrite::Delete(RecordQuery { + space: SPACE.to_owned(), + repo: DID.to_owned(), + collection: original.collection.clone(), + rkey: original.rkey.clone(), + }), + ] { + assert!(matches!( + database.apply_space_writes(&[SpaceWrite::Create(record("first", "new")), second], REV), + Err(Error::Xrpc { + name: "AccountDeactivated", + .. + }) + )); + let records = database + .list_space_records(SPACE, DID, None, None, 10, false) + .unwrap(); + assert_eq!(records.len(), 1); + assert_eq!(records[0].rkey, original.rkey); + assert_eq!(records[0].cid, original.cid); + assert_eq!(records[0].value, original.value); + let (_account, status) = database.hosted_account_by_identifier(DID).unwrap().unwrap(); + assert_eq!(status, AccountStatus::Active); + assert_eq!( + database + .list_space_ops(SPACE, DID, None, None, 10) + .unwrap() + .len(), + 1 + ); + } +} + +fn record(rkey: &str, text: &str) -> SpaceRecord { + let value = json!({"$type": "com.example.record", "text": text}); + SpaceRecord { + space: SPACE.to_owned(), + did: DID.to_owned(), + collection: "com.example.record".to_owned(), + rkey: rkey.to_owned(), + cid: cpds_protocol::record_cid(&value).unwrap(), + value, + } +} diff --git a/src/space/domain.rs b/src/space/domain.rs index fb27f6a..a2dd00b 100644 --- a/src/space/domain.rs +++ b/src/space/domain.rs @@ -1,14 +1,20 @@ //! Domain values owned by the `ATProto` Spaces feature. use cid::Cid; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use serde_json::Value; -#[derive(Clone, Debug)] +use crate::domain::record::RecordView; + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] pub struct Space { pub uri: String, + #[serde(skip)] pub authority: String, + #[serde(skip)] pub kind: String, + #[serde(skip)] pub skey: String, pub policy: Value, pub app_access: Value, @@ -25,6 +31,10 @@ pub struct SpaceRecord { } impl SpaceRecord { + pub(crate) fn path(&self) -> String { + format!("{}/{}", self.collection, self.rkey) + } + #[must_use] pub fn uri(&self) -> String { format!( @@ -34,14 +44,7 @@ impl SpaceRecord { } } -#[derive(Debug, Serialize)] -pub struct SpaceRecordView { - pub uri: String, - pub cid: String, - pub value: Value, -} - -impl From for SpaceRecordView { +impl From for RecordView { fn from(record: SpaceRecord) -> Self { Self { uri: record.uri(), @@ -51,38 +54,37 @@ impl From for SpaceRecordView { } } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Serialize)] pub struct SpaceOp { + #[serde(skip)] pub id: i64, pub rev: String, pub collection: String, pub rkey: String, pub cid: Option, pub prev: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub value: Option, } -#[derive(Clone, Debug)] -pub struct SpaceWriter { +#[derive(Clone, Debug, Serialize)] +pub struct SpaceWriter> { pub did: String, pub rev: String, - pub hash: Vec, + pub hash: Hash, } -#[derive(Clone, Debug)] -pub struct NotifyRegistration { - pub service: String, - pub endpoint: String, +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct RecordQuery { + pub(crate) space: String, + pub(crate) repo: String, + pub(crate) collection: String, + pub(crate) rkey: String, } #[derive(Clone, Debug)] -pub enum SpaceWrite { +pub(crate) enum SpaceWrite { Create(SpaceRecord), Update(SpaceRecord), - Delete { - space: String, - did: String, - collection: String, - rkey: String, - }, + Delete(RecordQuery), } diff --git a/src/space/http.rs b/src/space/http.rs index 153f19d..0690410 100644 --- a/src/space/http.rs +++ b/src/space/http.rs @@ -1,1082 +1,27 @@ -//! HTTP/XRPC adapters for the `ATProto` Spaces alpha lexicons. +//! HTTP adapters for the Spaces alpha lexicons, grouped by endpoint purpose. -use std::str::FromStr; +mod credentials; +mod management; +mod notifications; +mod records; +mod sync; -use axum::{ - Json, - extract::{OriginalUri, Query, State}, - http::{HeaderMap, HeaderValue, header::CONTENT_TYPE}, - response::{IntoResponse, Response}, -}; -use cid::Cid; -use serde::{Deserialize, Serialize}; -use serde_json::{Value, json}; +use serde::Deserialize; -use crate::{ - AppState, - error::Error, - http::{ApiError, ApiResult, auth}, - space::{CreateRecord, ListRecords, NotifiedWrite, PutRecord, WriteRequest, WriteResult}, - space::{ - commit::LexBytes, - domain::{Space, SpaceRecord, SpaceRecordView}, - }, +pub(crate) use credentials::{get_delegation_token, get_space_credential}; +pub(crate) use management::{ + add_member, create_space, delete_space, get_space, list_members, list_spaces, remove_member, + update_space, }; - -#[derive(Serialize)] -pub struct DelegationOutput { - token: String, -} - -pub async fn get_delegation_token( - State(state): State, - headers: HeaderMap, - Query(query): Query, -) -> ApiResult> { - let oauth = auth::token(&headers, &state)?; - state.spaces.authorize_delegation(&oauth, &query.space)?; - Ok(Json(DelegationOutput { - token: state - .space_auth - .mint_delegation(&query.space, &oauth.account.did)?, - })) -} - -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct CredentialInput { - space: String, - client_attestation: Option, -} - -#[derive(Serialize)] -pub struct CredentialOutput { - credential: String, -} - -pub async fn get_space_credential( - State(state): State, - OriginalUri(uri): OriginalUri, - headers: HeaderMap, - Json(input): Json, -) -> ApiResult> { - let token = auth::bearer(&headers)?; - let proof = auth::dpop_proof(&headers)?; - let htu = auth::request_url(&state, &uri); - let (delegation, jkt) = state - .space_auth - .exchange_delegation(token, proof, "POST", &htu) - .await?; - if delegation.space != input.space { - return Err(ApiError(Error::Xrpc { - name: "InvalidDelegationToken", - message: "delegation token subject does not match the requested space".to_owned(), - })); - } - let space_ref = crate::syntax::space_ref(&input.space)?; - let client_id = if let Some(attestation) = input.client_attestation.as_deref() { - Some( - state - .space_auth - .verify_client_attestation( - attestation, - &format!("{}#atproto_space_host", space_ref.authority), - ) - .await?, - ) - } else { - None - }; - let managing_app = state.spaces.authorize_credential( - &input.space, - &delegation.user_did, - client_id.as_deref(), - )?; - if let Some(service) = managing_app - && !state - .space_auth - .check_user_access( - &service, - &input.space, - &delegation.user_did, - client_id.as_deref(), - ) - .await - { - return Err(ApiError(Error::Xrpc { - name: "UserNotAuthorized", - message: "managing app denied access".to_owned(), - })); - } - Ok(Json(CredentialOutput { - credential: state.space_auth.mint_credential(&input.space, jkt)?, - })) -} - -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct CreateSpaceInput { - #[serde(rename = "type")] - space_type: String, - skey: Option, - policy: Value, - app_access: Value, -} - -#[derive(Serialize)] -pub struct UriOutput { - uri: String, -} - -#[derive(Serialize)] -#[serde(rename_all = "camelCase")] -pub struct WriteOutput { - uri: String, - cid: String, - validation_status: &'static str, -} - -impl From for WriteOutput { - fn from(record: SpaceRecordView) -> Self { - Self { - uri: record.uri, - cid: record.cid, - validation_status: "unknown", - } - } -} - -pub async fn create_space( - State(state): State, - headers: HeaderMap, - Json(input): Json, -) -> ApiResult> { - let auth = auth::token(&headers, &state)?; - Ok(Json(UriOutput { - uri: state.spaces.create_space( - &auth, - &input.space_type, - input.skey.as_deref(), - input.policy, - input.app_access, - )?, - })) -} - -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct UpdateSpaceInput { - space: String, - policy: Option, - app_access: Option, -} - -pub async fn update_space( - State(state): State, - headers: HeaderMap, - Json(input): Json, -) -> ApiResult> { - let auth = auth::token(&headers, &state)?; - state.spaces.update_space( - &auth, - &input.space, - input.policy.as_ref(), - input.app_access.as_ref(), - )?; - Ok(Json(serde_json::json!({}))) -} - -#[derive(Deserialize)] -pub struct SpaceInput { - space: String, -} - -pub async fn delete_space( - State(state): State, - headers: HeaderMap, - Json(input): Json, -) -> ApiResult> { - let auth = auth::token(&headers, &state)?; - let registrations = state.spaces.notify_registrations(&input.space)?; - state.spaces.delete_space(&auth, &input.space)?; - let body = json!({"space": input.space}); - for registration in registrations { - let _ = state - .space_auth - .deliver_notification( - ®istration.endpoint, - ®istration.service, - "com.atproto.space.notifySpaceDeleted", - &body, - ) - .await; - } - Ok(Json(serde_json::json!({}))) -} - -#[derive(Deserialize)] -pub struct SpaceQuery { - space: String, -} - -#[derive(Serialize)] -#[serde(rename_all = "camelCase")] -pub struct SpaceOutput { - uri: String, - policy: Value, - app_access: Value, -} - -impl From for SpaceOutput { - fn from(space: Space) -> Self { - Self { - uri: space.uri, - policy: space.policy, - app_access: space.app_access, - } - } -} - -pub async fn get_space( - State(state): State, - OriginalUri(uri): OriginalUri, - headers: HeaderMap, - Query(query): Query, -) -> ApiResult> { - let auth = auth::space_read_token(&headers, &state, &uri, "GET").await?; - let space = match &auth { - auth::SpaceReadToken::OAuth(token) => state.spaces.get_space(token, &query.space)?, - auth::SpaceReadToken::Credential(credential) => state - .spaces - .get_space_with_credential(&credential.space, &query.space)?, - }; - Ok(Json(space.into())) -} - -#[derive(Deserialize)] -pub struct MemberInput { - space: String, - did: String, -} - -pub async fn add_member( - State(state): State, - headers: HeaderMap, - Json(input): Json, -) -> ApiResult> { - let auth = auth::token(&headers, &state)?; - state.spaces.add_member(&auth, &input.space, &input.did)?; - Ok(Json(serde_json::json!({}))) -} - -pub async fn remove_member( - State(state): State, - headers: HeaderMap, - Json(input): Json, -) -> ApiResult> { - let auth = auth::token(&headers, &state)?; - state - .spaces - .remove_member(&auth, &input.space, &input.did)?; - Ok(Json(serde_json::json!({}))) -} - -#[derive(Deserialize)] -pub struct ListMembersQuery { - space: String, - limit: Option, - cursor: Option, -} - -#[derive(Serialize)] -pub struct MemberView { - did: String, -} - -#[derive(Serialize)] -pub struct ListMembersOutput { - members: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - cursor: Option, -} - -pub async fn list_members( - State(state): State, - headers: HeaderMap, - Query(query): Query, -) -> ApiResult> { - let auth = auth::token(&headers, &state)?; - let limit = query.limit.unwrap_or(100).clamp(1, 1000); - let members = state - .spaces - .list_members(&auth, &query.space, query.cursor.as_deref(), limit)?; - let cursor = members.last().cloned(); - Ok(Json(ListMembersOutput { - members: members.into_iter().map(|did| MemberView { did }).collect(), - cursor, - })) -} - -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct RecordInput { - space: String, - repo: String, - collection: String, - rkey: Option, - #[allow(dead_code)] - validate: Option, - record: Value, -} - -pub async fn create_record( - State(state): State, - headers: HeaderMap, - Json(input): Json, -) -> ApiResult> { - let auth = auth::token(&headers, &state)?; - let output = state.spaces.create_record( - &auth, - CreateRecord { - space: &input.space, - repo: &input.repo, - collection: &input.collection, - rkey: input.rkey.as_deref(), - value: input.record, - validate: input.validate, - }, - )?; - publish_repo_position(&state, &input.space, &input.repo).await; - Ok(Json(output.into())) -} - -pub async fn put_record( - State(state): State, - headers: HeaderMap, - Json(input): Json, -) -> ApiResult> { - let rkey = input.rkey.as_deref().ok_or_else(|| { - ApiError(Error::InvalidRequest( - "putRecord requires `rkey`".to_owned(), - )) - })?; - let auth = auth::token(&headers, &state)?; - let output = state.spaces.put_record( - &auth, - PutRecord { - space: &input.space, - repo: &input.repo, - collection: &input.collection, - rkey, - value: input.record, - validate: input.validate, - }, - )?; - publish_repo_position(&state, &input.space, &input.repo).await; - Ok(Json(output.into())) -} - -#[derive(Deserialize)] -pub struct DeleteRecordInput { - space: String, - repo: String, - collection: String, - rkey: String, -} - -pub async fn delete_record( - State(state): State, - headers: HeaderMap, - Json(input): Json, -) -> ApiResult> { - let auth = auth::token(&headers, &state)?; - state.spaces.delete_record( - &auth, - &input.space, - &input.repo, - &input.collection, - &input.rkey, - )?; - publish_repo_position(&state, &input.space, &input.repo).await; - Ok(Json(serde_json::json!({}))) -} - -#[derive(Deserialize)] -#[serde(tag = "$type")] -pub enum WriteInput { - #[serde(rename = "com.atproto.space.applyWrites#create")] - Create { - collection: String, - rkey: Option, - value: Value, - }, - #[serde(rename = "com.atproto.space.applyWrites#update")] - Update { - collection: String, - rkey: String, - value: Value, - }, - #[serde(rename = "com.atproto.space.applyWrites#delete")] - Delete { collection: String, rkey: String }, -} - -impl From for WriteRequest { - fn from(write: WriteInput) -> Self { - match write { - WriteInput::Create { - collection, - rkey, - value, - } => Self::Create { - collection, - rkey, - value, - }, - WriteInput::Update { - collection, - rkey, - value, - } => Self::Update { - collection, - rkey, - value, - }, - WriteInput::Delete { collection, rkey } => Self::Delete { collection, rkey }, - } - } -} - -#[derive(Deserialize)] -pub struct ApplyWritesInput { - space: String, - repo: String, - validate: Option, - writes: Vec, -} - -#[derive(Serialize)] -#[serde(tag = "$type")] -pub enum AppliedWrite { - #[serde(rename = "com.atproto.space.applyWrites#createResult")] - Create { - uri: String, - cid: String, - #[serde(rename = "validationStatus")] - validation_status: &'static str, - }, - #[serde(rename = "com.atproto.space.applyWrites#updateResult")] - Update { - uri: String, - cid: String, - #[serde(rename = "validationStatus")] - validation_status: &'static str, - }, - #[serde(rename = "com.atproto.space.applyWrites#deleteResult")] - Delete, -} - -impl From for AppliedWrite { - fn from(result: WriteResult) -> Self { - match result { - WriteResult::Create(record) => Self::Create { - uri: record.uri, - cid: record.cid, - validation_status: "unknown", - }, - WriteResult::Update(record) => Self::Update { - uri: record.uri, - cid: record.cid, - validation_status: "unknown", - }, - WriteResult::Delete => Self::Delete, - } - } -} - -#[derive(Serialize)] -pub struct ApplyWritesOutput { - results: Vec, -} - -pub async fn apply_writes( - State(state): State, - headers: HeaderMap, - Json(input): Json, -) -> ApiResult> { - let auth = auth::token(&headers, &state)?; - let has_writes = !input.writes.is_empty(); - let results = state.spaces.apply_writes( - &auth, - &input.space, - &input.repo, - input.writes.into_iter().map(Into::into).collect(), - input.validate, - )?; - if has_writes { - publish_repo_position(&state, &input.space, &input.repo).await; - } - Ok(Json(ApplyWritesOutput { - results: results.into_iter().map(Into::into).collect(), - })) -} - -#[derive(Deserialize)] -pub struct RecordQuery { - space: String, - repo: String, - collection: String, - rkey: String, -} - -pub async fn get_record( - State(state): State, - OriginalUri(uri): OriginalUri, - headers: HeaderMap, - Query(query): Query, -) -> ApiResult> { - let auth = auth::space_read_token(&headers, &state, &uri, "GET").await?; - Ok(Json(state.spaces.get_record( - auth.authorization(), - &query.space, - &query.repo, - &query.collection, - &query.rkey, - )?)) -} - -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ListRecordsQuery { - space: String, - repo: String, - collection: Option, - limit: Option, - cursor: Option, - reverse: Option, - exclude_values: Option, -} - -#[derive(Serialize)] -pub struct ListedRecord { - collection: String, - rkey: String, - cid: String, - #[serde(skip_serializing_if = "Option::is_none")] - value: Option, -} - -impl ListedRecord { - fn from_record(record: SpaceRecord, exclude_values: bool) -> Self { - Self { - collection: record.collection, - rkey: record.rkey, - cid: record.cid.to_string(), - value: (!exclude_values).then_some(record.value), - } - } -} - -#[derive(Serialize)] -pub struct ListRecordsOutput { - records: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - cursor: Option, -} - -pub async fn list_records( - State(state): State, - OriginalUri(uri): OriginalUri, - headers: HeaderMap, - Query(query): Query, -) -> ApiResult> { - let auth = auth::space_read_token(&headers, &state, &uri, "GET").await?; - let exclude_values = query.exclude_values.unwrap_or(false); - let records = state.spaces.list_records( - auth.authorization(), - &ListRecords { - space: &query.space, - repo: &query.repo, - collection: query.collection.as_deref(), - cursor: query.cursor.as_deref(), - limit: query.limit.unwrap_or(50), - reverse: query.reverse.unwrap_or(false), - }, - )?; - let limit = query.limit.unwrap_or(50).clamp(1, 1000); - let cursor = (records.len() == limit) - .then(|| records.last().map(SpaceRecord::uri)) - .flatten(); - Ok(Json(ListRecordsOutput { - records: records - .into_iter() - .map(|record| ListedRecord::from_record(record, exclude_values)) - .collect(), - cursor, - })) -} - -#[derive(Deserialize)] -pub struct ListSpacesQuery { - #[serde(rename = "type")] - space_type: Option, - did: Option, - limit: Option, - cursor: Option, -} - -#[derive(Serialize)] -pub struct ListedSpace { - uri: String, -} - -#[derive(Serialize)] -pub struct ListSpacesOutput { - spaces: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - cursor: Option, -} - -pub async fn list_spaces( - State(state): State, - headers: HeaderMap, - Query(query): Query, -) -> ApiResult> { - let auth = auth::token(&headers, &state)?; - let limit = query.limit.unwrap_or(50).clamp(1, 100); - let spaces = state.spaces.list_spaces( - &auth, - query.space_type.as_deref(), - query.did.as_deref(), - query.cursor.as_deref(), - limit, - )?; - let cursor = (spaces.len() == limit) - .then(|| spaces.last().cloned()) - .flatten(); - Ok(Json(ListSpacesOutput { - spaces: spaces.into_iter().map(|uri| ListedSpace { uri }).collect(), - cursor, - })) -} - -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ListOpsQuery { - space: String, - repo: String, - since: Option, - limit: Option, - cursor: Option, - exclude_values: Option, -} - -#[derive(Serialize)] -pub struct OpView { - rev: String, - collection: String, - rkey: String, - cid: Option, - prev: Option, - #[serde(skip_serializing_if = "Option::is_none")] - value: Option, -} - -#[derive(Serialize)] -pub struct ListOpsOutput { - ops: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - commit: Option, - #[serde(skip_serializing_if = "Option::is_none")] - cursor: Option, -} - -pub async fn list_repo_ops( - State(state): State, - OriginalUri(uri): OriginalUri, - headers: HeaderMap, - Query(query): Query, -) -> ApiResult> { - let auth = auth::space_read_token(&headers, &state, &uri, "GET").await?; - let exclude_values = query.exclude_values.unwrap_or(false); - let page = state.spaces.list_repo_ops( - auth.authorization(), - &query.space, - &query.repo, - query.since.as_deref(), - query.cursor.as_deref(), - query.limit.unwrap_or(100), - )?; - Ok(Json(ListOpsOutput { - ops: page - .ops - .into_iter() - .map(|op| OpView { - rev: op.rev, - collection: op.collection, - rkey: op.rkey, - cid: op.cid, - prev: op.prev, - value: (!exclude_values).then_some(op.value).flatten(), - }) - .collect(), - commit: page.commit, - cursor: page.cursor, - })) -} - -#[derive(Deserialize)] -pub struct LatestCommitQuery { - space: String, - repo: String, -} - -#[derive(Serialize)] -pub struct LatestCommitOutput { - commit: super::commit::SignedCommit, -} - -pub async fn get_latest_commit( - State(state): State, - OriginalUri(uri): OriginalUri, - headers: HeaderMap, - Query(query): Query, -) -> ApiResult> { - let auth = auth::space_read_token(&headers, &state, &uri, "GET").await?; - Ok(Json(LatestCommitOutput { - commit: state - .spaces - .get_latest_commit(auth.authorization(), &query.space, &query.repo)?, - })) -} - -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct GetRepoQuery { - space: String, - repo: String, - exclude_values: Option, -} - -pub async fn get_repo( - State(state): State, - OriginalUri(uri): OriginalUri, - headers: HeaderMap, - Query(query): Query, -) -> ApiResult { - let auth = auth::space_read_token(&headers, &state, &uri, "GET").await?; - let car = state.spaces.get_repo( - auth.authorization(), - &query.space, - &query.repo, - query.exclude_values.unwrap_or(false), - )?; - Ok(([(CONTENT_TYPE, "application/vnd.ipld.car")], car).into_response()) -} - -#[derive(Deserialize)] -pub struct NotifyWriteInput { - space: String, - repo: String, - rev: String, - hash: LexBytes, -} - -pub async fn notify_write( - State(state): State, - headers: HeaderMap, - Json(input): Json, -) -> ApiResult> { - let service = auth::service(&headers, &state, "com.atproto.space.notifyWrite").await?; - let managing_app = state - .spaces - .managing_app_for_writer(&input.space, &input.repo)?; - let managing_app_authorized = if let Some(managing_app) = managing_app { - state - .space_auth - .check_user_access(&managing_app, &input.space, &input.repo, None) - .await - } else { - false - }; - state.spaces.record_notified_writer(&NotifiedWrite { - issuer: &service.iss, - audience: &service.aud, - space: &input.space, - repo: &input.repo, - rev: &input.rev, - hash: &input.hash.decode()?, - managing_app_authorized, - })?; - let notification = json!({ - "space": input.space, - "repo": input.repo, - "rev": input.rev, - "hash": input.hash, - }); - for registration in state - .spaces - .notify_registrations(notification["space"].as_str().unwrap_or_default())? - { - let _ = state - .space_auth - .deliver_notification( - ®istration.endpoint, - ®istration.service, - "com.atproto.space.notifyWrite", - ¬ification, - ) - .await; - } - Ok(Json(json!({}))) -} - -#[derive(Deserialize)] -pub struct ListReposQuery { - space: String, - limit: Option, - cursor: Option, -} - -#[derive(Serialize)] -pub struct RepoView { - did: String, - rev: String, - hash: LexBytes, -} - -#[derive(Serialize)] -pub struct ListReposOutput { - repos: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - cursor: Option, -} - -pub async fn list_repos( - State(state): State, - OriginalUri(uri): OriginalUri, - headers: HeaderMap, - Query(query): Query, -) -> ApiResult> { - let credential = auth::space_credential(&headers, &state, &uri, "GET").await?; - let limit = query.limit.unwrap_or(100).clamp(1, 1000); - let writers = state.spaces.list_repos( - &credential.space, - &query.space, - query.cursor.as_deref(), - limit, - )?; - let cursor = (writers.len() == limit) - .then(|| writers.last().map(|writer| writer.did.clone())) - .flatten(); - Ok(Json(ListReposOutput { - repos: writers - .into_iter() - .map(|writer| RepoView { - did: writer.did, - rev: writer.rev, - hash: LexBytes::new(writer.hash), - }) - .collect(), - cursor, - })) -} - -#[derive(Deserialize)] -pub struct NotifyRegistrationInput { - space: String, - service: String, -} - -#[derive(Serialize)] -#[serde(rename_all = "camelCase")] -pub struct NotifyRegistrationOutput { - expires_at: String, -} - -pub async fn register_notify( - State(state): State, - OriginalUri(uri): OriginalUri, - headers: HeaderMap, - Json(input): Json, -) -> ApiResult> { - let credential = auth::space_credential(&headers, &state, &uri, "POST").await?; - let endpoint = state - .space_auth - .resolve_service_endpoint(&input.service) - .await?; - let expires_at = state.spaces.register_notify( - &credential.space, - &input.space, - &input.service, - endpoint.as_str(), - )?; - Ok(Json(NotifyRegistrationOutput { - expires_at: datetime(expires_at)?, - })) -} - -pub async fn unregister_notify( - State(state): State, - OriginalUri(uri): OriginalUri, - headers: HeaderMap, - Json(input): Json, -) -> ApiResult> { - let credential = auth::space_credential(&headers, &state, &uri, "POST").await?; - state - .spaces - .unregister_notify(&credential.space, &input.space, &input.service)?; - Ok(Json(json!({}))) -} - -pub async fn notify_space_deleted( - State(state): State, - headers: HeaderMap, - Json(input): Json, -) -> ApiResult> { - let service = auth::service(&headers, &state, "com.atproto.space.notifySpaceDeleted").await?; - let space = crate::syntax::space_ref(&input.space)?; - if service.iss != space.authority { - return Err(ApiError(Error::Authorization( - "notifySpaceDeleted issuer is not the space authority".to_owned(), - ))); - } - if service.aud != state.config.account.did { - return Err(ApiError(Error::Authorization( - "notifySpaceDeleted audience does not match this service".to_owned(), - ))); - } - Ok(Json(json!({}))) -} - -async fn publish_repo_position(state: &AppState, space: &str, repo: &str) { - let Ok(Some((rev, hash))) = state.spaces.repo_position(space, repo) else { - return; - }; - let Ok(space_ref) = crate::syntax::space_ref(space) else { - return; - }; - if space_ref.authority == state.config.account.did { - let _ = state.spaces.record_notified_writer(&NotifiedWrite { - issuer: repo, - audience: &space_ref.authority, - space, - repo, - rev: &rev, - hash: &hash, - managing_app_authorized: false, - }); - return; - } - let Ok(endpoint) = state - .space_auth - .resolve_service_endpoint(&space_ref.authority) - .await - else { - return; - }; - let body = json!({ - "space": space, - "repo": repo, - "rev": rev, - "hash": LexBytes::new(hash), - }); - let _ = state - .space_auth - .deliver_notification( - endpoint.as_str(), - &space_ref.authority, - "com.atproto.space.notifyWrite", - &body, - ) - .await; -} - -fn datetime(timestamp: i64) -> Result { - let value = time::OffsetDateTime::from_unix_timestamp(timestamp) - .map_err(|error| ApiError(Error::Internal(format!("invalid timestamp: {error}"))))?; - Ok(format!( - "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}Z", - value.year(), - value.month() as u8, - value.day(), - value.hour(), - value.minute(), - value.second() - )) -} - -#[derive(Deserialize)] -pub struct BlobQuery { - space: String, - repo: String, - cid: String, -} - -pub async fn get_blob( - State(state): State, - OriginalUri(uri): OriginalUri, - headers: HeaderMap, - Query(query): Query, -) -> ApiResult { - let auth = auth::space_read_token(&headers, &state, &uri, "GET").await?; - let cid = Cid::from_str(&query.cid) - .map_err(|error| ApiError(Error::InvalidRequest(format!("invalid CID: {error}"))))?; - state - .spaces - .get_blob_allowed(auth.authorization(), &query.space, &query.repo, &cid)?; - let (blob, bytes) = state - .repository - .get_blob(&query.repo, &cid) - .await - .map_err(|error| match error { - Error::NotFound => ApiError(Error::Xrpc { - name: "BlobNotFound", - message: "blob data was not found".to_owned(), - }), - other => ApiError(other), - })?; - let content_type = HeaderValue::from_str(&blob.media_type).map_err(|error| { - ApiError(Error::Internal(format!( - "invalid stored MIME type: {error}" - ))) - })?; - Ok(([(CONTENT_TYPE, content_type)], bytes).into_response()) -} +pub(crate) use notifications::{ + list_repos, notify_space_deleted, notify_write, register_notify, unregister_notify, +}; +pub(crate) use records::{ + apply_writes, create_record, delete_record, get_record, list_records, put_record, +}; +pub(crate) use sync::{get_blob, get_latest_commit, get_repo, list_blobs, list_repo_ops}; #[derive(Deserialize)] -pub struct ListBlobsQuery { +pub(crate) struct SpaceInput { space: String, - repo: String, - since: Option, - limit: Option, - cursor: Option, -} - -#[derive(Serialize)] -pub struct ListBlobsOutput { - cids: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - cursor: Option, -} - -pub async fn list_blobs( - State(state): State, - OriginalUri(uri): OriginalUri, - headers: HeaderMap, - Query(query): Query, -) -> ApiResult> { - let auth = auth::space_read_token(&headers, &state, &uri, "GET").await?; - let limit = query.limit.unwrap_or(500).clamp(1, 1000); - let cids = state.spaces.list_blobs( - auth.authorization(), - &query.space, - &query.repo, - query.since.as_deref(), - query.cursor.as_deref(), - limit, - )?; - let cursor = (cids.len() == limit) - .then(|| cids.last().cloned()) - .flatten(); - Ok(Json(ListBlobsOutput { cids, cursor })) } diff --git a/src/space/http/credentials.rs b/src/space/http/credentials.rs new file mode 100644 index 0000000..d0d1e9f --- /dev/null +++ b/src/space/http/credentials.rs @@ -0,0 +1,137 @@ +//! Delegation exchange and DPoP-bound Space credential endpoints. + +use crate::http::request::{HeaderQuery, SpaceJson}; +use axum::{ + Json, + extract::{OriginalUri, Query, State}, +}; +use serde::Deserialize; +use serde_json::{Value, json}; + +use crate::{ + error::Error, + http::{ApiResult, auth}, + space::auth::SpaceClaims, +}; + +use super::SpaceInput; + +pub(crate) async fn get_delegation_token( + (State(state), headers, Query(query)): HeaderQuery, +) -> ApiResult> { + let oauth = auth::token(&headers, &state)?; + state.spaces.authorize_delegation(&oauth, &query.space)?; + Ok(Json(json!({ + "token": state + .space_auth + .mint_delegation(&query.space, &oauth.account.did)?, + }))) +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct CredentialInput { + space: String, + client_attestation: Option, +} + +pub(crate) async fn get_space_credential( + (State(state), OriginalUri(uri), headers, Json(input)): SpaceJson, +) -> ApiResult> { + let token = auth::bearer(&headers)?; + let proof = auth::dpop_proof(&headers)?; + let htu = auth::request_url(&state, &uri); + let (SpaceClaims { iss, sub, .. }, jkt) = state + .space_auth + .exchange_delegation(token, proof, "POST", &htu) + .await?; + if sub != input.space { + return Err(Error::xrpc( + "InvalidDelegationToken", + "delegation token subject does not match the requested space", + )); + } + let space_ref = input.space.parse::()?; + let client_id = if let Some(attestation) = input.client_attestation.as_deref() { + Some( + state + .space_auth + .verify_client_attestation( + attestation, + &format!("{}#atproto_space_host", space_ref.authority), + ) + .await?, + ) + } else { + None + }; + let managing_app = + state + .spaces + .authorize_credential(&input.space, &iss, client_id.as_deref())?; + if let Some(service) = managing_app + && !state + .space_auth + .check_user_access(&service, &input.space, &iss, client_id.as_deref()) + .await + { + return Err(Error::xrpc( + "UserNotAuthorized", + "managing app denied access", + )); + } + Ok(Json(json!({ + "credential": state.space_auth.mint_credential(&input.space, jkt)?, + }))) +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::{body::to_bytes, response::IntoResponse}; + use serde::Serialize; + + #[tokio::test] + async fn single_string_credential_outputs_keep_full_response_bytes() { + #[derive(Serialize)] + struct PreviousDelegationOutput { + token: String, + } + #[derive(Serialize)] + struct PreviousCredentialOutput { + credential: String, + } + let values = [ + String::new(), + "header.payload.signature".to_owned(), + "quote-\"-backslash-\\-雪-🦀".to_owned(), + (0u8..=31).map(char::from).collect(), + "a".repeat(4096), + ]; + for value in values { + for (previous, actual) in [ + ( + Json(PreviousDelegationOutput { + token: value.clone(), + }) + .into_response(), + Json(json!({"token": value})).into_response(), + ), + ( + Json(PreviousCredentialOutput { + credential: value.clone(), + }) + .into_response(), + Json(json!({"credential": value})).into_response(), + ), + ] { + assert_eq!(actual.status(), previous.status()); + assert_eq!(actual.headers(), previous.headers()); + assert_eq!( + to_bytes(actual.into_body(), usize::MAX).await.unwrap(), + to_bytes(previous.into_body(), usize::MAX).await.unwrap(), + ); + } + } + } +} diff --git a/src/space/http/management.rs b/src/space/http/management.rs new file mode 100644 index 0000000..847a936 --- /dev/null +++ b/src/space/http/management.rs @@ -0,0 +1,382 @@ +//! Simple-space lifecycle, membership, and Space discovery endpoints. + +use crate::http::request::{HeaderQuery, JsonRequest, SpaceQuery}; +use axum::{ + Json, + extract::{OriginalUri, Query, State}, +}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +use crate::{ + http::{ApiResult, auth}, + space::{CreateSpaceInput, ListSpacesQuery, SpacePageQuery, UpdateSpaceInput, domain::Space}, +}; + +use super::{SpaceInput, notifications::deliver_registrations}; + +pub(crate) async fn create_space( + (State(state), headers, Json(input)): JsonRequest, +) -> ApiResult> { + let auth = auth::token(&headers, &state)?; + Ok(Json(json!({ + "uri": state.spaces.create_space(&auth, input)?, + }))) +} + +pub(crate) async fn update_space( + (State(state), headers, Json(input)): JsonRequest, +) -> ApiResult> { + let auth = auth::token(&headers, &state)?; + state.spaces.update_space(&auth, &input)?; + Ok(Json(json!({}))) +} + +pub(crate) async fn delete_space( + (State(state), headers, Json(input)): JsonRequest, +) -> ApiResult> { + let auth = auth::token(&headers, &state)?; + let registrations = state.spaces.notify_registrations(&input.space)?; + state.spaces.delete_space(&auth, &input.space)?; + let body = json!({"space": input.space}); + deliver_registrations( + &state, + ®istrations, + "com.atproto.space.notifySpaceDeleted", + &body, + ) + .await; + Ok(Json(json!({}))) +} + +pub(crate) async fn get_space( + (State(state), OriginalUri(uri), headers, Query(query)): SpaceQuery, +) -> ApiResult> { + let auth = auth::space_read_token(&headers, &state, &uri, "GET").await?; + Ok(Json(state.spaces.get_space(&auth, &query.space)?)) +} + +#[derive(Deserialize)] +pub(crate) struct MemberInput { + space: String, + did: String, +} + +pub(crate) async fn add_member( + (State(state), headers, Json(input)): JsonRequest, +) -> ApiResult> { + let auth = auth::token(&headers, &state)?; + state.spaces.add_member(&auth, &input.space, &input.did)?; + Ok(Json(json!({}))) +} + +pub(crate) async fn remove_member( + (State(state), headers, Json(input)): JsonRequest, +) -> ApiResult> { + let auth = auth::token(&headers, &state)?; + state + .spaces + .remove_member(&auth, &input.space, &input.did)?; + Ok(Json(json!({}))) +} + +#[derive(Serialize)] +pub(crate) struct ListMembersOutput { + members: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + cursor: Option, +} + +pub(crate) async fn list_members( + (State(state), headers, Query(query)): HeaderQuery, +) -> ApiResult> { + let auth = auth::token(&headers, &state)?; + let members = state.spaces.list_members(&auth, &query)?; + let cursor = members.last().cloned(); + Ok(Json(ListMembersOutput { + members: members.into_iter().map(|did| json!({"did": did})).collect(), + cursor, + })) +} + +#[derive(Serialize)] +pub(crate) struct ListSpacesOutput { + spaces: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + cursor: Option, +} + +pub(crate) async fn list_spaces( + (State(state), headers, Query(query)): HeaderQuery, +) -> ApiResult> { + let auth = auth::token(&headers, &state)?; + let limit = query.limit.unwrap_or(50).clamp(1, 100); + let spaces = state.spaces.list_spaces(&auth, &query)?; + let cursor = (spaces.len() == limit) + .then(|| spaces.last().cloned()) + .flatten(); + Ok(Json(ListSpacesOutput { + spaces: spaces.into_iter().map(|uri| json!({"uri": uri})).collect(), + cursor, + })) +} + +#[cfg(test)] +mod tests { + use super::{ + CreateSpaceInput, Json, ListMembersOutput, ListSpacesOutput, ListSpacesQuery, Query, Space, + SpacePageQuery, UpdateSpaceInput, Value, json, + }; + use axum::{body::to_bytes, response::IntoResponse}; + use serde::{Deserialize, Serialize}; + + #[derive(Serialize)] + struct PreviousUriOutput { + uri: String, + } + + #[derive(Serialize)] + struct PreviousMemberView { + did: String, + } + + async fn assert_response_bytes(previous: impl Serialize, actual: impl Serialize) { + let previous = Json(previous).into_response(); + let actual = Json(actual).into_response(); + assert_eq!(actual.status(), previous.status()); + assert_eq!(actual.headers(), previous.headers()); + assert_eq!( + to_bytes(actual.into_body(), usize::MAX).await.unwrap(), + to_bytes(previous.into_body(), usize::MAX).await.unwrap(), + ); + } + + #[tokio::test] + async fn single_string_management_outputs_keep_full_response_bytes() { + let values = [ + String::new(), + "at://did:example:alice/space/app.test.space/one".to_owned(), + "did:example:alice".to_owned(), + "not a URI or DID".to_owned(), + "quote-\"-backslash-\\-雪-🦀".to_owned(), + (0u8..=31).map(char::from).collect(), + ]; + for value in values { + assert_response_bytes( + PreviousUriOutput { uri: value.clone() }, + json!({"uri": value}), + ) + .await; + assert_response_bytes( + PreviousMemberView { did: value.clone() }, + json!({"did": value}), + ) + .await; + } + } + + #[tokio::test] + async fn single_string_management_pages_keep_full_response_bytes() { + #[derive(Serialize)] + struct PreviousListMembersOutput { + members: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + cursor: Option, + } + #[derive(Serialize)] + struct PreviousListSpacesOutput { + spaces: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + cursor: Option, + } + for values in [vec![], vec![""], vec!["z", "a-\"\\\n\0-雪", ""]] { + for cursor in [ + None, + Some(String::new()), + Some("cursor-\"\\\n-雪".to_owned()), + ] { + assert_response_bytes( + PreviousListMembersOutput { + members: values + .iter() + .map(|did| PreviousMemberView { + did: (*did).to_owned(), + }) + .collect(), + cursor: cursor.clone(), + }, + ListMembersOutput { + members: values.iter().map(|did| json!({"did": did})).collect(), + cursor: cursor.clone(), + }, + ) + .await; + assert_response_bytes( + PreviousListSpacesOutput { + spaces: values + .iter() + .map(|uri| PreviousUriOutput { + uri: (*uri).to_owned(), + }) + .collect(), + cursor: cursor.clone(), + }, + ListSpacesOutput { + spaces: values.iter().map(|uri| json!({"uri": uri})).collect(), + cursor, + }, + ) + .await; + } + } + } + + #[test] + fn shared_management_inputs_keep_exact_json_parsing() { + #[derive(Deserialize)] + #[serde(rename_all = "camelCase")] + struct PreviousCreateSpaceInput { + #[serde(rename = "type")] + space_type: String, + skey: Option, + policy: Value, + app_access: Value, + } + #[derive(Deserialize)] + #[serde(rename_all = "camelCase")] + struct PreviousUpdateSpaceInput { + space: String, + policy: Option, + app_access: Option, + } + for input in [ + "{}", + r#"{"type":"app.test.space","policy":{},"appAccess":{},"unknown":true}"#, + r#"{"type":"app.test.space","skey":null,"policy":null,"appAccess":null}"#, + r#"{"type":"app.test.space","skey":7,"policy":{},"appAccess":{}}"#, + r#"{"type":"app.test.space","type":"duplicate","policy":{},"appAccess":{}}"#, + r#"{"type":"app.test.space","policy":{},"app_access":{}}"#, + r#"{"type":null,"policy":{},"appAccess":{}}"#, + r#"{"type":"app.test.space","appAccess":{}}"#, + r#"{"space":"space"}"#, + r#"{"space":"space","policy":null,"appAccess":null}"#, + r#"{"space":"space","policy":{},"appAccess":{},"unknown":true}"#, + r#"{"space":"space","policy":{},"policy":null}"#, + r#"{"space":"space","space":"duplicate"}"#, + r#"{"space":null,"app_access":{}}"#, + r#"{"space":"space","app_access":{}}"#, + ] { + let previous = serde_json::from_str::(input) + .map(|input| (input.space_type, input.skey, input.policy, input.app_access)) + .map_err(|error| error.to_string()); + let actual = serde_json::from_str::(input) + .map(|input| (input.space_type, input.skey, input.policy, input.app_access)) + .map_err(|error| error.to_string()); + assert_eq!(actual, previous, "create input {input}"); + let previous = serde_json::from_str::(input) + .map(|input| (input.space, input.policy, input.app_access)) + .map_err(|error| error.to_string()); + let actual = serde_json::from_str::(input) + .map(|input| (input.space, input.policy, input.app_access)) + .map_err(|error| error.to_string()); + assert_eq!(actual, previous, "update input {input}"); + } + } + + #[test] + fn shared_member_query_keeps_exact_values_and_rejections() { + #[derive(Deserialize)] + struct PreviousListMembersQuery { + space: String, + limit: Option, + cursor: Option, + } + let mut paths = vec!["/".to_owned(), "/?limit=1".to_owned()]; + paths.extend([ + "", "&limit=0", "&limit=1001", "&limit=-1", "&limit=", "&limit=no", + "&space=duplicate", "&limit=1&limit=2", "&cursor=one&cursor=two", + "&cursor=", "&cursor=not-a-did", "&unknown=one&unknown=two", + ].map(|suffix| format!("/?space=at%3A%2F%2Fdid%3Aexample%3Aalice%2Fspace%2Fapp.test.space%2Fone{suffix}"))); + paths.push(format!("/?space=space&limit={}", usize::MAX)); + paths.push(format!("/?space=space&limit={}0", usize::MAX)); + for path in paths { + let uri = path.parse().unwrap(); + let previous = Query::::try_from_uri(&uri) + .map(|Query(query)| (query.space, query.limit, query.cursor)) + .map_err(|error| (error.status(), error.body_text())); + let actual = Query::::try_from_uri(&uri) + .map(|Query(query)| (query.space, query.limit, query.cursor)) + .map_err(|error| (error.status(), error.body_text())); + assert_eq!(actual, previous, "member query {path}"); + } + } + + #[test] + fn shared_space_discovery_query_keeps_exact_values_and_rejections() { + #[derive(Deserialize)] + struct PreviousListSpacesQuery { + #[serde(rename = "type")] + space_type: Option, + did: Option, + limit: Option, + cursor: Option, + } + let mut paths = [ + "/", + "/?type=com.example.forum&did=did%3Aexample%3Aremote&cursor=one&limit=1", + "/?type=&did=&cursor=", + "/?type=invalid&did=invalid", + "/?space_type=ignored&unknown=one&unknown=two", + "/?type=one&type=two", + "/?did=one&did=two", + "/?cursor=one&cursor=two", + "/?limit=1&limit=2", + "/?limit=0", + "/?limit=101", + "/?limit=-1", + "/?limit=", + "/?limit=no", + ] + .map(str::to_owned) + .to_vec(); + paths.push(format!("/?limit={}", usize::MAX)); + paths.push(format!("/?limit={}0", usize::MAX)); + for path in paths { + let uri = path.parse().unwrap(); + let previous = Query::::try_from_uri(&uri) + .map(|Query(query)| (query.space_type, query.did, query.limit, query.cursor)) + .map_err(|error| (error.status(), error.body_text())); + let actual = Query::::try_from_uri(&uri) + .map(|Query(query)| (query.space_type, query.did, query.limit, query.cursor)) + .map_err(|error| (error.status(), error.body_text())); + assert_eq!(actual, previous, "space discovery query {path}"); + } + } + + #[test] + fn space_output_preserves_public_fields_and_omits_internal_metadata() { + let space = Space { + uri: "at://did:web:alice.test/com.atproto.space.test/key".to_owned(), + authority: "did:web:alice.test".to_owned(), + kind: "com.atproto.space.test".to_owned(), + skey: "key".to_owned(), + policy: json!({"read": "members"}), + app_access: json!(null), + }; + assert_eq!( + serde_json::to_value(&space).unwrap(), + json!({ + "uri": space.uri, + "policy": {"read": "members"}, + "appAccess": null, + }), + ); + assert_eq!( + serde_json::to_string(&space).unwrap(), + concat!( + r#"{"uri":"at://did:web:alice.test/com.atproto.space.test/key","#, + r#""policy":{"read":"members"},"appAccess":null}"#, + ), + ); + } +} diff --git a/src/space/http/notifications.rs b/src/space/http/notifications.rs new file mode 100644 index 0000000..0577feb --- /dev/null +++ b/src/space/http/notifications.rs @@ -0,0 +1,446 @@ +//! Writer discovery, authenticated notices, and subscription registration. + +use crate::http::request::{JsonRequest, SpaceJson, SpaceQuery}; +use axum::{ + Json, + extract::{OriginalUri, Query, State}, +}; +use futures::{StreamExt as _, stream}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +use crate::{ + AppState, + clock::format_utc_datetime, + error::{Error, ErrorContext as _}, + events::best_effort_notice, + http::{ApiResult, auth}, + space::{ + NotifiedWrite, NotifyRegistrationInput, SpacePageQuery, commit::LexBytes, + domain::SpaceWriter, + }, +}; + +use super::SpaceInput; + +#[derive(Deserialize, Serialize)] +pub(crate) struct NotifyWriteInput { + space: String, + repo: String, + rev: String, + hash: LexBytes, +} + +pub(crate) async fn notify_write( + (State(state), headers, Json(input)): JsonRequest, +) -> ApiResult> { + let (issuer, audience) = + auth::service(&headers, &state, "com.atproto.space.notifyWrite").await?; + let managing_app = state + .spaces + .managing_app_for_writer(&input.space, &input.repo)?; + let managing_app_authorized = if let Some(managing_app) = managing_app { + state + .space_auth + .check_user_access(&managing_app, &input.space, &input.repo, None) + .await + } else { + false + }; + let registrations = state.spaces.notify_registrations(&input.space)?; + state.spaces.record_notified_writer(&NotifiedWrite { + issuer: &issuer, + audience: &audience, + space: &input.space, + repo: &input.repo, + rev: &input.rev, + hash: &input.hash.decode()?, + managing_app_authorized, + })?; + let notification = json!(input); + deliver_registrations( + &state, + ®istrations, + "com.atproto.space.notifyWrite", + ¬ification, + ) + .await; + Ok(Json(json!({}))) +} + +#[derive(Serialize)] +pub(crate) struct ListReposOutput { + repos: Vec>, + #[serde(skip_serializing_if = "Option::is_none")] + cursor: Option, +} + +pub(crate) async fn list_repos( + (State(state), OriginalUri(uri), headers, Query(query)): SpaceQuery, +) -> ApiResult> { + let space = auth::space_credential(&headers, &state, &uri, "GET").await?; + let limit = query.limit.unwrap_or(100).clamp(1, 1000); + let writers = state.spaces.list_repos(&space, &query)?; + let cursor = (writers.len() == limit) + .then(|| writers.last().map(|writer| writer.did.clone())) + .flatten(); + Ok(Json(ListReposOutput { + repos: writers + .into_iter() + .map(|writer| SpaceWriter { + did: writer.did, + rev: writer.rev, + hash: LexBytes::new(writer.hash), + }) + .collect(), + cursor, + })) +} + +pub(crate) async fn register_notify( + (State(state), OriginalUri(uri), headers, Json(input)): SpaceJson, +) -> ApiResult> { + let space = auth::space_credential(&headers, &state, &uri, "POST").await?; + let endpoint = state + .space_auth + .resolve_service_endpoint(&input.service) + .await?; + let expires_at = state + .spaces + .register_notify(&space, &input, endpoint.as_str())?; + Ok(Json(json!({ + "expiresAt": format_utc_datetime(expires_at) + .context(Error::Internal, "invalid timestamp")?, + }))) +} + +pub(crate) async fn unregister_notify( + (State(state), OriginalUri(uri), headers, Json(input)): SpaceJson, +) -> ApiResult> { + let space = auth::space_credential(&headers, &state, &uri, "POST").await?; + state.spaces.unregister_notify(&space, &input)?; + Ok(Json(json!({}))) +} + +pub(crate) async fn notify_space_deleted( + (State(state), headers, Json(input)): JsonRequest, +) -> ApiResult> { + let (issuer, audience) = + auth::service(&headers, &state, "com.atproto.space.notifySpaceDeleted").await?; + let space = input.space.parse::()?; + if issuer != space.authority { + return Err(Error::Authorization( + "notifySpaceDeleted issuer is not the space authority".to_owned(), + )); + } + if audience != state.config.account.did { + return Err(Error::Authorization( + "notifySpaceDeleted audience does not match this service".to_owned(), + )); + } + Ok(Json(json!({}))) +} + +pub(super) async fn publish_repo_position(state: &AppState, space: &str, repo: &str) { + best_effort_notice("Space repository position", async { + let Some((rev, hash)) = state.spaces.repo_position(space, repo)? else { + return Ok(()); + }; + let space_ref = space.parse::()?; + if space_ref.authority == state.config.account.did { + return state.spaces.record_notified_writer(&NotifiedWrite { + issuer: repo, + audience: &space_ref.authority, + space, + repo, + rev: &rev, + hash: &hash, + managing_app_authorized: false, + }); + } + let endpoint = state + .space_auth + .resolve_service_endpoint(&space_ref.authority) + .await?; + let body = json!({ + "space": space, + "repo": repo, + "rev": rev, + "hash": LexBytes::new(hash), + }); + state + .space_auth + .deliver_notification( + endpoint.as_str(), + &space_ref.authority, + "com.atproto.space.notifyWrite", + &body, + ) + .await + }) + .await; +} + +pub(super) async fn deliver_registrations( + state: &AppState, + registrations: &[(String, String)], + method: &'static str, + body: &Value, +) { + best_effort_notice("Space subscriber notifications", async { + stream::iter(registrations).for_each_concurrent(4, |(service, endpoint)| async move { + if let Err(error) = state.space_auth.deliver_notification( + endpoint, service, method, body, + ).await { + tracing::warn!(service = %service, %method, %error, "Space notification failed"); + } + }).await; + Ok(()) + }) + .await; +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::{body::to_bytes, response::IntoResponse}; + + #[tokio::test] + async fn single_string_registration_output_keeps_full_response_bytes() { + #[derive(Serialize)] + #[serde(rename_all = "camelCase")] + struct PreviousNotifyRegistrationOutput { + expires_at: String, + } + let values = [ + String::new(), + "1970-01-01T00:00:00Z".to_owned(), + "-9999-01-01T00:00:00Z".to_owned(), + "quote-\"-backslash-\\-雪-🦀".to_owned(), + (0u8..=31).map(char::from).collect(), + ]; + for value in values { + let previous = Json(PreviousNotifyRegistrationOutput { + expires_at: value.clone(), + }) + .into_response(); + let actual = Json(json!({"expiresAt": value})).into_response(); + assert_eq!(actual.status(), previous.status()); + assert_eq!(actual.headers(), previous.headers()); + assert_eq!( + to_bytes(actual.into_body(), usize::MAX).await.unwrap(), + to_bytes(previous.into_body(), usize::MAX).await.unwrap(), + ); + } + } + + #[test] + fn registration_timestamps_preserve_exact_text_and_error_context() { + let mut cases = vec![(0, "1970-01-01T00:00:00Z"), (-1, "1969-12-31T23:59:59Z")]; + for (year, expected) in [ + (-9999, "-9999-01-01T00:00:00Z"), + (-1, "-001-01-01T00:00:00Z"), + (0, "0000-01-01T00:00:00Z"), + (1, "0001-01-01T00:00:00Z"), + (2000, "2000-01-01T00:00:00Z"), + (9999, "9999-01-01T00:00:00Z"), + ] { + let timestamp = time::Date::from_calendar_date(year, time::Month::January, 1) + .unwrap() + .midnight() + .assume_utc() + .unix_timestamp(); + cases.push((timestamp, expected)); + } + for (timestamp, expected) in cases { + let output = json!({ + "expiresAt": format_utc_datetime(timestamp) + .context(Error::Internal, "invalid timestamp") + .unwrap(), + }); + assert_eq!(output["expiresAt"], expected); + assert_eq!( + serde_json::to_vec(&output).unwrap(), + format!(r#"{{"expiresAt":"{expected}"}}"#).as_bytes(), + ); + } + for timestamp in [i64::MIN, i64::MAX] { + let previous = time::OffsetDateTime::from_unix_timestamp(timestamp) + .map_err(|error| Error::Internal(format!("invalid timestamp: {error}"))) + .unwrap_err(); + let actual = format_utc_datetime(timestamp) + .context(Error::Internal, "invalid timestamp") + .unwrap_err(); + assert!(matches!(actual, Error::Internal(_))); + assert_eq!(format!("{actual:?}"), format!("{previous:?}")); + assert_eq!(actual.to_string(), previous.to_string()); + } + } + + #[test] + fn writer_pages_keep_previous_json_bytes() { + #[derive(Serialize)] + struct PreviousRepoView { + did: String, + rev: String, + hash: LexBytes, + } + #[derive(Serialize)] + struct PreviousListReposOutput { + repos: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + cursor: Option, + } + let hashes = [ + Vec::new(), + (0..=u8::MAX).collect(), + vec![0], + vec![0, 1], + vec![0, 1, 2], + vec![0, 1, 2, 3], + vec![0; 31], + vec![u8::MAX; 32], + vec![128; 33], + ]; + for hash in hashes { + for (did, rev) in [ + ("", ""), + ("did:web:alice.test", "3jqfcqzm3fo2j"), + ("not a DID", "not a revision"), + ("did:example:café/\"\\\n\t\0", "rev-\"\\\n\t\0-雪"), + ] { + for cursor in [None, Some("cursor-\"\\\n-雪".to_owned())] { + let writer = SpaceWriter { + did: did.to_owned(), + rev: rev.to_owned(), + hash: hash.clone(), + }; + assert_eq!( + format!("{writer:?}"), + format!("SpaceWriter {{ did: {did:?}, rev: {rev:?}, hash: {hash:?} }}"), + ); + let previous = PreviousListReposOutput { + repos: vec![PreviousRepoView { + did: writer.did.clone(), + rev: writer.rev.clone(), + hash: LexBytes::new(&writer.hash), + }], + cursor: cursor.clone(), + }; + let actual = ListReposOutput { + repos: vec![SpaceWriter { + did: writer.did, + rev: writer.rev, + hash: LexBytes::new(writer.hash), + }], + cursor, + }; + assert_eq!( + serde_json::to_vec(&actual).unwrap(), + serde_json::to_vec(&previous).unwrap(), + "hash length {}: {did:?}, {rev:?}", + hash.len(), + ); + } + } + } + assert_eq!( + serde_json::to_vec(&ListReposOutput { + repos: Vec::new(), + cursor: None, + }) + .unwrap(), + br#"{"repos":[]}"#, + ); + } + + #[test] + fn shared_writer_query_keeps_previous_values_and_rejections() { + #[derive(Deserialize)] + struct PreviousListReposQuery { + space: String, + limit: Option, + cursor: Option, + } + for path in [ + "/", + "/?limit=1", + "/?space=", + "/?space=space", + "/?space=space&cursor=", + "/?space=space&limit=0", + "/?space=space&limit=1001", + "/?space=space&limit=", + "/?space=space&limit=-1", + "/?space=space&limit=no", + "/?space=space&space=other", + "/?space=space&limit=1&limit=2", + "/?space=space&cursor=one&cursor=two", + "/?space=space&unknown=one&unknown=two", + "/?space=at%3A%2F%2Fdid%3Aexample%3Aalice%2Fspace%2Fapp.test.space%2Fone&cursor=not-a-did", + ] { + let uri = path.parse().unwrap(); + let previous = Query::::try_from_uri(&uri) + .map(|Query(query)| (query.space, query.limit, query.cursor)) + .map_err(|error| (error.status(), error.body_text())); + let actual = Query::::try_from_uri(&uri) + .map(|Query(query)| (query.space, query.limit, query.cursor)) + .map_err(|error| (error.status(), error.body_text())); + assert_eq!(actual, previous, "writer query {path}"); + } + } + + #[test] + fn shared_registration_input_keeps_previous_json_parsing() { + #[derive(Deserialize)] + struct PreviousNotifyRegistrationInput { + space: String, + service: String, + } + for input in [ + "{}", + r#"{"space":"space"}"#, + r#"{"service":"service"}"#, + r#"{"space":"space","service":"service","unknown":true}"#, + r#"{"space":"","service":""}"#, + r#"{"space":null,"service":"service"}"#, + r#"{"space":"space","service":null}"#, + r#"{"space":"space","space":"duplicate","service":"service"}"#, + r#"{"space":"space","service":"service","service":"duplicate"}"#, + ] { + let previous = serde_json::from_str::(input) + .map(|input| (input.space, input.service)) + .map_err(|error| error.to_string()); + let actual = serde_json::from_str::(input) + .map(|input| (input.space, input.service)) + .map_err(|error| error.to_string()); + assert_eq!(actual, previous, "registration input {input}"); + } + } + + #[test] + fn notification_input_serializes_to_the_previous_notice_bytes_without_decoding_hashes() { + for hash in ["AAECAw", "AA==", "", "not valid base64!"] { + let input: NotifyWriteInput = serde_json::from_value(json!({ + "space": "at://did:example:alice/space/app.test.space/one", + "repo": "did:example:alice", "rev": "3jqfcqzm3fo2j", + "hash": {"$bytes": hash, "ignored": true}, "ignored": true, + })) + .unwrap(); + let previous = json!({ + "space": input.space, + "repo": input.repo, + "rev": input.rev, + "hash": input.hash, + }); + let actual = json!(input); + assert_eq!(actual, previous); + assert_eq!( + serde_json::to_vec(&actual).unwrap(), + serde_json::to_vec(&previous).unwrap() + ); + assert_eq!(actual["hash"], json!({"$bytes": hash})); + assert_eq!(actual.as_object().unwrap().len(), 4); + } + } +} diff --git a/src/space/http/records.rs b/src/space/http/records.rs new file mode 100644 index 0000000..be349ca --- /dev/null +++ b/src/space/http/records.rs @@ -0,0 +1,629 @@ +//! Permissioned record CRUD, ordered batches, and record listings. + +use crate::http::request::{JsonRequest, SpaceQuery}; +use axum::{ + Json, + extract::{OriginalUri, Query, State}, +}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::collections::BTreeMap; + +use crate::{ + domain::record::RecordView, + error::Error, + http::{ApiResult, auth}, + space::{ + ListRecords, RecordInput, RecordQuery, WriteRequest, + domain::{SpaceRecord, SpaceWrite}, + }, +}; + +use super::notifications::publish_repo_position; + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct WriteOutput { + uri: String, + cid: String, + validation_status: &'static str, +} + +impl From for WriteOutput { + fn from(record: RecordView) -> Self { + Self { + uri: record.uri, + cid: record.cid, + validation_status: "unknown", + } + } +} + +pub(crate) async fn create_record( + (State(state), headers, Json(input)): JsonRequest, +) -> ApiResult> { + let auth = auth::token(&headers, &state)?; + let output = state.spaces.create_record( + &auth, + RecordInput { + space: &input.space, + repo: &input.repo, + collection: &input.collection, + rkey: input.rkey.as_deref(), + record: input.record, + validate: input.validate, + }, + )?; + publish_repo_position(&state, &input.space, &input.repo).await; + Ok(Json(output.into())) +} + +pub(crate) async fn put_record( + (State(state), headers, Json(input)): JsonRequest, +) -> ApiResult> { + let rkey = input + .rkey + .as_deref() + .ok_or_else(|| Error::InvalidRequest("putRecord requires `rkey`".to_owned()))?; + let auth = auth::token(&headers, &state)?; + let output = state.spaces.put_record( + &auth, + RecordInput { + space: &input.space, + repo: &input.repo, + collection: &input.collection, + rkey, + record: input.record, + validate: input.validate, + }, + )?; + publish_repo_position(&state, &input.space, &input.repo).await; + Ok(Json(output.into())) +} + +pub(crate) async fn delete_record( + (State(state), headers, Json(input)): JsonRequest, +) -> ApiResult> { + let auth = auth::token(&headers, &state)?; + state.spaces.delete_record(&auth, &input)?; + publish_repo_position(&state, &input.space, &input.repo).await; + Ok(Json(json!({}))) +} + +#[derive(Deserialize)] +pub(crate) struct ApplyWritesInput { + space: String, + repo: String, + validate: Option, + writes: Vec, +} + +#[derive(Serialize)] +#[serde(tag = "$type")] +pub(crate) enum AppliedWrite { + #[serde(rename = "com.atproto.space.applyWrites#createResult")] + Create(WriteOutput), + #[serde(rename = "com.atproto.space.applyWrites#updateResult")] + Update(WriteOutput), + #[serde(rename = "com.atproto.space.applyWrites#deleteResult")] + Delete, +} + +impl From for AppliedWrite { + fn from(result: SpaceWrite) -> Self { + match result { + SpaceWrite::Create(record) => Self::Create(RecordView::from(record).into()), + SpaceWrite::Update(record) => Self::Update(RecordView::from(record).into()), + SpaceWrite::Delete(_) => Self::Delete, + } + } +} + +pub(crate) async fn apply_writes( + (State(state), headers, Json(input)): JsonRequest, +) -> ApiResult>>> { + let auth = auth::token(&headers, &state)?; + let has_writes = !input.writes.is_empty(); + let results = state.spaces.apply_writes( + &auth, + &input.space, + &input.repo, + input.writes, + input.validate, + )?; + if has_writes { + publish_repo_position(&state, &input.space, &input.repo).await; + } + Ok(Json(BTreeMap::from([( + "results", + results.into_iter().map(Into::into).collect(), + )]))) +} + +pub(crate) async fn get_record( + (State(state), OriginalUri(uri), headers, Query(query)): SpaceQuery, +) -> ApiResult> { + let auth = auth::space_read_token(&headers, &state, &uri, "GET").await?; + Ok(Json(state.spaces.get_record(&auth, &query)?)) +} + +#[derive(Serialize)] +pub(crate) struct ListedRecord { + collection: String, + rkey: String, + cid: String, + #[serde(skip_serializing_if = "Option::is_none")] + value: Option, +} + +#[derive(Serialize)] +pub(crate) struct ListRecordsOutput { + records: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + cursor: Option, +} + +pub(crate) async fn list_records( + (State(state), OriginalUri(uri), headers, Query(query)): SpaceQuery, +) -> ApiResult> { + let auth = auth::space_read_token(&headers, &state, &uri, "GET").await?; + let exclude_values = query.exclude_values.unwrap_or(false); + let records = state.spaces.list_records(&auth, &query)?; + let cursor = (records.len() == query.limit()) + .then(|| records.last().map(SpaceRecord::uri)) + .flatten(); + Ok(Json(ListRecordsOutput { + records: records + .into_iter() + .map(|record| ListedRecord { + collection: record.collection, + rkey: record.rkey, + cid: record.cid.to_string(), + value: (!exclude_values).then_some(record.value), + }) + .collect(), + cursor, + })) +} + +#[cfg(test)] +mod tests { + use axum::{ + Json, + body::{Body, to_bytes}, + extract::FromRequest as _, + http::{Request, StatusCode, header::CONTENT_TYPE}, + response::IntoResponse, + }; + + use super::{ + AppliedWrite, BTreeMap, ListRecords, Query, RecordInput, RecordQuery, SpaceRecord, + SpaceWrite, WriteOutput, WriteRequest, json, + }; + + #[tokio::test] + async fn singleton_batch_map_keeps_typed_union_response_bytes() { + #[derive(serde::Serialize)] + struct PreviousApplyWritesOutput { + results: Vec, + } + for (uri, cid) in [ + ( + "at://did:example:alice/app.test.record/key", + "bafkreigh2akiscaildc5tbdwz7cnfu7gcox26symljyv36xbunqmqkpylu", + ), + ("", ""), + ("uri-\"\\\n\0-雪", "not a CID-\"\\\n\0-雪"), + ] { + let result = |kind| { + let write = WriteOutput { + uri: uri.to_owned(), + cid: cid.to_owned(), + validation_status: "unknown", + }; + match kind { + 0 => AppliedWrite::Create(write), + 1 => AppliedWrite::Update(write), + _ => AppliedWrite::Delete, + } + }; + for kinds in [ + &[][..], + &[0][..], + &[1][..], + &[2][..], + &[0, 1, 2][..], + &[2, 1, 0][..], + &[0, 0, 1, 2][..], + ] { + let previous = Json(PreviousApplyWritesOutput { + results: kinds.iter().copied().map(result).collect(), + }) + .into_response(); + let actual = Json(BTreeMap::from([( + "results", + kinds.iter().copied().map(result).collect::>(), + )])) + .into_response(); + assert_eq!(actual.status(), previous.status()); + assert_eq!(actual.headers(), previous.headers()); + assert_eq!( + to_bytes(actual.into_body(), usize::MAX).await.unwrap(), + to_bytes(previous.into_body(), usize::MAX).await.unwrap(), + "URI: {uri:?}, CID: {cid:?}, variants: {kinds:?}", + ); + } + } + } + + mod previous { + #[derive(serde::Deserialize)] + #[serde(rename_all = "camelCase")] + pub(super) struct RecordInput { + pub(super) space: String, + pub(super) repo: String, + pub(super) collection: String, + pub(super) rkey: Option, + pub(super) validate: Option, + pub(super) record: serde_json::Value, + } + + #[derive(serde::Deserialize)] + pub(super) struct RecordQuery { + pub(super) space: String, + pub(super) repo: String, + pub(super) collection: String, + pub(super) rkey: String, + } + } + + impl From for RecordQuery { + fn from(query: previous::RecordQuery) -> Self { + Self { + space: query.space, + repo: query.repo, + collection: query.collection, + rkey: query.rkey, + } + } + } + + impl From for RecordInput { + fn from(input: previous::RecordInput) -> Self { + Self { + space: input.space, + repo: input.repo, + collection: input.collection, + rkey: input.rkey, + validate: input.validate, + record: input.record, + } + } + } + + fn record_input_fields(input: &RecordInput) -> serde_json::Value { + json!([ + input.space, + input.repo, + input.collection, + input.rkey, + input.validate, + input.record, + ]) + } + + fn record_fields(query: RecordQuery) -> (String, String, String, String) { + (query.space, query.repo, query.collection, query.rkey) + } + + #[test] + fn shared_record_query_preserves_raw_query_values_and_rejections() { + let valid = vec![ + ( + "space", + "at://did:example:alice/space/com.example.forum/default", + ), + ("repo", "did:example:alice"), + ("collection", "com.example.post"), + ("rkey", "quote '\" café & + /"), + ]; + let mut cases = vec![valid.clone(), Vec::new()]; + for index in 0..valid.len() { + let mut missing = valid.clone(); + missing.remove(index); + cases.push(missing); + let mut duplicate = valid.clone(); + duplicate.push(valid[index]); + cases.push(duplicate); + let mut empty = valid.clone(); + empty[index].1 = ""; + cases.push(empty); + } + let mut unknown = valid; + unknown.extend([("unknown", "one"), ("unknown", "two")]); + cases.push(unknown); + let mut queries: Vec<_> = cases + .into_iter() + .map(|pairs| { + url::form_urlencoded::Serializer::new(String::new()) + .extend_pairs(pairs) + .finish() + }) + .collect(); + queries.extend(["%", "space=%FF", "space=a&repo=%00"].map(str::to_owned)); + for query in queries { + let uri = format!("/?{query}").parse().unwrap(); + let previous = Query::::try_from_uri(&uri) + .map(|Query(query)| record_fields(query.into())) + .map_err(|error| (error.status(), error.body_text())); + let actual = Query::::try_from_uri(&uri) + .map(|Query(query)| record_fields(query)) + .map_err(|error| (error.status(), error.body_text())); + assert_eq!(actual, previous, "query {query}"); + } + } + + async fn extract_record_json( + body: &str, + ) -> Result { + let request = Request::post("/xrpc/com.atproto.space.deleteRecord") + .header(CONTENT_TYPE, "application/json") + .body(Body::from(body.to_owned())) + .unwrap(); + Json::::from_request(request, &()) + .await + .map(|Json(query)| query) + .map_err(|error| (error.status(), error.body_text())) + } + + #[tokio::test] + async fn shared_record_input_preserves_default_keys_maps_sequences_and_errors() { + let valid = json!({ + "space": "at://did:example:alice/space/com.example.forum/default", + "repo": "did:example:alice", + "collection": "com.example.post", + "rkey": "quote '\" café & + /", + "validate": false, + "record": {"$type": "com.example.post", "text": "private record value"}, + }); + let fields = ["space", "repo", "collection", "rkey", "validate", "record"]; + let alternatives = [ + json!(null), + json!(true), + json!(0), + json!([]), + json!({}), + json!(""), + ]; + let mut bodies = vec![valid.to_string()]; + for field in fields { + let mut missing = valid.clone(); + missing.as_object_mut().unwrap().remove(field); + bodies.push(missing.to_string()); + bodies.push(format!( + "{},\"{field}\":null}}", + valid.to_string().trim_end_matches('}') + )); + bodies.push(format!( + "{{\"{field}\":null,{}", + valid.to_string().trim_start_matches('{') + )); + for value in &alternatives { + let mut changed = valid.clone(); + changed[field] = value.clone(); + bodies.push(changed.to_string()); + } + } + let mut optional_missing = valid.clone(); + optional_missing.as_object_mut().unwrap().remove("rkey"); + optional_missing.as_object_mut().unwrap().remove("validate"); + bodies.push(optional_missing.to_string()); + bodies.push(format!( + "{},\"unknown\":1,\"unknown\":2}}", + valid.to_string().trim_end_matches('}') + )); + bodies.extend(["", "{", "null", "true", "0", "\"text\""].map(str::to_owned)); + let sequence: Vec<_> = fields.iter().map(|field| valid[field].clone()).collect(); + for length in 0..=sequence.len() { + bodies.push(serde_json::to_string(&sequence[..length]).unwrap()); + } + let mut extra = sequence.clone(); + extra.push(json!(true)); + bodies.push(serde_json::to_string(&extra).unwrap()); + for index in 0..sequence.len() { + for value in &alternatives { + let mut changed = sequence.clone(); + changed[index] = value.clone(); + bodies.push(serde_json::to_string(&changed).unwrap()); + } + } + for body in bodies { + let previous = extract_record_json::(&body) + .await + .map(|input| record_input_fields(&input.into())); + let actual = extract_record_json::(&body) + .await + .map(|input| record_input_fields(&input)); + assert_eq!(actual, previous, "JSON {body}"); + } + } + + #[tokio::test] + async fn shared_record_query_preserves_raw_json_values_and_rejections() { + let valid = json!({ + "space": "at://did:example:alice/space/com.example.forum/default", + "repo": "did:example:alice", + "collection": "com.example.post", + "rkey": "quote '\" café & + /", + }); + let mut bodies = vec![valid.to_string()]; + for field in ["space", "repo", "collection", "rkey"] { + let mut missing = valid.clone(); + missing.as_object_mut().unwrap().remove(field); + bodies.push(missing.to_string()); + bodies.push(format!( + "{},\"{field}\":null}}", + valid.to_string().trim_end_matches('}') + )); + for value in [ + json!(null), + json!(true), + json!(0), + json!([]), + json!({}), + json!(""), + ] { + let mut changed = valid.clone(); + changed[field] = value; + bodies.push(changed.to_string()); + } + } + let mut unknown = valid; + unknown["unknown"] = json!({"nested": [null, false]}); + bodies.push(unknown.to_string()); + bodies.extend( + [ + "", + "{", + "{}", + "null", + "true", + "1", + "\"text\"", + "[]", + "[\"s\",\"r\",\"c\",\"k\"]", + "[\"s\",\"r\",\"c\"]", + "[\"s\",\"r\",\"c\",\"k\",\"extra\"]", + ] + .map(str::to_owned), + ); + for body in bodies { + let previous = extract_record_json::(&body) + .await + .map(|query| record_fields(query.into())); + let actual = extract_record_json::(&body) + .await + .map(record_fields); + assert_eq!(actual, previous, "JSON {body}"); + } + } + + #[test] + fn record_queries_preserve_defaults_clamps_and_camel_case_options() { + let path = "/?space=at%3A%2F%2Fdid%3Aexample%3Aalice%2Fapp.test.space%2Fone&repo=did%3Aexample%3Aalice"; + let query = Query::::try_from_uri(&path.parse().unwrap()).unwrap(); + assert_eq!(query.space, "at://did:example:alice/app.test.space/one"); + assert_eq!(query.repo, "did:example:alice"); + assert_eq!(query.limit(), 50); + assert_eq!(query.collection, None); + assert_eq!(query.cursor, None); + assert_eq!(query.reverse, None); + assert_eq!(query.exclude_values, None); + + for (requested, limit) in [(0, 1), (1, 1), (50, 50), (1000, 1000), (1001, 1000)] { + let uri = format!( + "{path}&collection=app.test.record&cursor=record%2Fkey&limit={requested}&reverse=true&excludeValues=true&unknown=ignored" + ) + .parse() + .unwrap(); + let query = Query::::try_from_uri(&uri).unwrap(); + assert_eq!(query.limit(), limit); + assert_eq!(query.collection.as_deref(), Some("app.test.record")); + assert_eq!(query.cursor.as_deref(), Some("record/key")); + assert_eq!(query.reverse, Some(true)); + assert_eq!(query.exclude_values, Some(true)); + } + + for invalid in ["limit=-1", "limit=abc", "reverse=1", "excludeValues=yes"] { + let uri = format!("{path}&{invalid}").parse().unwrap(); + assert!(Query::::try_from_uri(&uri).is_err()); + } + for missing in ["/?space=space", "/?repo=repo"] { + assert!(Query::::try_from_uri(&missing.parse().unwrap()).is_err()); + } + } + + #[test] + fn batch_requests_keep_the_alpha_union_tags_and_optional_create_key() { + let create = json!({ + "$type": "com.atproto.space.applyWrites#create", + "collection": "app.test.record", + "value": {"content": "hello"}, + "unknownField": "still ignored", + }); + let write = serde_json::from_value::(create.clone()).unwrap(); + assert!( + matches!(write, WriteRequest::Create { collection, rkey: None, value } + if collection == "app.test.record" && value == create["value"]) + ); + + for kind in ["update", "delete"] { + let mut input = create.clone(); + input["$type"] = json!(format!("com.atproto.space.applyWrites#{kind}")); + assert!(serde_json::from_value::(input.clone()).is_err()); + input["rkey"] = json!("record-key"); + let write = serde_json::from_value::(input).unwrap(); + assert!(match write { + WriteRequest::Update { + collection, + rkey, + value, + } => { + kind == "update" + && collection == "app.test.record" + && rkey == "record-key" + && value == create["value"] + } + WriteRequest::Delete { collection, rkey } => { + kind == "delete" && collection == "app.test.record" && rkey == "record-key" + } + WriteRequest::Create { .. } => false, + }); + } + let mut invalid = create; + invalid["$type"] = json!("com.atproto.repo.applyWrites#create"); + assert!(serde_json::from_value::(invalid).is_err()); + } + + #[test] + fn batch_results_keep_the_alpha_union_tags_and_flat_record_fields() { + let record = || { + let value = json!({"notPartOfWriteResult": true}); + SpaceRecord { + space: "at://did:web:host.test/space/app.test.space/main".to_owned(), + did: "did:web:alice.test".to_owned(), + collection: "app.test.record".to_owned(), + rkey: "key".to_owned(), + cid: cpds_protocol::record_cid(&value).unwrap(), + value, + } + }; + let uri = record().uri(); + let cid = record().cid.to_string(); + for (result, tag) in [ + (SpaceWrite::Create(record()), "createResult"), + (SpaceWrite::Update(record()), "updateResult"), + ] { + assert_eq!( + serde_json::to_value(AppliedWrite::from(result)).unwrap(), + json!({ + "$type": format!("com.atproto.space.applyWrites#{tag}"), + "uri": uri, + "cid": cid, + "validationStatus": "unknown", + }), + ); + } + let record = record(); + assert_eq!( + serde_json::to_value(AppliedWrite::from(SpaceWrite::Delete(RecordQuery { + space: record.space, + repo: record.did, + collection: record.collection, + rkey: record.rkey, + }))) + .unwrap(), + json!({"$type": "com.atproto.space.applyWrites#deleteResult"}), + ); + } +} diff --git a/src/space/http/sync.rs b/src/space/http/sync.rs new file mode 100644 index 0000000..8681cf1 --- /dev/null +++ b/src/space/http/sync.rs @@ -0,0 +1,385 @@ +//! Authorized Space recovery, operation history, and blob reads. + +use crate::http::request::SpaceQuery; +use std::{collections::BTreeMap, str::FromStr}; + +use axum::{ + Json, + extract::{OriginalUri, Query, State}, + response::Response, +}; +use cid::Cid; +use serde::{Deserialize, Serialize}; + +use crate::{ + error::{Error, ErrorContext as _}, + http::{ApiResult, auth, blob_response, car_response}, + space::{ListBlobsQuery, ListOpsQuery, RepoOpsPage, car, commit::SignedCommit}, +}; + +pub(crate) async fn list_repo_ops( + (State(state), OriginalUri(uri), headers, Query(query)): SpaceQuery, +) -> ApiResult> { + let auth = auth::space_read_token(&headers, &state, &uri, "GET").await?; + let mut page = state.spaces.list_repo_ops(&auth, &query)?; + if query.exclude_values.unwrap_or(false) { + for op in &mut page.ops { + op.value = None; + } + } + Ok(Json(page)) +} + +#[derive(Deserialize)] +pub(crate) struct LatestCommitQuery { + space: String, + repo: String, +} + +pub(crate) async fn get_latest_commit( + (State(state), OriginalUri(uri), headers, Query(query)): SpaceQuery, +) -> ApiResult>> { + let auth = auth::space_read_token(&headers, &state, &uri, "GET").await?; + Ok(Json(BTreeMap::from([( + "commit", + state + .spaces + .signed_snapshot(&auth, &query.space, &query.repo)? + .0, + )]))) +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct GetRepoQuery { + space: String, + repo: String, + exclude_values: Option, +} + +pub(crate) async fn get_repo( + (State(state), OriginalUri(uri), headers, Query(query)): SpaceQuery, +) -> ApiResult { + let auth = auth::space_read_token(&headers, &state, &uri, "GET").await?; + let exclude_values = query.exclude_values.unwrap_or(false); + let (commit, records) = state + .spaces + .signed_snapshot(&auth, &query.space, &query.repo)?; + let car = car::serialize_repo(&commit, records, exclude_values)?; + Ok(car_response(car)) +} + +#[derive(Deserialize)] +pub(crate) struct BlobQuery { + space: String, + repo: String, + cid: String, +} + +pub(crate) async fn get_blob( + (State(state), OriginalUri(uri), headers, Query(query)): SpaceQuery, +) -> ApiResult { + let auth = auth::space_read_token(&headers, &state, &uri, "GET").await?; + let cid = Cid::from_str(&query.cid).context(Error::InvalidRequest, "invalid CID")?; + state + .spaces + .get_blob_allowed(&auth, &query.space, &query.repo, &cid)?; + let (blob, bytes) = state + .repository + .read_blob(&query.repo, &cid) + .await + .map_err(|error| match error { + Error::NotFound => Error::xrpc("BlobNotFound", "blob data was not found"), + other => other, + })?; + blob_response(&blob.media_type, bytes) +} + +#[derive(Serialize)] +pub(crate) struct ListBlobsOutput { + cids: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + cursor: Option, +} + +pub(crate) async fn list_blobs( + (State(state), OriginalUri(uri), headers, Query(query)): SpaceQuery, +) -> ApiResult> { + let auth = auth::space_read_token(&headers, &state, &uri, "GET").await?; + let limit = query.limit.unwrap_or(500).clamp(1, 1000); + let cids = state.spaces.list_blobs(&auth, &query)?; + let cursor = (cids.len() == limit) + .then(|| cids.last().cloned()) + .flatten(); + Ok(Json(ListBlobsOutput { cids, cursor })) +} + +#[cfg(test)] +mod tests { + use super::{BTreeMap, Json, ListBlobsQuery, ListOpsQuery, Query, RepoOpsPage}; + use crate::space::{ + commit::{LexBytes, SignedCommit}, + domain::SpaceOp, + }; + use axum::{body::to_bytes, response::IntoResponse}; + use serde::{Deserialize, Serialize}; + use serde_json::json; + + #[tokio::test] + async fn singleton_commit_map_keeps_typed_inner_response_bytes_without_decoding() { + #[derive(Serialize)] + struct PreviousLatestCommitOutput { + commit: SignedCommit, + } + for bytes in ["", "AAECAw", "AA==", "not valid base64!", "\"\\\n\0-雪"] { + for rev in ["", "3jqfcqzm3fo2j", "rev-\"\\\n\0-雪"] { + for ver in [1, u8::MAX] { + let commit = SignedCommit { + ver, + hash: LexBytes { + value: bytes.to_owned(), + }, + ikm: LexBytes { + value: bytes.to_owned(), + }, + sig: LexBytes { + value: bytes.to_owned(), + }, + mac: LexBytes { + value: bytes.to_owned(), + }, + rev: rev.to_owned(), + }; + let previous = Json(PreviousLatestCommitOutput { + commit: commit.clone(), + }) + .into_response(); + let actual = Json(BTreeMap::from([("commit", commit)])).into_response(); + assert_eq!(actual.status(), previous.status()); + assert_eq!(actual.headers(), previous.headers()); + assert_eq!( + to_bytes(actual.into_body(), usize::MAX).await.unwrap(), + to_bytes(previous.into_body(), usize::MAX).await.unwrap(), + "bytes: {bytes:?}, revision: {rev:?}, version: {ver}", + ); + } + } + } + } + + #[derive(Deserialize)] + #[serde(rename_all = "camelCase")] + struct PreviousListOpsQuery { + space: String, + repo: String, + since: Option, + limit: Option, + cursor: Option, + exclude_values: Option, + } + + #[derive(Deserialize)] + struct PreviousListBlobsQuery { + space: String, + repo: String, + since: Option, + limit: Option, + cursor: Option, + } + + #[test] + fn shared_recovery_queries_match_previous_extractors() { + let base = "/?space=at%3A%2F%2Fdid%3Aexample%3Aalice%2Fapp.test.space%2Fone&repo=did%3Aexample%3Aalice"; + let mut paths = vec![ + "/".to_owned(), + "/?space=space".to_owned(), + "/?repo=repo".to_owned(), + ]; + paths.extend( + [ + "", + "&limit=0", + "&limit=1001", + "&limit=", + "&limit=-1", + "&limit=abc", + "&excludeValues=true", + "&excludeValues=false", + "&excludeValues=1", + "&excludeValues=", + "&exclude_values=true", + "&space=another", + "&repo=another", + "&limit=1&limit=2", + "&since=one&since=two", + "&cursor=one&cursor=two", + "&excludeValues=true&excludeValues=false", + "&since=invalid&cursor=not-a-number", + "&since=&cursor=", + "&unknown=one&unknown=two", + "&limit=100&since=3jqfcqzm3fo2j&cursor=42&excludeValues=true", + ] + .map(|suffix| format!("{base}{suffix}")), + ); + paths.push(format!("{base}&limit={}", usize::MAX)); + paths.push(format!("{base}&limit={}0", usize::MAX)); + for path in paths { + let uri = path.parse().unwrap(); + let previous = Query::::try_from_uri(&uri) + .map(|Query(query)| { + ( + query.space, + query.repo, + query.since, + query.limit, + query.cursor, + query.exclude_values, + ) + }) + .map_err(|error| (error.status(), error.body_text())); + let actual = Query::::try_from_uri(&uri) + .map(|Query(query)| { + ( + query.space, + query.repo, + query.since, + query.limit, + query.cursor, + query.exclude_values, + ) + }) + .map_err(|error| (error.status(), error.body_text())); + assert_eq!(actual, previous, "operation query {path}"); + + let previous = Query::::try_from_uri(&uri) + .map(|Query(query)| { + ( + query.space, + query.repo, + query.since, + query.limit, + query.cursor, + ) + }) + .map_err(|error| (error.status(), error.body_text())); + let actual = Query::::try_from_uri(&uri) + .map(|Query(query)| { + ( + query.space, + query.repo, + query.since, + query.limit, + query.cursor, + ) + }) + .map_err(|error| (error.status(), error.body_text())); + assert_eq!(actual, previous, "blob query {path}"); + } + } + + #[test] + fn operation_output_preserves_null_links_and_omits_internal_ids_and_absent_values() { + let cid = cpds_protocol::record_cid(&json!({"$type": "app.test.record"})) + .unwrap() + .to_string(); + let ops = [ + (Some(cid.clone()), None, Some(json!({"text": "created"}))), + (Some(cid.clone()), Some(cid.clone()), Some(json!(null))), + (None, Some(cid.clone()), None), + ] + .into_iter() + .map(|(cid, prev, value)| SpaceOp { + id: 42, + rev: "3jqfcqzm3fo2j".to_owned(), + collection: "app.test.record".to_owned(), + rkey: "key".to_owned(), + cid, + prev, + value, + }) + .collect(); + let output = RepoOpsPage { + ops, + commit: None, + cursor: None, + }; + assert_eq!( + serde_json::to_value(output).unwrap(), + json!({"ops": [ + {"rev": "3jqfcqzm3fo2j", "collection": "app.test.record", "rkey": "key", + "cid": cid, "prev": null, "value": {"text": "created"}}, + {"rev": "3jqfcqzm3fo2j", "collection": "app.test.record", "rkey": "key", + "cid": cid, "prev": cid, "value": null}, + {"rev": "3jqfcqzm3fo2j", "collection": "app.test.record", "rkey": "key", + "cid": null, "prev": cid}, + ]}), + ); + } + + #[test] + fn shared_operation_pages_preserve_previous_json_bytes_and_lazy_commit_encoding() { + #[derive(Serialize)] + struct PreviousListOpsOutput { + ops: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + commit: Option, + #[serde(skip_serializing_if = "Option::is_none")] + cursor: Option, + } + + let cid = cpds_protocol::record_cid(&json!({"$type": "app.test.record"})) + .unwrap() + .to_string(); + let ops = [ + ( + Some(cid.clone()), + None, + Some(json!({"link": {"$link": cid}})), + ), + (Some(cid.clone()), Some(cid.clone()), Some(json!(null))), + (None, Some("stored-without-decoding".to_owned()), None), + ] + .into_iter() + .map(|(cid, prev, value)| SpaceOp { + id: 42, + rev: "3jqfcqzm3fo2j".to_owned(), + collection: "app.test.record".to_owned(), + rkey: "key".to_owned(), + cid, + prev, + value, + }) + .collect::>(); + let commit = SignedCommit { + ver: 1, + hash: LexBytes { + value: "not valid base64!".to_owned(), + }, + ikm: LexBytes::new([2]), + sig: LexBytes::new([3]), + mac: LexBytes::new([4]), + rev: "3jqfcqzm3fo2j".to_owned(), + }; + assert!(commit.hash.decode().is_err()); + for commit in [None, Some(commit)] { + for cursor in [None, Some("cursor-token".to_owned())] { + for ops in [Vec::new(), ops.clone()] { + let previous = PreviousListOpsOutput { + ops: ops.clone(), + commit: commit.clone(), + cursor: cursor.clone(), + }; + let output = RepoOpsPage { + ops, + commit: commit.clone(), + cursor: cursor.clone(), + }; + assert_eq!( + serde_json::to_vec(&output).unwrap(), + serde_json::to_vec(&previous).unwrap(), + ); + } + } + } + } +} diff --git a/src/space/mod.rs b/src/space/mod.rs index bbbcdeb..4404ae0 100644 --- a/src/space/mod.rs +++ b/src/space/mod.rs @@ -1,38 +1,41 @@ //! `ATProto` permissioned-data (Spaces) application service. pub(crate) mod auth; +mod authorization; mod car; mod commit; mod database; pub(crate) mod domain; pub(crate) mod http; +mod records; +pub(crate) mod routes; -use std::collections::{BTreeSet, HashSet}; - -use cid::Cid; +use atrium_crypto::keypair::Secp256k1Keypair; +use serde::Deserialize; use serde_json::Value; use crate::{ - clock::TidClock, + clock::{TidClock, now}, database::Database, - domain::{blob::referenced_blob_cids, oauth::AuthenticatedToken}, - error::{Error, Result}, - repository::{record_cid, validate_record_type}, + domain::oauth::AuthenticatedToken, + error::{Error, ErrorContext as _, Result}, syntax::{self, SpaceRef}, }; -use self::{ - commit::{CommitSigner, SignedCommit}, - domain::{ - NotifyRegistration, Space, SpaceOp, SpaceRecord, SpaceRecordView, SpaceWrite, SpaceWriter, - }, +use self::domain::{Space, SpaceWriter}; +pub(crate) use authorization::ReadAuthorization; +use authorization::{Capability, require_scope, union_type, validate_app_access, validate_policy}; +pub(crate) use domain::RecordQuery; +pub(crate) use records::{ + ListBlobsQuery, ListOpsQuery, ListRecords, ListSpacesQuery, RecordInput, RepoOpsPage, + WriteRequest, }; pub(crate) struct SpaceService { database: Database, tids: TidClock, hosted_did: String, - signer: CommitSigner, + signer: Secp256k1Keypair, } impl SpaceService { @@ -45,115 +48,43 @@ impl SpaceService { database, tids: TidClock::default(), hosted_did, - signer: CommitSigner::new(signing_key)?, + signer: Secp256k1Keypair::import(signing_key).context( + Error::Configuration, + "invalid account secp256k1 signing key", + )?, }) } pub(crate) fn create_space( &self, auth: &AuthenticatedToken, - space_type: &str, - skey: Option<&str>, - policy: Value, - app_access: Value, + input: CreateSpaceInput, ) -> Result { - syntax::collection(space_type)?; - validate_policy(&policy)?; - validate_app_access(&app_access)?; - let skey = skey.map_or_else(|| self.tids.next(), ToOwned::to_owned); + syntax::collection(&input.space_type)?; + validate_policy(&input.policy)?; + validate_app_access(&input.app_access)?; + let skey = input.skey.unwrap_or_else(|| self.tids.next()); syntax::record_key(&skey)?; let space_ref = SpaceRef { authority: auth.account.did.clone(), - space_type: space_type.to_owned(), + space_type: input.space_type.clone(), skey, }; - require_scope(auth, &space_ref, Capability::Manage("create"), None)?; + require_scope(auth, &space_ref, Capability::Manage("create"))?; let uri = space_ref.to_string(); self.database .create_space(&Space { uri: uri.clone(), authority: auth.account.did.clone(), - kind: space_type.to_owned(), + kind: input.space_type, skey: space_ref.skey, - policy, - app_access, + policy: input.policy, + app_access: input.app_access, }) .map_err(|error| named_conflict(error, "SpaceAlreadyExists"))?; Ok(uri) } - pub(crate) fn authorize_delegation( - &self, - auth: &AuthenticatedToken, - space: &str, - ) -> Result<()> { - if auth.account.did != self.hosted_did { - return Err(Error::Authorization( - "OAuth account is not hosted by this PDS".to_owned(), - )); - } - let space_ref = syntax::space_ref(space)?; - require_scope(auth, &space_ref, Capability::Read, None) - } - - pub(crate) fn authorize_credential( - &self, - space: &str, - user_did: &str, - client_id: Option<&str>, - ) -> Result> { - let space_ref = syntax::space_ref(space)?; - if space_ref.authority != self.hosted_did { - return Err(named("SpaceNotFound", "this PDS is not the space host")); - } - match self.database.space_is_deleted(space)? { - Some(true) => return Err(named("SpaceDeleted", "space has been deleted")), - None => return Err(named("SpaceNotFound", "space does not exist")), - Some(false) => {} - } - let local = self.database.space(space)?; - - if union_type(&local.app_access) == Some("com.atproto.simplespace.defs#allowList") { - let allowed = local - .app_access - .get("allowed") - .and_then(Value::as_array) - .expect("validated allowList policy"); - if !client_id.is_some_and(|client| { - allowed - .iter() - .any(|allowed| allowed.as_str() == Some(client)) - }) { - return Err(named( - "AppNotAuthorized", - "application is not authorized for this space", - )); - } - } - if user_did == local.authority { - return Ok(None); - } - match union_type(&local.policy) { - Some("com.atproto.simplespace.defs#publicPolicy") => Ok(None), - Some("com.atproto.simplespace.defs#memberListPolicy") - if self.database.is_space_member(space, user_did)? => - { - Ok(None) - } - Some("com.atproto.simplespace.defs#managingAppPolicy") => local - .policy - .get("managingApp") - .and_then(Value::as_str) - .map(ToOwned::to_owned) - .map(Some) - .ok_or_else(|| named("UserNotAuthorized", "managing app is not configured")), - _ => Err(named( - "UserNotAuthorized", - "user is not authorized for this space", - )), - } - } - pub(crate) fn record_notified_writer(&self, input: &NotifiedWrite<'_>) -> Result<()> { syntax::did(input.repo)?; syntax::tid(input.rev)?; @@ -162,7 +93,7 @@ impl SpaceService { "notifyWrite hash must be exactly 32 bytes".to_owned(), )); } - let space_ref = syntax::space_ref(input.space)?; + let space_ref = input.space.parse::()?; if input.issuer != input.repo { return Err(Error::Authorization( "notifyWrite issuer does not match the claimed writer".to_owned(), @@ -182,8 +113,7 @@ impl SpaceService { "notifyWrite writer is not authorized for the space".to_owned(), )); } - self.database - .record_space_writer(input.space, input.repo, input.rev, input.hash) + self.database.record_space_writer(input) } pub(crate) fn managing_app_for_writer( @@ -191,7 +121,7 @@ impl SpaceService { space: &str, repo: &str, ) -> Result> { - let space_ref = syntax::space_ref(space)?; + let space_ref = space.parse::()?; if space_ref.authority != self.hosted_did || repo == self.hosted_did { return Ok(None); } @@ -199,59 +129,43 @@ impl SpaceService { if union_type(&local.policy) != Some("com.atproto.simplespace.defs#managingAppPolicy") { return Ok(None); } - Ok(local - .policy - .get("managingApp") - .and_then(Value::as_str) - .map(ToOwned::to_owned)) + Ok(local.policy["managingApp"].as_str().map(ToOwned::to_owned)) } pub(crate) fn list_repos( &self, credential_space: &str, - space: &str, - cursor: Option<&str>, - limit: usize, + query: &SpacePageQuery, ) -> Result> { - if credential_space != space { - return Err(Error::Authorization( - "space credential does not cover the requested space".to_owned(), - )); - } - let space_ref = syntax::space_ref(space)?; - if space_ref.authority != self.hosted_did { - return Err(named("SpaceNotFound", "this PDS is not the space host")); - } - self.database.space(space).map_err(space_not_found)?; - self.database - .list_space_writers(space, cursor, limit.clamp(1, 1000)) + self.assert_local_credential_space(credential_space, &query.space)?; + self.database.list_space_writers(query) } pub(crate) fn register_notify( &self, credential_space: &str, - space: &str, - service: &str, + input: &NotifyRegistrationInput, endpoint: &str, ) -> Result { - self.assert_local_credential_space(credential_space, space)?; - let expires_at = time::OffsetDateTime::now_utc().unix_timestamp() + 86_400; + self.require_active_account(&self.hosted_did)?; + self.assert_local_credential_space(credential_space, &input.space)?; + let expires_at = now() + 86_400; self.database - .register_space_notify(space, service, endpoint, expires_at)?; + .register_space_notify(input, endpoint, expires_at)?; Ok(expires_at) } pub(crate) fn unregister_notify( &self, credential_space: &str, - space: &str, - service: &str, + input: &NotifyRegistrationInput, ) -> Result<()> { - self.assert_local_credential_space(credential_space, space)?; - self.database.unregister_space_notify(space, service) + self.require_active_account(&self.hosted_did)?; + self.assert_local_credential_space(credential_space, &input.space)?; + self.database.unregister_space_notify(input) } - pub(crate) fn notify_registrations(&self, space: &str) -> Result> { + pub(crate) fn notify_registrations(&self, space: &str) -> Result> { self.database.space_notify_registrations(space) } @@ -274,36 +188,44 @@ impl SpaceService { pub(crate) fn update_space( &self, auth: &AuthenticatedToken, - uri: &str, - policy: Option<&Value>, - app_access: Option<&Value>, + input: &UpdateSpaceInput, ) -> Result<()> { - let space_ref = syntax::space_ref(uri)?; - require_scope(auth, &space_ref, Capability::Manage("update"), None)?; + let space_ref = input.space.parse::()?; + require_scope(auth, &space_ref, Capability::Manage("update"))?; self.require_owner(auth, &space_ref)?; - if let Some(value) = policy { + if let Some(value) = &input.policy { validate_policy(value)?; } - if let Some(value) = app_access { + if let Some(value) = &input.app_access { validate_app_access(value)?; } - self.database - .update_space(uri, policy, app_access) - .map_err(space_not_found) + self.database.update_space(input).map_err(space_not_found) } pub(crate) fn delete_space(&self, auth: &AuthenticatedToken, uri: &str) -> Result<()> { - let space_ref = syntax::space_ref(uri)?; - require_scope(auth, &space_ref, Capability::Manage("delete"), None)?; + let space_ref = uri.parse::()?; + require_scope(auth, &space_ref, Capability::Manage("delete"))?; self.require_owner(auth, &space_ref)?; self.database.delete_space(uri, &auth.account.did) } - pub(crate) fn get_space(&self, auth: &AuthenticatedToken, uri: &str) -> Result { - let space_ref = syntax::space_ref(uri)?; - require_scope(auth, &space_ref, Capability::ReadSelf, None)?; + pub(crate) fn get_space(&self, auth: &ReadAuthorization, uri: &str) -> Result { + match auth { + ReadAuthorization::OAuth(auth) => { + let space_ref = uri.parse::()?; + require_scope(auth, &space_ref, Capability::ReadSelf)?; + } + ReadAuthorization::Credential(credential_space) if credential_space != uri => { + return Err(Error::Authorization( + "space credential does not cover the requested space".to_owned(), + )); + } + ReadAuthorization::Credential(_) => {} + } let space = self.database.space(uri).map_err(space_not_found)?; - if !self.user_can_read(&space, &auth.account.did)? { + if let ReadAuthorization::OAuth(auth) = auth + && !self.user_can_read(&space, &auth.account.did)? + { return Err(Error::Authorization( "the account is not allowed to read this space".to_owned(), )); @@ -311,23 +233,10 @@ impl SpaceService { Ok(space) } - pub(crate) fn get_space_with_credential( - &self, - credential_space: &str, - uri: &str, - ) -> Result { - if credential_space != uri { - return Err(Error::Authorization( - "space credential does not cover the requested space".to_owned(), - )); - } - self.database.space(uri).map_err(space_not_found) - } - pub(crate) fn add_member(&self, auth: &AuthenticatedToken, uri: &str, did: &str) -> Result<()> { syntax::did(did)?; - let space_ref = syntax::space_ref(uri)?; - require_scope(auth, &space_ref, Capability::Manage("update"), None)?; + let space_ref = uri.parse::()?; + require_scope(auth, &space_ref, Capability::Manage("update"))?; self.require_owner(auth, &space_ref)?; self.database.space(uri).map_err(space_not_found)?; self.database.add_space_member(uri, did) @@ -340,8 +249,8 @@ impl SpaceService { did: &str, ) -> Result<()> { syntax::did(did)?; - let space_ref = syntax::space_ref(uri)?; - require_scope(auth, &space_ref, Capability::Manage("update"), None)?; + let space_ref = uri.parse::()?; + require_scope(auth, &space_ref, Capability::Manage("update"))?; self.require_owner(auth, &space_ref)?; if did == auth.account.did { return Err(Error::InvalidRequest( @@ -355,551 +264,44 @@ impl SpaceService { pub(crate) fn list_members( &self, auth: &AuthenticatedToken, - uri: &str, - cursor: Option<&str>, - limit: usize, + query: &SpacePageQuery, ) -> Result> { - let space_ref = syntax::space_ref(uri)?; - require_scope(auth, &space_ref, Capability::ReadSelf, None)?; - self.database.space(uri).map_err(space_not_found)?; - self.database - .list_space_members(uri, cursor, limit.clamp(1, 1000)) - } - - pub(crate) fn create_record( - &self, - auth: &AuthenticatedToken, - input: CreateRecord<'_>, - ) -> Result { - reject_unavailable_strict_validation(input.validate)?; - let space_ref = - self.authorize_write(auth, input.space, input.repo, input.collection, "create")?; - validate_record_type(input.collection, &input.value)?; - self.database - .validate_blob_refs(&auth.account.did, &input.value)?; - let rkey = input - .rkey - .map_or_else(|| self.tids.next(), ToOwned::to_owned); - syntax::record_key(&rkey)?; - let record = SpaceRecord { - space: space_ref.to_string(), - did: auth.account.did.clone(), - collection: input.collection.to_owned(), - rkey, - cid: record_cid(&input.value)?, - value: input.value, - }; - self.database - .create_space_record(&record, &self.tids.next()) - .map_err(|error| named_conflict(error, "RecordAlreadyExists"))?; - Ok(record.into()) - } - - pub(crate) fn put_record( - &self, - auth: &AuthenticatedToken, - input: PutRecord<'_>, - ) -> Result { - reject_unavailable_strict_validation(input.validate)?; - let space_ref = - self.authorize_write(auth, input.space, input.repo, input.collection, "update")?; - syntax::record_key(input.rkey)?; - validate_record_type(input.collection, &input.value)?; - self.database - .validate_blob_refs(&auth.account.did, &input.value)?; - let record = SpaceRecord { - space: space_ref.to_string(), - did: auth.account.did.clone(), - collection: input.collection.to_owned(), - rkey: input.rkey.to_owned(), - cid: record_cid(&input.value)?, - value: input.value, - }; - self.database.put_space_record(&record, &self.tids.next())?; - Ok(record.into()) - } - - pub(crate) fn delete_record( - &self, - auth: &AuthenticatedToken, - space: &str, - repo: &str, - collection: &str, - rkey: &str, - ) -> Result<()> { - let space_ref = self.authorize_write(auth, space, repo, collection, "delete")?; - syntax::record_key(rkey)?; - self.database.delete_space_record( - &space_ref.to_string(), - &auth.account.did, - collection, - rkey, - &self.tids.next(), - ) - } - - pub(crate) fn apply_writes( - &self, - auth: &AuthenticatedToken, - space: &str, - repo: &str, - writes: Vec, - validate: Option, - ) -> Result> { - if writes.len() > 200 { - return Err(Error::InvalidRequest( - "Too many writes. Max: 200".to_owned(), - )); - } - if writes.iter().any(WriteRequest::has_value) { - reject_unavailable_strict_validation(validate)?; - } - - let mut prepared = Vec::with_capacity(writes.len()); - let mut results = Vec::with_capacity(writes.len()); - for write in writes { - match write { - WriteRequest::Create { - collection, - rkey, - value, - } => { - let space_ref = - self.authorize_write(auth, space, repo, &collection, "create")?; - validate_record_type(&collection, &value)?; - self.database - .validate_blob_refs(&auth.account.did, &value)?; - let rkey = rkey.unwrap_or_else(|| self.tids.next()); - syntax::record_key(&rkey)?; - let record = SpaceRecord { - space: space_ref.to_string(), - did: auth.account.did.clone(), - collection, - rkey, - cid: record_cid(&value)?, - value, - }; - results.push(WriteResult::Create(record.clone().into())); - prepared.push(SpaceWrite::Create(record)); - } - WriteRequest::Update { - collection, - rkey, - value, - } => { - let space_ref = - self.authorize_write(auth, space, repo, &collection, "update")?; - syntax::record_key(&rkey)?; - validate_record_type(&collection, &value)?; - self.database - .validate_blob_refs(&auth.account.did, &value)?; - let record = SpaceRecord { - space: space_ref.to_string(), - did: auth.account.did.clone(), - collection, - rkey, - cid: record_cid(&value)?, - value, - }; - results.push(WriteResult::Update(record.clone().into())); - prepared.push(SpaceWrite::Update(record)); - } - WriteRequest::Delete { collection, rkey } => { - let space_ref = - self.authorize_write(auth, space, repo, &collection, "delete")?; - syntax::record_key(&rkey)?; - prepared.push(SpaceWrite::Delete { - space: space_ref.to_string(), - did: auth.account.did.clone(), - collection, - rkey, - }); - results.push(WriteResult::Delete); - } - } - } - - self.database - .apply_space_writes(&prepared, &self.tids.next()) - .map_err(|error| match error { - Error::Conflict(_) => named_conflict(error, "RecordAlreadyExists"), - Error::NotFound => named_not_found(error, "RecordNotFound"), - other => other, - })?; - Ok(results) - } - - pub(crate) fn get_record( - &self, - auth: ReadAuthorization<'_>, - space: &str, - repo: &str, - collection: &str, - rkey: &str, - ) -> Result { - let space_ref = self.authorize_read(auth, space, repo)?; - syntax::collection(collection)?; - syntax::record_key(rkey)?; - self.ensure_repo(&space_ref.to_string(), repo)?; - Ok(self - .database - .get_space_record(&space_ref.to_string(), repo, collection, rkey) - .map_err(|error| named_not_found(error, "RecordNotFound"))? - .into()) - } - - pub(crate) fn list_records( - &self, - auth: ReadAuthorization<'_>, - query: &ListRecords<'_>, - ) -> Result> { - let space_ref = self.authorize_read(auth, query.space, query.repo)?; - if let Some(collection) = query.collection { - syntax::collection(collection)?; - } - self.ensure_repo(&space_ref.to_string(), query.repo)?; - self.database.list_space_records( - &space_ref.to_string(), - query.repo, - query.collection, - query.cursor, - query.limit.clamp(1, 1000), - query.reverse, - ) - } - - pub(crate) fn list_spaces( - &self, - auth: &AuthenticatedToken, - space_type: Option<&str>, - authority: Option<&str>, - cursor: Option<&str>, - limit: usize, - ) -> Result> { - if let Some(value) = space_type { - syntax::collection(value)?; - } - if let Some(value) = authority { - syntax::did(value)?; - } - let target = SpaceRef { - authority: authority.unwrap_or("*").to_owned(), - space_type: space_type.unwrap_or("*").to_owned(), - skey: "*".to_owned(), - }; - require_scope(auth, &target, Capability::ReadSelf, None)?; - let limit = limit.clamp(1, 100); - Ok(self - .database - .list_spaces_for_repo(&auth.account.did, None, None, cursor, 1000)? - .into_iter() - .filter(|uri| { - syntax::space_ref(uri).is_ok_and(|space| { - space_type.is_none_or(|value| value == space.space_type) - && authority.is_none_or(|value| value == space.authority) - }) - }) - .take(limit) - .collect()) - } - - pub(crate) fn list_repo_ops( - &self, - auth: ReadAuthorization<'_>, - space: &str, - repo: &str, - since: Option<&str>, - cursor: Option<&str>, - limit: usize, - ) -> Result { - let space_ref = self.authorize_read(auth, space, repo)?; - let cursor_id = cursor - .map(str::parse::) - .transpose() - .map_err(|_| named("MalformedCursor", "invalid listRepoOps cursor"))?; - if cursor.is_none() - && let Some(value) = since - { - syntax::tid(value)?; - } - self.ensure_repo(&space_ref.to_string(), repo)?; - let limit = limit.clamp(1, 1000); - let mut ops = self.database.list_space_ops( - &space_ref.to_string(), - repo, - since, - cursor_id, - limit.saturating_add(1), - )?; - let has_more = ops.len() > limit; - if has_more { - ops.truncate(limit); - } - let next_cursor = has_more - .then(|| ops.last().map(|op| op.id.to_string())) - .flatten(); - let commit = if has_more { - None - } else { - Some(self.get_latest_commit(auth, space, repo)?) - }; - Ok(RepoOpsPage { - ops, - commit, - cursor: next_cursor, - }) - } - - pub(crate) fn get_latest_commit( - &self, - auth: ReadAuthorization<'_>, - space: &str, - repo: &str, - ) -> Result { - let space_ref = self.authorize_read(auth, space, repo)?; - let rev = self - .database - .latest_space_rev(&space_ref.to_string(), repo) - .map_err(|error| named_not_found(error, "RepoNotFound"))?; - let records = self.database.list_space_records( - &space_ref.to_string(), - repo, - None, - None, - usize::MAX, - true, - )?; - self.signer - .sign(&space_ref.to_string(), repo, &rev, &records) - } - - pub(crate) fn get_repo( - &self, - auth: ReadAuthorization<'_>, - space: &str, - repo: &str, - exclude_values: bool, - ) -> Result> { - let space_ref = self.authorize_read(auth, space, repo)?; - let rev = self - .database - .latest_space_rev(&space_ref.to_string(), repo) - .map_err(|error| named_not_found(error, "RepoNotFound"))?; - let records = self.database.list_space_records( - &space_ref.to_string(), - repo, - None, - None, - usize::MAX, - true, - )?; - let commit = self - .signer - .sign(&space_ref.to_string(), repo, &rev, &records)?; - car::serialize_repo(&commit, records, exclude_values) - } - - pub(crate) fn public_key_multibase(&self) -> String { - self.signer.public_key_multibase().to_owned() - } - - pub(crate) fn get_blob_allowed( - &self, - auth: ReadAuthorization<'_>, - space: &str, - repo: &str, - cid: &Cid, - ) -> Result<()> { - let space_ref = self.authorize_read(auth, space, repo)?; - self.ensure_repo(&space_ref.to_string(), repo)?; - let cid_text = cid.to_string(); - let records = self.database.list_space_records( - &space_ref.to_string(), - repo, - None, - None, - 1000, - false, - )?; - if records - .iter() - .any(|record| referenced_blob_cids(&record.value).contains(&cid_text)) - { - Ok(()) - } else { - Err(named("BlobNotFound", "blob is not referenced by this repo")) - } - } - - pub(crate) fn list_blobs( - &self, - auth: ReadAuthorization<'_>, - space: &str, - repo: &str, - since: Option<&str>, - cursor: Option<&str>, - limit: usize, - ) -> Result> { - let space_ref = self.authorize_read(auth, space, repo)?; - if let Some(value) = since { - syntax::tid(value)?; - } - self.ensure_repo(&space_ref.to_string(), repo)?; - let changed_paths = if since.is_some() { - Some( - self.database - .list_space_ops(&space_ref.to_string(), repo, since, None, usize::MAX)? - .into_iter() - .map(|op| (op.collection, op.rkey)) - .collect::>(), - ) - } else { - None - }; - let records = self.database.list_space_records( - &space_ref.to_string(), - repo, - None, - None, - usize::MAX, - true, - )?; - let mut cids = BTreeSet::new(); - for record in records { - if changed_paths - .as_ref() - .is_some_and(|paths| !paths.contains(&(record.collection, record.rkey))) - { - continue; - } - cids.extend(referenced_blob_cids(&record.value)); - } - Ok(cids - .into_iter() - .filter(|cid| cursor.is_none_or(|cursor| cid.as_str() > cursor)) - .take(limit.clamp(1, 1000)) - .collect()) - } - - fn ensure_repo(&self, space: &str, repo: &str) -> Result<()> { - self.database - .latest_space_rev(space, repo) - .map(|_| ()) - .map_err(|error| named_not_found(error, "RepoNotFound")) - } - - fn authorize_write( - &self, - auth: &AuthenticatedToken, - space: &str, - repo: &str, - collection: &str, - action: &'static str, - ) -> Result { - if repo != auth.account.did { - return Err(Error::Authorization( - "a permissioned write must target the authenticated account DID".to_owned(), - )); - } - syntax::collection(collection)?; - let space_ref = syntax::space_ref(space)?; - require_scope( - auth, - &space_ref, - Capability::Record(action), - Some(collection), - )?; - if space_ref.authority == self.hosted_did { - let local = self.database.space(space).map_err(space_not_found)?; - if !self.user_can_read(&local, &auth.account.did)? { - return Err(Error::Authorization( - "the account is not a member of this space".to_owned(), - )); - } - } - Ok(space_ref) - } - - fn authorize_read( - &self, - auth: ReadAuthorization<'_>, - space: &str, - repo: &str, - ) -> Result { - let space_ref = syntax::space_ref(space)?; - match auth { - ReadAuthorization::OAuth(auth) => { - if repo != auth.account.did { - return Err(Error::Authorization( - "an OAuth credential can only read its account's own permissioned repo" - .to_owned(), - )); - } - require_scope(auth, &space_ref, Capability::ReadSelf, None)?; - if space_ref.authority == self.hosted_did { - let local = self.database.space(space).map_err(space_not_found)?; - if !self.user_can_read(&local, &auth.account.did)? { - return Err(Error::Authorization( - "the account is not a member of this space".to_owned(), - )); - } - } - } - ReadAuthorization::Credential(credential_space) => { - if credential_space != space { - return Err(Error::Authorization( - "space credential does not cover the requested space".to_owned(), - )); - } - } - } - Ok(space_ref) + let space_ref = query.space.parse::()?; + require_scope(auth, &space_ref, Capability::ReadSelf)?; + self.database.space(&query.space).map_err(space_not_found)?; + self.database.list_space_members(query) } +} - fn require_owner(&self, auth: &AuthenticatedToken, space: &SpaceRef) -> Result<()> { - if space.authority != auth.account.did || space.authority != self.hosted_did { - return Err(Error::Authorization( - "the authenticated account is not the space owner".to_owned(), - )); - } - Ok(()) - } +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct CreateSpaceInput { + #[serde(rename = "type")] + pub(crate) space_type: String, + pub(crate) skey: Option, + pub(crate) policy: Value, + pub(crate) app_access: Value, +} - fn assert_local_credential_space(&self, credential_space: &str, space: &str) -> Result<()> { - if credential_space != space { - return Err(Error::Authorization( - "space credential does not cover the requested space".to_owned(), - )); - } - let space_ref = syntax::space_ref(space)?; - if space_ref.authority != self.hosted_did { - return Err(named("SpaceNotFound", "this PDS is not the space host")); - } - self.database.space(space).map_err(space_not_found)?; - Ok(()) - } +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct UpdateSpaceInput { + pub(crate) space: String, + pub(crate) policy: Option, + pub(crate) app_access: Option, +} - fn user_can_read(&self, space: &Space, did: &str) -> Result { - match union_type(&space.policy) { - Some("com.atproto.simplespace.defs#publicPolicy") => Ok(true), - Some("com.atproto.simplespace.defs#memberListPolicy") => { - self.database.is_space_member(&space.uri, did) - } - // A managing-app callback is intentionally not guessed. Until a - // configured verifier exists, only the owner is admitted. - Some("com.atproto.simplespace.defs#managingAppPolicy") => Ok(space.authority == did), - _ => Ok(false), - } - } +#[derive(Deserialize)] +pub(crate) struct SpacePageQuery { + pub(crate) space: String, + pub(crate) limit: Option, + pub(crate) cursor: Option, } -pub(crate) struct ListRecords<'a> { - pub(crate) space: &'a str, - pub(crate) repo: &'a str, - pub(crate) collection: Option<&'a str>, - pub(crate) cursor: Option<&'a str>, - pub(crate) limit: usize, - pub(crate) reverse: bool, +#[derive(Deserialize)] +pub(crate) struct NotifyRegistrationInput { + pub(crate) space: String, + pub(crate) service: String, } pub(crate) struct NotifiedWrite<'a> { @@ -912,280 +314,23 @@ pub(crate) struct NotifiedWrite<'a> { pub(crate) managing_app_authorized: bool, } -#[derive(Clone, Copy)] -pub(crate) enum ReadAuthorization<'a> { - OAuth(&'a AuthenticatedToken), - Credential(&'a str), -} - -pub(crate) struct RepoOpsPage { - pub(crate) ops: Vec, - pub(crate) commit: Option, - pub(crate) cursor: Option, -} - -pub(crate) struct CreateRecord<'a> { - pub(crate) space: &'a str, - pub(crate) repo: &'a str, - pub(crate) collection: &'a str, - pub(crate) rkey: Option<&'a str>, - pub(crate) value: Value, - pub(crate) validate: Option, -} - -pub(crate) struct PutRecord<'a> { - pub(crate) space: &'a str, - pub(crate) repo: &'a str, - pub(crate) collection: &'a str, - pub(crate) rkey: &'a str, - pub(crate) value: Value, - pub(crate) validate: Option, -} - -pub(crate) enum WriteRequest { - Create { - collection: String, - rkey: Option, - value: Value, - }, - Update { - collection: String, - rkey: String, - value: Value, - }, - Delete { - collection: String, - rkey: String, - }, -} - -impl WriteRequest { - fn has_value(&self) -> bool { - matches!(self, Self::Create { .. } | Self::Update { .. }) - } -} - -pub(crate) enum WriteResult { - Create(SpaceRecordView), - Update(SpaceRecordView), - Delete, -} - -#[derive(Clone, Copy)] -enum Capability { - Read, - ReadSelf, - Record(&'static str), - Manage(&'static str), -} - -fn require_scope( - auth: &AuthenticatedToken, - target: &SpaceRef, - capability: Capability, - collection: Option<&str>, -) -> Result<()> { - if auth.scope.split_ascii_whitespace().any(|scope| { - parse_space_scope(scope).is_some_and(|grant| { - grant.matches(&auth.account.did, target) && grant.permits(capability, collection) - }) - }) { - return Ok(()); - } - Err(Error::Authorization(format!( - "OAuth token has no covering `space:` grant for {target}" - ))) -} - -struct SpaceGrant { - space_type: String, - authority: String, - skey: String, - collections: Vec, - actions: Vec, - manage: Vec, -} - -impl SpaceGrant { - fn matches(&self, actor: &str, target: &SpaceRef) -> bool { - (self.space_type == "*" || self.space_type == target.space_type) - && (self.authority == "*" - || self.authority == target.authority - || (self.authority == "self" && target.authority == actor)) - && (self.skey == "*" || self.skey == target.skey) - } - - fn permits(&self, capability: Capability, collection: Option<&str>) -> bool { - match capability { - Capability::Read => self.actions.iter().any(|value| value == "read"), - Capability::ReadSelf => { - self.actions.is_empty() - || self.actions.iter().any(|value| value == "read") - || self.actions.iter().any(|value| value == "read_self") - } - Capability::Record(action) => { - let action_allowed = - self.actions.is_empty() || self.actions.iter().any(|value| value == action); - let collection_allowed = collection.is_some_and(|target| { - self.collections - .iter() - .any(|value| value == "*" || value == target) - }); - action_allowed && collection_allowed - } - Capability::Manage(action) => self.manage.iter().any(|value| value == action), - } - } -} - -fn parse_space_scope(value: &str) -> Option { - let value = value.strip_prefix("space:")?; - let (space_type, query) = value.split_once('?').unwrap_or((value, "")); - if space_type != "*" && syntax::collection(space_type).is_err() { - return None; - } - let mut grant = SpaceGrant { - space_type: space_type.to_owned(), - authority: "self".to_owned(), - skey: "*".to_owned(), - collections: Vec::new(), - actions: Vec::new(), - manage: Vec::new(), - }; - for (name, value) in url::form_urlencoded::parse(query.as_bytes()) { - match name.as_ref() { - "authority" => grant.authority = value.into_owned(), - "skey" => grant.skey = value.into_owned(), - "collection" => grant.collections.push(value.into_owned()), - "action" => grant.actions.push(value.into_owned()), - "manage" => grant.manage.push(value.into_owned()), - _ => return None, - } - } - Some(grant) -} - -fn validate_policy(value: &Value) -> Result<()> { - match union_type(value) { - Some( - "com.atproto.simplespace.defs#publicPolicy" - | "com.atproto.simplespace.defs#memberListPolicy", - ) => Ok(()), - Some("com.atproto.simplespace.defs#managingAppPolicy") - if value.get("managingApp").and_then(Value::as_str).is_some() => - { - Ok(()) - } - _ => Err(Error::InvalidRequest( - "unsupported simplespace policy".to_owned(), - )), - } -} - -fn validate_app_access(value: &Value) -> Result<()> { - match union_type(value) { - Some("com.atproto.simplespace.defs#open") => Ok(()), - Some("com.atproto.simplespace.defs#allowList") - if value - .get("allowed") - .and_then(Value::as_array) - .is_some_and(|values| values.iter().all(|value| value.as_str().is_some())) => - { - Ok(()) - } - _ => Err(Error::InvalidRequest( - "unsupported simplespace appAccess policy".to_owned(), - )), - } -} - -fn union_type(value: &Value) -> Option<&str> { - value.as_object()?.get("$type")?.as_str() -} - fn space_not_found(error: Error) -> Error { named_not_found(error, "SpaceNotFound") } -fn named(name: &'static str, message: &str) -> Error { - Error::Xrpc { - name, - message: message.to_owned(), - } -} - fn named_not_found(error: Error, name: &'static str) -> Error { match error { - Error::NotFound => Error::Xrpc { - name, - message: name.to_owned(), - }, + Error::NotFound => Error::xrpc(name, name), other => other, } } fn named_conflict(error: Error, name: &'static str) -> Error { match error { - Error::Conflict(_) => Error::Xrpc { - name, - message: name.to_owned(), - }, + Error::Conflict(_) => Error::xrpc(name, name), other => other, } } -fn reject_unavailable_strict_validation(validate: Option) -> Result<()> { - if validate == Some(true) { - return Err(Error::InvalidRequest( - "record validation was required, but this PDS has no matching Lexicon".to_owned(), - )); - } - Ok(()) -} - #[cfg(test)] -mod tests { - use crate::{ - domain::{account::Account, oauth::AuthenticatedToken}, - syntax::space_ref, - }; - - use super::{Capability, require_scope}; - - fn auth(scope: &str) -> AuthenticatedToken { - AuthenticatedToken { - account: Account { - did: "did:example:alice".to_owned(), - handle: "alice.test".to_owned(), - }, - client_id: "https://client.test/metadata.json".to_owned(), - scope: scope.to_owned(), - } - } - - #[test] - fn space_scopes_match_authority_action_and_collection() { - let target = space_ref("at://did:example:owner/space/com.example.forum/default").unwrap(); - let grant = auth( - "atproto space:com.example.forum?authority=did%3Aexample%3Aowner&collection=com.example.post&action=create", - ); - assert!( - require_scope( - &grant, - &target, - Capability::Record("create"), - Some("com.example.post") - ) - .is_ok() - ); - assert!( - require_scope( - &grant, - &target, - Capability::Record("delete"), - Some("com.example.post") - ) - .is_err() - ); - } -} +mod tests; diff --git a/src/space/records.rs b/src/space/records.rs new file mode 100644 index 0000000..f1a1610 --- /dev/null +++ b/src/space/records.rs @@ -0,0 +1,416 @@ +//! Permissioned record operations, inventories, and signed recovery views. +use super::{ + ReadAuthorization, SpaceService, + authorization::{Capability, require_scope}, + car, + commit::{self, SignedCommit}, + domain::{RecordQuery, SpaceOp, SpaceRecord, SpaceWrite}, + named_conflict, named_not_found, +}; +pub(crate) use crate::domain::record::WriteRequest; +use crate::syntax::SpaceRef; +use crate::{ + domain::{oauth::AuthenticatedToken, record::RecordView}, + error::{Error, Result}, + syntax, +}; +use cid::Cid; +use cpds_protocol::{record_cid, validate_record_type}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +impl SpaceService { + pub(crate) fn create_record( + &self, + auth: &AuthenticatedToken, + input: RecordInput<&str, Option<&str>>, + ) -> Result { + let record = self.prepare_create_record(auth, input)?; + self.database + .create_space_record(&record, &self.tids.next()) + .map_err(|error| named_conflict(error, "RecordAlreadyExists"))?; + Ok(record.into()) + } + + fn prepare_create_record( + &self, + auth: &AuthenticatedToken, + input: RecordInput<&str, Option<&str>>, + ) -> Result { + reject_unavailable_strict_validation(input.validate)?; + let space_ref = + self.authorize_write(auth, input.space, input.repo, input.collection, "create")?; + validate_record_type(input.collection, &input.record)?; + self.database + .validate_blob_refs(&auth.account.did, &input.record)?; + let rkey = input + .rkey + .map_or_else(|| self.tids.next(), ToOwned::to_owned); + syntax::record_key(&rkey)?; + Ok(SpaceRecord { + space: space_ref.to_string(), + did: auth.account.did.clone(), + collection: input.collection.to_owned(), + rkey, + cid: record_cid(&input.record)?, + value: input.record, + }) + } + + pub(crate) fn put_record( + &self, + auth: &AuthenticatedToken, + input: RecordInput<&str, &str>, + ) -> Result { + let record = self.prepare_update_record(auth, input)?; + self.database.put_space_record(&record, &self.tids.next())?; + Ok(record.into()) + } + + fn prepare_update_record( + &self, + auth: &AuthenticatedToken, + input: RecordInput<&str, &str>, + ) -> Result { + reject_unavailable_strict_validation(input.validate)?; + let space_ref = + self.authorize_write(auth, input.space, input.repo, input.collection, "update")?; + syntax::record_key(input.rkey)?; + validate_record_type(input.collection, &input.record)?; + self.database + .validate_blob_refs(&auth.account.did, &input.record)?; + Ok(SpaceRecord { + space: space_ref.to_string(), + did: auth.account.did.clone(), + collection: input.collection.to_owned(), + rkey: input.rkey.to_owned(), + cid: record_cid(&input.record)?, + value: input.record, + }) + } + + pub(crate) fn delete_record( + &self, + auth: &AuthenticatedToken, + query: &RecordQuery, + ) -> Result<()> { + self.authorize_write(auth, &query.space, &query.repo, &query.collection, "delete")?; + syntax::record_key(&query.rkey)?; + self.database.delete_space_record(query, &self.tids.next()) + } + + pub(crate) fn apply_writes( + &self, + auth: &AuthenticatedToken, + space: &str, + repo: &str, + writes: Vec, + validate: Option, + ) -> Result> { + if writes.len() > 200 { + return Err(Error::InvalidRequest( + "Too many writes. Max: 200".to_owned(), + )); + } + if writes.iter().any(|write| { + matches!( + write, + WriteRequest::Create { .. } | WriteRequest::Update { .. } + ) + }) { + reject_unavailable_strict_validation(validate)?; + } + + let mut prepared = Vec::with_capacity(writes.len()); + for write in writes { + match write { + WriteRequest::Create { + collection, + rkey, + value, + } => { + let record = self.prepare_create_record( + auth, + RecordInput { + space, + repo, + collection: &collection, + rkey: rkey.as_deref(), + record: value, + validate: None, + }, + )?; + prepared.push(SpaceWrite::Create(record)); + } + WriteRequest::Update { + collection, + rkey, + value, + } => { + let record = self.prepare_update_record( + auth, + RecordInput { + space, + repo, + collection: &collection, + rkey: &rkey, + record: value, + validate: None, + }, + )?; + prepared.push(SpaceWrite::Update(record)); + } + WriteRequest::Delete { collection, rkey } => { + let space_ref = + self.authorize_write(auth, space, repo, &collection, "delete")?; + syntax::record_key(&rkey)?; + prepared.push(SpaceWrite::Delete(RecordQuery { + space: space_ref.to_string(), + repo: auth.account.did.clone(), + collection, + rkey, + })); + } + } + } + + self.database + .apply_space_writes(&prepared, &self.tids.next()) + .map_err(|error| match error { + Error::Conflict(_) => named_conflict(error, "RecordAlreadyExists"), + Error::NotFound => named_not_found(error, "RecordNotFound"), + other => other, + })?; + Ok(prepared) + } + + pub(crate) fn get_record( + &self, + auth: &ReadAuthorization, + query: &RecordQuery, + ) -> Result { + self.authorize_read(auth, &query.space, &query.repo)?; + syntax::collection(&query.collection)?; + syntax::record_key(&query.rkey)?; + self.ensure_repo(&query.space, &query.repo)?; + Ok(self + .database + .get_space_record(query) + .map_err(|error| named_not_found(error, "RecordNotFound"))? + .into()) + } + + pub(crate) fn list_records( + &self, + auth: &ReadAuthorization, + query: &ListRecords, + ) -> Result> { + let space_ref = self.authorize_read(auth, &query.space, &query.repo)?; + if let Some(collection) = &query.collection { + syntax::collection(collection)?; + } + self.ensure_repo(&space_ref.to_string(), &query.repo)?; + self.database.list_space_records( + &space_ref.to_string(), + &query.repo, + query.collection.as_deref(), + query.cursor.as_deref(), + query.limit(), + query.reverse.unwrap_or(false), + ) + } + + pub(crate) fn list_spaces( + &self, + auth: &AuthenticatedToken, + query: &ListSpacesQuery, + ) -> Result> { + if let Some(value) = query.space_type.as_deref() { + syntax::collection(value)?; + } + if let Some(value) = query.did.as_deref() { + syntax::did(value)?; + } + let target = SpaceRef { + authority: query.did.as_deref().unwrap_or("*").to_owned(), + space_type: query.space_type.as_deref().unwrap_or("*").to_owned(), + skey: "*".to_owned(), + }; + require_scope(auth, &target, Capability::ReadSelf)?; + self.database.list_spaces_for_repo(&auth.account.did, query) + } + + pub(crate) fn list_repo_ops( + &self, + auth: &ReadAuthorization, + query: &ListOpsQuery, + ) -> Result { + let space_ref = self.authorize_read(auth, &query.space, &query.repo)?; + let cursor_id = query + .cursor + .as_deref() + .map(str::parse::) + .transpose() + .map_err(|_| Error::xrpc("MalformedCursor", "invalid listRepoOps cursor"))?; + if query.cursor.is_none() + && let Some(value) = query.since.as_deref() + { + syntax::tid(value)?; + } + self.ensure_repo(&space_ref.to_string(), &query.repo)?; + let limit = query.limit.unwrap_or(100).clamp(1, 1000); + let mut ops = self.database.list_space_ops( + &space_ref.to_string(), + &query.repo, + query.since.as_deref(), + cursor_id, + limit.saturating_add(1), + )?; + let has_more = ops.len() > limit; + if has_more { + ops.truncate(limit); + } + let next_cursor = has_more + .then(|| ops.last().map(|op| op.id.to_string())) + .flatten(); + let commit = if has_more { + None + } else { + Some(self.signed_snapshot(auth, &query.space, &query.repo)?.0) + }; + Ok(RepoOpsPage { + ops, + commit, + cursor: next_cursor, + }) + } + + pub(crate) fn signed_snapshot( + &self, + auth: &ReadAuthorization, + space: &str, + repo: &str, + ) -> Result<(SignedCommit, Vec)> { + let space_ref = self.authorize_read(auth, space, repo)?.to_string(); + let (rev, records) = self + .database + .space_snapshot(&space_ref, repo, car::MAX_REPO_EXPORT_BYTES) + .map_err(|error| named_not_found(error, "RepoNotFound"))?; + let commit = commit::sign(&self.signer, &space_ref, repo, &rev, &records)?; + Ok((commit, records)) + } + + pub(crate) fn get_blob_allowed( + &self, + auth: &ReadAuthorization, + space: &str, + repo: &str, + cid: &Cid, + ) -> Result<()> { + let space_ref = self.authorize_read(auth, space, repo)?; + self.ensure_repo(&space_ref.to_string(), repo)?; + if self.database.space_references_blob(space, repo, cid)? { + Ok(()) + } else { + Err(Error::xrpc( + "BlobNotFound", + "blob is not referenced by this repo", + )) + } + } + + pub(crate) fn list_blobs( + &self, + auth: &ReadAuthorization, + query: &ListBlobsQuery, + ) -> Result> { + self.authorize_read(auth, &query.space, &query.repo)?; + if let Some(value) = query.since.as_deref() { + syntax::tid(value)?; + } + self.ensure_repo(&query.space, &query.repo)?; + self.database.space_blob_cids(query) + } + + fn ensure_repo(&self, space: &str, repo: &str) -> Result<()> { + self.database + .latest_space_rev(space, repo) + .map(|_| ()) + .map_err(|error| named_not_found(error, "RepoNotFound")) + } +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ListOpsQuery { + pub(crate) space: String, + pub(crate) repo: String, + pub(crate) since: Option, + pub(crate) limit: Option, + pub(crate) cursor: Option, + pub(crate) exclude_values: Option, +} + +#[derive(Deserialize)] +pub(crate) struct ListBlobsQuery { + pub(crate) space: String, + pub(crate) repo: String, + pub(crate) since: Option, + pub(crate) limit: Option, + pub(crate) cursor: Option, +} + +#[derive(Deserialize)] +pub(crate) struct ListSpacesQuery { + #[serde(rename = "type")] + pub(crate) space_type: Option, + pub(crate) did: Option, + pub(crate) limit: Option, + pub(crate) cursor: Option, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ListRecords { + pub(crate) space: String, + pub(crate) repo: String, + pub(crate) collection: Option, + pub(crate) cursor: Option, + pub(crate) limit: Option, + pub(crate) reverse: Option, + pub(crate) exclude_values: Option, +} + +impl ListRecords { + pub(crate) fn limit(&self) -> usize { + self.limit.unwrap_or(50).clamp(1, 1000) + } +} +#[derive(Serialize)] +pub(crate) struct RepoOpsPage { + pub(crate) ops: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) commit: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) cursor: Option, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct RecordInput> { + pub(crate) space: Text, + pub(crate) repo: Text, + pub(crate) collection: Text, + pub(crate) rkey: Key, + pub(crate) validate: Option, + pub(crate) record: Value, +} + +fn reject_unavailable_strict_validation(validate: Option) -> Result<()> { + if validate == Some(true) { + return Err(Error::InvalidRequest( + "record validation was required, but this PDS has no matching Lexicon".to_owned(), + )); + } + Ok(()) +} diff --git a/src/space/routes.rs b/src/space/routes.rs new file mode 100644 index 0000000..928bd78 --- /dev/null +++ b/src/space/routes.rs @@ -0,0 +1,51 @@ +//! Space endpoint registration lives with the feature; shared middleware stays in HTTP. + +use super::http as space; +use crate::{ + AppState, + http::{ + rate_limit::{expensive, protected_routes}, + xrpc_routes, + }, +}; +use axum::{ + Router, + routing::{get, post}, +}; + +pub(crate) fn routes() -> Router { + protected_routes(xrpc_routes([ + ("simplespace.createSpace", post(space::create_space)), + ("simplespace.updateSpace", post(space::update_space)), + ("simplespace.deleteSpace", post(space::delete_space)), + ("simplespace.getSpace", get(space::get_space)), + ("simplespace.addMember", post(space::add_member)), + ("simplespace.removeMember", post(space::remove_member)), + ("simplespace.listMembers", get(space::list_members)), + ("space.createRecord", post(space::create_record)), + ("space.getDelegationToken", get(space::get_delegation_token)), + ( + "space.getSpaceCredential", + expensive(post(space::get_space_credential)), + ), + ("space.putRecord", post(space::put_record)), + ("space.deleteRecord", post(space::delete_record)), + ("space.applyWrites", post(space::apply_writes)), + ("space.getRecord", get(space::get_record)), + ("space.getRepo", expensive(get(space::get_repo))), + ("space.notifyWrite", post(space::notify_write)), + ("space.listRepos", get(space::list_repos)), + ("space.registerNotify", post(space::register_notify)), + ("space.unregisterNotify", post(space::unregister_notify)), + ( + "space.notifySpaceDeleted", + post(space::notify_space_deleted), + ), + ("space.listRecords", get(space::list_records)), + ("space.listSpaces", get(space::list_spaces)), + ("space.listRepoOps", get(space::list_repo_ops)), + ("space.getLatestCommit", get(space::get_latest_commit)), + ("space.getBlob", expensive(get(space::get_blob))), + ("space.listBlobs", get(space::list_blobs)), + ])) +} diff --git a/src/space/tests.rs b/src/space/tests.rs new file mode 100644 index 0000000..9d102ae --- /dev/null +++ b/src/space/tests.rs @@ -0,0 +1,1538 @@ +use crate::{ + domain::{account::Account, oauth::AuthenticatedToken}, + syntax::SpaceRef, +}; + +use super::domain::SpaceRecord; +use super::*; +use cpds_protocol::record_cid; + +fn validation_guard_write(kind: usize) -> WriteRequest { + let collection = "invalid collection".to_owned(); + let rkey = "/".to_owned(); + match kind { + 0 => WriteRequest::Create { + collection, + rkey: Some(rkey), + value: Value::Null, + }, + 1 => WriteRequest::Update { + collection, + rkey, + value: Value::Null, + }, + 2 => WriteRequest::Delete { collection, rkey }, + _ => unreachable!(), + } +} + +#[test] +fn batch_value_validation_guard_preserves_empty_delete_and_mixed_priority() { + let auth = auth("atproto"); + let service = SpaceService::new( + Database::open_in_memory().unwrap(), + auth.account.did.clone(), + &[7; 32], + ) + .unwrap(); + for length in 0..=3 { + for ordinal in 0..3_usize.pow(length) { + let kinds = (0..length) + .map(|index| (ordinal / 3_usize.pow(index)) % 3) + .collect::>(); + for validate in [None, Some(false), Some(true)] { + let expected = if kinds.is_empty() { + Ok(0) + } else if validate == Some(true) && kinds.iter().any(|kind| *kind < 2) { + Err("the request is invalid: record validation was required, but this PDS has no matching Lexicon".to_owned()) + } else { + Err("authorization failed: a permissioned write must target the authenticated account DID".to_owned()) + }; + let actual = service + .apply_writes( + &auth, + "invalid space", + "another repo", + kinds.iter().copied().map(validation_guard_write).collect(), + validate, + ) + .map(|writes| writes.len()) + .map_err(|error| error.to_string()); + assert_eq!(actual, expected, "{kinds:?}, {validate:?}"); + } + } + } + for kind in 0..3 { + for validate in [None, Some(false), Some(true)] { + let error = service + .apply_writes( + &auth, + "invalid space", + "another repo", + (0..201).map(|_| validation_guard_write(kind)).collect(), + validate, + ) + .unwrap_err(); + assert!( + matches!(error, Error::InvalidRequest(message) if message == "Too many writes. Max: 200") + ); + } + } +} + +#[test] +fn invalid_signing_keys_keep_configuration_error_context() { + let database = Database::open_in_memory().unwrap(); + for secret in [[0; 32], [u8::MAX; 32]] { + let expected = format!( + "invalid account secp256k1 signing key: {}", + Secp256k1Keypair::import(&secret) + .err() + .expect("fixture must not be a valid signing key") + ); + let error = SpaceService::new(database.clone(), "did:example:alice".to_owned(), &secret) + .err() + .expect("invalid signing key must prevent construction"); + assert!(matches!(error, Error::Configuration(message) if message == expected)); + } +} + +fn management_input() -> CreateSpaceInput { + CreateSpaceInput { + space_type: "com.example.forum".to_owned(), + skey: Some("default".to_owned()), + policy: serde_json::json!({"$type": "com.atproto.simplespace.defs#memberListPolicy"}), + app_access: serde_json::json!({"$type": "com.atproto.simplespace.defs#open"}), + } +} + +#[test] +fn shared_management_requests_preserve_validation_and_authorization_priority() { + let database = Database::open_in_memory().unwrap(); + let denied = auth("atproto"); + let allowed = auth("atproto space:*?authority=*&manage=create&manage=update&action=read_self"); + database + .ensure_account(&allowed.account.did, "alice.test", "password") + .unwrap(); + let service = + SpaceService::new(database.clone(), allowed.account.did.clone(), &[7; 32]).unwrap(); + let uri = "at://did:example:alice/space/com.example.forum/default"; + for (stage, expected) in [ + ( + 0, + "the request is invalid: `invalid` is not a valid collection NSID".to_owned(), + ), + ( + 1, + "the request is invalid: unsupported simplespace policy".to_owned(), + ), + ( + 2, + "the request is invalid: unsupported simplespace appAccess policy".to_owned(), + ), + ( + 3, + "the request is invalid: `/` is not a valid record key".to_owned(), + ), + ( + 4, + format!("authorization failed: OAuth token has no covering `space:` grant for {uri}"), + ), + ] { + let mut input = management_input(); + if stage == 0 { + input.space_type = "invalid".to_owned(); + } + if stage <= 1 { + input.policy = Value::Null; + } + if stage <= 2 { + input.app_access = Value::Null; + } + if stage <= 3 { + input.skey = Some("/".to_owned()); + } + assert_eq!( + service + .create_space(&denied, input) + .unwrap_err() + .to_string(), + expected + ); + } + assert!(matches!(database.space(uri), Err(Error::NotFound))); + assert_eq!( + service.create_space(&allowed, management_input()).unwrap(), + uri + ); + assert!(matches!( + service.create_space(&allowed, management_input()), + Err(Error::Xrpc { + name: "SpaceAlreadyExists", + .. + }) + )); +} + +#[test] +fn shared_space_updates_preserve_validation_and_authorization_priority() { + let database = Database::open_in_memory().unwrap(); + let denied = auth("atproto"); + let allowed = auth("atproto space:*?authority=*&manage=create&manage=update&action=read_self"); + database + .ensure_account(&allowed.account.did, "alice.test", "password") + .unwrap(); + let service = + SpaceService::new(database.clone(), allowed.account.did.clone(), &[7; 32]).unwrap(); + let uri = "at://did:example:alice/space/com.example.forum/default"; + service.create_space(&allowed, management_input()).unwrap(); + let mut input = UpdateSpaceInput { + space: "invalid".to_owned(), + policy: Some(Value::Null), + app_access: Some(Value::Null), + }; + assert_eq!( + service + .update_space(&denied, &input) + .unwrap_err() + .to_string(), + "the request is invalid: `invalid` is not a valid space reference" + ); + input.space = uri.to_owned(); + assert_eq!( + service + .update_space(&denied, &input) + .unwrap_err() + .to_string(), + format!("authorization failed: OAuth token has no covering `space:` grant for {uri}") + ); + input.space = "at://did:example:other/space/com.example.forum/default".to_owned(); + assert_eq!( + service + .update_space(&allowed, &input) + .unwrap_err() + .to_string(), + "authorization failed: the authenticated account is not the space owner" + ); + input.space = uri.to_owned(); + assert_eq!( + service + .update_space(&allowed, &input) + .unwrap_err() + .to_string(), + "the request is invalid: unsupported simplespace policy" + ); + input.policy = None; + assert_eq!( + service + .update_space(&allowed, &input) + .unwrap_err() + .to_string(), + "the request is invalid: unsupported simplespace appAccess policy" + ); + input.app_access = None; + service.update_space(&allowed, &input).unwrap(); + let stored = database.space(uri).unwrap(); + assert_eq!(stored.policy, management_input().policy); + assert_eq!(stored.app_access, management_input().app_access); + input.space = "at://did:example:alice/space/com.example.forum/missing".to_owned(); + assert!(matches!( + service.update_space(&allowed, &input), + Err(Error::Xrpc { + name: "SpaceNotFound", + .. + }) + )); +} + +#[test] +fn member_queries_keep_default_limits_clamps_cursors_and_error_order() { + let database = Database::open_in_memory().unwrap(); + let allowed = auth("atproto space:*?authority=*&manage=create&action=read_self"); + database + .ensure_account(&allowed.account.did, "alice.test", "password") + .unwrap(); + let service = + SpaceService::new(database.clone(), allowed.account.did.clone(), &[7; 32]).unwrap(); + let uri = service.create_space(&allowed, management_input()).unwrap(); + let mut members = vec![allowed.account.did.clone()]; + for index in 0..1001 { + let did = format!("did:example:member{index:04}"); + database.add_space_member(&uri, &did).unwrap(); + members.push(did); + } + members.sort(); + let mut query = SpacePageQuery { + space: uri, + limit: None, + cursor: None, + }; + for (limit, expected) in [ + (None, 100), + (Some(0), 1), + (Some(1001), 1000), + (Some(usize::MAX), 1000), + ] { + query.limit = limit; + assert_eq!( + service.list_members(&allowed, &query).unwrap(), + members[..expected] + ); + } + query.limit = Some(2); + query.cursor = Some(members[50].clone()); + assert_eq!( + service.list_members(&allowed, &query).unwrap(), + members[51..53] + ); + query.cursor = Some("not-a-did".to_owned()); + assert!(service.list_members(&allowed, &query).unwrap().is_empty()); + query.space = "invalid".to_owned(); + assert_eq!( + service + .list_members(&auth("atproto"), &query) + .unwrap_err() + .to_string(), + "the request is invalid: `invalid` is not a valid space reference" + ); + query.space = "at://did:example:alice/space/com.example.forum/missing".to_owned(); + assert!(matches!( + service.list_members(&auth("atproto"), &query), + Err(Error::Authorization(_)) + )); + assert!(matches!( + service.list_members(&allowed, &query), + Err(Error::Xrpc { + name: "SpaceNotFound", + .. + }) + )); +} + +#[test] +fn shared_space_reads_preserve_credential_lookup_and_oauth_checks() { + let database = Database::open_in_memory().unwrap(); + let grant = auth("atproto space:*?authority=*&action=read_self"); + let service = SpaceService::new(database.clone(), grant.account.did.clone(), &[7; 32]).unwrap(); + assert!(matches!( + service.get_space( + &ReadAuthorization::Credential("invalid".to_owned()), + "invalid" + ), + Err(Error::Xrpc { + name: "SpaceNotFound", + .. + }) + )); + assert_eq!( + service + .get_space( + &ReadAuthorization::Credential("different".to_owned()), + "invalid" + ) + .unwrap_err() + .to_string(), + "authorization failed: space credential does not cover the requested space" + ); + assert_eq!( + service + .get_space(&ReadAuthorization::OAuth(grant.clone()), "invalid") + .unwrap_err() + .to_string(), + "the request is invalid: `invalid` is not a valid space reference" + ); + let uri = "at://did:example:alice/space/com.example.forum/default"; + assert!(matches!( + service.get_space(&ReadAuthorization::OAuth(auth("atproto")), uri), + Err(Error::Authorization(_)) + )); + assert!(matches!( + service.get_space(&ReadAuthorization::OAuth(grant.clone()), uri), + Err(Error::Xrpc { + name: "SpaceNotFound", + .. + }) + )); + database + .create_space(&Space { + uri: uri.to_owned(), + authority: grant.account.did.clone(), + kind: "com.example.forum".to_owned(), + skey: "default".to_owned(), + policy: management_input().policy, + app_access: management_input().app_access, + }) + .unwrap(); + assert!( + service + .get_space(&ReadAuthorization::OAuth(grant.clone()), uri) + .is_ok() + ); + assert!( + service + .get_space(&ReadAuthorization::Credential(uri.to_owned()), uri) + .is_ok() + ); + let mut outsider = grant; + outsider.account.did = "did:example:bob".to_owned(); + assert_eq!( + service + .get_space(&ReadAuthorization::OAuth(outsider.clone()), uri) + .unwrap_err() + .to_string(), + "authorization failed: the account is not allowed to read this space" + ); +} + +#[test] +fn owned_read_authorization_reuses_both_variants_and_preserves_denial_priority() { + let database = Database::open_in_memory().unwrap(); + let grant = auth("atproto space:*?authority=*&manage=create&action=read_self"); + database + .ensure_account(&grant.account.did, "alice.test", "password") + .unwrap(); + let service = SpaceService::new(database.clone(), grant.account.did.clone(), &[7; 32]).unwrap(); + let uri = service.create_space(&grant, management_input()).unwrap(); + let expected = serde_json::to_value(database.space(&uri).unwrap()).unwrap(); + let authorizations = [ + ReadAuthorization::OAuth(grant), + ReadAuthorization::Credential(uri.clone()), + ]; + let query = RecordQuery { + space: uri.clone(), + repo: "did:example:alice".to_owned(), + collection: "invalid".to_owned(), + rkey: "/".to_owned(), + }; + for authorization in &authorizations { + for _ in 0..2 { + assert_eq!( + serde_json::to_value(service.get_space(authorization, &uri).unwrap()).unwrap(), + expected + ); + assert_eq!( + service + .get_record(authorization, &query) + .unwrap_err() + .to_string(), + "the request is invalid: `invalid` is not a valid collection NSID" + ); + } + } + database + .connection() + .unwrap() + .execute_batch("CREATE TEMP TABLE spaces (uri)") + .unwrap(); + for authorization in &authorizations { + assert!(matches!( + service.get_space(authorization, &uri), + Err(Error::Database(_)) + )); + let expected = match authorization { + ReadAuthorization::OAuth(_) => { + "the request is invalid: `invalid` is not a valid space reference" + } + ReadAuthorization::Credential(_) => { + "authorization failed: space credential does not cover the requested space" + } + }; + assert_eq!( + service + .get_space(authorization, "invalid") + .unwrap_err() + .to_string(), + expected + ); + } + let denied = ReadAuthorization::OAuth(auth("atproto")); + for _ in 0..2 { + assert_eq!( + service.get_space(&denied, &uri).unwrap_err().to_string(), + format!("authorization failed: OAuth token has no covering `space:` grant for {uri}") + ); + } +} + +fn credential_policy_fixture() -> (Database, SpaceService, String) { + let database = Database::open_in_memory().unwrap(); + let did = "did:example:alice"; + database + .ensure_account(did, "alice.test", "password") + .unwrap(); + let uri = "at://did:example:alice/space/com.example.forum/default".to_owned(); + database + .create_space(&Space { + uri: uri.clone(), + authority: did.to_owned(), + kind: "com.example.forum".to_owned(), + skey: "default".to_owned(), + policy: Value::Null, + app_access: management_input().app_access, + }) + .unwrap(); + let service = SpaceService::new(database.clone(), did.to_owned(), &[7; 32]).unwrap(); + (database, service, uri) +} + +fn previous_local_credential_gate( + service: &SpaceService, + credential_space: &str, + space: &str, +) -> Result<()> { + if credential_space != space { + return Err(Error::Authorization( + "space credential does not cover the requested space".to_owned(), + )); + } + let space_ref = space.parse::()?; + if space_ref.authority != service.hosted_did { + return Err(Error::xrpc( + "SpaceNotFound", + "this PDS is not the space host", + )); + } + service.database.space(space).map_err(space_not_found)?; + Ok(()) +} + +fn stored_notify_rows(database: &Database) -> Vec<(String, String, String, i64, i64)> { + database + .query( + "SELECT space_uri, service, endpoint, expires_at, updated_at + FROM space_notify_registrations ORDER BY space_uri, service", + [], + |row| Ok(<(_, _, _, _, _)>::try_from(row)?), + ) + .unwrap() +} + +fn prepare_credential_gate_space<'a>(database: &Database, uri: &'a str, state: &str) -> &'a str { + database + .connection() + .unwrap() + .execute( + "UPDATE spaces SET deleted_at = NULL, policy_json = 'null', + app_access_json = '{}' WHERE uri = ?1", + [uri], + ) + .unwrap(); + match state { + "remote" => "at://did:example:remote/space/com.example.forum/default", + "invalid" => "not a space reference", + "missing" => "at://did:example:alice/space/com.example.forum/missing", + "deleted" | "json" | "sql" => { + let sql = match state { + "deleted" => "UPDATE spaces SET deleted_at = 1 WHERE uri = ?1", + "json" => "UPDATE spaces SET policy_json = '{' WHERE uri = ?1", + _ => { + "UPDATE spaces SET policy_json = '{', app_access_json = X'00' + WHERE uri = ?1" + } + }; + database.connection().unwrap().execute(sql, [uri]).unwrap(); + uri + } + _ => uri, + } +} + +fn assert_notification_gate_case( + service: &SpaceService, + credential: &str, + requested: &str, + local_uri: &str, + account_error: Option<&str>, + case: &str, +) { + let database = &service.database; + let expected_notification = service + .require_active_account(&service.hosted_did) + .and_then(|()| previous_local_credential_gate(service, credential, requested)); + if let Some(expected_name) = account_error { + assert!( + matches!(&expected_notification, Err(Error::Xrpc { name, .. }) + if *name == expected_name), + "{case}" + ); + } + let expected_notification = expected_notification.map_err(|error| format!("{error:?}")); + database + .register_space_notify( + &NotifyRegistrationInput { + space: local_uri.to_owned(), + service: "did:example:subscriber".to_owned(), + }, + "before", + 42, + ) + .unwrap(); + let before = stored_notify_rows(database); + let input = NotifyRegistrationInput { + space: requested.to_owned(), + service: "did:example:subscriber".to_owned(), + }; + let registered = service.register_notify(credential, &input, "after"); + assert_eq!( + registered + .as_ref() + .map(|_| ()) + .map_err(|error| format!("{error:?}")), + expected_notification, + "register: {case}" + ); + let after_register = stored_notify_rows(database); + if let Ok(expires_at) = registered { + assert_eq!(after_register.len(), 1, "{case}"); + assert_eq!(after_register[0].2, "after", "{case}"); + assert_eq!(after_register[0].3, expires_at, "{case}"); + } else { + assert_eq!(after_register, before, "register mutation: {case}"); + } + let unregistered = service.unregister_notify(credential, &input); + assert_eq!( + unregistered.map_err(|error| format!("{error:?}")), + expected_notification, + "unregister: {case}" + ); + if expected_notification.is_ok() { + assert!(stored_notify_rows(database).is_empty(), "{case}"); + } else { + assert_eq!( + stored_notify_rows(database), + after_register, + "unregister mutation: {case}" + ); + } +} + +#[test] +fn shared_local_credential_gate_preserves_errors_and_notification_mutations() { + let (database, service, uri) = credential_policy_fixture(); + database + .record_space_writer(&NotifiedWrite { + issuer: "", + audience: "", + space: &uri, + repo: "did:example:writer", + rev: "raw revision", + hash: &[0, 255], + managing_app_authorized: false, + }) + .unwrap(); + for (status, account_error) in [ + ("active", None), + ("deactivated", Some("RepoDeactivated")), + ("suspended", Some("RepoSuspended")), + ("takendown", Some("RepoTakendown")), + ("deleted", Some("RepoNotFound")), + ("missing", Some("RepoNotFound")), + ] { + if status == "missing" { + database + .connection() + .unwrap() + .execute("DELETE FROM accounts", []) + .unwrap(); + } else { + database + .connection() + .unwrap() + .execute("UPDATE accounts SET status = ?1", [status]) + .unwrap(); + } + for state in [ + "local", "remote", "invalid", "missing", "deleted", "json", "sql", + ] { + let requested = prepare_credential_gate_space(&database, &uri, state); + for matching in [true, false] { + let credential = if matching { + requested + } else { + "different space" + }; + let case = format!("{status}/{state}/matching={matching}"); + let query = SpacePageQuery { + space: requested.to_owned(), + limit: None, + cursor: None, + }; + let expected_list = previous_local_credential_gate(&service, credential, requested) + .and_then(|()| database.list_space_writers(&query)); + let actual_list = service.list_repos(credential, &query); + assert_eq!( + format!("{actual_list:?}"), + format!("{expected_list:?}"), + "{case}" + ); + if matching && state == "sql" { + assert!(matches!(&actual_list, Err(Error::Database(_))), "{case}"); + } + if matching && state == "json" { + assert!(matches!(&actual_list, Err(Error::Json(_))), "{case}"); + } + if state == "local" && matching { + assert_eq!(actual_list.unwrap().len(), 1, "{case}"); + } + + assert_notification_gate_case( + &service, + credential, + requested, + &uri, + account_error, + &case, + ); + } + } + } +} + +#[test] +fn credential_read_policy_preserves_owner_member_and_managing_app_results() { + let (database, service, uri) = credential_policy_fixture(); + database.add_space_member(&uri, "did:example:bob").unwrap(); + let denied = Err("user is not authorized for this space"); + let unconfigured = Err("managing app is not configured"); + for (policy, member, outsider) in [ + ( + r#"{"$type":"com.atproto.simplespace.defs#publicPolicy"}"#, + Ok(None), + Ok(None), + ), + ( + r#"{"$type":"com.atproto.simplespace.defs#memberListPolicy"}"#, + Ok(None), + denied, + ), + ( + r#"{"$type":"com.atproto.simplespace.defs#managingAppPolicy","managingApp":"did:example:manager"}"#, + Ok(Some("did:example:manager")), + Ok(Some("did:example:manager")), + ), + ( + r#"{"$type":"com.atproto.simplespace.defs#managingAppPolicy","managingApp":""}"#, + Ok(Some("")), + Ok(Some("")), + ), + ( + r#"{"$type":"com.atproto.simplespace.defs#managingAppPolicy"}"#, + unconfigured, + unconfigured, + ), + ( + r#"{"$type":"com.atproto.simplespace.defs#managingAppPolicy","managingApp":null}"#, + unconfigured, + unconfigured, + ), + ( + r#"{"$type":"com.atproto.simplespace.defs#managingAppPolicy","managingApp":7}"#, + unconfigured, + unconfigured, + ), + (r#"{"$type":"unsupported"}"#, denied, denied), + ("null", denied, denied), + ] { + let policy: Value = serde_json::from_str(policy).unwrap(); + database + .update_space(&UpdateSpaceInput { + space: uri.clone(), + policy: Some(policy.clone()), + app_access: None, + }) + .unwrap(); + for (did, expected) in [ + ("did:example:alice", Ok(None)), + ("did:example:bob", member), + ("did:example:carol", outsider), + ] { + let actual = service.authorize_credential(&uri, did, None); + match expected { + Ok(expected) => assert_eq!(actual.unwrap().as_deref(), expected, "{policy}: {did}"), + Err(expected) => assert!( + matches!(actual, Err(Error::Xrpc { name: "UserNotAuthorized", message }) if message == expected), + "{policy}: {did}" + ), + } + } + } +} + +#[test] +fn credential_read_policy_only_queries_membership_for_nonowner_member_policy() { + let (database, service, uri) = credential_policy_fixture(); + database + .connection() + .unwrap() + .execute_batch("DROP TABLE space_members") + .unwrap(); + for (kind, manager) in [ + ("publicPolicy", None), + ("memberListPolicy", None), + ("managingAppPolicy", Some("did:example:manager")), + ] { + let policy = serde_json::json!({ + "$type": format!("com.atproto.simplespace.defs#{kind}"), + "managingApp": manager, + }); + database + .update_space(&UpdateSpaceInput { + space: uri.clone(), + policy: Some(policy), + app_access: None, + }) + .unwrap(); + assert_eq!( + service + .authorize_credential(&uri, "did:example:alice", None) + .unwrap(), + None + ); + let actual = service.authorize_credential(&uri, "did:example:bob", None); + if kind == "memberListPolicy" { + assert!(matches!(actual, Err(Error::Database(_)))); + } else { + assert_eq!(actual.unwrap().as_deref(), manager); + } + } +} + +#[test] +fn credential_app_allowlist_is_checked_before_owner_and_read_policy() { + let (database, service, uri) = credential_policy_fixture(); + let app_access = serde_json::json!({ + "$type": "com.atproto.simplespace.defs#allowList", + "allowed": ["https://client.test/metadata.json"], + }); + database + .update_space(&UpdateSpaceInput { + space: uri.clone(), + policy: None, + app_access: Some(app_access), + }) + .unwrap(); + for did in ["did:example:alice", "did:example:bob"] { + for client in [None, Some("https://other.test/metadata.json")] { + assert!(matches!( + service.authorize_credential(&uri, did, client), + Err(Error::Xrpc { name: "AppNotAuthorized", message }) + if message == "application is not authorized for this space" + )); + } + } + assert_eq!( + service + .authorize_credential( + &uri, + "did:example:alice", + Some("https://client.test/metadata.json") + ) + .unwrap(), + None + ); +} + +fn seed_record(database: &Database, space: &str, rkey: &str, value: Value) { + database + .create_space_record( + &SpaceRecord { + space: space.to_owned(), + did: "did:example:alice".to_owned(), + collection: "com.example.post".to_owned(), + rkey: rkey.to_owned(), + cid: record_cid(&value).unwrap(), + value, + }, + "3kbcq3p7ad422", + ) + .unwrap(); +} + +fn seed_blob_records(database: &Database, space: &str, count: u32) -> Vec { + use crate::domain::blob::Blob; + + let mut cids = Vec::new(); + for index in 0..count { + let blob = Blob { + cid: cpds_protocol::sha256_cid(0x55, &index.to_le_bytes()), + media_type: "application/octet-stream".to_owned(), + size: 4, + }; + database + .begin_blob_upload("did:example:alice", &blob) + .unwrap(); + database + .finish_blob_upload("did:example:alice", &blob.cid) + .unwrap(); + cids.push(blob.cid.to_string()); + seed_record( + database, + space, + &format!("{index:04}"), + serde_json::json!({ + "$type": "com.example.post", + "blob": {"$type": "blob", "ref": {"$link": blob.cid.to_string()}, + "mimeType": blob.media_type, "size": blob.size}, + }), + ); + } + cids.sort(); + cids +} + +#[test] +fn shared_sync_queries_keep_default_limits_clamps_and_operation_lookahead() { + let database = Database::open_in_memory().unwrap(); + let did = "did:example:alice"; + let space = "at://did:example:remote/space/com.example.forum/default"; + database + .ensure_account(did, "alice.test", "password") + .unwrap(); + let cids = seed_blob_records(&database, space, 1001); + let service = SpaceService::new(database, did.to_owned(), &[7; 32]).unwrap(); + let mut ops = ListOpsQuery { + space: space.to_owned(), + repo: did.to_owned(), + since: None, + limit: None, + cursor: None, + exclude_values: None, + }; + let mut blobs = ListBlobsQuery { + space: space.to_owned(), + repo: did.to_owned(), + since: None, + limit: None, + cursor: None, + }; + let mut next_page_cursor = None; + for (limit, expected_ops, expected_blobs) in [ + (None, 100, 500), + (Some(0), 1, 1), + (Some(1001), 1000, 1000), + (Some(usize::MAX), 1000, 1000), + ] { + ops.limit = limit; + blobs.limit = limit; + let page = service + .list_repo_ops(&ReadAuthorization::Credential(space.to_owned()), &ops) + .unwrap(); + assert_eq!(page.ops.len(), expected_ops); + assert_eq!(page.cursor, page.ops.last().map(|op| op.id.to_string())); + assert!(page.commit.is_none()); + next_page_cursor = page.cursor; + assert_eq!( + service + .list_blobs(&ReadAuthorization::Credential(space.to_owned()), &blobs) + .unwrap(), + cids[..expected_blobs], + ); + } + ops.cursor = next_page_cursor; + ops.limit = Some(0); + let page = service + .list_repo_ops(&ReadAuthorization::Credential(space.to_owned()), &ops) + .unwrap(); + assert_eq!(page.ops.len(), 1); + assert!(page.cursor.is_none()); + assert!(page.commit.is_some()); + blobs.cursor = Some(cids[0].clone()); + blobs.limit = Some(0); + assert_eq!( + service + .list_blobs(&ReadAuthorization::Credential(space.to_owned()), &blobs) + .unwrap(), + cids[1..2], + ); +} + +#[test] +fn shared_sync_queries_preserve_authorization_and_validation_priority() { + let database = Database::open_in_memory().unwrap(); + let did = "did:example:alice"; + let space = "at://did:example:remote/space/com.example.forum/default"; + database + .ensure_account(did, "alice.test", "password") + .unwrap(); + let service = SpaceService::new(database, did.to_owned(), &[7; 32]).unwrap(); + let mut query = ListOpsQuery { + space: space.to_owned(), + repo: did.to_owned(), + since: Some("invalid-revision".to_owned()), + limit: None, + cursor: Some("invalid-cursor".to_owned()), + exclude_values: None, + }; + assert!(matches!( + service.list_repo_ops(&ReadAuthorization::Credential("another-space".to_owned()), &query), + Err(Error::Authorization(message)) if message == "space credential does not cover the requested space" + )); + assert!(matches!( + service.list_repo_ops(&ReadAuthorization::Credential(space.to_owned()), &query), + Err(Error::Xrpc { name: "MalformedCursor", message }) if message == "invalid listRepoOps cursor" + )); + query.cursor = None; + assert!(matches!( + service.list_repo_ops(&ReadAuthorization::Credential(space.to_owned()), &query), + Err(Error::InvalidRequest(message)) if message == "`invalid-revision` is not a valid TID" + )); + query.cursor = Some("0".to_owned()); + assert!(matches!( + service.list_repo_ops(&ReadAuthorization::Credential(space.to_owned()), &query), + Err(Error::Xrpc { + name: "RepoNotFound", + .. + }) + )); + + let mut query = ListBlobsQuery { + space: space.to_owned(), + repo: did.to_owned(), + since: Some("invalid-revision".to_owned()), + limit: Some(usize::MAX), + cursor: Some("any-cursor".to_owned()), + }; + assert!(matches!( + service.list_blobs(&ReadAuthorization::Credential("another-space".to_owned()), &query), + Err(Error::Authorization(message)) if message == "space credential does not cover the requested space" + )); + assert!(matches!( + service.list_blobs(&ReadAuthorization::Credential(space.to_owned()), &query), + Err(Error::InvalidRequest(message)) if message == "`invalid-revision` is not a valid TID" + )); + query.since = None; + assert!(matches!( + service.list_blobs(&ReadAuthorization::Credential(space.to_owned()), &query), + Err(Error::Xrpc { + name: "RepoNotFound", + .. + }) + )); +} + +#[test] +fn record_query_space_uris_round_trip_without_normalizing_raw_components() { + for (position, fixture) in [ + ( + 0, + include_str!("../../tests/fixtures/atproto-did-syntax-valid.txt"), + ), + ( + 2, + include_str!("../../tests/fixtures/atproto-nsid-syntax-valid.txt"), + ), + ( + 3, + include_str!("../../tests/fixtures/atproto-record-key-syntax-valid.txt"), + ), + ] { + for value in fixture + .lines() + .filter(|line| !line.is_empty() && !line.starts_with('#')) + { + let mut segments = ["did:example:alice", "space", "com.example.forum", "default"]; + segments[position] = value; + let original = format!("at://{}", segments.join("/")); + assert_eq!(original.parse::().unwrap().to_string(), original); + } + } + for original in [ + "at://did:web:localhost%3A1234/space/com.example.fooBar/literal:self", + "at://did:method:val%BB/space/one.two.FiVe/~1.2-3_", + "at://did:method:val%bb/space/one.two.FiVe/...", + ] { + assert_eq!(original.parse::().unwrap().to_string(), original); + } + for invalid in [ + "", + "at:///space/com.example.forum/default", + "at://did:example:alice/space//default", + "at://did:example:alice/space/com.example.forum/", + "at://did:example:alice/space/com.example.forum/default/extra", + "at://did:example:🦀/space/com.example.forum/default", + "at://did:example:alice/space/com.example.é/default", + "at://did:example:alice/space/com.example.forum/🦀", + ] { + assert!(invalid.parse::().is_err(), "{invalid}"); + } +} + +#[test] +fn shared_record_query_preserves_read_and_delete_validation_priority() { + let database = Database::open_in_memory().unwrap(); + let denied = auth("atproto"); + let allowed = auth("atproto space:*?authority=*&collection=*&action=delete"); + database + .ensure_account(&allowed.account.did, "alice.test", "password") + .unwrap(); + let service = SpaceService::new(database, allowed.account.did.clone(), &[7; 32]).unwrap(); + let space = "at://did:example:remote/space/com.example.forum/default"; + let mut query = RecordQuery { + space: space.to_owned(), + repo: allowed.account.did.clone(), + collection: "invalid".to_owned(), + rkey: "/".to_owned(), + }; + assert_eq!( + service + .get_record( + &ReadAuthorization::Credential("wrong-space".to_owned()), + &query + ) + .unwrap_err() + .to_string(), + "authorization failed: space credential does not cover the requested space" + ); + for (collection, rkey, expected) in [ + ( + "invalid", + "/", + "the request is invalid: `invalid` is not a valid collection NSID", + ), + ( + "com.example.post", + "/", + "the request is invalid: `/` is not a valid record key", + ), + ("com.example.post", "one", "RepoNotFound"), + ] { + query.collection = collection.to_owned(); + query.rkey = rkey.to_owned(); + let error = service + .get_record(&ReadAuthorization::Credential(space.to_owned()), &query) + .unwrap_err(); + if expected == "RepoNotFound" { + assert!(matches!( + error, + Error::Xrpc { + name: "RepoNotFound", + .. + } + )); + } else { + assert_eq!(error.to_string(), expected); + } + } + query.repo = "did:example:other".to_owned(); + query.collection = "invalid".to_owned(); + query.rkey = "/".to_owned(); + assert_eq!( + service + .delete_record(&denied, &query) + .unwrap_err() + .to_string(), + "authorization failed: a permissioned write must target the authenticated account DID" + ); + query.repo = allowed.account.did.clone(); + assert_eq!( + service + .delete_record(&denied, &query) + .unwrap_err() + .to_string(), + "the request is invalid: `invalid` is not a valid collection NSID" + ); + query.collection = "com.example.post".to_owned(); + assert_eq!( + service + .delete_record(&denied, &query) + .unwrap_err() + .to_string(), + format!("authorization failed: OAuth token has no covering `space:` grant for {space}") + ); + assert_eq!( + service + .delete_record(&allowed, &query) + .unwrap_err() + .to_string(), + "the request is invalid: `/` is not a valid record key" + ); +} + +#[test] +fn space_credentials_cannot_read_inactive_hosted_repositories() { + use crate::domain::account::AccountStatus; + let database = Database::open_in_memory().unwrap(); + let did = "did:example:alice"; + let space = "at://did:example:remote/space/com.example.forum/default"; + database + .ensure_account(did, "alice.test", "password") + .unwrap(); + let service = SpaceService::new(database.clone(), did.to_owned(), &[7; 32]).unwrap(); + seed_record( + &database, + space, + "one", + serde_json::json!({"$type": "com.example.post"}), + ); + let query = RecordQuery { + space: space.to_owned(), + repo: did.to_owned(), + collection: "com.example.post".to_owned(), + rkey: "one".to_owned(), + }; + service + .get_record(&ReadAuthorization::Credential(space.to_owned()), &query) + .unwrap(); + database + .transition_account( + did, + AccountStatus::Active.as_str(), + crate::database::LifecycleEvent::Account(AccountStatus::Deactivated), + 100, + ) + .unwrap(); + let error = service + .get_record(&ReadAuthorization::Credential(space.to_owned()), &query) + .unwrap_err(); + assert!(matches!( + error, + Error::Xrpc { + name: "RepoDeactivated", + .. + } + )); +} + +#[test] +fn blob_authorization_checks_references_beyond_the_first_record_page() { + use crate::domain::blob::Blob; + use serde_json::json; + + let database = Database::open_in_memory().unwrap(); + let did = "did:example:alice"; + let space = "at://did:example:alice/space/com.example.forum/default"; + database + .ensure_account(did, "alice.test", "password") + .unwrap(); + let service = SpaceService::new(database.clone(), did.to_owned(), &[7; 32]).unwrap(); + let blob = Blob { + cid: cpds_protocol::sha256_cid(0x55, b"private"), + media_type: "image/png".to_owned(), + size: 7, + }; + database.begin_blob_upload(did, &blob).unwrap(); + database.finish_blob_upload(did, &blob.cid).unwrap(); + seed_record( + &database, + space, + "0000", + json!({ + "$type": "com.example.post", + "image": {"$type": "blob", "ref": {"$link": blob.cid.to_string()}, "mimeType": blob.media_type, "size": blob.size} + }), + ); + for index in 1..=1000 { + seed_record( + &database, + space, + &format!("{index:04}"), + json!({"$type": "com.example.post"}), + ); + } + + service + .get_blob_allowed( + &ReadAuthorization::Credential(space.to_owned()), + space, + did, + &blob.cid, + ) + .unwrap(); + let unknown = cpds_protocol::sha256_cid(0x55, b"unknown"); + assert!( + service + .get_blob_allowed( + &ReadAuthorization::Credential(space.to_owned()), + space, + did, + &unknown + ) + .is_err() + ); + assert!( + service + .get_blob_allowed( + &ReadAuthorization::Credential("another-space".to_owned()), + space, + did, + &blob.cid + ) + .is_err() + ); +} + +#[test] +fn filtered_space_pages_include_remote_spaces_after_a_thousand_nonmatches() { + use serde_json::json; + + let database = Database::open_in_memory().unwrap(); + let grant = auth("atproto space:*?authority=*&action=read_self"); + database + .ensure_account(&grant.account.did, "alice.test", "password") + .unwrap(); + let service = SpaceService::new(database.clone(), grant.account.did.clone(), &[7; 32]).unwrap(); + for index in 0..1001 { + seed_record( + &database, + &format!("at://did:example:remote/space/com.example.aaa/{index}"), + "entry", + json!({"$type": "com.example.post"}), + ); + } + let first = "at://did:example:remote/space/com.example.zzz/a"; + let second = "at://did:example:remote/space/com.example.zzz/b"; + for space in [first, second] { + seed_record( + &database, + space, + "entry", + json!({"$type": "com.example.post"}), + ); + } + + let mut query = ListSpacesQuery { + space_type: Some("com.example.zzz".to_owned()), + did: Some("did:example:remote".to_owned()), + cursor: None, + limit: Some(1), + }; + assert_eq!(service.list_spaces(&grant, &query).unwrap(), [first]); + query.cursor = Some(first.to_owned()); + assert_eq!(service.list_spaces(&grant, &query).unwrap(), [second]); + query.cursor = Some(second.to_owned()); + assert!(service.list_spaces(&grant, &query).unwrap().is_empty()); +} + +#[test] +fn shared_space_discovery_query_preserves_validation_priority_and_limits() { + let database = Database::open_in_memory().unwrap(); + let denied = auth("atproto"); + let allowed = auth("atproto space:*?authority=*&action=read_self"); + database + .ensure_account(&allowed.account.did, "alice.test", "password") + .unwrap(); + let service = + SpaceService::new(database.clone(), allowed.account.did.clone(), &[7; 32]).unwrap(); + let mut query = ListSpacesQuery { + space_type: Some("invalid-type".to_owned()), + did: Some("invalid-did".to_owned()), + cursor: Some("zzzz".to_owned()), + limit: Some(0), + }; + assert_eq!( + service + .list_spaces(&denied, &query) + .unwrap_err() + .to_string(), + "the request is invalid: `invalid-type` is not a valid collection NSID" + ); + query.space_type = Some("com.example.forum".to_owned()); + assert_eq!( + service + .list_spaces(&denied, &query) + .unwrap_err() + .to_string(), + "the request is invalid: `invalid-did` is not a valid DID" + ); + query.did = Some("did:example:remote".to_owned()); + assert_eq!( + service + .list_spaces(&denied, &query) + .unwrap_err() + .to_string(), + "authorization failed: OAuth token has no covering `space:` grant for at://did:example:remote/space/com.example.forum/*" + ); + let spaces = (0..101) + .map(|index| format!("at://did:example:remote/space/com.example.forum/{index:03}")) + .collect::>(); + for space in &spaces { + seed_record( + &database, + space, + "entry", + serde_json::json!({"$type": "com.example.post"}), + ); + } + query.space_type = None; + query.did = None; + query.cursor = None; + for (limit, expected) in [ + (None, 50), + (Some(0), 1), + (Some(1), 1), + (Some(100), 100), + (Some(101), 100), + (Some(usize::MAX), 100), + ] { + query.limit = limit; + assert_eq!( + service.list_spaces(&allowed, &query).unwrap(), + spaces[..expected], + "limit {limit:?}" + ); + } +} + +fn auth(scope: &str) -> AuthenticatedToken { + AuthenticatedToken { + account: Account { + did: "did:example:alice".to_owned(), + handle: "alice.test".to_owned(), + }, + client_id: "https://client.test/metadata.json".to_owned(), + scope: scope.to_owned(), + } +} + +#[test] +fn space_scopes_match_authority_action_and_collection() { + let target = "at://did:example:owner/space/com.example.forum/default" + .parse::() + .unwrap(); + let grant = auth( + "atproto space:com.example.forum?authority=did%3Aexample%3Aowner&collection=com.example.post&action=create", + ); + assert!( + require_scope( + &grant, + &target, + Capability::Record("create", "com.example.post") + ) + .is_ok() + ); + assert!( + require_scope( + &grant, + &target, + Capability::Record("delete", "com.example.post") + ) + .is_err() + ); +} + +#[test] +fn capabilities_keep_collection_action_and_read_defaults_together() { + let target = "at://did:example:owner/space/com.example.forum/default" + .parse::() + .unwrap(); + let capabilities = [ + Capability::Read, + Capability::ReadSelf, + Capability::Record("create", "com.example.post"), + Capability::Record("delete", "com.example.post"), + Capability::Record("create", "com.example.other"), + Capability::Manage("create"), + Capability::Manage("update"), + ]; + for (scope, expected) in [ + ("atproto", [false; 7]), + ( + "space:*?authority=*", + [false, true, false, false, false, false, false], + ), + ( + "space:*?authority=*&collection=*", + [false, true, true, true, true, false, false], + ), + ( + "space:*?authority=*&collection=com.example.post&action=create", + [false, false, true, false, false, false, false], + ), + ( + "space:*?authority=*&action=read", + [true, true, false, false, false, false, false], + ), + ( + "space:*?authority=*&action=read_self", + [false, true, false, false, false, false, false], + ), + ( + "space:*?authority=*&collection=com.example.post&action=create&action=delete", + [false, false, true, true, false, false, false], + ), + ( + "space:*?authority=*&collection=com.example.other&collection=com.example.post&action=create", + [false, false, true, false, true, false, false], + ), + ( + "space:*?authority=*&manage=create", + [false, true, false, false, false, true, false], + ), + ( + "space:*?authority=*&manage=update&action=unrelated", + [false, false, false, false, false, false, true], + ), + ( + "space:*?authority=did:example:other&authority=*&collection=*", + [false, true, true, true, true, false, false], + ), + ( + "space:*?authority=*&authority=did:example:other&collection=*", + [false; 7], + ), + ("space:*?authority=*&collection=*&unknown=value", [false; 7]), + ("space:*?collection=*", [false; 7]), + ( + "space:com.example.other?authority=*&collection=*", + [false; 7], + ), + ] { + for (index, (capability, expected)) in capabilities.into_iter().zip(expected).enumerate() { + let result = require_scope(&auth(scope), &target, capability); + assert_eq!( + result.is_ok(), + expected, + "scope {scope}, capability {index}" + ); + if !expected { + assert_eq!( + result.unwrap_err().to_string(), + format!( + "authorization failed: OAuth token has no covering `space:` grant for {target}" + ), + ); + } + } + } +} + +#[test] +fn native_json_policy_field_reads_preserve_missing_and_wrong_types() { + let values = [ + Value::Null, + serde_json::json!(false), + serde_json::json!(0), + serde_json::json!(""), + serde_json::json!("application"), + serde_json::json!([]), + serde_json::json!(["application"]), + serde_json::json!(["application", null]), + serde_json::json!({}), + serde_json::json!({"other": "application"}), + ]; + let mut documents = values.to_vec(); + for value in &values { + documents.push(serde_json::json!({"managingApp": value, "allowed": value})); + } + for document in &documents { + for field in ["managingApp", "allowed"] { + assert_eq!( + document.get(field).and_then(Value::as_str), + document[field].as_str(), + ); + assert_eq!( + document.get(field).and_then(Value::as_array), + document[field].as_array(), + ); + assert_eq!( + document + .get(field) + .and_then(Value::as_array) + .is_some_and(|values| values.iter().all(|value| value.as_str().is_some())), + document[field] + .as_array() + .is_some_and(|values| values.iter().all(Value::is_string)), + ); + } + } +} diff --git a/src/syntax.rs b/src/syntax.rs index b2aa38d..174caa1 100644 --- a/src/syntax.rs +++ b/src/syntax.rs @@ -27,16 +27,19 @@ impl fmt::Display for SpaceRef { impl FromStr for SpaceRef { type Err = Error; + /// Parse and validate a canonical permissioned-data space reference. + /// + /// # Errors + /// + /// Returns `InvalidRequest` unless all four required segments are valid. fn from_str(value: &str) -> Result { - let rest = value.strip_prefix("at://").ok_or_else(|| { - Error::InvalidRequest(format!("`{value}` is not a valid space reference")) - })?; - let segments: Vec<_> = rest.split('/').collect(); - if segments.len() != 4 || segments[1] != "space" { - return Err(Error::InvalidRequest(format!( - "`{value}` is not a valid space reference" - ))); - } + let segments = value + .strip_prefix("at://") + .map(|rest| rest.split('/').collect::>()) + .filter(|segments| segments.len() == 4 && segments[1] == "space") + .ok_or_else(|| { + Error::InvalidRequest(format!("`{value}` is not a valid space reference")) + })?; did(segments[0])?; collection(segments[2])?; record_key(segments[3])?; @@ -54,9 +57,7 @@ impl FromStr for SpaceRef { /// /// Returns `InvalidRequest` when the value is not a bounded, path-safe DID. pub fn did(value: &str) -> Result<&str> { - Did::new(value.to_owned()) - .map(|_| value) - .map_err(|_| Error::InvalidRequest(format!("`{value}` is not a valid DID"))) + parse::(value, "DID").map(|_| value) } /// Validate an AT Protocol timestamp identifier. @@ -65,9 +66,7 @@ pub fn did(value: &str) -> Result<&str> { /// /// Returns `InvalidRequest` unless `value` is a canonical 13-character TID. pub fn tid(value: &str) -> Result<&str> { - Tid::new(value.to_owned()) - .map(|_| value) - .map_err(|_| Error::InvalidRequest(format!("`{value}` is not a valid TID"))) + parse::(value, "TID").map(|_| value) } /// Parse an AT Protocol datetime without allowing an offset to normalize it @@ -77,9 +76,7 @@ pub fn tid(value: &str) -> Result<&str> { /// /// Returns `InvalidRequest` when the value is not a valid Lexicon datetime. pub fn datetime(value: &str) -> Result { - let datetime = value - .parse::() - .map_err(|_| Error::InvalidRequest(format!("`{value}` is not a valid datetime")))?; + let datetime = parse::(value, "datetime")?; let minimum = "0000-01-01T00:00:00Z" .parse::() .map_err(|_| Error::Internal("AT Protocol datetime invariant is invalid".to_owned()))?; @@ -101,54 +98,36 @@ pub fn datetime(value: &str) -> Result { /// Returns `InvalidRequest` when the tag syntax is invalid or a variant or /// extension singleton occurs more than once. pub fn language(value: &str) -> Result { - let language = value - .parse::() - .map_err(|_| Error::InvalidRequest(format!("`{value}` is not a valid language tag")))?; - let mut variants = HashSet::new(); - let mut extensions = HashSet::new(); + let language = parse::(value, "language tag")?; + let mut seen = HashSet::new(); let mut extension_started = false; for subtag in value.split('-').skip(1) { - if subtag.len() == 1 { - if subtag.eq_ignore_ascii_case("x") { - break; - } - extension_started = true; - if !extensions.insert(subtag.to_ascii_lowercase()) { - return Err(Error::InvalidRequest(format!( - "`{value}` contains a repeated language extension" - ))); + let kind = match subtag.as_bytes() { + [b'x' | b'X'] => break, + [_] => { + extension_started = true; + "extension" } - } else if !extension_started - && ((4..=8).contains(&subtag.len()) - && (subtag.len() != 4 || subtag.as_bytes().first().is_some_and(u8::is_ascii_digit))) - && !variants.insert(subtag.to_ascii_lowercase()) - { + [first, _, _, _] if !extension_started && first.is_ascii_digit() => "variant", + [_, _, _, _, _, ..] if !extension_started && subtag.len() <= 8 => "variant", + _ => continue, + }; + if !seen.insert(subtag.to_ascii_lowercase()) { return Err(Error::InvalidRequest(format!( - "`{value}` contains a repeated language variant" + "`{value}` contains a repeated language {kind}" ))); } } Ok(language) } -/// Parse and validate a canonical permissioned-data space reference. -/// -/// # Errors -/// -/// Returns `InvalidRequest` unless all four required segments are valid. -pub fn space_ref(value: &str) -> Result { - value.parse() -} - /// Validate the subset of NSID syntax used for record collections. /// /// # Errors /// /// Returns `InvalidRequest` when `value` is not a normalized collection NSID. pub fn collection(value: &str) -> Result<&str> { - Nsid::new(value.to_owned()) - .map(|_| value) - .map_err(|_| Error::InvalidRequest(format!("`{value}` is not a valid collection NSID"))) + parse::(value, "collection NSID").map(|_| value) } /// Validate an AT Protocol record key. @@ -158,16 +137,20 @@ pub fn collection(value: &str) -> Result<&str> { /// Returns `InvalidRequest` when `value` is empty, reserved, too long, or /// contains a disallowed character. pub fn record_key(value: &str) -> Result<&str> { - RecordKey::new(value.to_owned()) - .map(|_| value) - .map_err(|_| Error::InvalidRequest(format!("`{value}` is not a valid record key"))) + parse::(value, "record key").map(|_| value) +} + +pub(crate) fn parse(value: &str, kind: &str) -> Result { + value + .parse() + .map_err(|_| Error::InvalidRequest(format!("`{value}` is not a valid {kind}"))) } #[cfg(test)] mod tests { use atrium_api::types::string::{AtIdentifier, Handle}; - use super::{SpaceRef, collection, datetime, did, language, record_key, space_ref, tid}; + use super::{SpaceRef, collection, datetime, did, language, record_key, tid}; fn fixture_cases(value: &str) -> impl Iterator { value.lines().filter(|line| { @@ -176,6 +159,238 @@ mod tests { }) } + #[test] + fn shared_native_parsing_preserves_borrowed_values_and_error_labels() { + let fixtures = concat!( + include_str!("../tests/fixtures/atproto-did-syntax-valid.txt"), + include_str!("../tests/fixtures/atproto-did-syntax-invalid.txt"), + include_str!("../tests/fixtures/atproto-tid-syntax-valid.txt"), + include_str!("../tests/fixtures/atproto-tid-syntax-invalid.txt"), + include_str!("../tests/fixtures/atproto-nsid-syntax-valid.txt"), + include_str!("../tests/fixtures/atproto-nsid-syntax-invalid.txt"), + include_str!("../tests/fixtures/atproto-record-key-syntax-valid.txt"), + include_str!("../tests/fixtures/atproto-record-key-syntax-invalid.txt"), + ); + for value in + fixture_cases(fixtures).chain(["", "a\0b", "café", "\"quoted\"", "line\nbreak"]) + { + for (actual, previous, kind) in [ + ( + did(value), + super::Did::new(value.to_owned()).map(|_| ()), + "DID", + ), + ( + tid(value), + super::Tid::new(value.to_owned()).map(|_| ()), + "TID", + ), + ( + collection(value), + super::Nsid::new(value.to_owned()).map(|_| ()), + "collection NSID", + ), + ( + record_key(value), + super::RecordKey::new(value.to_owned()).map(|_| ()), + "record key", + ), + ] { + let expected = previous.map(|()| value).map_err(|_| { + super::Error::InvalidRequest(format!("`{value}` is not a valid {kind}")) + }); + assert_eq!( + actual.as_ref().map_err(|error| format!("{error:?}")), + expected.as_ref().map_err(|error| format!("{error:?}")), + "{kind}: {value:?}" + ); + if let Ok(actual) = actual { + assert!(std::ptr::eq(actual, value)); + } + } + } + } + + #[test] + fn shared_datetime_parser_preserves_text_and_error_priority() { + fn previous_datetime(value: &str) -> super::Result { + let datetime = value.parse::().map_err(|_| { + super::Error::InvalidRequest(format!("`{value}` is not a valid datetime")) + })?; + let minimum = "0000-01-01T00:00:00Z" + .parse::() + .map_err(|_| { + super::Error::Internal("AT Protocol datetime invariant is invalid".to_owned()) + })?; + if datetime < minimum { + return Err(super::Error::InvalidRequest(format!( + "`{value}` normalizes before the minimum AT Protocol datetime" + ))); + } + Ok(datetime) + } + for value in fixture_cases(concat!( + include_str!("../tests/fixtures/atproto-datetime-syntax-valid.txt"), + include_str!("../tests/fixtures/atproto-datetime-syntax-invalid.txt"), + include_str!("../tests/fixtures/atproto-datetime-parse-invalid.txt"), + )) + .chain(["", "0000-01-01T00:00:00+01:00", "2026-09-05T12:34:56.1200Z"]) + { + let shape = |result: super::Result| { + result + .map(|datetime| serde_json::to_value(datetime).unwrap()) + .map_err(|error| format!("{error:?}")) + }; + assert_eq!( + shape(datetime(value)), + shape(previous_datetime(value)), + "{value}" + ); + } + } + + #[test] + fn shared_space_shape_gate_preserves_component_error_priority() { + fn previous(value: &str) -> super::Result { + let rest = value.strip_prefix("at://").ok_or_else(|| { + super::Error::InvalidRequest(format!("`{value}` is not a valid space reference")) + })?; + let segments: Vec<_> = rest.split('/').collect(); + if segments.len() != 4 || segments[1] != "space" { + return Err(super::Error::InvalidRequest(format!( + "`{value}` is not a valid space reference" + ))); + } + did(segments[0])?; + collection(segments[2])?; + record_key(segments[3])?; + Ok(SpaceRef { + authority: segments[0].to_owned(), + space_type: segments[2].to_owned(), + skey: segments[3].to_owned(), + }) + } + for prefix in ["at://", "AT://", "", "at:/", " at://"] { + for authority in [ + "did:example:alice", + "did:web:example.com%3A8443", + "invalid", + "", + ] { + for marker in ["space", "SPACE", ""] { + for kind in ["com.example.forum", "bad", ""] { + for key in ["default", "..", "", "with/slash", "a:b"] { + let value = format!("{prefix}{authority}/{marker}/{kind}/{key}"); + assert_eq!( + value + .parse::() + .map_err(|error| format!("{error:?}")), + previous(&value).map_err(|error| format!("{error:?}")), + "{value}" + ); + } + } + } + } + } + for value in [ + "", + "at://", + "at://did:example:alice", + "at://did:example:alice/space", + ] { + assert_eq!( + value + .parse::() + .map_err(|error| format!("{error:?}")), + previous(value).map_err(|error| format!("{error:?}")), + "{value}" + ); + } + } + + fn previous_language(value: &str) -> super::Result { + let language = value.parse::().map_err(|_| { + super::Error::InvalidRequest(format!("`{value}` is not a valid language tag")) + })?; + let mut variants = std::collections::HashSet::new(); + let mut extensions = std::collections::HashSet::new(); + let mut extension_started = false; + for subtag in value.split('-').skip(1) { + if subtag.len() == 1 { + if subtag.eq_ignore_ascii_case("x") { + break; + } + extension_started = true; + if !extensions.insert(subtag.to_ascii_lowercase()) { + return Err(super::Error::InvalidRequest(format!( + "`{value}` contains a repeated language extension" + ))); + } + } else if !extension_started + && ((4..=8).contains(&subtag.len()) + && (subtag.len() != 4 + || subtag.as_bytes().first().is_some_and(u8::is_ascii_digit))) + && !variants.insert(subtag.to_ascii_lowercase()) + { + return Err(super::Error::InvalidRequest(format!( + "`{value}` contains a repeated language variant" + ))); + } + } + Ok(language) + } + + #[test] + fn language_subtag_patterns_preserve_values_and_error_priority() { + let values = [ + "", + "en", + "en-US", + "en-Latn-US", + "sl-rozaj-biske-1994", + "de-1901-1901", + "sl-rozaj-ROZAJ", + "en-a-foo-A-bar", + "en-abcde-ABCDE-a-foo-a-bar", + "en-abcde-ABCDE-a-foo-a-bar_", + "en-a-foo-abcde-abcde", + "en-a-foo-x-a-a", + "en-X-abcde-ABCDE", + "x-abcde-ABCDE", + "X-abcde-ABCDE", + "x-a-foo-A-bar", + "x-x-abcde-ABCDE", + "x-1abc-1ABC", + "x-abcd-ABCD", + "x-abcdefgh-ABCDEFGH", + "x-abcdefghi-abcdefghi", + "i-klingon", + "en-GB-oed", + "sgn-BE-FR", + "en--US", + "en-1234-1234", + "en-abcde-abcde-", + "en-ééé", + ]; + let outcome = |result: super::Result| { + result + .map(|language| serde_json::to_string(&language).unwrap()) + .map_err(|error| format!("{error:?}")) + }; + for value in values.into_iter().chain(fixture_cases(concat!( + include_str!("../tests/fixtures/atproto-language-syntax-valid.txt"), + include_str!("../tests/fixtures/atproto-language-syntax-invalid.txt"), + include_str!("../tests/fixtures/atproto-language-parse-invalid.txt") + ))) { + assert_eq!( + outcome(language(value)), + outcome(previous_language(value)), + "{value}" + ); + } + } + #[test] fn accepts_protocol_shaped_identifiers() { assert!(collection("app.bsky.feed.post").is_ok()); @@ -354,17 +569,29 @@ mod tests { #[test] fn parses_only_complete_canonical_space_references() { assert_eq!( - space_ref("at://did:web:alice.test/space/com.example.forum/default").unwrap(), + "at://did:web:alice.test/space/com.example.forum/default" + .parse::() + .unwrap(), SpaceRef { authority: "did:web:alice.test".to_owned(), space_type: "com.example.forum".to_owned(), skey: "default".to_owned(), } ); - assert!(space_ref("at://did:web:alice.test/com.example.forum/default").is_err()); assert!( - space_ref("at://did:web:alice.test/space/com.example.forum/default/extra").is_err() + "at://did:web:alice.test/com.example.forum/default" + .parse::() + .is_err() + ); + assert!( + "at://did:web:alice.test/space/com.example.forum/default/extra" + .parse::() + .is_err() + ); + assert!( + "at://did:web:alice.test/space/com.example.forum/../oops" + .parse::() + .is_err() ); - assert!(space_ref("at://did:web:alice.test/space/com.example.forum/../oops").is_err()); } } diff --git a/tests/backup_restore.rs b/tests/backup_restore.rs index e8d9c01..d19d1e1 100644 --- a/tests/backup_restore.rs +++ b/tests/backup_restore.rs @@ -4,7 +4,7 @@ use atrium_api::com::atproto::sync::subscribe_repos; use cpds::{ AppState, backup, blob_store::FileBlobStore, - config::{AccountConfig, Config, DeploymentMode, PublicOrigin}, + config::{AccountConfig, Config, DeploymentMode, HttpOrigin}, database::Database, domain::{account::Account, oauth::OAuthClient}, events::NoopEventSink, @@ -49,7 +49,7 @@ async fn populated_backup_restores_repository_blob_index_and_event_chain() { "size": blob.size } }); - let written = state + let (written_record, _) = state .repository .create_record( &actor, @@ -88,7 +88,7 @@ async fn populated_backup_restores_repository_blob_index_and_event_chain() { .repository .get_record(DID, "app.bsky.feed.post", "backup", None) .unwrap(); - assert_eq!(record.cid, written.record.cid); + assert_eq!(record.cid, written_record.cid); assert_eq!(record.value, value); assert_eq!( restored @@ -126,7 +126,7 @@ fn test_config(database_path: std::path::PathBuf, blob_directory: std::path::Pat Config { deployment_mode: DeploymentMode::Local, listen_address: "127.0.0.1:3000".parse().unwrap(), - public_origin: PublicOrigin::parse("http://pds.test/", DeploymentMode::Local).unwrap(), + public_origin: HttpOrigin::parse_public("http://pds.test/", DeploymentMode::Local).unwrap(), database_path, blob_directory, blob_orphan_ttl: std::time::Duration::from_secs(86_400), diff --git a/tests/blob_store_contract.rs b/tests/blob_store_contract.rs index f75c743..d6c157a 100644 --- a/tests/blob_store_contract.rs +++ b/tests/blob_store_contract.rs @@ -32,7 +32,7 @@ async fn contract(store: &dyn BlobStore) { #[tokio::test] async fn memory_store_satisfies_the_blob_contract() { - contract(&MemoryBlobStore::new()).await; + contract(&MemoryBlobStore::default()).await; } #[tokio::test] diff --git a/tests/http_errors.rs b/tests/http_errors.rs index cdf4b55..173e09a 100644 --- a/tests/http_errors.rs +++ b/tests/http_errors.rs @@ -7,6 +7,28 @@ use axum::{ use serde_json::Value; use support::TestApp; +#[tokio::test] +async fn server_errors_do_not_disclose_internal_diagnostics() { + use axum::response::IntoResponse as _; + use cpds::error::Error; + + for error in [ + Error::Internal("private repository details".to_owned()), + Error::Configuration("private configuration details".to_owned()), + Error::Io(std::io::Error::other("private filesystem path")), + Error::Database(rusqlite::Error::InvalidParameterName( + "private SQL".to_owned(), + )), + Error::Upstream("private upstream URL".to_owned()), + ] { + let response = error.into_response(); + assert!(response.status().is_server_error()); + let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + let body: Value = serde_json::from_slice(&body).unwrap(); + assert!(!body["message"].as_str().unwrap().contains("private")); + } +} + #[tokio::test] async fn unknown_xrpc_routes_and_wrong_methods_use_json_errors() { let app = TestApp::new(); diff --git a/tests/http_security.rs b/tests/http_security.rs index 5ca24a3..41698bd 100644 --- a/tests/http_security.rs +++ b/tests/http_security.rs @@ -20,10 +20,25 @@ async fn blob_downloads_use_the_official_atproto_browser_isolation_headers() { .unwrap(), ) .await; - let cid = json(upload).await["blob"]["ref"]["$link"] - .as_str() - .unwrap() - .to_owned(); + let blob = json(upload).await["blob"].clone(); + let cid = blob["ref"]["$link"].as_str().unwrap().to_owned(); + let published = app + .send( + Request::post("/xrpc/com.atproto.repo.createRecord") + .header(header::AUTHORIZATION, format!("Bearer {token}")) + .header(header::CONTENT_TYPE, "application/json") + .body(Body::from( + serde_json::json!({ + "repo": "did:web:alice.test", + "collection": "com.example.post", + "record": {"$type": "com.example.post", "image": blob} + }) + .to_string(), + )) + .unwrap(), + ) + .await; + assert_eq!(published.status(), StatusCode::OK); let response = app .send( diff --git a/tests/oauth_flow.rs b/tests/oauth_flow.rs index 4129bb3..ffe0483 100644 --- a/tests/oauth_flow.rs +++ b/tests/oauth_flow.rs @@ -9,6 +9,34 @@ use cpds::oauth::challenge_for_verifier; use support::{TestApp, VERIFIER, form, json}; use url::Url; +#[tokio::test] +async fn unsupported_token_grant_is_rejected_before_pkce_and_code_redemption() { + let app = TestApp::new(); + let body = form(&[ + ("grant_type", "unsupported"), + ("code", "not-a-code"), + ("client_id", "http://client.test/metadata.json"), + ("redirect_uri", "http://client.test/callback"), + ("code_verifier", "invalid"), + ]); + let response = app + .send( + Request::post("/oauth/token") + .header(CONTENT_TYPE, "application/x-www-form-urlencoded") + .body(Body::from(body)) + .unwrap(), + ) + .await; + assert_eq!(response.status(), 400); + assert_eq!( + json(response).await, + serde_json::json!({ + "error": "InvalidRequest", + "message": "the request is invalid: grant_type must be `authorization_code`", + }) + ); +} + #[tokio::test] async fn authorization_code_flow_issues_a_working_bearer_token() { let app = TestApp::new(); diff --git a/tests/process_lifecycle.rs b/tests/process_lifecycle.rs index a2031a2..83674ed 100644 --- a/tests/process_lifecycle.rs +++ b/tests/process_lifecycle.rs @@ -273,13 +273,13 @@ async fn offline_operator_status_updates_existing_storage_without_binding() { "{}", String::from_utf8_lossy(&output.stderr) ); - let account = Database::open_existing(&database) + let (account, status) = Database::open_existing(&database) .unwrap() .hosted_account_by_identifier("did:web:localhost") .unwrap() .unwrap(); - assert_eq!(account.status, expected_status); - assert_eq!(account.account.handle, expected_handle); + assert_eq!(status, expected_status); + assert_eq!(account.handle, expected_handle); } drop(listener); diff --git a/tests/record_flow.rs b/tests/record_flow.rs index 5c36b40..7e39532 100644 --- a/tests/record_flow.rs +++ b/tests/record_flow.rs @@ -1,12 +1,19 @@ mod support; +use std::{io::Cursor, time::Duration}; + use atrium_api::{ com::atproto::repo::{ - create_record, delete_record, get_record, list_missing_blobs, list_records, put_record, - upload_blob, + apply_writes, create_record, delete_record, get_record, list_missing_blobs, list_records, + put_record, upload_blob, }, + com::atproto::sync::subscribe_repos, types::{BlobRef, TypedBlobRef}, }; +use atrium_repo::{ + Repository, + blockstore::{AsyncBlockStoreRead as _, CarStore}, +}; use axum::{ body::Body, http::{Request, header::CONTENT_TYPE}, @@ -15,6 +22,193 @@ use cpds::domain::event::RepositoryEvent; use serde_json::json; use support::{TestApp, json as response_json}; +#[tokio::test] +async fn public_apply_writes_is_atomic_ordered_and_emits_one_commit() { + let app = TestApp::new(); + let token = app.access_token().await; + let response = app + .send( + Request::post("/xrpc/com.atproto.repo.applyWrites") + .header(CONTENT_TYPE, "application/json") + .header("authorization", format!("Bearer {token}")) + .body(Body::from( + json!({ + "repo": "did:web:alice.test", + "writes": [ + {"$type":"com.atproto.repo.applyWrites#create", "collection":"app.bsky.feed.post", "rkey":"dependent", "value":{"$type":"app.bsky.feed.post", "text":"first"}}, + {"$type":"com.atproto.repo.applyWrites#update", "collection":"app.bsky.feed.post", "rkey":"dependent", "value":{"$type":"app.bsky.feed.post", "text":"second"}}, + {"$type":"com.atproto.repo.applyWrites#create", "collection":"app.bsky.feed.post", "rkey":"survivor", "value":{"$type":"app.bsky.feed.post", "text":"survivor"}}, + {"$type":"com.atproto.repo.applyWrites#delete", "collection":"app.bsky.feed.post", "rkey":"dependent"} + ] + }) + .to_string(), + )) + .unwrap(), + ) + .await; + let status = response.status(); + let body = response_json(response).await; + assert_eq!(status, 200, "{body}"); + let output: apply_writes::Output = serde_json::from_value(body).unwrap(); + let results = output.results.as_ref().unwrap(); + assert_eq!(results.len(), 4); + assert!(output.commit.is_some()); + let apply_writes::OutputResultsItem::CreateResult(created) = &results[0] else { + panic!("first result should be a create"); + }; + let apply_writes::OutputResultsItem::UpdateResult(updated) = &results[1] else { + panic!("second result should be an update"); + }; + let apply_writes::OutputResultsItem::CreateResult(survivor_result) = &results[2] else { + panic!("third result should be a create"); + }; + let transient_cids = [ + created.cid.as_ref().to_owned(), + updated.cid.as_ref().to_owned(), + ]; + let survivor_cid = survivor_result.cid.as_ref().to_owned(); + + let survivor = app + .repository + .get_record("did:web:alice.test", "app.bsky.feed.post", "survivor", None) + .unwrap(); + assert_eq!(survivor.value["text"], "survivor"); + assert!( + app.repository + .get_record( + "did:web:alice.test", + "app.bsky.feed.post", + "dependent", + None, + ) + .is_err() + ); + + let messages = app.repository.repo_messages_after(0, 10).await.unwrap(); + let commit = messages + .iter() + .find_map(|message| match message { + subscribe_repos::Message::Commit(commit) => Some(commit), + _ => None, + }) + .unwrap(); + assert_eq!(commit.ops.len(), 4); + assert_eq!(commit.ops[0].action, "create"); + assert_eq!(commit.ops[1].action, "update"); + assert_eq!(commit.ops[2].path, "app.bsky.feed.post/survivor"); + assert_eq!(commit.ops[3].action, "delete"); + let mut car = CarStore::open(Cursor::new(commit.blocks.clone())) + .await + .unwrap(); + assert_eq!(car.roots().collect::>(), vec![commit.commit.0]); + assert!(car.read_block(survivor_cid).await.is_ok()); + for cid in transient_cids { + assert!(car.read_block(cid).await.is_err()); + } + let mut repo = Repository::open(&mut car, commit.commit.0).await.unwrap(); + assert!( + repo.get_raw::("app.bsky.feed.post/dependent") + .await + .unwrap() + .is_none() + ); + assert_eq!( + repo.get_raw::("app.bsky.feed.post/survivor") + .await + .unwrap() + .unwrap()["text"], + "survivor" + ); + app.repository.verify().await.unwrap(); + assert_failed_apply_writes_roll_back(&app, &token).await; +} + +async fn assert_failed_apply_writes_roll_back(app: &TestApp, token: &str) { + for (rkey, swap_commit) in [ + ("rolled-back", None), + ( + "bad-swap", + Some("bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a"), + ), + ] { + let writes = if swap_commit.is_none() { + json!([ + {"$type":"com.atproto.repo.applyWrites#create", "collection":"app.bsky.feed.post", "rkey":rkey, "value":{"$type":"app.bsky.feed.post", "text":"must disappear"}}, + {"$type":"com.atproto.repo.applyWrites#update", "collection":"app.bsky.feed.post", "rkey":"missing", "value":{"$type":"app.bsky.feed.post", "text":"fails"}} + ]) + } else { + json!([ + {"$type":"com.atproto.repo.applyWrites#create", "collection":"app.bsky.feed.post", "rkey":rkey, "value":{"$type":"app.bsky.feed.post", "text":"must disappear"}} + ]) + }; + let response = app + .send( + Request::post("/xrpc/com.atproto.repo.applyWrites") + .header(CONTENT_TYPE, "application/json") + .header("authorization", format!("Bearer {token}")) + .body(Body::from( + json!({ + "repo": "did:web:alice.test", + "writes": writes, + "swapCommit": swap_commit + }) + .to_string(), + )) + .unwrap(), + ) + .await; + assert_eq!( + response.status(), + if swap_commit.is_some() { 400 } else { 404 } + ); + assert!( + app.repository + .get_record("did:web:alice.test", "app.bsky.feed.post", rkey, None,) + .is_err() + ); + } +} + +#[tokio::test] +async fn public_apply_writes_rejects_invalid_batch_controls_before_mutation() { + let app = TestApp::new(); + let token = app.access_token().await; + let write = json!({ + "$type":"com.atproto.repo.applyWrites#create", + "collection":"app.bsky.feed.post", + "rkey":"bounded", + "value":{"$type":"app.bsky.feed.post", "text":"bounded"} + }); + for (writes, validate) in [ + (Vec::new(), None), + (vec![write.clone(); 201], None), + (vec![write.clone()], Some(true)), + ] { + let response = app + .send( + Request::post("/xrpc/com.atproto.repo.applyWrites") + .header(CONTENT_TYPE, "application/json") + .header("authorization", format!("Bearer {token}")) + .body(Body::from( + json!({ + "repo": "did:web:alice.test", + "writes": writes, + "validate": validate + }) + .to_string(), + )) + .unwrap(), + ) + .await; + assert_eq!(response.status(), 400); + } + assert!( + app.repository + .get_record("did:web:alice.test", "app.bsky.feed.post", "bounded", None,) + .is_err() + ); +} + #[tokio::test] async fn oversized_dag_cbor_records_fail_before_repository_mutation() { let app = TestApp::new(); @@ -603,7 +797,7 @@ async fn get_note(app: &TestApp, rkey: &str) -> axum::response::Response { } #[tokio::test] -async fn authenticated_clients_can_upload_and_publicly_fetch_blobs() { +async fn blobs_become_public_only_when_referenced_by_a_public_record() { let app = TestApp::new(); let token = app.access_token().await; let bytes = b"a tiny image-like payload"; @@ -623,6 +817,27 @@ async fn authenticated_clients_can_upload_and_publicly_fetch_blobs() { }; let cid = blob.r#ref.0.to_string(); + let unpublished = get_blob(&app, &cid).await; + assert_eq!(unpublished.status(), 400); + assert_eq!(response_json(unpublished).await["error"], "BlobNotFound"); + let published = app + .send( + Request::post("/xrpc/com.atproto.repo.createRecord") + .header(CONTENT_TYPE, "application/json") + .header("authorization", format!("Bearer {token}")) + .body(Body::from( + json!({ + "repo": "did:web:alice.test", + "collection": "app.bsky.feed.post", + "record": {"$type": "app.bsky.feed.post", "image": body.blob} + }) + .to_string(), + )) + .unwrap(), + ) + .await; + assert_eq!(published.status(), 200); + let fetched = app .send( Request::get(format!( @@ -640,6 +855,77 @@ async fn authenticated_clients_can_upload_and_publicly_fetch_blobs() { assert_eq!(&fetched_bytes[..], bytes); } +#[tokio::test] +async fn apply_writes_updates_blob_references_atomically_for_the_final_state() { + let app = TestApp::new(); + let token = app.access_token().await; + let (survivor_blob, survivor_cid) = upload_blob_value(&app, &token, b"survivor").await; + let (transient_blob, transient_cid) = upload_blob_value(&app, &token, b"transient").await; + + let response = app + .send( + Request::post("/xrpc/com.atproto.repo.applyWrites") + .header(CONTENT_TYPE, "application/json") + .header("authorization", format!("Bearer {token}")) + .body(Body::from( + json!({ + "repo": "did:web:alice.test", + "writes": [ + {"$type":"com.atproto.repo.applyWrites#create", "collection":"app.bsky.feed.post", "rkey":"transient-blob", "value":{"$type":"app.bsky.feed.post", "image":transient_blob}}, + {"$type":"com.atproto.repo.applyWrites#create", "collection":"app.bsky.feed.post", "rkey":"surviving-blob", "value":{"$type":"app.bsky.feed.post", "image":survivor_blob}}, + {"$type":"com.atproto.repo.applyWrites#delete", "collection":"app.bsky.feed.post", "rkey":"transient-blob"} + ] + }) + .to_string(), + )) + .unwrap(), + ) + .await; + assert_eq!(response.status(), 200); + assert_eq!( + app.repository + .collect_orphan_blobs(Duration::ZERO) + .await + .unwrap(), + 1 + ); + let missing = get_blob(&app, &transient_cid).await; + assert_eq!(missing.status(), 400); + assert_eq!(response_json(missing).await["error"], "BlobNotFound"); + assert_eq!(get_blob(&app, &survivor_cid).await.status(), 200); + + let response = app + .send( + Request::post("/xrpc/com.atproto.repo.applyWrites") + .header(CONTENT_TYPE, "application/json") + .header("authorization", format!("Bearer {token}")) + .body(Body::from( + json!({ + "repo": "did:web:alice.test", + "writes": [ + {"$type":"com.atproto.repo.applyWrites#create", "collection":"app.bsky.feed.post", "rkey":"must-roll-back", "value":{"$type":"app.bsky.feed.post", "image":survivor_blob}}, + {"$type":"com.atproto.repo.applyWrites#create", "collection":"app.bsky.feed.post", "rkey":"missing-blob", "value":{"$type":"app.bsky.feed.post", "image":transient_blob}} + ] + }) + .to_string(), + )) + .unwrap(), + ) + .await; + assert_eq!(response.status(), 400); + assert!( + app.repository + .get_record( + "did:web:alice.test", + "app.bsky.feed.post", + "must-roll-back", + None, + ) + .is_err() + ); + app.repository.verify().await.unwrap(); +} + #[tokio::test] async fn record_writes_validate_owned_atrium_blob_metadata() { let app = TestApp::new(); @@ -666,15 +952,12 @@ async fn record_writes_validate_owned_atrium_blob_metadata() { .unwrap(), ) .await; - let duplicate: upload_blob::Output = - serde_json::from_value(response_json(duplicate).await).unwrap(); - let conflicting = create_blob_record( - &app, - &token, - "conflicting-reupload", - serde_json::to_value(duplicate.blob.clone()).unwrap(), - ) - .await; + assert_eq!(duplicate.status(), 409); + assert_eq!(response_json(duplicate).await["error"], "InvalidRequest"); + let mut conflicting_blob = blob.clone(); + conflicting_blob["mimeType"] = json!("image/jpeg"); + let conflicting = + create_blob_record(&app, &token, "conflicting-reupload", conflicting_blob).await; assert_eq!(conflicting.status(), 400); let accepted = create_blob_record(&app, &token, "accepted-blob", blob.clone()).await; @@ -756,6 +1039,43 @@ async fn create_blob_record( .await } +async fn upload_blob_value( + app: &TestApp, + token: &str, + bytes: &'static [u8], +) -> (serde_json::Value, String) { + let response = app + .send( + Request::post("/xrpc/com.atproto.repo.uploadBlob") + .header(CONTENT_TYPE, "application/octet-stream") + .header("authorization", format!("Bearer {token}")) + .body(Body::from(bytes)) + .unwrap(), + ) + .await; + assert_eq!(response.status(), 200); + let output: upload_blob::Output = + serde_json::from_value(response_json(response).await).unwrap(); + let BlobRef::Typed(TypedBlobRef::Blob(blob)) = &output.blob else { + panic!("uploadBlob must return the current typed blob shape"); + }; + ( + serde_json::to_value(&output.blob).unwrap(), + blob.r#ref.0.to_string(), + ) +} + +async fn get_blob(app: &TestApp, cid: &str) -> axum::response::Response { + app.send( + Request::get(format!( + "/xrpc/com.atproto.sync.getBlob?did=did%3Aweb%3Aalice.test&cid={cid}" + )) + .body(Body::empty()) + .unwrap(), + ) + .await +} + #[tokio::test] async fn blob_download_rejects_noncanonical_dids_and_cids() { let app = TestApp::new(); diff --git a/tests/space_flow.rs b/tests/space_flow.rs index 5d3885f..cd6bf68 100644 --- a/tests/space_flow.rs +++ b/tests/space_flow.rs @@ -327,6 +327,64 @@ async fn list_blobs_only_exposes_references_in_the_selected_space_repo() { ); let listed = get_json(&app, &token, &uri).await; assert_eq!(listed["cids"], json!([cid])); + + let private_uri = format!( + "/xrpc/com.atproto.space.getBlob?space={}&repo=did%3Aweb%3Aalice.test&cid={cid}", + urlencoding(SPACE) + ); + let private = get(&app, &token, &private_uri).await; + assert_eq!(private.status(), StatusCode::OK); + assert_eq!(private.headers()["cache-control"], "no-store"); + assert_eq!( + private.headers()["content-security-policy"], + "default-src 'none'; sandbox" + ); + assert_eq!( + to_bytes(private.into_body(), usize::MAX).await.unwrap(), + "first-image" + ); + + for uri in [ + private_uri, + format!("/xrpc/com.atproto.sync.getBlob?did=did%3Aweb%3Aalice.test&cid={cid}"), + ] { + let response = app + .send(Request::get(uri).body(Body::empty()).unwrap()) + .await; + assert!(!response.status().is_success()); + assert_eq!(response.headers()["cache-control"], "no-store"); + } + + call_json( + &app, + &token, + "/xrpc/com.atproto.repo.createRecord", + json!({ + "repo": "did:web:alice.test", + "collection": "com.example.post", + "rkey": "shared-blob", + "record": {"$type": "com.example.post", "image": blob} + }), + ) + .await; + let public_uri = format!("/xrpc/com.atproto.sync.getBlob?did=did%3Aweb%3Aalice.test&cid={cid}"); + assert_eq!( + app.send(Request::get(&public_uri).body(Body::empty()).unwrap()) + .await + .status(), + StatusCode::OK + ); + call_json( + &app, + &token, + "/xrpc/com.atproto.repo.deleteRecord", + json!({"repo": "did:web:alice.test", "collection": "com.example.post", "rkey": "shared-blob"}), + ) + .await; + let response = app + .send(Request::get(public_uri).body(Body::empty()).unwrap()) + .await; + assert_eq!(response_json(response).await["error"], "BlobNotFound"); } #[tokio::test] @@ -543,7 +601,12 @@ async fn get_repo_returns_two_root_verifiable_car_layouts() { "repo": "did:web:alice.test", "collection": "com.example.post", "rkey": rkey, - "record": {"$type": "com.example.post", "text": rkey} + "record": { + "$type": "com.example.post", + "text": rkey, + "data": {"$bytes": "AQID"}, + "link": {"$link": "bafkreibme22gw2h7y2h7tg2fhqotaqjucnbc24deqo72b6mkl2egezxhvy"} + } }), ) .await; @@ -691,6 +754,143 @@ async fn notification_registrations_are_resolved_renewable_and_idempotent() { } } +#[tokio::test] +async fn slow_subscribers_do_not_turn_committed_space_changes_into_request_timeouts() { + use std::{sync::atomic::Ordering, time::Duration}; + + let (app, calls, server) = stalled_notification_app().await; + let oauth = app.access_token_with_scope(SCOPE).await; + create_test_space(&app, &oauth).await; + let key = P256SigningKey::from_slice(&[9; 32]).unwrap(); + let credential = mint_credential(&app, &oauth, &key, "slow-notice-exchange").await; + + // Two independent registrations resolve to the test receiver. Both must be + // attempted within one notice budget, despite neither producing a response. + for (index, service) in ["did:web:alice.test", "did:web:alice.test#atproto_pds"] + .into_iter() + .enumerate() + { + let response = post_with_credential( + &app, + "/xrpc/com.atproto.space.registerNotify", + &credential, + &key, + &format!("slow-notice-register-{index}"), + &json!({"space": SPACE, "service": service}), + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + } + + let service_token = make_service_jwt( + "did:web:alice.test", + "did:web:alice.test", + "com.atproto.space.notifyWrite", + ); + let rev = "3kbcq3p7ad422"; + let response = tokio::time::timeout( + Duration::from_secs(4), + app.send( + Request::post("/xrpc/com.atproto.space.notifyWrite") + .header("authorization", format!("Bearer {service_token}")) + .header(CONTENT_TYPE, "application/json") + .body(Body::from( + json!({ + "space": SPACE, "repo": "did:web:alice.test", "rev": rev, + "hash": {"$bytes": STANDARD_NO_PAD.encode([42_u8; 32])}, + }) + .to_string(), + )) + .unwrap(), + ), + ) + .await + .expect("the committed writer notice must not await every subscriber's HTTP timeout"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(calls.load(Ordering::SeqCst), 2); + + let list_path = format!( + "/xrpc/com.atproto.space.listRepos?space={}", + urlencoding(SPACE) + ); + let proof = make_dpop( + &key, + "slow-notice-read", + "GET", + app.public_origin + .endpoint("/xrpc/com.atproto.space.listRepos") + .as_str(), + Some(&credential), + ); + let response = app + .send( + Request::get(list_path) + .header("authorization", format!("DPoP {credential}")) + .header("dpop", proof) + .body(Body::empty()) + .unwrap(), + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response_json(response).await["repos"][0]["rev"], rev); + + let response = tokio::time::timeout( + Duration::from_secs(4), + app.send( + Request::post("/xrpc/com.atproto.simplespace.deleteSpace") + .header("authorization", format!("Bearer {oauth}")) + .header(CONTENT_TYPE, "application/json") + .body(Body::from(json!({"space": SPACE}).to_string())) + .unwrap(), + ), + ) + .await + .expect("the committed deletion must not await every subscriber's HTTP timeout"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(calls.load(Ordering::SeqCst), 4); + let response = get( + &app, + &oauth, + &format!( + "/xrpc/com.atproto.simplespace.getSpace?space={}", + urlencoding(SPACE), + ), + ) + .await; + assert_eq!(response_json(response).await["error"], "SpaceNotFound"); + server.abort(); +} + +async fn stalled_notification_app() -> ( + TestApp, + std::sync::Arc, + tokio::task::JoinHandle<()>, +) { + use cpds::config::{DeploymentMode, HttpOrigin}; + use std::{ + future::pending, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + }; + + let calls = Arc::new(AtomicUsize::new(0)); + let handler_calls = calls.clone(); + let receiver = Router::new().fallback(move || { + handler_calls.fetch_add(1, Ordering::SeqCst); + async { pending::().await } + }); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let origin = HttpOrigin::parse_public( + &format!("http://{}/", listener.local_addr().unwrap()), + DeploymentMode::Local, + ) + .unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, receiver).await.unwrap() }); + (TestApp::with_public_origin(origin), calls, server) +} + #[tokio::test] async fn allow_list_spaces_require_verified_single_use_client_attestations() { let app_key = P256SigningKey::from_slice(&[11; 32]).unwrap(); @@ -849,7 +1049,7 @@ async fn post_with_credential( key, jti, "POST", - &format!("http://pds.test{path}"), + app.public_origin.endpoint(path).as_str(), Some(credential), ), ) @@ -874,7 +1074,7 @@ async fn mint_credential(app: &TestApp, oauth: &str, key: &P256SigningKey, jti: key, jti, "POST", - "http://pds.test/xrpc/com.atproto.space.getSpaceCredential", + app.public_origin.endpoint(exchange_path).as_str(), None, ); let response = app diff --git a/tests/subscribe_repos_flow.rs b/tests/subscribe_repos_flow.rs index 1cb16a1..4c5ff79 100644 --- a/tests/subscribe_repos_flow.rs +++ b/tests/subscribe_repos_flow.rs @@ -23,9 +23,9 @@ use axum::{Json, Router, extract::State, routing::post}; use cpds::{ AppState, blob_store::MemoryBlobStore, - config::{AccountConfig, Config, DeploymentMode, PublicOrigin}, + config::{AccountConfig, Config, DeploymentMode, HttpOrigin}, database::Database, - domain::account::Account, + domain::{account::Account, record::RecordView}, events::NoopEventSink, http, }; @@ -49,7 +49,7 @@ async fn subscribe_repos_backfills_streams_and_rejects_future_cursors() { did: DID.to_owned(), handle: "alice.test".to_owned(), }; - let first = create_post(&state, &actor, "first").await; + let (_, first_commit) = create_post(&state, &actor, "first").await; let (shutdown, stopped) = oneshot::channel(); let server = tokio::spawn(http::serve(listener, state.clone(), async move { let _ = stopped.await; @@ -62,7 +62,7 @@ async fn subscribe_repos_backfills_streams_and_rejects_future_cursors() { assert_eq!(header.op, 1); assert_eq!(header.t.as_deref(), Some("#commit")); assert_eq!(commit.seq, 3); - assert_eq!(commit.commit.0.to_string(), first.commit.cid); + assert_eq!(&commit.commit.0, first_commit.cid.as_ref()); let mut car = CarStore::open(Cursor::new(commit.blocks.clone())) .await .unwrap(); @@ -80,12 +80,12 @@ async fn subscribe_repos_backfills_streams_and_rejects_future_cursors() { drop(backfill); let (mut live, mut buffered) = open_websocket(address, "").await; - let second = create_post(&state, &actor, "second").await; + let (_, second_commit) = create_post(&state, &actor, "second").await; let frame = read_binary_frame(&mut live, &mut buffered).await; let (header, commit) = decode_message::(&frame); assert_eq!(header.t.as_deref(), Some("#commit")); assert_eq!(commit.seq, 4); - assert_eq!(commit.commit.0.to_string(), second.commit.cid); + assert_eq!(&commit.commit.0, second_commit.cid.as_ref()); drop(live); let (mut future, mut buffered) = open_websocket(address, "?cursor=999").await; @@ -117,7 +117,7 @@ async fn subscribe_repos_replays_committed_events_after_database_restart() { first_listener.local_addr().unwrap(), Database::open(&path).unwrap(), ); - let first = create_post(&first_state, &actor, "before-restart").await; + let (_, first_commit) = create_post(&first_state, &actor, "before-restart").await; drop(first_state); drop(first_listener); @@ -135,14 +135,14 @@ async fn subscribe_repos_replays_committed_events_after_database_restart() { let (header, commit) = decode_message::(&frame); assert_eq!(header.t.as_deref(), Some("#commit")); assert_eq!(commit.seq, 3); - assert_eq!(commit.commit.0.to_string(), first.commit.cid); + assert_eq!(&commit.commit.0, first_commit.cid.as_ref()); - let second = create_post(&state, &actor, "after-restart").await; + let (_, second_commit) = create_post(&state, &actor, "after-restart").await; let frame = read_binary_frame(&mut stream, &mut buffered).await; let (header, commit) = decode_message::(&frame); assert_eq!(header.t.as_deref(), Some("#commit")); assert_eq!(commit.seq, 4); - assert_eq!(commit.commit.0.to_string(), second.commit.cid); + assert_eq!(&commit.commit.0, second_commit.cid.as_ref()); drop(stream); shutdown.send(()).unwrap(); @@ -162,14 +162,14 @@ async fn subscribe_repos_reports_cursors_outside_the_retained_window() { did: DID.to_owned(), handle: "alice.test".to_owned(), }; - let first = create_post(&state, &actor, "first").await; + let (_, first_commit) = create_post(&state, &actor, "first").await; create_post(&state, &actor, "second").await; create_post(&state, &actor, "third").await; create_post(&state, &actor, "fourth").await; assert!( state .repository - .export_repo(DID, Some(&first.commit.rev)) + .export_repo(DID, Some(&first_commit.rev)) .await .is_err() ); @@ -212,7 +212,7 @@ async fn fake_relay_crawls_resolves_identity_repairs_gap_and_resumes_live() { for rkey in ["first", "second", "third"] { create_post(&state, &actor, rkey).await; } - let before_repair = create_post(&state, &actor, "fourth").await; + let (_, before_repair_commit) = create_post(&state, &actor, "fourth").await; let (shutdown, stopped) = oneshot::channel(); let server = tokio::spawn(http::serve(listener, state.clone(), async move { let _ = stopped.await; @@ -236,7 +236,7 @@ async fn fake_relay_crawls_resolves_identity_repairs_gap_and_resumes_live() { .await .unwrap(); let root = car.roots().next().unwrap(); - assert_eq!(root.to_string(), before_repair.commit.cid); + assert_eq!(&root, before_repair_commit.cid.as_ref()); let mut repo = Repository::open(&mut car, root).await.unwrap(); let commit = repo.commit(); verify_signature(&verification_key, &commit.bytes(), commit.sig()).unwrap(); @@ -252,16 +252,16 @@ async fn fake_relay_crawls_resolves_identity_repairs_gap_and_resumes_live() { drop(car); let (mut live, mut buffered) = open_websocket(address, "").await; - let after_repair = create_post(&state, &actor, "fifth").await; + let (_, after_repair_commit) = create_post(&state, &actor, "fifth").await; let frame = read_binary_frame(&mut live, &mut buffered).await; let (header, commit) = decode_message::(&frame); assert_eq!(header.t.as_deref(), Some("#commit")); assert_eq!(commit.seq, 7); assert_eq!( commit.since.as_deref(), - Some(before_repair.commit.rev.as_str()) + Some(before_repair_commit.rev.as_str()) ); - assert_eq!(commit.commit.0.to_string(), after_repair.commit.cid); + assert_eq!(&commit.commit.0, after_repair_commit.cid.as_ref()); let mut car = CarStore::open(Cursor::new(commit.blocks.clone())) .await .unwrap(); @@ -388,7 +388,7 @@ async fn subscribe_repos_disconnects_a_stalled_consumer_and_allows_replay() { })); let (stalled, _) = open_websocket_with_receive_buffer(address, "", Some(1_024)).await; - let write = state + let (_, written_commit) = state .repository .create_record( &actor, @@ -414,7 +414,7 @@ async fn subscribe_repos_disconnects_a_stalled_consumer_and_allows_replay() { let (header, commit) = decode_message::(&frame); assert_eq!(header.t.as_deref(), Some("#commit")); assert_eq!(commit.seq, 3); - assert_eq!(commit.commit.0.to_string(), write.commit.cid); + assert_eq!(&commit.commit.0, written_commit.cid.as_ref()); drop(replay); shutdown.send(()).unwrap(); @@ -509,8 +509,11 @@ fn state_with_options( let config = Config { deployment_mode: DeploymentMode::Local, listen_address: address, - public_origin: PublicOrigin::parse(&format!("http://{address}"), DeploymentMode::Local) - .unwrap(), + public_origin: HttpOrigin::parse_public( + &format!("http://{address}"), + DeploymentMode::Local, + ) + .unwrap(), database_path: ":memory:".into(), blob_directory: "unused".into(), blob_orphan_ttl: std::time::Duration::from_secs(86_400), @@ -530,7 +533,7 @@ fn state_with_options( AppState::assemble( config, database, - Arc::new(MemoryBlobStore::new()), + Arc::new(MemoryBlobStore::default()), Arc::new(NoopEventSink), ) .unwrap() @@ -540,7 +543,10 @@ async fn create_post( state: &AppState, actor: &Account, rkey: &str, -) -> cpds::domain::record::RecordWrite { +) -> ( + RecordView, + atrium_api::com::atproto::repo::defs::CommitMetaData, +) { state .repository .create_record( diff --git a/tests/support/mod.rs b/tests/support/mod.rs index ae2a860..b507148 100644 --- a/tests/support/mod.rs +++ b/tests/support/mod.rs @@ -13,7 +13,7 @@ use axum::{ use cpds::{ AppState, blob_store::{BlobStore, MemoryBlobStore}, - config::{AccountConfig, Config, DeploymentMode, PublicOrigin}, + config::{AccountConfig, Config, DeploymentMode, HttpOrigin}, database::Database, domain::oauth::OAuthClient, events::RecordingEventSink, @@ -32,18 +32,30 @@ pub struct TestApp { pub router: Router, pub events: Arc, pub repository: Arc, + pub public_origin: HttpOrigin, } impl TestApp { pub fn new() -> Self { - Self::with_blob_store(Arc::new(MemoryBlobStore::new())) + Self::with_blob_store(Arc::new(MemoryBlobStore::default())) } pub fn with_blob_store(blobs: Arc) -> Self { + Self::assemble( + blobs, + HttpOrigin::parse_public("http://pds.test/", DeploymentMode::Local).unwrap(), + ) + } + + pub fn with_public_origin(public_origin: HttpOrigin) -> Self { + Self::assemble(Arc::new(MemoryBlobStore::default()), public_origin) + } + + fn assemble(blobs: Arc, public_origin: HttpOrigin) -> Self { let config = Config { deployment_mode: DeploymentMode::Local, listen_address: "127.0.0.1:3000".parse().unwrap(), - public_origin: PublicOrigin::parse("http://pds.test/", DeploymentMode::Local).unwrap(), + public_origin: public_origin.clone(), database_path: ":memory:".into(), blob_directory: "unused".into(), blob_orphan_ttl: std::time::Duration::from_secs(86_400), @@ -77,6 +89,7 @@ impl TestApp { router: http::router(state), events, repository, + public_origin, } }