use fjall::Slice; use jacquard_common::IntoStatic; use jacquard_common::types::did::Did; use miette::{IntoDiagnostic, Result}; use std::sync::Arc; use tracing::{debug, error, warn}; use crate::backfill::admission::BackfillAdmission; use crate::backfill::client::ThrottledHttpClient; use crate::backfill::error::BackfillError; use crate::config::BackfillStrategy; use crate::db::{Txn as DbTxn, keys}; use crate::ingest::indexer::IndexerTx; use crate::ingest::indexer::repos::write_repo; use crate::state::AppState; use crate::types::{GaugeState, RepoState, RepoStatus, ResyncErrorKind, ResyncState}; use super::apply::Outcome; use super::process::process_did; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(super) enum TaskDisposition { Finished, Deferred, } /// what writing down a failed backfill came to enum Recorded { /// the repo already sits in resync as gone, so only its queue entry goes Gone, /// `applied` when the failure moved the repo from the queue into resync Retrying { applied: bool }, } fn stage_excluded_pending_cleanup( txn: &mut DbTxn<'_>, db: &crate::db::Db, did: &Did, pending_key: &[u8], ) -> Result { if !txn.is_excluded(did)? { return Ok(false); } txn.transition_pending_key(did, pending_key, GaugeState::Synced)?; db.indexer .stage_pending_remove(&mut txn.batch, did, pending_key); Ok(true) } async fn remove_pending_if_excluded( state: &Arc, did: &Did, pending_key: Slice, ) -> Result { let owner = did.clone(); let did = did.clone(); write_repo(state, &owner, move |db| { let mut txn = DbTxn::new(db); if !stage_excluded_pending_cleanup(&mut txn, db, &did, pending_key.as_ref())? { return Ok(false); } txn.commit()?; Ok(true) }) .await } async fn is_pending(state: &AppState, pending_key: Slice) -> Result { state .db .run(move |db| db.indexer.is_pending(&pending_key).into_diagnostic()) .await } /// takes `pending_key` off the queue. the gauge only moves when it's still the repo's current /// key, so an orphan or a key something else already took off leaves it alone async fn remove_pending(state: &Arc, did: &Did, pending_key: Slice) -> Result<()> { let owner = did.clone(); let did = did.clone(); write_repo(state, &owner, move |db| { let mut txn = DbTxn::new(db); txn.transition_pending_key(&did, &pending_key, GaugeState::Synced)?; db.indexer .stage_pending_remove(&mut txn.batch, &did, &pending_key); txn.commit() }) .await } pub(super) async fn did_task( state: &Arc, http: ThrottledHttpClient, buffer_tx: IndexerTx, did: &Did, pending_key: Slice, admission: BackfillAdmission, _permit: tokio::sync::OwnedSemaphorePermit, verify_signatures: bool, strategy: BackfillStrategy, ) -> Result { // Drop stale/excluded work before resolver, admission, or network I/O. A // repository drop may race task spawning, and orphan pending keys are not // necessarily the one named by current metadata. if remove_pending_if_excluded(state, did, pending_key.clone()).await? { return Ok(TaskDisposition::Finished); } // the worker hands out keys from a snapshot of the queue, so a key whose task finished after // the snapshot comes back once that task leaves in_flight. it removed the key first, so this // read sees it gone, and running again would write its stale repo state over the synced one if !is_pending(state, pending_key.clone()).await? { return Ok(TaskDisposition::Finished); } let outcome = process_did( state, &http, &buffer_tx, did, pending_key.clone(), &admission, verify_signatures, strategy, ) .await; match outcome { Ok(Outcome::Drained) => { state .backfill_drained .fetch_add(1, std::sync::atomic::Ordering::Relaxed); Ok(TaskDisposition::Finished) } Ok(Outcome::Skipped) => { remove_pending_if_excluded(state, did, pending_key).await?; Ok(TaskDisposition::Finished) } Ok(Outcome::Stopping) => Ok(TaskDisposition::Finished), // the discard and inactive paths already took the entry off, unless they found the repo // excluded and left it Ok(Outcome::Superseded | Outcome::Discarded | Outcome::Inactive) => { remove_pending(state, did, pending_key).await?; Ok(TaskDisposition::Finished) } Err(BackfillError::PdsBusy) => Ok(TaskDisposition::Deferred), Err(BackfillError::UnsafeEndpoint(reason)) => { // The identity document is the only authority that can repair a // permanently unsafe PDS. Park this queue item rather than hot // retrying the same invalid endpoint. A later identity event can // enqueue fresh work with new evidence. warn!(%reason, "unsafe PDS endpoint, parking backfill until identity evidence changes"); remove_pending(state, did, pending_key).await?; Ok(TaskDisposition::Finished) } Err(BackfillError::Deleted) => { warn!("orphaned pending entry, cleaning up"); remove_pending(state, did, pending_key).await?; state .backfill_drained .fetch_add(1, std::sync::atomic::Ordering::Relaxed); Ok(TaskDisposition::Finished) } Err(e) => { match &e { BackfillError::Ratelimited | BackfillError::PreemptivelyThrottled => { debug!("too many requests"); } BackfillError::Transport(reason) => { error!(%reason, "transport error"); } BackfillError::Generic(e) => { error!(err = %e, "failed"); } BackfillError::RepoNotFound => { warn!("repo not found, retrying through existing backoff"); } BackfillError::Deleted => unreachable!("already handled"), BackfillError::PdsBusy => unreachable!("already handled"), BackfillError::UnsafeEndpoint(_) => unreachable!("handled above"), } let error_kind = match &e { BackfillError::Ratelimited | BackfillError::PreemptivelyThrottled => { ResyncErrorKind::Ratelimited } BackfillError::Transport(_) => ResyncErrorKind::Transport, BackfillError::Generic(_) => ResyncErrorKind::Generic, BackfillError::RepoNotFound => ResyncErrorKind::Generic, BackfillError::Deleted => unreachable!("already handled"), BackfillError::PdsBusy => unreachable!("already handled"), BackfillError::UnsafeEndpoint(_) => unreachable!("handled above"), }; let error_string = e.to_string(); let throttled = matches!(e, BackfillError::PreemptivelyThrottled); let recorded = write_repo(state, did, { let did_key = keys::repo_key(did).into_static(); let did = did.clone(); let pending_key = pending_key.clone(); move |db| { let mut txn = DbTxn::new(db); let existing = db .indexer .resync .get(&did_key) .into_diagnostic()? .map(|bytes| rmp_serde::from_slice::(&bytes).into_diagnostic()) .transpose()?; let entry_retries = match existing { Some(ResyncState::Error { retry_count, .. }) => retry_count, Some(ResyncState::Gone { .. }) => { db.indexer .stage_pending_remove(&mut txn.batch, &did, &pending_key); txn.commit()?; return Ok::<_, miette::Report>(Recorded::Gone); } None => 0, }; if stage_excluded_pending_cleanup(&mut txn, db, &did, pending_key.as_ref())? { txn.commit()?; return Ok(Recorded::Retrying { applied: false }); } let metadata_key = keys::repo_metadata_key(&did); let mut metadata = db .repo_metadata .get(&metadata_key) .into_diagnostic()? .map(|bytes| crate::db::deser_repo_meta(&bytes)) .transpose()?; let mut retry_count = metadata .as_ref() .map_or(0, |metadata| metadata.retry_count) .max(entry_retries); let next_retry = if throttled { chrono::Utc::now().timestamp() + 60 } else { retry_count = retry_count.saturating_add(1); ResyncState::next_backoff(retry_count) }; let resync_state = ResyncState::Error { kind: error_kind, retry_count, next_retry, }; let applied = txn.transition_pending_key( &did, pending_key.as_ref(), GaugeState::Resync(Some(error_kind)), )?; db.indexer .stage_pending_remove(&mut txn.batch, &did, &pending_key); if applied { txn.batch.insert( &db.indexer.resync, &did_key, rmp_serde::to_vec(&resync_state).into_diagnostic()?, ); if let Some(metadata) = metadata.as_mut() { metadata.retry_count = retry_count; txn.batch.insert( &db.repo_metadata, &metadata_key, crate::db::ser_repo_meta(metadata)?, ); } if let Some(bytes) = db.repos.get(&did_key).into_diagnostic()? { let mut repo: RepoState = rmp_serde::from_slice(&bytes).into_diagnostic()?; // a failed fetch says nothing about the account, so one that // went inactive keeps its status if repo.active { repo.status = RepoStatus::Error(error_string.into()); } txn.put_repo(&did, &repo)?; } } txn.commit()?; Ok(Recorded::Retrying { applied }) } }) .await?; match recorded { Recorded::Gone => Ok(TaskDisposition::Finished), Recorded::Retrying { applied } => { if applied { state .backfill_drained .fetch_add(1, std::sync::atomic::Ordering::Relaxed); } Err(e) } } } } } #[cfg(test)] mod tests { use super::*; use crate::backfill::admission::BackfillAdmission; use crate::backfill::test_util::{public_client, serve, tangled_repo}; use crate::config::Config; #[cfg(feature = "indexer_stream")] use crate::crawler::{ReconcileOutcome, RepoListing, reconcile_listing}; use crate::db; use crate::db::types::DbRkey; use crate::ingest::indexer::IndexerMessage; use crate::ingest::indexer::{ FirehoseWorker, IndexerCommitData, IndexerEvent, IndexerEventData, }; use crate::ingest::stream::{Commit as StreamCommit, Datetime, RepoOp, RepoOpAction}; use crate::types::RepoMetadata; use axum::Router; use axum::routing::get; use bytes::Bytes; #[cfg(feature = "indexer_stream")] use crate::crawler::worker::ListedRepoStatus; use jacquard_common::CowStr; use jacquard_common::types::cid::{CidLink, IpldCid}; use crate::mst::AtpCommit; use jacquard_common::types::string::Handle; use jacquard_common::types::tid::Tid; use reqwest::StatusCode; use std::sync::atomic::{AtomicUsize, Ordering}; use tokio::sync::{RwLock, Semaphore}; use url::Url; #[tokio::test] async fn excluded_orphan_pending_entry_is_removed_before_work() -> miette::Result<()> { let tmp = tempfile::tempdir().into_diagnostic()?; let config = Config { database_path: tmp.path().to_path_buf(), ..Default::default() }; let state = Arc::new(AppState::new(&config)?); let did = Did::new("did:plc:ewvi7nxzyoun6zhxrhs64oiz")?.into_static(); let pending_key = keys::pending_key(7); let mut batch = state.db.inner.batch(); batch.insert( &state.db.filter, crate::db::filter::exclude_key(did.as_str())?, [], ); state .db .indexer .stage_pending_insert(&mut batch, &did, &pending_key); batch.commit().into_diagnostic()?; assert!( remove_pending_if_excluded(&state, &did, fjall::Slice::from(pending_key.as_slice()),) .await? ); assert!( !state .db .indexer .is_pending(&pending_key) .into_diagnostic()? ); Ok(()) } // --- full-backfill `RepoNotFound` boundary --------------------------------------------- // // `getRepo` `RepoNotFound` is a fetch result, not lifecycle evidence. these tests pin the // non-destructive policy at the task boundary: the snapshot survives, the queue parks a // retryable error, and only a later valid CAR or an authoritative deletion moves state. const DID: &str = "did:plc:ewvi7nxzyoun6zhxrhs64oiz"; const NOT_FOUND_BODY: &[u8] = br#"{"error":"RepoNotFound","message":"not here"}"#; const COLLECTION: &str = "app.bsky.feed.post"; fn post(text: &str) -> Vec { serde_ipld_dagcbor::to_vec(&serde_json::json!({ "$type": COLLECTION, "text": text, })) .unwrap() } struct PdsStub { base: Url, address: std::net::SocketAddr, response: Arc>, /// getRepo requests requests: Arc, get_repo: Gate, blocks: Arc>, block_requests: Arc, get_blocks: Gate, latest_commit: Arc>, } /// where a request to one route waits while a test changes something under the task #[derive(Clone, Default)] struct Gate { arrived: Arc, hold: Arc>, } impl Gate { async fn pass(&self) { self.arrived.notify_waiters(); drop(self.hold.lock().await); } } /// a stub route answering with whatever `response` holds, counting requests and passing /// `gate` first fn stub_route( response: &Arc>, requests: &Arc, gate: &Gate, ) -> axum::routing::MethodRouter { let (response, requests, gate) = (response.clone(), requests.clone(), gate.clone()); get(move || { let (response, requests, gate) = (response.clone(), requests.clone(), gate.clone()); async move { requests.fetch_add(1, Ordering::SeqCst); gate.pass().await; response.read().await.clone() } }) } impl PdsStub { async fn spawn() -> Self { let response = Arc::new(RwLock::new(( StatusCode::BAD_REQUEST, Bytes::from_static(NOT_FOUND_BODY), ))); let requests = Arc::new(AtomicUsize::new(0)); let get_repo = Gate::default(); // no getBlocks until a test serves one, like the atcr hold let blocks = Arc::new(RwLock::new((StatusCode::NOT_FOUND, Bytes::new()))); let block_requests = Arc::new(AtomicUsize::new(0)); let get_blocks = Gate::default(); // no getLatestCommit either, so a sparse backfill falls back to getRepo let latest_commit = Arc::new(RwLock::new((StatusCode::NOT_FOUND, Bytes::new()))); let app = Router::new() .route( "/xrpc/com.atproto.sync.getRepo", stub_route(&response, &requests, &get_repo), ) .route( "/xrpc/com.atproto.sync.getBlocks", stub_route(&blocks, &block_requests, &get_blocks), ) .route( "/xrpc/com.atproto.sync.getLatestCommit", stub_route(&latest_commit, &Arc::default(), &Gate::default()), ); let address = serve(app).await; // public by name, so the backfill client takes it, and pinned to the local socket by // the test client, so nothing resolves it let base = Url::parse("http://pds.example/").unwrap(); Self { base, address, response, requests, get_repo, blocks, block_requests, get_blocks, latest_commit, } } async fn serve_car(&self, car: Bytes) { *self.response.write().await = (StatusCode::OK, car); } fn requests(&self) -> usize { self.requests.load(Ordering::SeqCst) } async fn serve_blocks(&self, status: StatusCode, body: Bytes) { *self.blocks.write().await = (status, body); } fn block_requests(&self) -> usize { self.block_requests.load(Ordering::SeqCst) } /// answers a sparse backfill from `car`: its commit as the latest one, and every block /// it asks getBlocks for, since the whole car holds them all async fn serve_sparse(&self, car: &BuiltCar) -> miette::Result<()> { let commit = crate::car::parse_car(car.bytes.clone(), false)?.root; let latest = serde_json::json!({ "cid": commit.to_string(), "rev": car.rev.as_str() }); *self.latest_commit.write().await = (StatusCode::OK, Bytes::from(latest.to_string())); self.serve_blocks(StatusCode::OK, car.bytes.clone()).await; Ok(()) } } async fn spawn_doc_stub(doc: String) -> Url { let app = Router::new().fallback({ move |_uri: axum::http::Uri| { let doc = doc.clone(); async move { ( [(axum::http::header::CONTENT_TYPE, "application/json")], doc, ) } } }); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let base = Url::parse(&format!("http://{}/", listener.local_addr().unwrap())).unwrap(); tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); base } fn did_doc(base: &Url) -> String { serde_json::json!({ "@context": ["https://www.w3.org/ns/did/v1"], "id": DID, "service": [{ "id": "#atproto_pds", "type": "AtprotoPersonalDataServer", "serviceEndpoint": base.as_str(), }], }) .to_string() } fn rooted(rev: &str) -> miette::Result> { let mut state = RepoState::synced(); state.root = Some(crate::types::Commit { version: 3, rev: crate::db::types::DbTid::from(&Tid::new(rev).into_diagnostic()?), data: crate::car::compute_cid(b"seeded-root"), prev: None, sig: Bytes::from_static(b"signature"), }); state.pds = Some(CowStr::Borrowed("https://old-pds.example")); state.handle = Some(Handle::new_static("alice.test").into_diagnostic()?); state.last_message_time = Some(101); state.last_updated_at = 5; Ok(state.into_static()) } struct BuiltCar { bytes: Bytes, mst_root: IpldCid, rev: Tid, } async fn build_car(rev: &str, records: &[(&str, &[u8])]) -> miette::Result { build_car_for(DID, rev, records).await } async fn build_car_for( commit_did: &'static str, rev: &str, records: &[(&str, &[u8])], ) -> miette::Result { let records = records .iter() .map(|&(rkey, body)| (format!("{COLLECTION}/{rkey}"), body)) .collect::>(); build_car_keyed(commit_did, rev, &records).await } /// a car whose records sit under full `collection/rkey` keys async fn build_car_keyed( commit_did: &'static str, rev: &str, records: &[(String, &[u8])], ) -> miette::Result { let (mst_root, mst_nodes) = crate::mst::build_tree( records .iter() .map(|(key, body)| (key.as_str(), crate::car::compute_cid(body))), )?; let rev = Tid::new(rev).into_diagnostic()?; let did: Did = Did::new_static(commit_did).into_diagnostic()?; let commit = AtpCommit { did, version: 3, data: mst_root, rev: rev.clone(), prev: None, sig: Bytes::from_static(b"signature"), }; let commit_cid = commit.to_cid()?; let commit_cbor = commit.to_cbor()?; let mut bytes = Vec::new(); let mut writer = crate::car::CarWriter::new(crate::car::CarHeader::new_v1(vec![commit_cid]), &mut bytes); writer .write(commit_cid, &commit_cbor) .await .into_diagnostic()?; for (cid, node_bytes) in mst_nodes { writer.write(cid, &node_bytes).await.into_diagnostic()?; } let mut seen = std::collections::HashSet::new(); for (_, body) in records { let cid = crate::car::compute_cid(body); if seen.insert(cid) { writer.write(cid, body).await.into_diagnostic()?; } } writer.finish().await.into_diagnostic()?; Ok(BuiltCar { bytes: Bytes::from(bytes), mst_root, rev, }) } struct Fixture { state: Arc, did: Did, pds: PdsStub, /// one slot per pds, so a test can see whether a task still holds it admission: BackfillAdmission, strategy: BackfillStrategy, /// what [`Self::reopen`] opens the database with again #[cfg_attr(not(feature = "indexer_stream"), allow(dead_code))] config: Config, #[cfg_attr(not(feature = "indexer_stream"), allow(dead_code))] doc_base: Url, _tmp: tempfile::TempDir, } // whatever a test does to the queue, its two keyspaces have to end in step impl Drop for Fixture { fn drop(&mut self) { if !std::thread::panicking() { self.state.db.indexer.assert_pending_dids_match(); } } } /// records per backfill step in these tests: one, so every repo with more than one record /// goes through several steps, unless `HYDRANT_TEST_BACKFILL_CHUNK_RECORDS` says otherwise /// (0 writes each repo in one step) fn test_chunk_records() -> usize { std::env::var("HYDRANT_TEST_BACKFILL_CHUNK_RECORDS") .ok() .and_then(|records| records.parse().ok()) .unwrap_or(1) } /// how big a car gets in memory before it spills in these tests, the default unless /// `HYDRANT_TEST_BACKFILL_SPILL_BYTES` says otherwise (0 spills every whole car) fn test_spill_bytes() -> usize { std::env::var("HYDRANT_TEST_BACKFILL_SPILL_BYTES") .ok() .and_then(|bytes| bytes.parse().ok()) .unwrap_or(Config::default().backfill_spill_bytes) } impl Fixture { async fn new() -> miette::Result { Self::with_chunk_records(test_chunk_records()).await } async fn with_chunk_records(records: usize) -> miette::Result { let pds = PdsStub::spawn().await; let endpoint = pds.base.clone(); Self::new_with(pds, endpoint, records, test_spill_bytes()).await } /// a fixture whose backfills spill every whole car, at one record a step async fn spilling() -> miette::Result { let pds = PdsStub::spawn().await; let endpoint = pds.base.clone(); Self::new_with(pds, endpoint, 1, 0).await } async fn new_with_pds(pds: PdsStub, pds_endpoint: Url) -> miette::Result { Self::new_with(pds, pds_endpoint, test_chunk_records(), test_spill_bytes()).await } async fn new_with( pds: PdsStub, pds_endpoint: Url, chunk_records: usize, spill_bytes: usize, ) -> miette::Result { let tmp = tempfile::tempdir().into_diagnostic()?; let doc_base = spawn_doc_stub(did_doc(&pds_endpoint)).await; let config = Config { database_path: tmp.path().to_path_buf(), plc_urls: vec![Url::parse("http://public.example/").unwrap()], identity_cache_size: 64, backfill_chunk_records: chunk_records, backfill_spill_bytes: spill_bytes, ..Default::default() }; Ok(Self { state: Arc::new(Self::open(&config, &doc_base)?), did: Did::new_static(DID).into_diagnostic()?, pds, admission: BackfillAdmission::new(4, 1), strategy: BackfillStrategy::Full, config, doc_base, _tmp: tmp, }) } fn open(config: &Config, doc_base: &Url) -> miette::Result { let mut app_state = AppState::new(config)?; let public_client = reqwest::Client::builder() .no_proxy() .resolve( "public.example", std::net::SocketAddr::from(( std::net::Ipv4Addr::LOCALHOST, doc_base.port().unwrap(), )), ) .build() .into_diagnostic()?; app_state.resolver = crate::resolver::Resolver::with_plc_urls_and_client( config.plc_urls.clone(), 64, crate::net::PublicHttpClient::test(public_client), ); Ok(app_state) } /// drops the database without a shutdown, the way a crash would, and opens it again #[cfg(feature = "indexer_stream")] fn reopen(&mut self) -> miette::Result<()> { // a database opens once at a time, so another one stands in while this one closes let spare = tempfile::tempdir().into_diagnostic()?; let stand_in = AppState::new(&Config { database_path: spare.path().to_path_buf(), ..Default::default() })?; let state = std::mem::replace(&mut self.state, Arc::new(stand_in)); drop(Arc::into_inner(state).expect("nothing else holds the app state")); self.state = Arc::new(Self::open(&self.config, &self.doc_base)?); Ok(()) } fn seed( &self, repo: &RepoState<'static>, index_id: u64, records: &[(&str, &[u8])], ) -> miette::Result<()> { let mut batch = self.state.db.inner.batch(); batch.insert( &self.state.db.repos, keys::repo_key(&self.did), db::ser_repo_state(repo)?, ); batch.insert( &self.state.db.repo_metadata, keys::repo_metadata_key(&self.did), db::ser_repo_meta(&RepoMetadata::backfilling(index_id))?, ); self.state.db.indexer.stage_pending_insert( &mut batch, &self.did, &keys::pending_key(index_id), ); for &(rkey, body) in records { self.state.db.indexer.stage_head( &mut batch, keys::record_key(&self.did, COLLECTION, &DbRkey::new(rkey)), body, ); } batch.commit().into_diagnostic() } /// mirrors the retry worker: a completed queue entry gives way to a fresh pending key. fn reindex_pending(&self, from: u64, to: u64) -> miette::Result<()> { let mut batch = self.state.db.inner.batch(); let indexer = &self.state.db.indexer; indexer.stage_pending_remove(&mut batch, &self.did, &keys::pending_key(from)); indexer.stage_pending_insert(&mut batch, &self.did, &keys::pending_key(to)); batch.insert( &self.state.db.repo_metadata, keys::repo_metadata_key(&self.did), db::ser_repo_meta(&RepoMetadata::backfilling(to))?, ); batch.commit().into_diagnostic() } fn clear_resync(&self) -> miette::Result<()> { self.state .db .indexer .resync .remove(keys::repo_key(&self.did)) .into_diagnostic()?; Ok(()) } /// the authoritative deletion primitive plus its `Deleted` tombstone: the same pair the /// firehose and list-repos paths apply under the repository lock, then the wipe's /// steps the way the owner shard runs them fn delete_authoritatively(&self, repo: &RepoState<'static>) -> miette::Result<()> { self.start_deletion(repo)?; FirehoseWorker::run_job(&self.state, &self.did); Ok(()) } /// [`Self::delete_authoritatively`] up to its wipe's marker, which a shard runs from fn start_deletion(&self, repo: &RepoState<'static>) -> miette::Result<()> { let mut txn = DbTxn::new(&self.state.db); crate::ops::start_wipe(&mut txn, &self.state.db, &self.did, repo)?; crate::ops::transition_repo( &mut txn, &mut Vec::new(), &self.did, repo.clone(), RepoStatus::Deleted, )?; txn.commit() } /// the bulk job the repo's marker names, if any fn marker(&self) -> miette::Result> { db::read_bulk_job(&self.state.db, &self.did) } /// starts a shard, which picks up every marker before anything else, and waits for it /// to run out of work async fn start_a_shard(&self) -> miette::Result<()> { let (tx, shard) = FirehoseWorker::spawn_one_shard(self.state.clone()); drop(tx); tokio::task::spawn_blocking(move || shard.join()) .await .into_diagnostic()? .map_err(|_| miette::miette!("indexer shard panicked")) } async fn run(&self, pending_key: [u8; 8]) -> Result { let (tx, shard) = FirehoseWorker::spawn_one_shard(self.state.clone()); let result = self.run_with(tx, pending_key).await; // the task held the shard's only sender, so the shard stops now tokio::task::spawn_blocking(move || shard.join()) .await .into_diagnostic()? .map_err(|_| miette::miette!("indexer shard panicked"))?; result } /// `run`, handing the finished backfill to `tx`, which may have no shard behind it async fn run_with( &self, tx: IndexerTx, pending_key: [u8; 8], ) -> Result { let (_, http) = public_client(self.pds.address, &self.state.throttler); let permit = Arc::new(Semaphore::new(1)).acquire_owned().await.unwrap(); did_task( &self.state, http, tx, &self.did, Slice::from(pending_key), self.admission.clone(), permit, false, self.strategy, ) .await } /// a live commit creating `rkey` out of `car` fn live_commit( &self, car: &BuiltCar, rkey: &str, body: &[u8], ) -> miette::Result> { let root = crate::car::parse_car(car.bytes.clone(), false)?.root; let cid = crate::car::compute_cid(body); Ok(StreamCommit { blobs: Vec::new(), blocks: car.bytes.clone(), commit: CidLink::from(root), ops: vec![RepoOp { action: RepoOpAction::Create, cid: Some(CidLink::from(cid)), path: CowStr::Owned(format!("{COLLECTION}/{rkey}").into()), prev: None, }], prev_data: None, rebase: false, repo: self.did.clone(), rev: car.rev.clone(), seq: 1, since: None, time: Datetime::try_from("2026-10-05T12:00:00Z".to_owned()).into_diagnostic()?, too_big: false, }) } /// [`Self::live_commit`] for `c`, followed by an op whose rkey doesn't parse, so applying /// it fails halfway fn half_bad_commit( &self, car: &BuiltCar, c: &[u8], ) -> miette::Result> { let mut commit = self.live_commit(car, "c", c)?; let mut bad = commit.ops[0].clone(); bad.path = CowStr::Owned(format!("{COLLECTION}/bad rkey").into()); commit.ops.push(bad); Ok(commit) } /// hands `commit`, whose blocks are `car`, to a shard as a live event and waits for it async fn apply_live( &self, car: &BuiltCar, commit: StreamCommit<'static>, ) -> miette::Result<()> { let (tx, shard) = FirehoseWorker::spawn_one_shard(self.state.clone()); tx.send(live_event(car, commit, false)?) .await .map_err(|_| miette::miette!("the shard is gone"))?; drop(tx); tokio::task::spawn_blocking(move || shard.join()) .await .into_diagnostic()? .map_err(|_| miette::miette!("indexer shard panicked")) } /// runs the task and `change` while it waits after its `after`th backfill step. the /// change gets the shard's sender, so it can hand the shard live events in between async fn run_pausing( &self, pending_key: [u8; 8], after: usize, change: F, ) -> miette::Result> where F: FnOnce(IndexerTx) -> Fut, Fut: Future>, { let (tx, shard) = FirehoseWorker::spawn_one_shard(self.state.clone()); // the api hands an erase to the same shard self.state.indexer.lock().replace(tx.clone()); let (paused, mut waiting) = tokio::sync::mpsc::unbounded_channel(); let (resume, resumed) = std::sync::mpsc::channel::<()>(); let resumed = std::sync::Mutex::new(resumed); *self.state.after_chunk.lock() = Some(Arc::new(move |sent| { if sent == after { let _ = paused.send(()); let _ = resumed.lock().unwrap().recv(); } })); let run = async { let result = self.run_with(tx.clone(), pending_key).await; // lets go of `paused`, so a change waiting on a step that never came ends *self.state.after_chunk.lock() = None; result }; let change = async { if waiting.recv().await.is_none() { return Err(miette::miette!("the task never got to step {after}")); } let changed = change(tx.clone()).await; let _ = resume.send(()); changed }; let (result, changed) = tokio::join!(run, change); self.state.indexer.lock().take(); drop(tx); tokio::task::spawn_blocking(move || shard.join()) .await .into_diagnostic()? .map_err(|_| miette::miette!("indexer shard panicked"))?; changed?; Ok(result) } /// [`Self::live_commit`], buffered the way the shard does while the repo is queued fn buffer(&self, car: &BuiltCar, rkey: &str, body: &[u8]) -> miette::Result<()> { self.buffer_commit(&self.live_commit(car, rkey, body)?) } fn buffer_commit(&self, commit: &StreamCommit<'static>) -> miette::Result<()> { let mut txn = DbTxn::new(&self.state.db); crate::ops::persist_to_resync_buffer(&mut txn, &self.state.db, &self.did, commit)?; txn.commit() } /// queues the repo under 7 and serves a car with record `a`, while the repo has moved on /// to a commit adding `b`. returns that commit's car and `b` async fn fetch_behind_a_live_commit(&self) -> miette::Result<(BuiltCar, Vec)> { self.seed(&rooted("3jzfcijpj2z2a")?, 7, &[])?; let a = post("a"); let b = post("b"); let fetched = build_car("3jzfcijpj2z2b", &[("a", a.as_slice())]).await?; let newer = build_car("3jzfcijpj2z2c", &[("a", a.as_slice()), ("b", b.as_slice())]).await?; self.pds.serve_car(fetched.bytes).await; Ok((newer, b)) } /// [`Self::fetch_behind_a_live_commit`], after the shard buffered that commit during the /// fetch. returns its car async fn fetch_with_a_buffered_commit(&self) -> miette::Result { let (newer, b) = self.fetch_behind_a_live_commit().await?; self.buffer(&newer, "b", &b)?; Ok(newer) } /// the repo is synced at `newer`, which added `b` to the fetched `a`, with nothing left /// queued or buffered fn assert_caught_up_to(&self, newer: &BuiltCar) -> miette::Result<()> { assert_eq!(self.record_rkeys()?, ["a", "b"]); let after = self.repo()?; assert_eq!( after.root.map(|root| root.rev), Some(crate::db::types::DbTid::from(&newer.rev)) ); assert_eq!(after.status, RepoStatus::Synced); assert!(self.pending_keys()?.is_empty()); assert_eq!(self.buffered(), 0); Ok(()) } /// a channel with no shard behind it, so nobody answers the handoff fn unanswered(&self) -> IndexerTx { FirehoseWorker::new(self.state.clone(), 1).0 } fn buffered(&self) -> usize { self.state .db .indexer .resync_buffer .prefix(keys::resync_buffer_prefix(&self.did)) .count() } /// runs the task for `pending_key` and applies `change` while the task waits on getRepo, /// the way another writer can move the queue under a task that's already in flight async fn run_changing( &self, pending_key: [u8; 8], change: impl Future>, ) -> miette::Result> { self.run_changing_at(&self.pds.get_repo, pending_key, change) .await } /// `run_changing` with the finished backfill handed to `tx`, a shard the test made, so the /// change can hand it work of its own async fn run_changing_on( &self, tx: IndexerTx, pending_key: [u8; 8], change: impl Future>, ) -> miette::Result> { let held = self.pds.get_repo.hold.lock().await; let arrived = self.pds.get_repo.arrived.notified(); tokio::pin!(arrived); arrived.as_mut().enable(); let (result, changed) = tokio::join!(self.run_with(tx, pending_key), async { arrived.await; let changed = change.await; drop(held); changed }); changed?; Ok(result) } /// `run_changing`, with the change applied while the task waits at `gate` async fn run_changing_at( &self, gate: &Gate, pending_key: [u8; 8], change: impl Future>, ) -> miette::Result> { let held = gate.hold.lock().await; // registered before the task starts, because `notify_waiters` only wakes // waiters that already exist and keeps nothing for a later one, so a // request from an earlier run can't let this change in early let arrived = gate.arrived.notified(); tokio::pin!(arrived); arrived.as_mut().enable(); let (result, changed) = tokio::join!(self.run(pending_key), async { arrived.await; let changed = change.await; drop(held); changed }); changed?; Ok(result) } fn put_repo(&self, repo: &RepoState<'_>) -> miette::Result<()> { let mut batch = self.state.db.inner.batch(); batch.insert( &self.state.db.repos, keys::repo_key(&self.did), db::ser_repo_state(repo)?, ); batch.commit().into_diagnostic() } fn repo(&self) -> miette::Result> { let bytes = self .state .db .repos .get(keys::repo_key(&self.did)) .into_diagnostic()? .expect("repository state row") .to_vec(); Ok(db::deser_repo_state(&bytes)?.into_static()) } fn resync(&self) -> miette::Result> { self.state .db .indexer .resync .get(keys::repo_key(&self.did)) .into_diagnostic()? .map(|bytes| rmp_serde::from_slice::(&bytes).into_diagnostic()) .transpose() } fn pending_keys(&self) -> miette::Result>> { self.state .db .indexer .pending_iter() .map(|guard| Ok(guard.into_inner().into_diagnostic()?.0.to_vec())) .collect() } fn record_rkeys(&self) -> miette::Result> { let prefix = keys::record_prefix_did(&self.did); self.state .db .indexer .head_prefix(&prefix) .map(|guard| { let (key, _) = guard.into_inner().into_diagnostic()?; let (_, rkey) = keys::split_record_suffix(&key[prefix.len()..])?; Ok(rkey.to_string()) }) .collect() } } /// `commit`, whose blocks are `car`, as the relay hands it to the indexer fn live_event( car: &BuiltCar, commit: StreamCommit<'static>, chain_break: bool, ) -> miette::Result { Ok(IndexerMessage::Event(Box::new(IndexerEvent { seq: 1, firehose: Url::parse("wss://relay.example").into_diagnostic()?, ticket: None, row: None, data: IndexerEventData::Commit(IndexerCommitData { commit, chain_break, parsed_blocks: crate::car::parse_commit_car(car.bytes.clone(), false)?, }), }))) } #[tokio::test] async fn unsafe_pds_endpoint_is_parked_until_fresh_identity_evidence() -> miette::Result<()> { let pds = PdsStub::spawn().await; let fixture = Fixture::new_with_pds(pds, Url::parse("http://127.0.0.1/").unwrap()).await?; fixture.seed(&RepoState::backfilling().into_static(), 13, &[])?; assert_eq!( fixture.run(keys::pending_key(13)).await?, TaskDisposition::Finished, "unsafe endpoint work must park instead of entering retry backoff" ); assert!(fixture.pending_keys()?.is_empty()); assert!(fixture.resync()?.is_none()); assert_eq!( fixture.pds.requests(), 0, "unsafe endpoint must not connect" ); Ok(()) } #[tokio::test] async fn car_whose_commit_names_another_did_is_rejected() -> miette::Result<()> { let fixture = Fixture::new().await?; fixture.seed(&RepoState::backfilling().into_static(), 9, &[])?; let body = post("a"); let other = "did:plc:bbbbbbbbbbbbbbbbbbbbbbbb"; let car = build_car_for(other, "3jzfcijpj2z2a", &[("a", body.as_slice())]).await?; fixture.pds.serve_car(car.bytes).await; let err = fixture.run(keys::pending_key(9)).await.unwrap_err(); assert!(err.to_string().contains("commit is for"), "{err}"); assert!(fixture.record_rkeys()?.is_empty()); Ok(()) } const WANTED: &str = "sh.tangled.repo"; fn only_collection(fixture: &Fixture, collection: &str) { let mut filter = crate::filter::FilterConfig::new(crate::filter::FilterMode::Filter); filter.collections = vec![collection.into()]; fixture.state.filter.store(Arc::new(filter)); } fn stored(fixture: &Fixture, collection: &str, rkey: &str) -> miette::Result>> { let key = keys::record_key(&fixture.did, collection, &DbRkey::new(rkey)); Ok(fixture .state .db .indexer .head(key) .into_diagnostic()? .map(|body| body.to_vec())) } #[tokio::test] async fn filtered_full_backfill_stores_what_a_whole_car_would_for_the_wanted_collection() -> miette::Result<()> { let (core, site) = (tangled_repo("core"), tangled_repo("site")); let posts = (0..300) .map(|i| post(&format!("post {i}"))) .collect::>(); let mut records = posts .iter() .enumerate() .map(|(i, body)| (format!("{COLLECTION}/3jzfcijpj{i:04}"), body.as_slice())) .collect::>(); records.push((format!("{WANTED}/core"), core.as_slice())); records.push((format!("{WANTED}/site"), site.as_slice())); let car = build_car_keyed(DID, "3jzfcijpj2z2a", &records).await?; let filtered = Fixture::new().await?; only_collection(&filtered, WANTED); filtered.seed(&RepoState::backfilling().into_static(), 9, &[])?; filtered.pds.serve_car(car.bytes.clone()).await; let whole = Fixture::new().await?; whole.seed(&RepoState::backfilling().into_static(), 9, &[])?; whole.pds.serve_car(car.bytes).await; assert_eq!( filtered.run(keys::pending_key(9)).await?, TaskDisposition::Finished ); assert_eq!( whole.run(keys::pending_key(9)).await?, TaskDisposition::Finished ); assert_eq!(filtered.record_rkeys()?, vec!["core", "site"]); assert_eq!(stored(&filtered, WANTED, "core")?, Some(core.clone())); for rkey in ["core", "site"] { assert_eq!( stored(&filtered, WANTED, rkey)?, stored(&whole, WANTED, rkey)? ); } assert_eq!( filtered.repo()?.root.map(|root| root.data), whole.repo()?.root.map(|root| root.data) ); assert_eq!(filtered.pds.requests(), 1); Ok(()) } #[tokio::test] async fn full_backfill_stores_every_record_a_whole_parse_finds() -> miette::Result<()> { let core = tangled_repo("core"); let posts = (0..500) .map(|i| post(&format!("post {i}"))) .collect::>(); let mut records = posts .iter() .enumerate() .map(|(i, body)| (format!("{COLLECTION}/3jzfcijpj{i:04}"), body.as_slice())) .collect::>(); records.push((format!("{WANTED}/core"), core.as_slice())); // the same body under two keys is one block in the car records.push((format!("{WANTED}/copy"), core.as_slice())); let car = build_car_keyed(DID, "3jzfcijpj2z2a", &records).await?; let parsed = crate::car::parse_car(car.bytes.clone(), true)?; let want = crate::mst::leaves(parsed.blocks.map(), car.mst_root)? .into_iter() .map(|(key, cid)| (key.to_string(), parsed.blocks.get(&cid).unwrap().to_vec())) .collect::>(); assert_eq!(want.len(), records.len()); let fixture = Fixture::new().await?; fixture.seed(&RepoState::backfilling().into_static(), 9, &[])?; fixture.pds.serve_car(car.bytes).await; assert_eq!( fixture.run(keys::pending_key(9)).await?, TaskDisposition::Finished ); let prefix = keys::record_prefix_did(&fixture.did); let stored = fixture .state .db .indexer .head_prefix(&prefix) .map(|guard| { let (key, body) = guard.into_inner().into_diagnostic()?; let (collection, rkey) = keys::split_record_suffix(&key[prefix.len()..])?; Ok((format!("{collection}/{rkey}"), body.to_vec())) }) .collect::>>()?; assert_eq!(stored, want); Ok(()) } /// `car` with the block under `cid` left out async fn car_without(car: &BuiltCar, cid: IpldCid) -> miette::Result { let parsed = crate::car::parse_car(car.bytes.clone(), false)?; let mut bytes = Vec::new(); let mut writer = crate::car::CarWriter::new(crate::car::CarHeader::new_v1(vec![parsed.root]), &mut bytes); for (block_cid, body) in parsed.blocks.map() { if *block_cid != cid { writer.write(*block_cid, body).await.into_diagnostic()?; } } writer.finish().await.into_diagnostic()?; Ok(Bytes::from(bytes)) } #[tokio::test] async fn a_car_missing_an_mst_node_writes_nothing() -> miette::Result<()> { let posts = (0..300) .map(|i| post(&format!("post {i}"))) .collect::>(); let records = posts .iter() .enumerate() .map(|(i, body)| (format!("{COLLECTION}/3jzfcijpj{i:04}"), body.as_slice())) .collect::>(); let car = build_car_keyed(DID, "3jzfcijpj2z2a", &records).await?; // the root's last subtree, so the walk hands out leaves before it finds the gap let parsed = crate::car::parse_car(car.bytes.clone(), false)?; let root = parsed.blocks.get(&car.mst_root).expect("root node"); let subtree = crate::mst::decode_node(root)? .into_iter() .rev() .find_map(|entry| match entry { crate::mst::FlatEntry::Tree { cid } => Some(cid), crate::mst::FlatEntry::Leaf { .. } => None, }) .expect("300 records need more than one node"); let fixture = Fixture::new().await?; let old = post("old"); fixture.seed( &RepoState::backfilling().into_static(), 9, &[("old", old.as_slice())], )?; fixture .pds .serve_car(car_without(&car, subtree).await?) .await; let err = fixture.run(keys::pending_key(9)).await.unwrap_err(); assert!(err.to_string().contains("missing from car"), "{err}"); assert_eq!(fixture.record_rkeys()?, ["old"]); assert_eq!(stored(&fixture, COLLECTION, "old")?, Some(old)); Ok(()) } #[tokio::test] async fn a_car_with_a_key_that_doesnt_parse_writes_nothing() -> miette::Result<()> { let (a, bad) = (post("a"), post("bad")); // a fine mst key, but its collection isn't an nsid let records = [ (format!("{COLLECTION}/a"), a.as_slice()), ("notansid/b".to_string(), bad.as_slice()), ]; let car = build_car_keyed(DID, "3jzfcijpj2z2a", &records).await?; let fixture = Fixture::new().await?; let old = post("old"); fixture.seed( &RepoState::backfilling().into_static(), 9, &[("old", old.as_slice())], )?; fixture.pds.serve_car(car.bytes).await; let err = fixture.run(keys::pending_key(9)).await.unwrap_err(); assert!(err.to_string().contains("invalid collection NSID"), "{err}"); assert_eq!(fixture.record_rkeys()?, ["old"]); Ok(()) } #[tokio::test] async fn a_record_whose_block_the_car_left_out_is_skipped() -> miette::Result<()> { let (a, b) = (post("a"), post("b")); let car = build_car("3jzfcijpj2z2a", &[("a", a.as_slice()), ("b", b.as_slice())]).await?; let fixture = Fixture::new().await?; fixture.seed(&RepoState::backfilling().into_static(), 9, &[])?; let b_cid = crate::car::compute_cid(&b); fixture.pds.serve_car(car_without(&car, b_cid).await?).await; assert_eq!( fixture.run(keys::pending_key(9)).await?, TaskDisposition::Finished ); assert_eq!(fixture.record_rkeys()?, ["a"]); Ok(()) } #[tokio::test] async fn a_repo_without_a_signal_record_writes_nothing() -> miette::Result<()> { let fixture = Fixture::new().await?; let mut filter = crate::filter::FilterConfig::new(crate::filter::FilterMode::Filter); filter.signals = vec![WANTED.into()]; fixture.state.filter.store(Arc::new(filter)); let old = post("old"); fixture.seed( &RepoState::backfilling().into_static(), 9, &[("old", old.as_slice())], )?; let a = post("a"); let car = build_car("3jzfcijpj2z2a", &[("a", a.as_slice())]).await?; fixture.pds.serve_car(car.bytes).await; assert_eq!( fixture.run(keys::pending_key(9)).await?, TaskDisposition::Finished ); assert_eq!(fixture.record_rkeys()?, ["old"]); assert_eq!(stored(&fixture, COLLECTION, "old")?, Some(old)); Ok(()) } #[tokio::test] async fn a_filtered_backfill_keeps_and_counts_only_the_wanted_collection() -> miette::Result<()> { let (a, core, site) = (post("a"), tangled_repo("core"), tangled_repo("site")); let records = [ (format!("{COLLECTION}/a"), a.as_slice()), (format!("{WANTED}/core"), core.as_slice()), (format!("{WANTED}/site"), site.as_slice()), ]; let car = build_car_keyed(DID, "3jzfcijpj2z2a", &records).await?; let fixture = Fixture::new().await?; only_collection(&fixture, WANTED); let old = post("old"); fixture.seed( &RepoState::backfilling().into_static(), 9, &[("old", old.as_slice())], )?; let mut batch = fixture.state.db.inner.batch(); db::set_record_count(&mut batch, &fixture.state.db, &fixture.did, COLLECTION, 1); batch.commit().into_diagnostic()?; fixture.pds.serve_car(car.bytes).await; assert_eq!( fixture.run(keys::pending_key(9)).await?, TaskDisposition::Finished ); // a stored record the filter no longer wants goes, like one the car doesn't have assert_eq!(fixture.record_rkeys()?, ["core", "site"]); let count = |collection| db::get_record_count(&fixture.state.db, &fixture.did, collection); assert_eq!(count(WANTED)?, 2); assert_eq!(count(COLLECTION)?, 0); Ok(()) } #[cfg(feature = "indexer_stream")] /// the stream's record events as (collection/rkey, action), in stream order fn record_events( fixture: &Fixture, ) -> miette::Result> { fixture .state .db .stream .event_range::<&[u8], _>(..) .map(|guard| { let value = guard.value().into_diagnostic()?; let event: crate::types::StoredEvent<'_> = rmp_serde::from_slice(&value).into_diagnostic()?; let key = format!("{}/{}", event.collection, event.rkey.to_smolstr()); Ok((key, event.action)) }) .collect() } #[cfg(feature = "indexer_stream")] #[tokio::test] async fn a_backfill_over_stored_records_writes_only_what_changed() -> miette::Result<()> { let (a_old, a_new, b, c, d) = (post("a"), post("a2"), post("b"), post("c"), post("d")); let fixture = Fixture::new().await?; fixture.seed( &RepoState::backfilling().into_static(), 9, &[ ("a", a_old.as_slice()), ("b", b.as_slice()), ("c", c.as_slice()), ], )?; let car = build_car( "3jzfcijpj2z2a", &[ ("a", a_new.as_slice()), ("b", b.as_slice()), ("d", d.as_slice()), ], ) .await?; fixture.pds.serve_car(car.bytes).await; assert_eq!( fixture.run(keys::pending_key(9)).await?, TaskDisposition::Finished ); assert_eq!(fixture.record_rkeys()?, ["a", "b", "d"]); assert_eq!(stored(&fixture, COLLECTION, "a")?, Some(a_new)); // in key order, so the delete of c sits between the other two assert_eq!( record_events(&fixture)?, [ ( format!("{COLLECTION}/a"), crate::db::types::DbAction::Update ), ( format!("{COLLECTION}/c"), crate::db::types::DbAction::Delete ), ( format!("{COLLECTION}/d"), crate::db::types::DbAction::Create ), ] ); assert_eq!( db::get_record_count(&fixture.state.db, &fixture.did, COLLECTION)?, 3 ); Ok(()) } #[cfg(feature = "indexer_stream")] async fn car_of(records: &[&(String, Vec)], rev: &str) -> miette::Result { let keyed = records .iter() .map(|(key, body)| (key.clone(), body.as_slice())) .collect::>(); build_car_keyed(DID, rev, &keyed).await } #[cfg(feature = "indexer_stream")] #[tokio::test] async fn rebackfilling_over_prefix_related_collections_writes_only_what_changed() -> miette::Result<()> { // collections that prefix each other or differ right before the `/`, and both kinds // of rkey, which is where heads key order and mst order could part let collections = ["a.b.c", "a.b.c.d", "a.b.cd", "a.b-c.d"]; let rkeys = [ "3jzfcijpj2z2a", "3k2akgqbabc2x", "self", "a.b", "a-b", "~", "z:z", ]; let mut records = collections .iter() .flat_map(|collection| { rkeys.iter().map(move |rkey| { let body = serde_ipld_dagcbor::to_vec(&serde_json::json!({ "$type": collection, "rkey": rkey, })) .unwrap(); (format!("{collection}/{rkey}"), body) }) }) .collect::>(); records.sort(); let every = records.iter().collect::>(); // a third of them go in the last car, spread over every collection let (dropped, kept): (Vec<_>, Vec<_>) = every.iter().enumerate().partition(|(i, _)| i % 3 == 1); let kept = kept .into_iter() .map(|(_, record)| *record) .collect::>(); let fixture = Fixture::new().await?; fixture.seed(&RepoState::backfilling().into_static(), 9, &[])?; let heads = || -> miette::Result, Vec)>> { fixture .state .db .indexer .head_prefix(keys::record_prefix_did(&fixture.did)) .map(|guard| { let (key, value) = guard.into_inner().into_diagnostic()?; Ok((key.to_vec(), value.to_vec())) }) .collect() }; let counts = || -> miette::Result> { collections .iter() .map(|collection| db::get_record_count(&fixture.state.db, &fixture.did, collection)) .collect() }; let run = |from: u64, to: u64, car: BuiltCar| { let fixture = &fixture; async move { if from != to { fixture.reindex_pending(from, to)?; } fixture.pds.serve_car(car.bytes).await; assert_eq!( fixture.run(keys::pending_key(to)).await?, TaskDisposition::Finished ); Ok::<_, miette::Report>(()) } }; run(9, 9, car_of(&every, "3jzfcijpj2z2a").await?).await?; let (events, stored, counted) = (record_events(&fixture)?, heads()?, counts()?); assert_eq!(events.len(), records.len()); assert_eq!(stored.len(), records.len()); assert_eq!(counted, [rkeys.len() as u64; 4]); // the same car again changes nothing run(9, 10, car_of(&every, "3jzfcijpj2z2a").await?).await?; assert_eq!(record_events(&fixture)?, events); assert_eq!(heads()?, stored); assert_eq!(counts()?, counted); // a car without some of them deletes just those, in key order run(10, 11, car_of(&kept, "3jzfcijpj2z2b").await?).await?; let deletes = dropped .iter() .map(|(_, (key, _))| (key.clone(), crate::db::types::DbAction::Delete)) .collect::>(); assert_eq!(record_events(&fixture)?[events.len()..], deletes); let kept_heads = stored .iter() .enumerate() .filter(|(i, _)| i % 3 != 1) .map(|(_, head)| head.clone()) .collect::>(); assert_eq!(heads()?, kept_heads); assert_eq!(counts()?.iter().sum::(), kept.len() as u64); Ok(()) } #[tokio::test] async fn a_full_backfill_holds_its_budget_until_its_last_step() -> miette::Result<()> { let body = post("a"); let car = build_car("3jzfcijpj2z2a", &[("a", body.as_slice())]).await?; let fixture = Fixture::new().await?; fixture.seed(&RepoState::backfilling().into_static(), 9, &[])?; fixture.pds.serve_car(car.bytes.clone()).await; // the repo's shard, kept busy so the write waits behind it after the fetch is done let (tx, shard) = FirehoseWorker::spawn_one_shard(fixture.state.clone()); let (busy_tx, busy) = tokio::sync::oneshot::channel(); let (release, release_rx) = std::sync::mpsc::channel::<()>(); let release_rx = parking_lot::Mutex::new(release_rx); let did = fixture.did.clone(); let busy_shard = async { let work = move |_: &AppState, _: &mut crate::ingest::indexer::bulk::ShardState| { busy_tx.send(()).unwrap(); let _ = release_rx.lock().recv(); }; tx.inner .send(IndexerMessage::OnShard( crate::ingest::indexer::message::OnShard { did, work: Box::new(work), }, )) .await .map_err(|_| miette::miette!("the shard is gone")) }; let budget = fixture.state.backfill_budget.clone(); let check = async { busy.await.unwrap(); tokio::time::sleep(std::time::Duration::from_millis(300)).await; let held = budget.used(); release.send(()).unwrap(); held }; let (result, held) = tokio::join!( fixture.run_changing_on(tx.clone(), keys::pending_key(9), busy_shard), check ); drop(tx); tokio::task::spawn_blocking(move || shard.join()) .await .into_diagnostic()? .map_err(|_| miette::miette!("indexer shard panicked"))?; assert_eq!(result??, TaskDisposition::Finished); // a spilled car holds its index and a step, not the car let peak = if fixture.config.backfill_spill_bytes < car.bytes.len() { crate::backfill::budget::spilled_peak_for(car.bytes.len()) } else { crate::backfill::budget::peak_for(car.bytes.len()) }; assert_eq!(held, peak); assert_eq!(budget.used(), 0); Ok(()) } /// a repo with a record filed under a collection other than its `$type`, which a filtered /// car drops and the scan then finds missing async fn car_with_a_misfiled_record() -> miette::Result<(BuiltCar, Vec)> { let odd = post("filed under the wrong collection"); let core = tangled_repo("core"); let records = [ (format!("{WANTED}/core"), core.as_slice()), (format!("{WANTED}/odd"), odd.as_slice()), (format!("{COLLECTION}/3jzfcijpj0001"), odd.as_slice()), ]; let car = build_car_keyed(DID, "3jzfcijpj2z2a", &records).await?; Ok((car, odd)) } /// a getBlocks answer holding `bodies` async fn blocks_car(bodies: &[&[u8]]) -> miette::Result { let cids = bodies .iter() .map(|body| crate::car::compute_cid(body)) .collect::>(); let mut bytes = Vec::new(); let mut writer = crate::car::CarWriter::new(crate::car::CarHeader::new_v1(cids.clone()), &mut bytes); for (cid, body) in cids.iter().zip(bodies) { writer.write(*cid, body).await.into_diagnostic()?; } writer.finish().await.into_diagnostic()?; Ok(Bytes::from(bytes)) } #[tokio::test] async fn get_blocks_brings_back_what_the_filtered_car_dropped() -> miette::Result<()> { let (car, odd) = car_with_a_misfiled_record().await?; let fixture = Fixture::new().await?; only_collection(&fixture, WANTED); fixture.seed(&RepoState::backfilling().into_static(), 9, &[])?; fixture.pds.serve_car(car.bytes).await; fixture .pds .serve_blocks(StatusCode::OK, blocks_car(&[&odd]).await?) .await; // the refetch still counts against the pds, so nothing else takes its slot meanwhile let result = fixture .run_changing_at(&fixture.pds.get_blocks, keys::pending_key(9), async { let slot = fixture.admission.try_acquire(&fixture.pds.base).await; assert!(slot.is_none(), "the refetch let go of the pds admission"); Ok(()) }) .await?; assert_eq!(result?, TaskDisposition::Finished); assert_eq!(fixture.record_rkeys()?, vec!["core", "odd"]); assert_eq!(stored(&fixture, WANTED, "odd")?, Some(odd)); assert_eq!( (fixture.pds.requests(), fixture.pds.block_requests()), (1, 1) ); let slot = fixture.admission.try_acquire(&fixture.pds.base).await; assert!(slot.is_some(), "the finished task kept the pds admission"); Ok(()) } #[tokio::test] async fn a_ratelimited_get_blocks_backs_off_instead_of_fetching_the_whole_car() -> miette::Result<()> { let (car, _) = car_with_a_misfiled_record().await?; let fixture = Fixture::new().await?; only_collection(&fixture, WANTED); fixture.seed(&RepoState::backfilling().into_static(), 9, &[])?; fixture.pds.serve_car(car.bytes).await; fixture .pds .serve_blocks(StatusCode::TOO_MANY_REQUESTS, Bytes::new()) .await; let err = fixture.run(keys::pending_key(9)).await.unwrap_err(); assert!(matches!(err, BackfillError::Ratelimited), "{err}"); assert_eq!( (fixture.pds.requests(), fixture.pds.block_requests()), (1, 1) ); assert!(fixture.record_rkeys()?.is_empty()); Ok(()) } #[tokio::test] async fn a_repo_that_moved_on_before_the_refetch_is_retried() -> miette::Result<()> { let (car, _) = car_with_a_misfiled_record().await?; let core = tangled_repo("core"); let moved = build_car_keyed( DID, "3jzfcijpj2z2b", &[(format!("{WANTED}/core"), core.as_slice())], ) .await?; let fixture = Fixture::new().await?; only_collection(&fixture, WANTED); fixture.seed(&RepoState::backfilling().into_static(), 9, &[])?; fixture.pds.serve_car(car.bytes).await; // the pds has no getBlocks, and by the time it says so the repo has a new commit let result = fixture .run_changing_at(&fixture.pds.get_blocks, keys::pending_key(9), async { fixture.pds.serve_car(moved.bytes.clone()).await; Ok(()) }) .await?; let err = result.unwrap_err(); assert!(err.to_string().contains("moved on"), "{err}"); assert_eq!(fixture.pds.requests(), 2); assert!(fixture.record_rkeys()?.is_empty()); Ok(()) } #[tokio::test] async fn a_repo_gone_before_the_refetch_is_retried_as_not_found() -> miette::Result<()> { let (car, _) = car_with_a_misfiled_record().await?; let fixture = Fixture::new().await?; only_collection(&fixture, WANTED); fixture.seed(&RepoState::backfilling().into_static(), 9, &[])?; fixture.pds.serve_car(car.bytes).await; let result = fixture .run_changing_at(&fixture.pds.get_blocks, keys::pending_key(9), async { *fixture.pds.response.write().await = (StatusCode::BAD_REQUEST, Bytes::from_static(NOT_FOUND_BODY)); Ok(()) }) .await?; assert!( matches!(result, Err(BackfillError::RepoNotFound)), "{result:?}" ); assert!(fixture.record_rkeys()?.is_empty()); Ok(()) } #[tokio::test] async fn record_filed_under_another_type_comes_back_from_a_whole_car() -> miette::Result<()> { // the stub has no getBlocks, like the atcr hold, so the dropped record can only come // from a second, whole car let (car, odd) = car_with_a_misfiled_record().await?; let fixture = Fixture::new().await?; only_collection(&fixture, WANTED); fixture.seed(&RepoState::backfilling().into_static(), 9, &[])?; fixture.pds.serve_car(car.bytes).await; assert_eq!( fixture.run(keys::pending_key(9)).await?, TaskDisposition::Finished ); assert_eq!(fixture.record_rkeys()?, vec!["core", "odd"]); assert_eq!(stored(&fixture, WANTED, "odd")?, Some(odd)); assert_eq!(fixture.pds.requests(), 2); Ok(()) } #[tokio::test] async fn repo_not_found_keeps_a_rooted_snapshot_and_parks_a_retry() -> miette::Result<()> { let fixture = Fixture::new().await?; let seeded = rooted("3jzfcijpj2z2a")?; let body = post("a"); fixture.seed(&seeded, 7, &[("a", body.as_slice())])?; let err = fixture.run(keys::pending_key(7)).await.unwrap_err(); assert!( matches!(err, BackfillError::RepoNotFound), "unexpected error: {err}" ); let after = fixture.repo()?; assert_eq!( after.root.as_ref().map(|root| root.data), seeded.root.as_ref().map(|root| root.data), "a non-authoritative fetch result must not clear the root" ); assert!( matches!(&after.status, RepoStatus::Error(_)), "expected a retryable error status, got {:?}", after.status ); assert!(after.active); assert_eq!(after.last_message_time, Some(101)); assert_eq!(fixture.record_rkeys()?, ["a"]); assert!( fixture .state .db .repo_metadata .get(keys::repo_metadata_key(&fixture.did)) .into_diagnostic()? .is_some(), "metadata must survive" ); assert!( fixture.pending_keys()?.is_empty(), "parked work must leave pending, or the worker spins" ); let Some(ResyncState::Error { kind, retry_count, next_retry, }) = fixture.resync()? else { panic!("expected a durable retryable error"); }; assert_eq!(kind, ResyncErrorKind::Generic); assert_eq!(retry_count, 1); assert!( next_retry > chrono::Utc::now().timestamp(), "the retry worker may only requeue a due error" ); Ok(()) } #[cfg(feature = "indexer_stream")] /// what a relay's listRepos pass does to the repo when the relay lists it as taken down fn relay_lists_it_takendown(fixture: &Fixture) -> miette::Result { let listing = RepoListing::inactive( fixture.did.clone(), crate::db::types::DbTid::from(&Tid::new("3jzfcijpj2z2b").into_diagnostic()?), ListedRepoStatus::Takendown, ); let mut txn = DbTxn::new(&fixture.state.db); let outcome = reconcile_listing(&fixture.state, &mut txn, &listing)?; txn.commit()?; Ok(outcome) } #[cfg(feature = "indexer_stream")] fn account_events( events: &mut tokio::sync::broadcast::Receiver, ) -> Vec<(bool, Option)> { std::iter::from_fn(|| events.try_recv().ok()) .filter_map(|event| match event { crate::types::BroadcastEvent::Ephemeral(event) => event.account, _ => None, }) .map(|account| (account.active, account.status.map(|s| s.to_string()))) .collect() } #[cfg(feature = "indexer_stream")] /// the pending key the repo's metadata names right now fn queued_key(fixture: &Fixture) -> miette::Result<[u8; 8]> { let bytes = fixture .state .db .repo_metadata .get(keys::repo_metadata_key(&fixture.did)) .into_diagnostic()? .expect("metadata row"); Ok(keys::pending_key(db::deser_repo_meta(&bytes)?.index_id)) } #[cfg(feature = "indexer_stream")] #[tokio::test] async fn a_relay_takedown_does_not_beat_a_pds_that_still_serves_the_repo() -> miette::Result<()> { let fixture = Fixture::new().await?; fixture.seed(&RepoState::backfilling().into_static(), 7, &[])?; let body = post("a"); let car = build_car("3jzfcijpj2z2b", &[("a", body.as_slice())]).await?; fixture.pds.serve_car(car.bytes).await; assert_eq!( fixture.run(keys::pending_key(7)).await?, TaskDisposition::Finished ); let synced = fixture.repo()?; assert!(synced.active); assert_eq!(synced.status, RepoStatus::Synced); let mut events = fixture.state.db.stream.event_tx.subscribe(); // the first pass asks the pds, which serves the repo assert_eq!( relay_lists_it_takendown(&fixture)?, ReconcileOutcome::StateOnly ); let asked = fixture.repo()?; assert!(asked.active); assert_eq!(asked.status, RepoStatus::Synced); assert!(fixture.resync()?.is_none()); assert_eq!(fixture.pending_keys()?.len(), 1); assert_eq!( fixture.run(queued_key(&fixture)?).await?, TaskDisposition::Finished ); assert_eq!(fixture.pds.requests(), 2); // later passes and a restart leave it synced without fetching it again for _ in 0..2 { assert_eq!( relay_lists_it_takendown(&fixture)?, ReconcileOutcome::Unchanged ); crate::backfill::manager::queue_gone_backfills(&fixture.state)?; assert!(fixture.pending_keys()?.is_empty()); assert!(fixture.resync()?.is_none()); let after = fixture.repo()?; assert!(after.active); assert_eq!(after.status, RepoStatus::Synced); } assert_eq!(fixture.pds.requests(), 2); assert_eq!(account_events(&mut events), []); assert_eq!(fixture.record_rkeys()?, ["a"]); Ok(()) } #[cfg(feature = "indexer_stream")] #[tokio::test] async fn a_relay_takedown_the_pds_confirms_makes_the_repo_inactive() -> miette::Result<()> { let fixture = Fixture::new().await?; fixture.seed(&RepoState::backfilling().into_static(), 7, &[])?; let body = post("a"); let car = build_car("3jzfcijpj2z2b", &[("a", body.as_slice())]).await?; fixture.pds.serve_car(car.bytes).await; assert_eq!( fixture.run(keys::pending_key(7)).await?, TaskDisposition::Finished ); let mut events = fixture.state.db.stream.event_tx.subscribe(); assert_eq!( relay_lists_it_takendown(&fixture)?, ReconcileOutcome::StateOnly ); assert_eq!(account_events(&mut events), []); *fixture.pds.response.write().await = ( StatusCode::BAD_REQUEST, Bytes::from_static(br#"{"error":"RepoTakendown","message":"gone"}"#), ); assert_eq!( fixture.run(queued_key(&fixture)?).await?, TaskDisposition::Finished ); let after = fixture.repo()?; assert!(!after.active); assert_eq!(after.status, RepoStatus::Takendown); assert!(matches!( fixture.resync()?, Some(ResyncState::Gone { status: RepoStatus::Takendown }) )); assert!(fixture.pending_keys()?.is_empty()); assert_eq!( account_events(&mut events), [(false, Some("takendown".to_string()))] ); assert_eq!( relay_lists_it_takendown(&fixture)?, ReconcileOutcome::Unchanged ); assert_eq!(account_events(&mut events), []); Ok(()) } #[cfg(feature = "indexer_stream")] #[tokio::test] async fn a_relay_listing_does_not_reactivate_a_repo_the_pds_refuses() -> miette::Result<()> { let fixture = Fixture::new().await?; fixture.seed(&RepoState::backfilling().into_static(), 7, &[])?; let body = post("a"); let car = build_car("3jzfcijpj2z2b", &[("a", body.as_slice())]).await?; fixture.pds.serve_car(car.bytes).await; assert_eq!( fixture.run(keys::pending_key(7)).await?, TaskDisposition::Finished ); *fixture.pds.response.write().await = ( StatusCode::BAD_REQUEST, Bytes::from_static(br#"{"error":"RepoDeactivated","message":"gone"}"#), ); fixture.reindex_pending(7, 8)?; assert_eq!( fixture.run(keys::pending_key(8)).await?, TaskDisposition::Finished ); let refused = fixture.repo()?; assert!(!refused.active); assert_eq!(refused.status, RepoStatus::Deactivated); let mut events = fixture.state.db.stream.event_tx.subscribe(); let relay_lists_it_active = || -> miette::Result { let listing = RepoListing::active(fixture.did.clone(), crate::db::types::DbTid::from(&car.rev)); let mut txn = DbTxn::new(&fixture.state.db); let outcome = reconcile_listing(&fixture.state, &mut txn, &listing)?; txn.commit()?; Ok(outcome) }; // the listing only asks the pds, which still refuses assert_eq!(relay_lists_it_active()?, ReconcileOutcome::StateOnly); assert!(!fixture.repo()?.active); assert_eq!(account_events(&mut events), []); assert_eq!( fixture.run(queued_key(&fixture)?).await?, TaskDisposition::Finished ); assert_eq!(fixture.pds.requests(), 3); // the refused fetch says inactive again, like every gone retry does assert_eq!( account_events(&mut events), [(false, Some("deactivated".to_string()))] ); for _ in 0..2 { assert_eq!(relay_lists_it_active()?, ReconcileOutcome::Unchanged); let after = fixture.repo()?; assert!(!after.active); assert_eq!(after.status, RepoStatus::Deactivated); } assert_eq!(fixture.pds.requests(), 3); assert_eq!(account_events(&mut events), []); Ok(()) } #[tokio::test] async fn backoff_grows_across_requeues_until_a_fetch_succeeds() -> miette::Result<()> { let fixture = Fixture::new().await?; fixture.seed(&RepoState::backfilling().into_static(), 9, &[])?; let metadata = || -> miette::Result { let bytes = fixture .state .db .repo_metadata .get(keys::repo_metadata_key(&fixture.did)) .into_diagnostic()? .expect("metadata row"); db::deser_repo_meta(&bytes) }; for (failures, minutes) in [(1, 2), (2, 4), (3, 8), (4, 16), (5, 32), (6, 60), (7, 60)] { let before = chrono::Utc::now().timestamp(); let pending_key = keys::pending_key(metadata()?.index_id); assert!(matches!( fixture.run(pending_key).await, Err(BackfillError::RepoNotFound) )); let Some(ResyncState::Error { retry_count, next_retry, .. }) = fixture.resync()? else { panic!("expected a retryable error after failure {failures}"); }; assert_eq!(retry_count, failures); assert_eq!(metadata()?.retry_count, failures); let delay = next_retry - before; let expected = minutes * 60; assert!( (expected * 9 / 10 - 1..=expected * 11 / 10 + 5).contains(&delay), "failure {failures} waits {delay}s, expected about {expected}s" ); assert_eq!(fixture.state.db.get_count_sync("resync"), 1); assert_eq!(fixture.state.db.get_count_sync("error_generic"), 1); // what the retry worker does once the entry is due assert!(crate::backfill::manager::queue_resync_if( &fixture.state, &fixture.did, |_| true )?); assert!(fixture.resync()?.is_none()); assert_eq!(fixture.pending_keys()?.len(), 1); assert_eq!(fixture.state.db.get_count_sync("resync"), 0); assert_eq!(fixture.state.db.get_count_sync("error_generic"), 0); } let body = post("a"); let car = build_car("3jzfcijpj2z2b", &[("a", body.as_slice())]).await?; fixture.pds.serve_car(car.bytes).await; let pending_key = keys::pending_key(metadata()?.index_id); assert_eq!(fixture.run(pending_key).await?, TaskDisposition::Finished); assert_eq!(metadata()?.retry_count, 0); assert!(fixture.resync()?.is_none()); assert!(fixture.pending_keys()?.is_empty()); Ok(()) } #[tokio::test] async fn a_fetch_that_succeeds_leaves_the_refetch_allowance_as_it_was() -> miette::Result<()> { use crate::ingest::indexer::refetch::tests as refetch; let fixture = Fixture::new().await?; fixture.seed(&RepoState::backfilling().into_static(), 9, &[])?; let allowance = crate::types::Allowance { since: chrono::Utc::now().timestamp(), used: 3, }; let mut metadata = refetch::metadata_of(&fixture.state, &fixture.did); metadata.refetch = Some(allowance); refetch::put_metadata(&fixture.state, &fixture.did, &metadata); let body = post("a"); let car = build_car("3jzfcijpj2z2b", &[("a", body.as_slice())]).await?; fixture.pds.serve_car(car.bytes).await; assert_eq!( fixture.run(keys::pending_key(metadata.index_id)).await?, TaskDisposition::Finished ); let after = refetch::metadata_of(&fixture.state, &fixture.did); assert_eq!(after.retry_count, 0); assert_eq!(after.refetch, Some(allowance)); Ok(()) } #[tokio::test] async fn repo_not_found_on_a_never_rooted_repo_uses_the_same_retry_policy() -> miette::Result<()> { let fixture = Fixture::new().await?; fixture.seed(&RepoState::backfilling().into_static(), 9, &[])?; let err = fixture.run(keys::pending_key(9)).await.unwrap_err(); assert!( matches!(err, BackfillError::RepoNotFound), "unexpected error: {err}" ); let after = fixture.repo()?; assert!(after.root.is_none(), "no root may be invented"); assert!( matches!(&after.status, RepoStatus::Error(_)), "expected a retryable error status, got {:?}", after.status ); assert!(fixture.record_rkeys()?.is_empty()); assert!(fixture.pending_keys()?.is_empty()); assert!(matches!( fixture.resync()?, Some(ResyncState::Error { kind: ResyncErrorKind::Generic, .. }) )); Ok(()) } #[tokio::test] async fn a_key_whose_task_already_finished_is_not_run_again() -> miette::Result<()> { let fixture = Fixture::new().await?; let seeded = rooted("3jzfcijpj2z2a")?; let body = post("a"); fixture.seed(&seeded, 7, &[("a", body.as_slice())])?; // the task that owned key 7 already finished and removed it let mut batch = fixture.state.db.inner.batch(); fixture.state.db.indexer.stage_pending_remove( &mut batch, &fixture.did, &keys::pending_key(7), ); batch.commit().into_diagnostic()?; assert_eq!( fixture.run(keys::pending_key(7)).await?, TaskDisposition::Finished ); assert_eq!(fixture.pds.requests(), 0); let after = fixture.repo()?; assert_eq!(after.status, seeded.status); assert_eq!( after.root.as_ref().map(|root| root.data), seeded.root.as_ref().map(|root| root.data) ); Ok(()) } #[tokio::test] async fn repo_not_found_cannot_overwrite_a_replaced_pending_entry() -> miette::Result<()> { let fixture = Fixture::new().await?; let seeded = rooted("3jzfcijpj2z2a")?; let body = post("a"); fixture.seed(&seeded, 7, &[("a", body.as_slice())])?; let err = fixture .run_changing(keys::pending_key(7), async { fixture.reindex_pending(7, 11) }) .await? .unwrap_err(); assert!( matches!(err, BackfillError::RepoNotFound), "unexpected error: {err}" ); assert_eq!(fixture.pending_keys()?, [keys::pending_key(11).to_vec()]); assert!( fixture.resync()?.is_none(), "stale work must not park a retry against a newer queue entry" ); let after = fixture.repo()?; assert_eq!(after.status, RepoStatus::Synced); assert_eq!( after.root.as_ref().map(|root| root.data), seeded.root.as_ref().map(|root| root.data) ); assert_eq!(fixture.record_rkeys()?, ["a"]); Ok(()) } #[tokio::test] async fn authoritative_deletion_is_not_resurrected_by_a_late_repo_not_found() -> miette::Result<()> { let fixture = Fixture::new().await?; let seeded = rooted("3jzfcijpj2z2a")?; let body = post("a"); fixture.seed(&seeded, 7, &[("a", body.as_slice())])?; let err = fixture .run_changing(keys::pending_key(7), async { fixture.delete_authoritatively(&seeded) }) .await? .unwrap_err(); assert!( matches!(err, BackfillError::RepoNotFound), "unexpected error: {err}" ); let after = fixture.repo()?; assert_eq!(after.status, RepoStatus::Deleted); assert!(!after.active, "the tombstone must stay inactive"); assert!( fixture.record_rkeys()?.is_empty(), "erased records must stay erased" ); assert!(fixture.pending_keys()?.is_empty()); assert!( fixture.resync()?.is_none(), "a deleted repo must not be re-queued" ); Ok(()) } #[tokio::test] async fn authoritative_deletion_is_not_resurrected_by_a_late_successful_backfill() -> miette::Result<()> { let fixture = Fixture::new().await?; let seeded = rooted("3jzfcijpj2z2a")?; let body = post("a"); fixture.seed(&seeded, 7, &[("a", body.as_slice())])?; let newer = post("b"); let car = build_car( "3jzfcijpj2z2b", &[("a", body.as_slice()), ("b", newer.as_slice())], ) .await?; fixture.pds.serve_car(car.bytes).await; let result = fixture .run_changing(keys::pending_key(7), async { fixture.delete_authoritatively(&seeded) }) .await?; assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); let after = fixture.repo()?; assert_eq!(after.status, RepoStatus::Deleted); assert!(!after.active, "the tombstone must stay inactive"); assert!( fixture.record_rkeys()?.is_empty(), "erased records must stay erased" ); assert!(fixture.pending_keys()?.is_empty()); Ok(()) } #[tokio::test] async fn an_untrack_during_the_fetch_is_not_undone() -> miette::Result<()> { let fixture = Fixture::new().await?; let seeded = rooted("3jzfcijpj2z2a")?; let body = post("a"); fixture.seed(&seeded, 7, &[("a", body.as_slice())])?; let newer = post("b"); let car = build_car( "3jzfcijpj2z2b", &[("a", body.as_slice()), ("b", newer.as_slice())], ) .await?; fixture.pds.serve_car(car.bytes).await; let repos = crate::control::ReposControl::new(fixture.state.clone()); let result = fixture .run_changing(keys::pending_key(7), async { let untracked = repos.untrack([fixture.did.clone()]).await?; assert_eq!(untracked, std::slice::from_ref(&fixture.did)); Ok(()) }) .await?; assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); let metadata = fixture .state .db .repo_metadata .get(keys::repo_metadata_key(&fixture.did)) .into_diagnostic()? .expect("repository metadata row"); assert!(!db::deser_repo_meta(&metadata)?.tracked); assert_eq!( fixture.record_rkeys()?, ["a"], "the fetched car must not land" ); assert_eq!( fixture.repo()?.root.map(|root| root.data), seeded.root.as_ref().map(|root| root.data) ); assert!(fixture.pending_keys()?.is_empty()); assert!(fixture.resync()?.is_none()); Ok(()) } #[tokio::test] async fn an_exclude_during_the_fetch_takes_the_repo_out_of_the_pending_count() -> miette::Result<()> { let fixture = Fixture::new().await?; let repos = crate::control::ReposControl::new(fixture.state.clone()); assert_eq!(repos.track([fixture.did.clone()]).await?.len(), 1); assert_eq!(fixture.state.db.get_count("pending").await, 1); let pending_key: [u8; 8] = fixture.pending_keys()?[0] .as_slice() .try_into() .into_diagnostic()?; let a = post("a"); let car = build_car("3jzfcijpj2z2a", &[("a", a.as_slice())]).await?; fixture.pds.serve_car(car.bytes).await; // after the task's own exclusion check, so it's the first step that finds it let result = fixture .run_changing(pending_key, async { fixture .state .db .filter .insert(crate::db::filter::exclude_key(fixture.did.as_str())?, []) .into_diagnostic() }) .await?; assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); assert!(fixture.record_rkeys()?.is_empty()); assert!(fixture.pending_keys()?.is_empty()); assert_eq!(fixture.state.db.get_count("pending").await, 0); Ok(()) } #[tokio::test] async fn a_deactivation_during_the_fetch_survives_it() -> miette::Result<()> { let fixture = Fixture::new().await?; let seeded = rooted("3jzfcijpj2z2a")?; let body = post("a"); fixture.seed(&seeded, 7, &[("a", body.as_slice())])?; let newer = post("b"); let car = build_car( "3jzfcijpj2z2b", &[("a", body.as_slice()), ("b", newer.as_slice())], ) .await?; fixture.pds.serve_car(car.bytes).await; let result = fixture .run_changing(keys::pending_key(7), async { // what the relay writes for an account event that lands while getRepo streams let mut row = fixture.repo()?; row.active = false; row.status = RepoStatus::Deactivated; fixture.put_repo(&row) }) .await?; assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); let after = fixture.repo()?; assert_eq!(after.status, RepoStatus::Deactivated); assert!(!after.active); assert_eq!( after.root.map(|root| root.rev), Some(crate::db::types::DbTid::from(&car.rev)) ); assert_eq!( after.pds.as_deref(), Some(fixture.pds.base.as_str()), "nothing else touched the identity, so the doc the backfill resolved lands" ); assert_eq!(fixture.record_rkeys()?, ["a", "b"]); Ok(()) } #[tokio::test] async fn a_backfill_brings_back_a_repo_marked_inactive_before_it() -> miette::Result<()> { let fixture = Fixture::new().await?; // say a reactivation went by while hydrant was down let mut seeded = rooted("3jzfcijpj2z2a")?; seeded.active = false; seeded.status = RepoStatus::Deactivated; fixture.seed(&seeded, 7, &[])?; let body = post("a"); let car = build_car("3jzfcijpj2z2b", &[("a", body.as_slice())]).await?; fixture.pds.serve_car(car.bytes).await; assert_eq!( fixture.run(keys::pending_key(7)).await?, TaskDisposition::Finished ); let after = fixture.repo()?; assert!(after.active); assert_eq!(after.status, RepoStatus::Synced); assert_eq!(fixture.record_rkeys()?, ["a"]); Ok(()) } #[tokio::test] async fn a_status_written_during_the_fetch_beats_what_the_fetch_saw() -> miette::Result<()> { let fixture = Fixture::new().await?; let mut seeded = rooted("3jzfcijpj2z2a")?; seeded.active = false; seeded.status = RepoStatus::Deactivated; fixture.seed(&seeded, 7, &[])?; let body = post("a"); let car = build_car("3jzfcijpj2z2b", &[("a", body.as_slice())]).await?; fixture.pds.serve_car(car.bytes).await; let result = fixture .run_changing(keys::pending_key(7), async { let mut row = fixture.repo()?; row.status = RepoStatus::Takendown; fixture.put_repo(&row) }) .await?; assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); let after = fixture.repo()?; assert!(!after.active); assert_eq!(after.status, RepoStatus::Takendown); Ok(()) } #[tokio::test] async fn a_commit_buffered_during_the_fetch_is_applied_before_the_repo_leaves_the_queue() -> miette::Result<()> { let fixture = Fixture::new().await?; let newer = fixture.fetch_with_a_buffered_commit().await?; assert_eq!( fixture.run(keys::pending_key(7)).await?, TaskDisposition::Finished ); fixture.assert_caught_up_to(&newer) } #[tokio::test] async fn a_commit_arriving_before_the_last_step_waits_for_the_drain() -> miette::Result<()> { let fixture = Fixture::new().await?; let (newer, b) = fixture.fetch_behind_a_live_commit().await?; // the last step comes here instead of the shard, so a live commit can reach the shard // first let (handoff_tx, mut handoff_worker) = FirehoseWorker::new(fixture.state.clone(), 1); let mut handoff = handoff_worker .bulk_rxs .pop() .expect("single shard receiver"); let (shard_tx, shard) = FirehoseWorker::spawn_one_shard(fixture.state.clone()); let gone = |_| miette::miette!("the shard is gone"); let forward = async { let last = handoff.recv().await.expect("the task sends its last step"); let event = IndexerEvent { seq: 1, firehose: Url::parse("wss://relay.example").into_diagnostic()?, ticket: None, row: None, data: IndexerEventData::Commit(IndexerCommitData { commit: fixture.live_commit(&newer, "b", &b)?, chain_break: false, parsed_blocks: crate::car::parse_commit_car(newer.bytes.clone(), false)?, }), }; shard_tx .send(IndexerMessage::Event(Box::new(event))) .await .map_err(gone)?; // the shard serves its live messages in order, so once this answers the commit is in shard_tx.on_owner(&fixture.did, |_, _| ()).await?; assert_eq!(fixture.buffered(), 1, "applied before the drain"); assert!(fixture.record_rkeys()?.is_empty()); shard_tx .bulk .send(last) .await .map_err(|_| miette::miette!("the shard is gone")) }; let (result, forwarded) = tokio::join!(fixture.run_with(handoff_tx, keys::pending_key(7)), forward); forwarded?; drop(shard_tx); tokio::task::spawn_blocking(move || shard.join()) .await .into_diagnostic()? .map_err(|_| miette::miette!("indexer shard panicked"))?; assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); fixture.assert_caught_up_to(&newer) } #[tokio::test] async fn a_buffered_commit_the_car_already_covers_is_dropped() -> miette::Result<()> { let fixture = Fixture::new().await?; fixture.seed(&rooted("3jzfcijpj2z2a")?, 7, &[])?; let a = post("a"); let b = post("b"); // left over from an earlier failed backfill, and `b` was deleted again before the car let older = build_car("3jzfcijpj2z2b", &[("a", a.as_slice()), ("b", b.as_slice())]).await?; fixture.buffer(&older, "b", &b)?; let fetched = build_car("3jzfcijpj2z2c", &[("a", a.as_slice())]).await?; fixture.pds.serve_car(fetched.bytes).await; assert_eq!( fixture.run(keys::pending_key(7)).await?, TaskDisposition::Finished ); assert_eq!(fixture.record_rkeys()?, ["a"]); let after = fixture.repo()?; assert_eq!( after.root.map(|root| root.rev), Some(crate::db::types::DbTid::from(&fetched.rev)) ); assert_eq!(after.status, RepoStatus::Synced); assert_eq!(fixture.buffered(), 0); Ok(()) } #[tokio::test] async fn a_commit_the_relay_is_already_past_keeps_the_newer_root() -> miette::Result<()> { let fixture = Fixture::new().await?; fixture.seed(&rooted("3jzfcijpj2z2a")?, 7, &[])?; let a = post("a"); let b = post("b"); let fetched = build_car("3jzfcijpj2z2b", &[("a", a.as_slice())]).await?; fixture.pds.serve_car(fetched.bytes).await; assert_eq!( fixture.run(keys::pending_key(7)).await?, TaskDisposition::Finished ); // the relay validated this commit and the one after it before the shard got to it let live = build_car("3jzfcijpj2z2c", &[("a", a.as_slice()), ("b", b.as_slice())]).await?; let mut row = fixture.repo()?; row.root = rooted("3jzfcijpj2z2d")?.root; fixture .state .db .repos .insert(keys::repo_key(&fixture.did), db::ser_repo_state(&row)?) .into_diagnostic()?; fixture .apply_live(&live, fixture.live_commit(&live, "b", &b)?) .await?; assert_eq!(fixture.record_rkeys()?, ["a", "b"]); assert_eq!(fixture.repo()?.root, rooted("3jzfcijpj2z2d")?.root); Ok(()) } /// queues the repo under 7 and serves a car at 2c that edited `a` and deleted `b` after a /// commit at 2b, which the relay validated before the fetch and which put `b` and the /// first `a` down. returns that commit's car and the commit async fn fetch_past_a_validated_commit( fixture: &Fixture, ) -> miette::Result<(BuiltCar, StreamCommit<'static>)> { fixture.seed(&rooted("3jzfcijpj2z2a")?, 7, &[])?; let a = post("a"); let b = post("b"); let older = build_car("3jzfcijpj2z2b", &[("a", a.as_slice()), ("b", b.as_slice())]).await?; let mut commit = fixture.live_commit(&older, "b", &b)?; commit.ops.push(RepoOp { action: RepoOpAction::Update, cid: Some(CidLink::from(crate::car::compute_cid(&a))), path: CowStr::Owned(format!("{COLLECTION}/a").into()), prev: None, }); let fetched = build_car("3jzfcijpj2z2c", &[("a", post("a, edited").as_slice())]).await?; fixture.pds.serve_car(fetched.bytes).await; Ok((older, commit)) } fn assert_records_are_the_cars(fixture: &Fixture) -> miette::Result<()> { assert_eq!(stored(fixture, COLLECTION, "a")?, Some(post("a, edited"))); assert_eq!(fixture.record_rkeys()?, ["a"]); Ok(()) } #[tokio::test] async fn a_live_commit_older_than_the_drained_car_leaves_its_records_alone() -> miette::Result<()> { let fixture = Fixture::new().await?; let (older, commit) = fetch_past_a_validated_commit(&fixture).await?; assert_eq!( fixture.run(keys::pending_key(7)).await?, TaskDisposition::Finished ); assert_records_are_the_cars(&fixture)?; // the backfill's handoff got to the shard first, so this reaches a repo that isn't queued fixture.apply_live(&older, commit).await?; assert_records_are_the_cars(&fixture)?; let c = post("c"); let newer = build_car( "3jzfcijpj2z2d", &[("a", post("a, edited").as_slice()), ("c", c.as_slice())], ) .await?; fixture .apply_live(&newer, fixture.live_commit(&newer, "c", &c)?) .await?; assert_eq!(fixture.record_rkeys()?, ["a", "c"]); Ok(()) } #[tokio::test] async fn a_commit_older_than_the_car_buffered_before_its_last_step_is_not_drained() -> miette::Result<()> { let fixture = Fixture::new().await?; let (older, commit) = fetch_past_a_validated_commit(&fixture).await?; // the last step comes here instead of the shard, so the older commit reaches the shard // first and gets buffered let (handoff_tx, mut handoff_worker) = FirehoseWorker::new(fixture.state.clone(), 1); let mut handoff = handoff_worker .bulk_rxs .pop() .expect("single shard receiver"); let (shard_tx, shard) = FirehoseWorker::spawn_one_shard(fixture.state.clone()); let gone = |_| miette::miette!("the shard is gone"); let forward = async { let last = handoff.recv().await.expect("the task sends its last step"); let event = IndexerEvent { seq: 1, firehose: Url::parse("wss://relay.example").into_diagnostic()?, ticket: None, row: None, data: IndexerEventData::Commit(IndexerCommitData { commit, chain_break: false, parsed_blocks: crate::car::parse_commit_car(older.bytes.clone(), false)?, }), }; shard_tx .send(IndexerMessage::Event(Box::new(event))) .await .map_err(gone)?; shard_tx .bulk .send(last) .await .map_err(|_| miette::miette!("the shard is gone")) }; let (result, forwarded) = tokio::join!(fixture.run_with(handoff_tx, keys::pending_key(7)), forward); forwarded?; drop(shard_tx); tokio::task::spawn_blocking(move || shard.join()) .await .into_diagnostic()? .map_err(|_| miette::miette!("indexer shard panicked"))?; assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); assert_records_are_the_cars(&fixture)?; assert_eq!(fixture.buffered(), 0); assert!(fixture.pending_keys()?.is_empty()); Ok(()) } #[tokio::test] async fn a_live_commit_to_a_repo_that_never_backfilled_applies() -> miette::Result<()> { let fixture = Fixture::new().await?; fixture.seed(&rooted("3jzfcijpj2z2a")?, 7, &[])?; let mut batch = fixture.state.db.inner.batch(); fixture.state.db.indexer.stage_pending_remove( &mut batch, &fixture.did, &keys::pending_key(7), ); batch.commit().into_diagnostic()?; let b = post("b"); let live = build_car("3jzfcijpj2z2b", &[("b", b.as_slice())]).await?; fixture .apply_live(&live, fixture.live_commit(&live, "b", &b)?) .await?; assert_eq!(fixture.record_rkeys()?, ["b"]); Ok(()) } #[tokio::test] async fn a_buffered_commit_failing_halfway_writes_none_of_its_records() -> miette::Result<()> { let fixture = Fixture::new().await?; fixture.fetch_with_a_buffered_commit().await?; let c = post("c"); let bad = build_car("3jzfcijpj2z2d", &[("c", c.as_slice())]).await?; let commit = fixture.half_bad_commit(&bad, &c)?; let mut txn = DbTxn::new(&fixture.state.db); crate::ops::persist_to_resync_buffer(&mut txn, &fixture.state.db, &fixture.did, &commit)?; txn.commit()?; let result = fixture.run(keys::pending_key(7)).await; assert!( matches!(result, Err(BackfillError::Generic(_))), "{result:?}" ); assert_eq!(fixture.record_rkeys()?, ["a", "b"]); assert_eq!(fixture.state.db.get_count_sync("records"), 2); Ok(()) } #[tokio::test] async fn a_live_commit_failing_halfway_writes_none_of_its_records() -> miette::Result<()> { let fixture = Fixture::new().await?; fixture.seed(&rooted("3jzfcijpj2z2a")?, 7, &[])?; let a = post("a"); let c = post("c"); let fetched = build_car("3jzfcijpj2z2b", &[("a", a.as_slice())]).await?; fixture.pds.serve_car(fetched.bytes).await; assert_eq!( fixture.run(keys::pending_key(7)).await?, TaskDisposition::Finished ); let live = build_car("3jzfcijpj2z2c", &[("a", a.as_slice()), ("c", c.as_slice())]).await?; fixture .apply_live(&live, fixture.half_bad_commit(&live, &c)?) .await?; assert_eq!(fixture.record_rkeys()?, ["a"]); assert_eq!(fixture.state.db.get_count_sync("records"), 1); Ok(()) } #[tokio::test] async fn an_orphaned_key_doesnt_write_its_car_over_live_records() -> miette::Result<()> { let fixture = Fixture::new().await?; // metadata names 8, which isn't queued, so the shard applied z live. 7 is an orphan let z = post("z"); fixture.seed(&rooted("3jzfcijpj2z2z")?, 7, &[("z", z.as_slice())])?; let mut batch = fixture.state.db.inner.batch(); batch.insert( &fixture.state.db.repo_metadata, keys::repo_metadata_key(&fixture.did), db::ser_repo_meta(&RepoMetadata::backfilling(8))?, ); batch.commit().into_diagnostic()?; let a = post("a"); let car = build_car("3jzfcijpj2z2b", &[("a", a.as_slice())]).await?; fixture.pds.serve_car(car.bytes).await; assert_eq!( fixture.run(keys::pending_key(7)).await?, TaskDisposition::Finished ); assert_eq!(fixture.record_rkeys()?, ["z"]); assert!(fixture.pending_keys()?.is_empty()); Ok(()) } #[tokio::test] async fn a_buffered_commit_that_cant_apply_costs_one_refetch() -> miette::Result<()> { let fixture = Fixture::new().await?; fixture.fetch_with_a_buffered_commit().await?; // a later live commit the drain can never apply, since its rkey doesn't parse let c = post("c"); let bad = build_car("3jzfcijpj2z2d", &[("c", c.as_slice())]).await?; fixture.buffer(&bad, "bad rkey", &c)?; let first = fixture.run(keys::pending_key(7)).await; let Err(BackfillError::Generic(e)) = &first else { panic!("{first:?}"); }; // the shard says why the drain failed assert!(e.to_string().contains("bad rkey"), "{e}"); // the commit before it was drained anyway assert_eq!(fixture.record_rkeys()?, ["a", "b"]); assert_eq!(fixture.buffered(), 0); assert!(matches!( fixture.resync()?, Some(ResyncState::Error { retry_count: 1, .. }) )); // the retry worker's requeue fixture.clear_resync()?; fixture.reindex_pending(7, 8)?; let again = fixture.run(keys::pending_key(8)).await; assert!(matches!(again, Ok(TaskDisposition::Finished)), "{again:?}"); assert_eq!(fixture.pds.requests(), 2); assert!(fixture.pending_keys()?.is_empty()); assert!(fixture.resync()?.is_none()); Ok(()) } /// a commit the firehose skipped leaves the next one not following the car, and applied it /// would land on records it doesn't go with #[tokio::test] async fn a_buffered_commit_that_doesnt_follow_the_car_costs_one_refetch() -> miette::Result<()> { let fixture = Fixture::new().await?; let newer = fixture.fetch_with_a_buffered_commit().await?; let (a, b, d) = (post("a"), post("b"), post("d")); let after_gap = build_car( "3jzfcijpj2z2e", &[ ("a", a.as_slice()), ("b", b.as_slice()), ("d", d.as_slice()), ], ) .await?; let mut commit = fixture.live_commit(&after_gap, "d", &d)?; commit.since = Some(Tid::new("3jzfcijpj2z2d").into_diagnostic()?); fixture.buffer_commit(&commit)?; let result = fixture.run(keys::pending_key(7)).await; let Err(BackfillError::Generic(e)) = &result else { panic!("{result:?}"); }; assert!(e.to_string().contains("doesn't follow"), "{e}"); // the commit before the gap still went in assert_eq!(fixture.record_rkeys()?, ["a", "b"]); assert_eq!( fixture.repo()?.root.map(|root| root.rev), Some(crate::db::types::DbTid::from(&newer.rev)) ); assert_eq!(fixture.buffered(), 0); assert!(matches!( fixture.resync()?, Some(ResyncState::Error { retry_count: 1, .. }) )); Ok(()) } /// the relay puts every commit it forwards on the row, so one buffered during the fetch can /// already be the root when the drain gets to it, without its records #[tokio::test] async fn a_buffered_commit_the_relay_already_put_on_the_row_is_drained() -> miette::Result<()> { let fixture = Fixture::new().await?; let (newer, b) = fixture.fetch_behind_a_live_commit().await?; let mut commit = fixture.live_commit(&newer, "b", &b)?; commit.since = Some(Tid::new("3jzfcijpj2z2b").into_diagnostic()?); fixture.buffer_commit(&commit)?; let result = fixture .run_changing(keys::pending_key(7), async { let mut row = fixture.repo()?; row.root = Some(crate::types::Commit { version: 3, rev: crate::db::types::DbTid::from(&newer.rev), data: newer.mst_root, prev: None, sig: Bytes::from_static(b"signature"), }); fixture.put_repo(&row) }) .await?; assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); fixture.assert_caught_up_to(&newer) } /// the pds is the authority on the repo, so a car older than what we had replaces the /// records, and the root goes back with them so the next commit shows the gap #[tokio::test] async fn a_car_behind_the_stored_root_takes_the_root_back_too() -> miette::Result<()> { let fixture = Fixture::new().await?; let (a, b) = (post("a"), post("b")); fixture.seed( &rooted("3jzfcijpj2z2c")?, 7, &[("a", a.as_slice()), ("b", b.as_slice())], )?; let behind = build_car("3jzfcijpj2z2b", &[("a", a.as_slice())]).await?; fixture.pds.serve_car(behind.bytes).await; let result = fixture.run(keys::pending_key(7)).await; assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); assert_eq!(fixture.record_rkeys()?, ["a"]); assert_eq!( fixture.repo()?.root.map(|root| root.rev), Some(crate::db::types::DbTid::from(&behind.rev)) ); Ok(()) } #[tokio::test] async fn a_backfill_the_shard_never_answers_is_retried_with_its_buffer_kept() -> miette::Result<()> { let fixture = Fixture::new().await?; fixture.fetch_with_a_buffered_commit().await?; let result = fixture .run_with(fixture.unanswered(), keys::pending_key(7)) .await; let Err(BackfillError::Generic(e)) = &result else { panic!("{result:?}"); }; assert!(e.to_string().contains("shard is gone"), "{e}"); assert!(matches!( fixture.resync()?, Some(ResyncState::Error { retry_count: 1, .. }) )); assert!(fixture.pending_keys()?.is_empty()); assert_eq!(fixture.buffered(), 1); Ok(()) } #[tokio::test] async fn a_backfill_stopped_before_its_drain_stays_queued() -> miette::Result<()> { let fixture = Fixture::new().await?; fixture.seed(&rooted("3jzfcijpj2z2a")?, 7, &[])?; let a = post("a"); let fetched = build_car("3jzfcijpj2z2b", &[("a", a.as_slice())]).await?; fixture.pds.serve_car(fetched.bytes).await; fixture.state.stop.trigger(); assert_eq!( fixture .run_with(fixture.unanswered(), keys::pending_key(7)) .await?, TaskDisposition::Finished ); assert_eq!(fixture.pending_keys()?, [keys::pending_key(7).to_vec()]); assert!(fixture.resync()?.is_none()); Ok(()) } #[tokio::test] async fn a_newer_event_repeating_the_old_value_beats_what_the_fetch_saw() -> miette::Result<()> { let fixture = Fixture::new().await?; let mut seeded = rooted("3jzfcijpj2z2a")?; seeded.active = false; seeded.status = RepoStatus::Deactivated; seeded.last_account_time = Some(1_000); fixture.seed(&seeded, 7, &[])?; let body = post("a"); let car = build_car("3jzfcijpj2z2b", &[("a", body.as_slice())]).await?; fixture.pds.serve_car(car.bytes).await; let result = fixture .run_changing(keys::pending_key(7), async { // what the relay writes for a newer deactivation that says what the row // already says let mut row = fixture.repo()?; row.last_account_time = Some(2_000); fixture.put_repo(&row) }) .await?; assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); let after = fixture.repo()?; assert!(!after.active); assert_eq!(after.status, RepoStatus::Deactivated); assert_eq!(after.last_account_time, Some(2_000)); Ok(()) } #[tokio::test] async fn an_identity_clock_moved_without_a_doc_keeps_the_backfills_doc() -> miette::Result<()> { let fixture = Fixture::new().await?; fixture.seed(&rooted("3jzfcijpj2z2a")?, 7, &[])?; let body = post("a"); let car = build_car("3jzfcijpj2z2b", &[("a", body.as_slice())]).await?; fixture.pds.serve_car(car.bytes).await; let result = fixture .run_changing(keys::pending_key(7), async { // the relay handling an identity event it got no fresh doc for let mut row = fixture.repo()?; row.last_identity_time = Some(2_000); fixture.put_repo(&row) }) .await?; assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); let after = fixture.repo()?; assert_eq!(after.pds.as_deref(), Some(fixture.pds.base.as_str())); assert_eq!(after.last_identity_time, Some(2_000)); Ok(()) } #[tokio::test] async fn an_identity_written_during_the_fetch_outlives_the_backfills_doc() -> miette::Result<()> { let fixture = Fixture::new().await?; let seeded = rooted("3jzfcijpj2z2a")?; fixture.seed(&seeded, 7, &[])?; let body = post("a"); let car = build_car("3jzfcijpj2z2b", &[("a", body.as_slice())]).await?; fixture.pds.serve_car(car.bytes).await; let result = fixture .run_changing(keys::pending_key(7), async { let mut row = fixture.repo()?; row.handle = Some(Handle::new_static("bob.test").into_diagnostic()?); fixture.put_repo(&row) }) .await?; assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); let after = fixture.repo()?; assert_eq!(after.handle.as_ref().map(Handle::as_str), Some("bob.test")); assert_eq!(after.pds.as_deref(), seeded.pds.as_deref()); assert_eq!( after.root.map(|root| root.rev), Some(crate::db::types::DbTid::from(&car.rev)) ); assert_eq!(after.status, seeded.status); Ok(()) } #[tokio::test] async fn a_failed_fetch_leaves_a_repo_that_went_inactive_inactive() -> miette::Result<()> { let fixture = Fixture::new().await?; let seeded = rooted("3jzfcijpj2z2a")?; fixture.seed(&seeded, 7, &[])?; // the stub answers getRepo with RepoNotFound, which the task retries let result = fixture .run_changing(keys::pending_key(7), async { let mut row = fixture.repo()?; row.active = false; row.status = RepoStatus::Deactivated; fixture.put_repo(&row) }) .await?; assert!( matches!(result, Err(BackfillError::RepoNotFound)), "{result:?}" ); let after = fixture.repo()?; assert_eq!(after.status, RepoStatus::Deactivated); assert!(!after.active); assert!(matches!( fixture.resync()?, Some(ResyncState::Error { retry_count: 1, .. }) )); assert!(fixture.pending_keys()?.is_empty()); Ok(()) } #[tokio::test] async fn a_later_valid_car_replaces_the_error_after_requeue() -> miette::Result<()> { let fixture = Fixture::new().await?; let seeded = rooted("3jzfcijpj2z2a")?; let first = post("a"); fixture.seed(&seeded, 7, &[("a", first.as_slice())])?; assert!(matches!( fixture.run(keys::pending_key(7)).await.unwrap_err(), BackfillError::RepoNotFound )); assert_eq!(fixture.pds.requests(), 1); fixture.clear_resync()?; fixture.reindex_pending(7, 21)?; let second = post("b"); let car = build_car( "3jzfcijpj2z2b", &[("a", first.as_slice()), ("b", second.as_slice())], ) .await?; fixture.pds.serve_car(car.bytes.clone()).await; assert_eq!( fixture.run(keys::pending_key(21)).await.unwrap(), TaskDisposition::Finished ); let after = fixture.repo()?; assert_eq!( after.root.as_ref().map(|root| root.data), Some(car.mst_root) ); assert_eq!( after.root.as_ref().map(|root| root.rev.to_tid()), Some(car.rev.clone()) ); assert_eq!(fixture.record_rkeys()?, ["a", "b"]); assert!( fixture.resync()?.is_none(), "a successful import clears the error row" ); assert!(fixture.pending_keys()?.is_empty()); assert_eq!(fixture.pds.requests(), 2); Ok(()) } // --- a backfill written in steps ------------------------------------------------------ /// three posts, one per step at one record a step, with the last one the last step async fn three_posts(fixture: &Fixture) -> miette::Result { fixture.seed(&rooted("3jzfcijpj2z2a")?, 7, &[])?; let (a, b, c) = (post("a"), post("b"), post("c")); let car = build_car( "3jzfcijpj2z2b", &[ ("a", a.as_slice()), ("b", b.as_slice()), ("c", c.as_slice()), ], ) .await?; fixture.pds.serve_car(car.bytes.clone()).await; Ok(car) } fn untrack(fixture: &Fixture) -> impl Future>> { let repos = crate::control::repos::ReposControl::new(fixture.state.clone()); let did = fixture.did.clone(); async move { repos.untrack([did]).await } } #[tokio::test] async fn a_live_commit_during_the_steps_lands_after_the_last() -> miette::Result<()> { let fixture = Fixture::with_chunk_records(1).await?; three_posts(&fixture).await?; let d = post("d"); let newer = build_car( "3jzfcijpj2z2c", &[ ("a", post("a").as_slice()), ("b", post("b").as_slice()), ("c", post("c").as_slice()), ("d", d.as_slice()), ], ) .await?; let (fixture, newer, d) = (&fixture, &newer, &d); let result = fixture .run_pausing(keys::pending_key(7), 1, |tx| async move { let commit = fixture.live_commit(newer, "d", d)?; tx.send(live_event(newer, commit, false)?) .await .map_err(|_| miette::miette!("the shard is gone")) }) .await?; assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); assert_eq!(fixture.record_rkeys()?, ["a", "b", "c", "d"]); assert_eq!(fixture.buffered(), 0); assert!(fixture.pending_keys()?.is_empty()); Ok(()) } #[tokio::test] async fn untrack_during_the_steps_is_refused_and_works_after() -> miette::Result<()> { let fixture = Fixture::with_chunk_records(1).await?; three_posts(&fixture).await?; let result = fixture .run_pausing(keys::pending_key(7), 1, |_| async { assert!(untrack(&fixture).await?.is_empty()); Ok(()) }) .await?; assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); assert_eq!(fixture.record_rkeys()?, ["a", "b", "c"]); assert_eq!(untrack(&fixture).await?, std::slice::from_ref(&fixture.did)); Ok(()) } #[tokio::test] async fn resync_during_the_steps_does_nothing() -> miette::Result<()> { let fixture = Fixture::with_chunk_records(1).await?; three_posts(&fixture).await?; let repos = crate::control::repos::ReposControl::new(fixture.state.clone()); let result = fixture .run_pausing(keys::pending_key(7), 1, |_| async { assert!(repos.resync([fixture.did.clone()]).await?.is_empty()); assert_eq!(fixture.pending_keys()?, [keys::pending_key(7).to_vec()]); Ok(()) }) .await?; assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); assert_eq!(fixture.record_rkeys()?, ["a", "b", "c"]); assert!(fixture.pending_keys()?.is_empty()); Ok(()) } #[tokio::test] async fn a_chain_break_during_the_steps_queues_the_repo_again_after_them() -> miette::Result<()> { let fixture = Fixture::with_chunk_records(1).await?; three_posts(&fixture).await?; let d = post("d"); let newer = build_car("3jzfcijpj2z2c", &[("d", d.as_slice())]).await?; let (fixture, newer, d) = (&fixture, &newer, &d); let result = fixture .run_pausing(keys::pending_key(7), 1, |tx| async move { let commit = fixture.live_commit(newer, "d", d)?; tx.send(live_event(newer, commit, true)?) .await .map_err(|_| miette::miette!("the shard is gone")) }) .await?; assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); // the steps all landed instead of being thrown away assert_eq!(fixture.record_rkeys()?, ["a", "b", "c"]); // and the repo is queued again under a new key, with the commit kept for that backfill let pending = fixture.pending_keys()?; assert_eq!(pending.len(), 1); assert_ne!(pending[0], keys::pending_key(7)); assert_eq!(fixture.buffered(), 1); assert!(fixture.marker()?.is_none()); Ok(()) } #[tokio::test] async fn an_erase_during_the_steps_leaves_nothing_behind() -> miette::Result<()> { let fixture = Fixture::with_chunk_records(1).await?; three_posts(&fixture).await?; let repos = crate::control::repos::ReposControl::new(fixture.state.clone()); let result = fixture .run_pausing(keys::pending_key(7), 1, |_| async { repos.erase_repo(&fixture.did, false).await?; Ok(()) }) .await?; assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); assert!(fixture.record_rkeys()?.is_empty()); assert!(fixture.pending_keys()?.is_empty()); assert_eq!( db::get_record_count(&fixture.state.db, &fixture.did, COLLECTION)?, 0 ); assert!(fixture.marker()?.is_none()); Ok(()) } #[tokio::test] async fn an_account_deletion_during_the_steps_leaves_nothing_behind() -> miette::Result<()> { let fixture = Fixture::with_chunk_records(1).await?; three_posts(&fixture).await?; let fixture = &fixture; let result = fixture .run_pausing(keys::pending_key(7), 1, |tx| async move { let deleted = IndexerEvent { seq: 1, firehose: Url::parse("wss://relay.example").into_diagnostic()?, ticket: None, row: None, data: IndexerEventData::Account(crate::ingest::indexer::IndexerAccountData { account: crate::ingest::stream::Account { active: false, did: fixture.did.clone(), seq: 1, status: Some(crate::ingest::stream::types::AccountStatus::Deleted), time: Datetime::try_from("2026-10-05T12:00:00Z".to_owned()) .into_diagnostic()?, }, was_active: true, changed: true, }), }; tx.send(IndexerMessage::Event(Box::new(deleted))) .await .map_err(|_| miette::miette!("the shard is gone")) }) .await?; assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); assert!(fixture.record_rkeys()?.is_empty()); assert!(fixture.pending_keys()?.is_empty()); assert!(fixture.marker()?.is_none()); Ok(()) } #[tokio::test] async fn a_sparse_backfill_lets_its_pds_slot_go_before_its_steps() -> miette::Result<()> { let mut fixture = Fixture::with_chunk_records(1).await?; fixture.strategy = BackfillStrategy::SparseFilter; only_collection(&fixture, COLLECTION); fixture.seed(&RepoState::backfilling().into_static(), 7, &[])?; let (a, b) = (post("a"), post("b")); let car = build_car("3jzfcijpj2z2a", &[("a", a.as_slice()), ("b", b.as_slice())]).await?; fixture.pds.serve_sparse(&car).await?; let pds = fixture.pds.base.clone(); let result = fixture .run_pausing(keys::pending_key(7), 1, |_| async { // one slot per pds, so this only gets it if the backfill gave it back let slot = fixture.admission.try_acquire(&pds).await; assert!(slot.is_some(), "the backfill still holds its pds slot"); Ok(()) }) .await?; assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); assert_eq!(fixture.record_rkeys()?, ["a", "b"]); // it went sparse rather than falling back to a whole car assert_eq!(fixture.pds.requests(), 0); Ok(()) } #[tokio::test] async fn an_exclude_during_the_steps_doesnt_stop_them() -> miette::Result<()> { let fixture = Fixture::with_chunk_records(1).await?; three_posts(&fixture).await?; let result = fixture .run_pausing(keys::pending_key(7), 1, |_| async { fixture .state .db .filter .insert(crate::db::filter::exclude_key(fixture.did.as_str())?, []) .into_diagnostic() }) .await?; assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); assert_eq!(fixture.record_rkeys()?, ["a", "b", "c"]); // the queue entry still goes, once the steps are done assert!(fixture.pending_keys()?.is_empty()); Ok(()) } #[tokio::test] async fn a_repo_that_fits_in_a_step_goes_in_one() -> miette::Result<()> { let fixture = Fixture::with_chunk_records(1000).await?; three_posts(&fixture).await?; let steps = Arc::new(AtomicUsize::new(0)); let counted = steps.clone(); *fixture.state.after_chunk.lock() = Some(Arc::new(move |_| { counted.fetch_add(1, Ordering::SeqCst); })); assert_eq!( fixture.run(keys::pending_key(7)).await?, TaskDisposition::Finished ); // the hook only runs between steps, and there were none assert_eq!(steps.load(Ordering::SeqCst), 0); assert_eq!(fixture.record_rkeys()?, ["a", "b", "c"]); Ok(()) } #[tokio::test] async fn big_records_end_a_step_before_its_chunk_of_records() -> miette::Result<()> { let fixture = Fixture::with_chunk_records(1000).await?; fixture.seed(&RepoState::backfilling().into_static(), 7, &[])?; // a quarter of a step each, about as big as a record gets let big = |name: &str| post(&name.repeat(super::super::apply::STEP_BYTES / 4)); let records = ["a", "b", "c", "d", "e"].map(|name| (name, big(name))); let car = build_car( "3jzfcijpj2z2a", &records .each_ref() .map(|(name, body)| (*name, body.as_slice())), ) .await?; fixture.pds.serve_car(car.bytes).await; let steps = Arc::new(AtomicUsize::new(0)); let counted = steps.clone(); *fixture.state.after_chunk.lock() = Some(Arc::new(move |_| { counted.fetch_add(1, Ordering::SeqCst); })); assert_eq!( fixture.run(keys::pending_key(7)).await?, TaskDisposition::Finished ); // four records pass the step's bytes, so the fifth goes in a step of its own. the hook // runs between steps only assert_eq!(steps.load(Ordering::SeqCst), 1); assert_eq!(fixture.record_rkeys()?, ["a", "b", "c", "d", "e"]); Ok(()) } #[cfg(feature = "indexer_stream")] /// the did's records as key -> cid fn heads(fixture: &Fixture) -> miette::Result> { let prefix = keys::record_prefix_did(&fixture.did); fixture .state .db .indexer .head_prefix(&prefix) .map(|guard| { let (key, value) = guard.into_inner().into_diagnostic()?; let (collection, rkey) = keys::split_record_suffix(&key[prefix.len()..])?; let cid = crate::db::ipld_cid_from_record_value(&value)?; Ok((format!("{collection}/{rkey}"), cid)) }) .collect() } #[cfg(feature = "indexer_stream")] /// the did's history, without the death revs, which go by the clock fn history(fixture: &Fixture) -> miette::Result, Vec)>> { let prefix = keys::record_prefix_did(&fixture.did); let mut end = prefix.clone(); *end.last_mut().unwrap() += 1; let mut entries = fixture .state .db .indexer .history_range(prefix..end) .map(|guard| { let (key, value) = guard.into_inner().into_diagnostic()?; Ok((key[..key.len() - 9].to_vec(), value.to_vec())) }) .collect::>>()?; entries.sort(); Ok(entries) } #[cfg(feature = "indexer_stream")] /// `start` with every record event in the stream played over it fn replayed( fixture: &Fixture, mut start: std::collections::BTreeMap, ) -> miette::Result> { for guard in fixture.state.db.stream.event_range::<&[u8], _>(..) { let value = guard.value().into_diagnostic()?; let event: crate::types::StoredEvent<'_> = rmp_serde::from_slice(&value).into_diagnostic()?; let key = format!("{}/{}", event.collection, event.rkey.to_smolstr()); match (event.action, event.data) { (crate::db::types::DbAction::Delete, _) => { start.remove(&key); } (_, crate::types::StoredData::Ptr(cid)) => { start.insert(key, cid); } (action, data) => panic!("{key}: {action:?} carries {data:?}"), } } Ok(start) } #[cfg(feature = "indexer_stream")] #[tokio::test] async fn a_backfill_stopped_after_any_step_comes_out_the_same_once_it_runs_again() -> miette::Result<()> { let (a, a2, b, c, d, x) = ( post("a"), post("a2"), post("b"), post("c"), post("d"), post("x"), ); let (a, a2, b, c, d, x) = (&a[..], &a2[..], &b[..], &c[..], &d[..], &x[..]); let stored: &[(&str, &[u8])] = &[("a", a), ("b", b), ("x", x)]; // an update, two creates and a delete, so four steps at one record a step let car = build_car("3jzfcijpj2z2b", &[("a", a2), ("b", b), ("c", c), ("d", d)]).await?; let newer = build_car("3jzfcijpj2z2c", &[("b", b), ("d", d), ("e", a)]).await?; let cids = |records: &[(&str, &[u8])]| { records .iter() .map(|(rkey, body)| { let cid = crate::car::compute_cid(body); (format!("{COLLECTION}/{rkey}"), cid) }) .collect::>() }; let start = cids(stored); let seeded = || async { let fixture = Fixture::with_chunk_records(1).await?; fixture.seed(&rooted("3jzfcijpj2z2a")?, 7, stored)?; miette::Result::<_>::Ok(fixture) }; let clean = seeded().await?; clean.pds.serve_car(car.bytes.clone()).await; assert_eq!( clean.run(keys::pending_key(7)).await?, TaskDisposition::Finished ); let want = cids(&[("a", a2), ("b", b), ("c", c), ("d", d)]); assert_eq!(heads(&clean)?, want); for stopped_after in 1..4 { for redo in [&car, &newer] { let mut fixture = seeded().await?; fixture.pds.serve_car(car.bytes.clone()).await; let state = Arc::downgrade(&fixture.state); *fixture.state.after_chunk.lock() = Some(Arc::new(move |sent| { if sent == stopped_after && let Some(state) = state.upgrade() { state.stop.trigger(); } })); assert_eq!( fixture.run(keys::pending_key(7)).await?, TaskDisposition::Finished ); assert_eq!(fixture.pending_keys()?, [keys::pending_key(7).to_vec()]); *fixture.state.after_chunk.lock() = None; fixture.reopen()?; // a spilled car was unlinked as soon as it was made, so the crash left nothing let mut spills = std::fs::read_dir(fixture.state.spills.dir()).into_diagnostic()?; assert!(spills.next().is_none()); fixture.pds.serve_car(redo.bytes.clone()).await; assert_eq!( fixture.run(keys::pending_key(7)).await?, TaskDisposition::Finished, "stopped after step {stopped_after}" ); let wanted = if std::ptr::eq(redo, &car) { want.clone() } else { cids(&[("b", b), ("d", d), ("e", a)]) }; assert_eq!( heads(&fixture)?, wanted, "stopped after step {stopped_after}" ); assert_eq!( db::get_record_count(&fixture.state.db, &fixture.did, COLLECTION)?, wanted.len() as u64 ); assert_eq!(replayed(&fixture, start.clone())?, wanted); if std::ptr::eq(redo, &car) { assert_eq!(history(&fixture)?, history(&clean)?); } assert!(fixture.pending_keys()?.is_empty()); assert!(fixture.resync()?.is_none()); } } Ok(()) } // --- the drain after the last step, in steps of its own ------------------------------- /// queues the repo under 7 with a car holding `a`, and buffers `n` later commits that each /// add a post, the way the shard does while the repo is queued. returns the posts' rkeys and /// the car the pds has once they're all in async fn fetch_with_buffered_posts( fixture: &Fixture, n: usize, ) -> miette::Result<(Vec, BuiltCar)> { use smol_str::ToSmolStr; fixture.seed(&rooted("3jzfcijpj2z2a")?, 7, &[])?; let a = post("a"); let fetched = build_car("3jzfcijpj2z2b", &[("a", a.as_slice())]).await?; fixture.pds.serve_car(fetched.bytes).await; let rev = |i: usize| { let base = crate::db::types::s32decode("3jzfcijpj2z2b"); crate::db::types::DbTid::new_from_bytes((base + 1 + i as u64).to_be_bytes()) .to_smolstr() .to_string() }; let rkeys = (0..n).map(|i| format!("p{i:03}")).collect::>(); let bodies = rkeys.iter().map(|rkey| post(rkey)).collect::>(); let mut records = vec![("a", a.as_slice())]; for (i, (rkey, body)) in rkeys.iter().zip(&bodies).enumerate() { records.push((rkey.as_str(), body.as_slice())); let car = build_car(&rev(i), &records).await?; fixture.buffer(&car, rkey, body)?; } let after_all = build_car(&rev(n), &records).await?; Ok((rkeys, after_all)) } /// runs the task on its own shard with `hook` running on the shard after each drain step, /// given the step and a sender to the shard async fn run_draining( fixture: &Fixture, pending_key: [u8; 8], hook: impl Fn(usize, &IndexerTx) + Send + Sync + 'static, ) -> Result { let (tx, shard) = FirehoseWorker::spawn_one_shard(fixture.state.clone()); let to_shard = tx.clone(); *fixture.state.after_drain_step.lock() = Some(Arc::new(move |step| hook(step, &to_shard))); let result = fixture.run_with(tx, pending_key).await; // the hook holds a sender, so the shard only stops once it's gone *fixture.state.after_drain_step.lock() = None; tokio::task::spawn_blocking(move || shard.join()) .await .into_diagnostic()? .map_err(|_| miette::miette!("indexer shard panicked"))?; result } #[tokio::test] async fn a_big_drain_runs_in_steps_with_other_repos_served_between() -> miette::Result<()> { let fixture = Fixture::with_chunk_records(1).await?; let (rkeys, _) = fetch_with_buffered_posts(&fixture, 4).await?; let other = Did::new_static("did:plc:bbbbbbbbbbbbbbbbbbbbbbbb").into_diagnostic()?; let seen = Arc::new(parking_lot::Mutex::new(Vec::new())); let (state, recorded) = (Arc::downgrade(&fixture.state), seen.clone()); let queued_other = other.clone(); let result = run_draining(&fixture, keys::pending_key(7), move |step, tx| { let state = state.upgrade().unwrap(); let other_queued = state .db .repo_metadata .get(keys::repo_metadata_key(&queued_other)) .unwrap() .is_some(); recorded.lock().push((step, other_queued)); if step == 1 { tx.blocking_send(IndexerMessage::new_repo_for_test(queued_other.clone())) .unwrap(); } }) .await; assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); // one commit a step, and the other repo's message went in right after the first assert_eq!(*seen.lock(), [(1, false), (2, true), (3, true), (4, true)]); let mut want = vec!["a".to_string()]; want.extend(rkeys); assert_eq!(fixture.record_rkeys()?, want); assert_eq!(fixture.buffered(), 0); assert_eq!( fixture.pending_keys()?.len(), 1, "only the other repo is queued" ); Ok(()) } #[cfg(feature = "indexer_stream")] #[tokio::test] async fn a_commit_arriving_during_the_drain_lands_after_it_in_order() -> miette::Result<()> { let fixture = Fixture::with_chunk_records(1).await?; let (rkeys, after_all) = fetch_with_buffered_posts(&fixture, 3).await?; let q = post("q"); let later = { use smol_str::ToSmolStr; let rev = crate::db::types::s32decode(after_all.rev.as_str()) + 1; crate::db::types::DbTid::new_from_bytes(rev.to_be_bytes()).to_smolstr() }; let with_q = build_car(&later, &[("q", q.as_slice())]).await?; let commit = fixture.live_commit(&with_q, "q", &q)?; let event = std::sync::Mutex::new(Some(live_event(&with_q, commit, false)?)); let result = run_draining(&fixture, keys::pending_key(7), move |step, tx| { if step == 1 && let Some(event) = event.lock().unwrap().take() { tx.blocking_send(event).unwrap(); } }) .await; assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); let mut want = vec![format!("{COLLECTION}/a")]; want.extend(rkeys.iter().map(|rkey| format!("{COLLECTION}/{rkey}"))); want.push(format!("{COLLECTION}/q")); let events = record_events(&fixture)? .into_iter() .map(|(key, _)| key) .collect::>(); assert_eq!(events, want); assert_eq!(fixture.buffered(), 0); assert!(fixture.pending_keys()?.is_empty()); Ok(()) } #[tokio::test] async fn untrack_during_the_drain_is_refused() -> miette::Result<()> { let fixture = Fixture::with_chunk_records(1).await?; fetch_with_buffered_posts(&fixture, 3).await?; let repos = crate::control::repos::ReposControl::new(fixture.state.clone()); let did = fixture.did.clone(); let answers = Arc::new(parking_lot::Mutex::new(Vec::new())); let recorded = answers.clone(); let handle = tokio::runtime::Handle::current(); let result = run_draining(&fixture, keys::pending_key(7), move |step, _| { if step == 1 { let untracked = handle.block_on(repos.untrack([did.clone()])).unwrap(); recorded.lock().push(untracked); } }) .await; assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); assert_eq!(*answers.lock(), [Vec::::new()]); assert!(fixture.pending_keys()?.is_empty()); assert_eq!(untrack(&fixture).await?, std::slice::from_ref(&fixture.did)); Ok(()) } #[tokio::test] async fn a_chain_break_during_the_drain_queues_the_repo_again_after_it() -> miette::Result<()> { let fixture = Fixture::with_chunk_records(1).await?; let (_, after_all) = fetch_with_buffered_posts(&fixture, 3).await?; let q = post("q"); let commit = fixture.live_commit(&after_all, "q", &q)?; let event = std::sync::Mutex::new(Some(live_event(&after_all, commit, true)?)); let result = run_draining(&fixture, keys::pending_key(7), move |step, tx| { if step == 1 && let Some(event) = event.lock().unwrap().take() { tx.blocking_send(event).unwrap(); } }) .await; assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); // the first step drained one commit, then the drain stopped and the repo is queued // under a new key with the rest of the buffer, the chain break's commit included assert_eq!(fixture.record_rkeys()?, ["a", "p000"]); let pending = fixture.pending_keys()?; assert_eq!(pending.len(), 1); assert_ne!(pending[0], keys::pending_key(7)); assert_eq!(fixture.buffered(), 3); assert!(fixture.marker()?.is_none()); Ok(()) } #[tokio::test] async fn a_drain_that_never_catches_up_takes_the_rest_in_one_step() -> miette::Result<()> { use crate::ingest::indexer::bulk::DRAIN_STEPS_BEFORE_ALL; let fixture = Fixture::with_chunk_records(1).await?; let (rkeys, _) = fetch_with_buffered_posts(&fixture, DRAIN_STEPS_BEFORE_ALL + 6).await?; let steps = Arc::new(AtomicUsize::new(0)); let counted = steps.clone(); let result = run_draining(&fixture, keys::pending_key(7), move |step, _| { counted.store(step, Ordering::SeqCst); }) .await; assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); assert_eq!(steps.load(Ordering::SeqCst), DRAIN_STEPS_BEFORE_ALL + 1); assert_eq!(fixture.record_rkeys()?.len(), rkeys.len() + 1); assert_eq!(fixture.buffered(), 0); assert!(fixture.pending_keys()?.is_empty()); Ok(()) } #[cfg(feature = "indexer_stream")] #[tokio::test] async fn a_drain_stopped_after_any_step_comes_out_the_same_once_it_runs_again() -> miette::Result<()> { const POSTS: usize = 4; let clean = Fixture::with_chunk_records(1).await?; let (_, after_all) = fetch_with_buffered_posts(&clean, POSTS).await?; assert_eq!( clean.run(keys::pending_key(7)).await?, TaskDisposition::Finished ); let want = heads(&clean)?; assert_eq!(want.len(), POSTS + 1); for stopped_after in 1..POSTS { let mut fixture = Fixture::with_chunk_records(1).await?; fetch_with_buffered_posts(&fixture, POSTS).await?; let state = Arc::downgrade(&fixture.state); let result = run_draining(&fixture, keys::pending_key(7), move |step, _| { if step == stopped_after && let Some(state) = state.upgrade() { state.stop.trigger(); } }) .await; assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); // the queue entry and what the drain hadn't reached yet are still there assert_eq!(fixture.pending_keys()?, [keys::pending_key(7).to_vec()]); assert_eq!(fixture.buffered(), POSTS - stopped_after); fixture.reopen()?; fixture.pds.serve_car(after_all.bytes.clone()).await; assert_eq!( fixture.run(keys::pending_key(7)).await?, TaskDisposition::Finished, "stopped after drain step {stopped_after}" ); assert_eq!( heads(&fixture)?, want, "stopped after drain step {stopped_after}" ); assert_eq!( replayed(&fixture, Default::default())?, want, "stopped after drain step {stopped_after}" ); assert_eq!(fixture.buffered(), 0); assert!(fixture.pending_keys()?.is_empty()); } Ok(()) } // --- a whole car spilled to disk ------------------------------------------------------- #[tokio::test] async fn a_spilled_block_that_rots_on_disk_fails_its_step_and_retries() -> miette::Result<()> { let fixture = Fixture::spilling().await?; three_posts(&fixture).await?; let c = crate::car::compute_cid(&post("c")); fixture .state .spills .after_spill(move |spilled| spilled.corrupt(&c).unwrap()); let err = fixture.run(keys::pending_key(7)).await.unwrap_err(); assert!(err.to_string().contains("no longer hashes"), "{err}"); // the steps before the one with the rotten block went in, and that one didn't assert_eq!(fixture.record_rkeys()?, ["a", "b"]); assert!(fixture.pending_keys()?.is_empty()); assert!(matches!( fixture.resync()?, Some(ResyncState::Error { retry_count: 1, .. }) )); Ok(()) } #[tokio::test] async fn a_car_with_no_disk_to_spill_to_writes_nothing_and_retries() -> miette::Result<()> { let fixture = Fixture::spilling().await?; three_posts(&fixture).await?; fixture.state.spills.set_free_bytes(0); let err = fixture.run(keys::pending_key(7)).await.unwrap_err(); assert!(err.to_string().contains("bytes free"), "{err}"); assert!(fixture.record_rkeys()?.is_empty()); assert!(fixture.marker()?.is_none()); assert!(matches!( fixture.resync()?, Some(ResyncState::Error { retry_count: 1, .. }) )); let mut spills = std::fs::read_dir(fixture.state.spills.dir()).into_diagnostic()?; assert!(spills.next().is_none()); Ok(()) } #[tokio::test] async fn a_spill_that_runs_out_of_disk_partway_writes_nothing_and_retries() -> miette::Result<()> { let fixture = Fixture::spilling().await?; three_posts(&fixture).await?; // room to start, but not for the car let spills = &fixture.state.spills; spills.set_free_bytes(crate::backfill::spill::Spills::HEADROOM + 10); spills.set_check_every(1); let err = fixture.run(keys::pending_key(7)).await.unwrap_err(); assert!(err.to_string().contains("bytes free"), "{err}"); assert!(fixture.record_rkeys()?.is_empty()); assert!(fixture.marker()?.is_none()); assert!(matches!( fixture.resync()?, Some(ResyncState::Error { retry_count: 1, .. }) )); let mut spilled = std::fs::read_dir(spills.dir()).into_diagnostic()?; assert!(spilled.next().is_none()); Ok(()) } #[tokio::test] async fn get_repo_of_a_repo_still_backfilling_answers_not_found() -> miette::Result<()> { let fixture = Fixture::with_chunk_records(1).await?; three_posts(&fixture).await?; let repos = crate::control::repos::ReposControl::new(fixture.state.clone()); let result = fixture .run_pausing(keys::pending_key(7), 1, |_| async { // its records are a mix of the old repo and the new one, which no root matches assert!(repos.get(&fixture.did).generate_car().await?.is_none()); Ok(()) }) .await?; assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); use futures::TryStreamExt; let car = repos .get(&fixture.did) .generate_car() .await? .expect("a backfilled repo has a car") .map_ok(|chunk| chunk.to_vec()) .try_concat() .await .into_diagnostic()?; let parsed = crate::car::parse_car(car.into(), true)?; let root = parsed.blocks.get(&parsed.root).expect("the commit"); let data = AtpCommit::from_cbor(root)?.data; assert_eq!(crate::mst::leaves(parsed.blocks.map(), data)?.len(), 3); // a resync entry alone doesn't hide a repo whose records match its root let resync = ResyncState::Error { kind: ResyncErrorKind::Generic, retry_count: 1, next_retry: 0, }; fixture .state .db .indexer .resync .insert( keys::repo_key(&fixture.did), rmp_serde::to_vec(&resync).into_diagnostic()?, ) .into_diagnostic()?; assert!(repos.get(&fixture.did).generate_car().await?.is_some()); Ok(()) } // --- erases and wipes, a step at a time on the owner shard ----------------------------- /// the repo synced with posts `a`, `b` and `c` and counted, so a wipe or an erase at one /// record a step takes several fn three_stored_posts(fixture: &Fixture) -> miette::Result> { let seeded = rooted("3jzfcijpj2z2a")?; let (a, b, c) = (post("a"), post("b"), post("c")); fixture.seed( &seeded, 7, &[ ("a", a.as_slice()), ("b", b.as_slice()), ("c", c.as_slice()), ], )?; let mut batch = fixture.state.db.inner.batch(); db::set_record_count(&mut batch, &fixture.state.db, &fixture.did, COLLECTION, 3); batch.commit().into_diagnostic()?; Ok(seeded) } /// stops hydrant once the running erase or wipe has done `steps` steps #[cfg(feature = "indexer_stream")] fn stop_after_job_step(fixture: &Fixture, steps: usize) { let state = Arc::downgrade(&fixture.state); *fixture.state.after_job_step.lock() = Some(Arc::new(move |step| { if step == steps && let Some(state) = state.upgrade() { state.stop.trigger(); } })); } impl Fixture { /// asks the api to erase the repo, on a shard that's gone once the answer is in #[cfg(feature = "indexer_stream")] async fn erase_on_a_shard( &self, ) -> miette::Result> { let (tx, shard) = FirehoseWorker::spawn_one_shard(self.state.clone()); self.state.indexer.lock().replace(tx); let repos = crate::control::repos::ReposControl::new(self.state.clone()); let erased = repos.erase_repo(&self.did, false).await; self.state.indexer.lock().take(); tokio::task::spawn_blocking(move || shard.join()) .await .into_diagnostic()? .map_err(|_| miette::miette!("indexer shard panicked"))?; Ok(erased) } /// the did's history, with the death revs #[cfg(feature = "indexer_stream")] fn wiped_history(&self) -> miette::Result, Vec)>> { let prefix = keys::record_prefix_did(&self.did); let mut end = prefix.clone(); *end.last_mut().unwrap() += 1; self.state .db .indexer .history_range(prefix..end) .map(|guard| { let (key, value) = guard.into_inner().into_diagnostic()?; Ok((key.to_vec(), value.to_vec())) }) .collect() } } #[cfg(feature = "indexer_stream")] #[tokio::test] async fn a_wipe_stopped_after_any_step_comes_out_the_same_once_it_runs_again() -> miette::Result<()> { // a head a step, and the third step also finds there are no more. none at all is a // crash right after the deletion's own txn for stopped_after in 0..=3 { let mut fixture = Fixture::with_chunk_records(1).await?; let seeded = three_stored_posts(&fixture)?; fixture.start_deletion(&seeded)?; let Some(crate::types::BulkJobV1::Wipe { death_rev }) = fixture.marker()? else { panic!("the deletion left no wipe marker"); }; if stopped_after > 0 { stop_after_job_step(&fixture, stopped_after); fixture.start_a_shard().await?; *fixture.state.after_job_step.lock() = None; } fixture.reopen()?; fixture.start_a_shard().await?; let want = ["a", "b", "c"] .map(|rkey| { let record = keys::record_key(&fixture.did, COLLECTION, &DbRkey::new(rkey)); (keys::history_key(&record, &death_rev), post(rkey)) }) .to_vec(); assert_eq!( fixture.wiped_history()?, want, "stopped after step {stopped_after}" ); assert!(fixture.record_rkeys()?.is_empty()); assert_eq!( db::get_record_count(&fixture.state.db, &fixture.did, COLLECTION)?, 0 ); assert!(fixture.marker()?.is_none()); assert!(fixture.pending_keys()?.is_empty()); assert_eq!(fixture.repo()?.status, RepoStatus::Deleted); } Ok(()) } #[cfg(feature = "indexer_stream")] #[tokio::test] async fn an_erase_stopped_after_any_step_comes_out_the_same_once_it_runs_again() -> miette::Result<()> { let mut stopped_after = 0; loop { stopped_after += 1; let mut fixture = Fixture::with_chunk_records(1).await?; three_stored_posts(&fixture)?; let old = post("old"); let record = keys::record_key(&fixture.did, COLLECTION, &DbRkey::new("a")); let mut batch = fixture.state.db.inner.batch(); fixture.state.db.indexer.stage_history( &mut batch, keys::history_key(&record, &crate::db::types::DbTid::new_from_bytes([0; 8])), &old, ); batch.commit().into_diagnostic()?; fixture.buffer(&build_car("3jzfcijpj2z2b", &[]).await?, "d", &post("d"))?; stop_after_job_step(&fixture, stopped_after); let finished = fixture.erase_on_a_shard().await?.is_ok(); *fixture.state.after_job_step.lock() = None; fixture.reopen()?; fixture.start_a_shard().await?; let gone = format!("stopped after step {stopped_after}"); assert!(fixture.record_rkeys()?.is_empty(), "{gone}"); assert!(fixture.wiped_history()?.is_empty(), "{gone}"); assert_eq!(fixture.buffered(), 0, "{gone}"); assert!( fixture .state .db .repos .get(keys::repo_key(&fixture.did)) .into_diagnostic()? .is_none(), "{gone}" ); assert!(fixture.pending_keys()?.is_empty(), "{gone}"); assert!(fixture.marker()?.is_none(), "{gone}"); assert_eq!( db::get_record_count(&fixture.state.db, &fixture.did, COLLECTION)?, 0 ); if finished { assert!( stopped_after > 3, "every kind of entry takes a step of its own" ); break; } } Ok(()) } #[tokio::test] async fn markers_whose_state_moved_on_are_dropped() -> miette::Result<()> { let stage = |fixture: &Fixture, job: crate::types::BulkJobV1| { let mut batch = fixture.state.db.inner.batch(); db::stage_bulk_job(&mut batch, &fixture.state.db, &fixture.did, &job)?; batch.commit().into_diagnostic() }; // a wipe for an account that's active again let fixture = Fixture::with_chunk_records(1).await?; three_stored_posts(&fixture)?; stage( &fixture, crate::types::BulkJobV1::Wipe { death_rev: crate::db::types::DbTid::new_from_bytes([0; 8]), }, )?; fixture.start_a_shard().await?; assert!(fixture.marker()?.is_none()); assert_eq!(fixture.record_rkeys()?, ["a", "b", "c"]); // a backfill whose queue entry went away let fixture = Fixture::with_chunk_records(1).await?; three_stored_posts(&fixture)?; stage( &fixture, crate::types::BulkJobV1::Apply { pending_key: keys::pending_key(9).to_vec(), }, )?; fixture.start_a_shard().await?; assert!(fixture.marker()?.is_none()); assert_eq!(untrack(&fixture).await?, std::slice::from_ref(&fixture.did)); // an erase of a repo that was tracked again let fixture = Fixture::with_chunk_records(1).await?; three_stored_posts(&fixture)?; stage(&fixture, crate::types::BulkJobV1::Erase)?; fixture.start_a_shard().await?; assert!(fixture.marker()?.is_none()); assert_eq!(fixture.record_rkeys()?, ["a", "b", "c"]); Ok(()) } #[cfg(feature = "indexer_stream")] #[tokio::test] async fn a_live_commit_during_a_wipe_or_an_erase_is_dropped_and_other_repos_go_between() -> miette::Result<()> { for erase in [false, true] { let fixture = Fixture::with_chunk_records(1).await?; let seeded = three_stored_posts(&fixture)?; let q = post("q"); let car = build_car("3jzfcijpj2z2b", &[("q", q.as_slice())]).await?; let event = live_event(&car, fixture.live_commit(&car, "q", &q)?, false)?; let other = Did::new_static("did:plc:bbbbbbbbbbbbbbbbbbbbbbbb").into_diagnostic()?; let (tx, shard) = FirehoseWorker::spawn_one_shard(fixture.state.clone()); fixture.state.indexer.lock().replace(tx.clone()); let seen = Arc::new(parking_lot::Mutex::new(Vec::new())); let (state, recorded) = (Arc::downgrade(&fixture.state), seen.clone()); // the hook lets go of its sender after the first step, so the shard can stop let sends = parking_lot::Mutex::new(Some((tx.clone(), event, other.clone()))); *fixture.state.after_job_step.lock() = Some(Arc::new(move |step| { let state = state.upgrade().unwrap(); let other_queued = state .db .repo_metadata .get(keys::repo_metadata_key(&other)) .unwrap() .is_some(); recorded.lock().push((step, other_queued)); if let Some((tx, event, other)) = sends.lock().take() { tx.blocking_send(event).unwrap(); tx.blocking_send(IndexerMessage::new_repo_for_test(other)) .unwrap(); } })); if erase { let repos = crate::control::repos::ReposControl::new(fixture.state.clone()); repos.erase_repo(&fixture.did, false).await?; } else { fixture.start_deletion(&seeded)?; let did = fixture.did.clone(); tx.on_owner(&fixture.did, move |_, shard| shard.mark(did)) .await?; } fixture.state.indexer.lock().take(); drop(tx); tokio::task::spawn_blocking(move || shard.join()) .await .into_diagnostic()? .map_err(|_| miette::miette!("indexer shard panicked"))?; *fixture.state.after_job_step.lock() = None; let seen = seen.lock().clone(); assert!(seen.len() > 1, "{seen:?}"); // the other repo's message went in right after the first step assert_eq!(seen[0], (1, false), "erase: {erase}"); assert!(seen[1..].iter().all(|&(_, queued)| queued), "{seen:?}"); assert!(fixture.record_rkeys()?.is_empty()); let q_event = format!("{COLLECTION}/q"); assert!( record_events(&fixture)? .iter() .all(|(key, _)| *key != q_event), "erase: {erase}" ); } Ok(()) } #[cfg(feature = "indexer_stream")] #[tokio::test] async fn untrack_after_a_restart_partway_through_the_steps_waits_for_them_to_run_again() -> miette::Result<()> { let mut fixture = Fixture::with_chunk_records(1).await?; let car = three_posts(&fixture).await?; let state = Arc::downgrade(&fixture.state); *fixture.state.after_chunk.lock() = Some(Arc::new(move |sent| { if sent == 1 && let Some(state) = state.upgrade() { state.stop.trigger(); } })); assert_eq!( fixture.run(keys::pending_key(7)).await?, TaskDisposition::Finished ); *fixture.state.after_chunk.lock() = None; fixture.reopen()?; fixture.start_a_shard().await?; assert!(untrack(&fixture).await?.is_empty()); assert_eq!(fixture.pending_keys()?, [keys::pending_key(7).to_vec()]); fixture.pds.serve_car(car.bytes).await; assert_eq!( fixture.run(keys::pending_key(7)).await?, TaskDisposition::Finished ); assert_eq!(fixture.record_rkeys()?, ["a", "b", "c"]); assert!(fixture.marker()?.is_none()); assert_eq!(untrack(&fixture).await?, std::slice::from_ref(&fixture.did)); Ok(()) } }