Something went wrong. Try again.
very fast at protocol indexer with flexible filtering, xrpc queries, cursor-backed event stream, and more, built on fjall
rust fjall at-protocol atproto indexer
Something went wrong. Try again.
Rust
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458use fjall::Slice;use jacquard_common::IntoStatic;use jacquard_common::types::did::Did;use miette::{IntoDiagnostic, Result};use std::sync::Arc;use tracing::{debug, error, warn};
use crate::backfill::admission::BackfillAdmission;use crate::backfill::client::ThrottledHttpClient;use crate::backfill::error::BackfillError;use crate::config::BackfillStrategy;use crate::db::{Txn as DbTxn, keys};use crate::ingest::indexer::IndexerTx;use crate::ingest::indexer::repos::write_repo;use crate::state::AppState;use crate::types::{GaugeState, RepoState, RepoStatus, ResyncErrorKind, ResyncState};
use super::apply::Outcome;use super::process::process_did;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]pub(super) enum TaskDisposition { Finished, Deferred,}
/// what writing down a failed backfill came toenum Recorded { /// the repo already sits in resync as gone, so only its queue entry goes Gone, /// `applied` when the failure moved the repo from the queue into resync Retrying { applied: bool },}
fn stage_excluded_pending_cleanup( txn: &mut DbTxn<'_>, db: &crate::db::Db, did: &Did, pending_key: &[u8],) -> Result<bool> { if !txn.is_excluded(did)? { return Ok(false); } txn.transition_pending_key(did, pending_key, GaugeState::Synced)?; db.indexer .stage_pending_remove(&mut txn.batch, did, pending_key); Ok(true)}
async fn remove_pending_if_excluded( state: &Arc<AppState>, did: &Did, pending_key: Slice,) -> Result<bool> { let owner = did.clone(); let did = did.clone(); write_repo(state, &owner, move |db| { let mut txn = DbTxn::new(db); if !stage_excluded_pending_cleanup(&mut txn, db, &did, pending_key.as_ref())? { return Ok(false); } txn.commit()?; Ok(true) }) .await}
async fn is_pending(state: &AppState, pending_key: Slice) -> Result<bool> { state .db .run(move |db| db.indexer.is_pending(&pending_key).into_diagnostic()) .await}
/// takes `pending_key` off the queue. the gauge only moves when it's still the repo's current/// key, so an orphan or a key something else already took off leaves it aloneasync fn remove_pending(state: &Arc<AppState>, did: &Did, pending_key: Slice) -> Result<()> { let owner = did.clone(); let did = did.clone(); write_repo(state, &owner, move |db| { let mut txn = DbTxn::new(db); txn.transition_pending_key(&did, &pending_key, GaugeState::Synced)?; db.indexer .stage_pending_remove(&mut txn.batch, &did, &pending_key); txn.commit() }) .await}
pub(super) async fn did_task( state: &Arc<AppState>, http: ThrottledHttpClient, buffer_tx: IndexerTx, did: &Did, pending_key: Slice, admission: BackfillAdmission, _permit: tokio::sync::OwnedSemaphorePermit, verify_signatures: bool, strategy: BackfillStrategy,) -> Result<TaskDisposition, BackfillError> { // Drop stale/excluded work before resolver, admission, or network I/O. A // repository drop may race task spawning, and orphan pending keys are not // necessarily the one named by current metadata. if remove_pending_if_excluded(state, did, pending_key.clone()).await? { return Ok(TaskDisposition::Finished); } // the worker hands out keys from a snapshot of the queue, so a key whose task finished after // the snapshot comes back once that task leaves in_flight. it removed the key first, so this // read sees it gone, and running again would write its stale repo state over the synced one if !is_pending(state, pending_key.clone()).await? { return Ok(TaskDisposition::Finished); }
let outcome = process_did( state, &http, &buffer_tx, did, pending_key.clone(), &admission, verify_signatures, strategy, ) .await; match outcome { Ok(Outcome::Drained) => { state .backfill_drained .fetch_add(1, std::sync::atomic::Ordering::Relaxed); Ok(TaskDisposition::Finished) } Ok(Outcome::Skipped) => { remove_pending_if_excluded(state, did, pending_key).await?; Ok(TaskDisposition::Finished) } Ok(Outcome::Stopping) => Ok(TaskDisposition::Finished), // the discard and inactive paths already took the entry off, unless they found the repo // excluded and left it Ok(Outcome::Superseded | Outcome::Discarded | Outcome::Inactive) => { remove_pending(state, did, pending_key).await?; Ok(TaskDisposition::Finished) } Err(BackfillError::PdsBusy) => Ok(TaskDisposition::Deferred), Err(BackfillError::UnsafeEndpoint(reason)) => { // The identity document is the only authority that can repair a // permanently unsafe PDS. Park this queue item rather than hot // retrying the same invalid endpoint. A later identity event can // enqueue fresh work with new evidence. warn!(%reason, "unsafe PDS endpoint, parking backfill until identity evidence changes"); remove_pending(state, did, pending_key).await?; Ok(TaskDisposition::Finished) } Err(BackfillError::Deleted) => { warn!("orphaned pending entry, cleaning up"); remove_pending(state, did, pending_key).await?; state .backfill_drained .fetch_add(1, std::sync::atomic::Ordering::Relaxed); Ok(TaskDisposition::Finished) } Err(e) => { match &e { BackfillError::Ratelimited | BackfillError::PreemptivelyThrottled => { debug!("too many requests"); } BackfillError::Transport(reason) => { error!(%reason, "transport error"); } BackfillError::Generic(e) => { error!(err = %e, "failed"); } BackfillError::RepoNotFound => { warn!("repo not found, retrying through existing backoff"); } BackfillError::Deleted => unreachable!("already handled"), BackfillError::PdsBusy => unreachable!("already handled"), BackfillError::UnsafeEndpoint(_) => unreachable!("handled above"), }
let error_kind = match &e { BackfillError::Ratelimited | BackfillError::PreemptivelyThrottled => { ResyncErrorKind::Ratelimited } BackfillError::Transport(_) => ResyncErrorKind::Transport, BackfillError::Generic(_) => ResyncErrorKind::Generic, BackfillError::RepoNotFound => ResyncErrorKind::Generic, BackfillError::Deleted => unreachable!("already handled"), BackfillError::PdsBusy => unreachable!("already handled"), BackfillError::UnsafeEndpoint(_) => unreachable!("handled above"), };
let error_string = e.to_string(); let throttled = matches!(e, BackfillError::PreemptivelyThrottled); let recorded = write_repo(state, did, { let did_key = keys::repo_key(did).into_static(); let did = did.clone(); let pending_key = pending_key.clone(); move |db| { let mut txn = DbTxn::new(db); let existing = db .indexer .resync .get(&did_key) .into_diagnostic()? .map(|bytes| rmp_serde::from_slice::<ResyncState>(&bytes).into_diagnostic()) .transpose()?; let entry_retries = match existing { Some(ResyncState::Error { retry_count, .. }) => retry_count, Some(ResyncState::Gone { .. }) => { db.indexer .stage_pending_remove(&mut txn.batch, &did, &pending_key); txn.commit()?; return Ok::<_, miette::Report>(Recorded::Gone); } None => 0, }; if stage_excluded_pending_cleanup(&mut txn, db, &did, pending_key.as_ref())? { txn.commit()?; return Ok(Recorded::Retrying { applied: false }); }
let metadata_key = keys::repo_metadata_key(&did); let mut metadata = db .repo_metadata .get(&metadata_key) .into_diagnostic()? .map(|bytes| crate::db::deser_repo_meta(&bytes)) .transpose()?; let mut retry_count = metadata .as_ref() .map_or(0, |metadata| metadata.retry_count) .max(entry_retries); let next_retry = if throttled { chrono::Utc::now().timestamp() + 60 } else { retry_count = retry_count.saturating_add(1); ResyncState::next_backoff(retry_count) }; let resync_state = ResyncState::Error { kind: error_kind, retry_count, next_retry, }; let applied = txn.transition_pending_key( &did, pending_key.as_ref(), GaugeState::Resync(Some(error_kind)), )?; db.indexer .stage_pending_remove(&mut txn.batch, &did, &pending_key); if applied { txn.batch.insert( &db.indexer.resync, &did_key, rmp_serde::to_vec(&resync_state).into_diagnostic()?, ); if let Some(metadata) = metadata.as_mut() { metadata.retry_count = retry_count; txn.batch.insert( &db.repo_metadata, &metadata_key, crate::db::ser_repo_meta(metadata)?, ); } if let Some(bytes) = db.repos.get(&did_key).into_diagnostic()? { let mut repo: RepoState = rmp_serde::from_slice(&bytes).into_diagnostic()?; // a failed fetch says nothing about the account, so one that // went inactive keeps its status if repo.active { repo.status = RepoStatus::Error(error_string.into()); } txn.put_repo(&did, &repo)?; } } txn.commit()?; Ok(Recorded::Retrying { applied }) } }) .await?;
match recorded { Recorded::Gone => Ok(TaskDisposition::Finished), Recorded::Retrying { applied } => { if applied { state .backfill_drained .fetch_add(1, std::sync::atomic::Ordering::Relaxed); } Err(e) } } } }}
#[cfg(test)]mod tests { use super::*; use crate::backfill::admission::BackfillAdmission; use crate::backfill::test_util::{public_client, serve, tangled_repo}; use crate::config::Config; #[cfg(feature = "indexer_stream")] use crate::crawler::{ReconcileOutcome, RepoListing, reconcile_listing}; use crate::db; use crate::db::types::DbRkey; use crate::ingest::indexer::IndexerMessage; use crate::ingest::indexer::{ FirehoseWorker, IndexerCommitData, IndexerEvent, IndexerEventData, }; use crate::ingest::stream::{Commit as StreamCommit, Datetime, RepoOp, RepoOpAction}; use crate::types::RepoMetadata; use axum::Router; use axum::routing::get; use bytes::Bytes; #[cfg(feature = "indexer_stream")] use crate::crawler::worker::ListedRepoStatus; use jacquard_common::CowStr; use jacquard_common::types::cid::{CidLink, IpldCid}; use crate::mst::AtpCommit; use jacquard_common::types::string::Handle; use jacquard_common::types::tid::Tid; use reqwest::StatusCode; use std::sync::atomic::{AtomicUsize, Ordering}; use tokio::sync::{RwLock, Semaphore}; use url::Url;
#[tokio::test] async fn excluded_orphan_pending_entry_is_removed_before_work() -> miette::Result<()> { let tmp = tempfile::tempdir().into_diagnostic()?; let config = Config { database_path: tmp.path().to_path_buf(), ..Default::default() }; let state = Arc::new(AppState::new(&config)?); let did = Did::new("did:plc:ewvi7nxzyoun6zhxrhs64oiz")?.into_static(); let pending_key = keys::pending_key(7); let mut batch = state.db.inner.batch(); batch.insert( &state.db.filter, crate::db::filter::exclude_key(did.as_str())?, [], ); state .db .indexer .stage_pending_insert(&mut batch, &did, &pending_key); batch.commit().into_diagnostic()?;
assert!( remove_pending_if_excluded(&state, &did, fjall::Slice::from(pending_key.as_slice()),) .await? ); assert!( !state .db .indexer .is_pending(&pending_key) .into_diagnostic()? ); Ok(()) }
// --- full-backfill `RepoNotFound` boundary --------------------------------------------- // // `getRepo` `RepoNotFound` is a fetch result, not lifecycle evidence. these tests pin the // non-destructive policy at the task boundary: the snapshot survives, the queue parks a // retryable error, and only a later valid CAR or an authoritative deletion moves state.
const DID: &str = "did:plc:ewvi7nxzyoun6zhxrhs64oiz"; const NOT_FOUND_BODY: &[u8] = br#"{"error":"RepoNotFound","message":"not here"}"#; const COLLECTION: &str = "app.bsky.feed.post";
fn post(text: &str) -> Vec<u8> { serde_ipld_dagcbor::to_vec(&serde_json::json!({ "$type": COLLECTION, "text": text, })) .unwrap() }
struct PdsStub { base: Url, address: std::net::SocketAddr, response: Arc<RwLock<(StatusCode, Bytes)>>, /// getRepo requests requests: Arc<AtomicUsize>, get_repo: Gate, blocks: Arc<RwLock<(StatusCode, Bytes)>>, block_requests: Arc<AtomicUsize>, get_blocks: Gate, latest_commit: Arc<RwLock<(StatusCode, Bytes)>>, }
/// where a request to one route waits while a test changes something under the task #[derive(Clone, Default)] struct Gate { arrived: Arc<tokio::sync::Notify>, hold: Arc<tokio::sync::Mutex<()>>, }
impl Gate { async fn pass(&self) { self.arrived.notify_waiters(); drop(self.hold.lock().await); } }
/// a stub route answering with whatever `response` holds, counting requests and passing /// `gate` first fn stub_route( response: &Arc<RwLock<(StatusCode, Bytes)>>, requests: &Arc<AtomicUsize>, gate: &Gate, ) -> axum::routing::MethodRouter { let (response, requests, gate) = (response.clone(), requests.clone(), gate.clone()); get(move || { let (response, requests, gate) = (response.clone(), requests.clone(), gate.clone()); async move { requests.fetch_add(1, Ordering::SeqCst); gate.pass().await; response.read().await.clone() } }) }
impl PdsStub { async fn spawn() -> Self { let response = Arc::new(RwLock::new(( StatusCode::BAD_REQUEST, Bytes::from_static(NOT_FOUND_BODY), ))); let requests = Arc::new(AtomicUsize::new(0)); let get_repo = Gate::default(); // no getBlocks until a test serves one, like the atcr hold let blocks = Arc::new(RwLock::new((StatusCode::NOT_FOUND, Bytes::new()))); let block_requests = Arc::new(AtomicUsize::new(0)); let get_blocks = Gate::default(); // no getLatestCommit either, so a sparse backfill falls back to getRepo let latest_commit = Arc::new(RwLock::new((StatusCode::NOT_FOUND, Bytes::new()))); let app = Router::new() .route( "/xrpc/com.atproto.sync.getRepo", stub_route(&response, &requests, &get_repo), ) .route( "/xrpc/com.atproto.sync.getBlocks", stub_route(&blocks, &block_requests, &get_blocks), ) .route( "/xrpc/com.atproto.sync.getLatestCommit", stub_route(&latest_commit, &Arc::default(), &Gate::default()), ); let address = serve(app).await; // public by name, so the backfill client takes it, and pinned to the local socket by // the test client, so nothing resolves it let base = Url::parse("http://pds.example/").unwrap(); Self { base, address, response, requests, get_repo, blocks, block_requests, get_blocks, latest_commit, } }
async fn serve_car(&self, car: Bytes) { *self.response.write().await = (StatusCode::OK, car); }
fn requests(&self) -> usize { self.requests.load(Ordering::SeqCst) }
async fn serve_blocks(&self, status: StatusCode, body: Bytes) { *self.blocks.write().await = (status, body); }
fn block_requests(&self) -> usize { self.block_requests.load(Ordering::SeqCst) }
/// answers a sparse backfill from `car`: its commit as the latest one, and every block /// it asks getBlocks for, since the whole car holds them all async fn serve_sparse(&self, car: &BuiltCar) -> miette::Result<()> { let commit = crate::car::parse_car(car.bytes.clone(), false)?.root; let latest = serde_json::json!({ "cid": commit.to_string(), "rev": car.rev.as_str() }); *self.latest_commit.write().await = (StatusCode::OK, Bytes::from(latest.to_string())); self.serve_blocks(StatusCode::OK, car.bytes.clone()).await; Ok(()) } }
async fn spawn_doc_stub(doc: String) -> Url { let app = Router::new().fallback({ move |_uri: axum::http::Uri| { let doc = doc.clone(); async move { ( [(axum::http::header::CONTENT_TYPE, "application/json")], doc, ) } } }); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let base = Url::parse(&format!("http://{}/", listener.local_addr().unwrap())).unwrap(); tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); base }
fn did_doc(base: &Url) -> String { serde_json::json!({ "@context": ["https://www.w3.org/ns/did/v1"], "id": DID, "service": [{ "id": "#atproto_pds", "type": "AtprotoPersonalDataServer", "serviceEndpoint": base.as_str(), }], }) .to_string() }
fn rooted(rev: &str) -> miette::Result<RepoState<'static>> { let mut state = RepoState::synced(); state.root = Some(crate::types::Commit { version: 3, rev: crate::db::types::DbTid::from(&Tid::new(rev).into_diagnostic()?), data: crate::car::compute_cid(b"seeded-root"), prev: None, sig: Bytes::from_static(b"signature"), }); state.pds = Some(CowStr::Borrowed("https://old-pds.example")); state.handle = Some(Handle::new_static("alice.test").into_diagnostic()?); state.last_message_time = Some(101); state.last_updated_at = 5; Ok(state.into_static()) }
struct BuiltCar { bytes: Bytes, mst_root: IpldCid, rev: Tid, }
async fn build_car(rev: &str, records: &[(&str, &[u8])]) -> miette::Result<BuiltCar> { build_car_for(DID, rev, records).await }
async fn build_car_for( commit_did: &'static str, rev: &str, records: &[(&str, &[u8])], ) -> miette::Result<BuiltCar> { let records = records .iter() .map(|&(rkey, body)| (format!("{COLLECTION}/{rkey}"), body)) .collect::<Vec<_>>(); build_car_keyed(commit_did, rev, &records).await }
/// a car whose records sit under full `collection/rkey` keys async fn build_car_keyed( commit_did: &'static str, rev: &str, records: &[(String, &[u8])], ) -> miette::Result<BuiltCar> { let (mst_root, mst_nodes) = crate::mst::build_tree( records .iter() .map(|(key, body)| (key.as_str(), crate::car::compute_cid(body))), )?; let rev = Tid::new(rev).into_diagnostic()?; let did: Did = Did::new_static(commit_did).into_diagnostic()?; let commit = AtpCommit { did, version: 3, data: mst_root, rev: rev.clone(), prev: None, sig: Bytes::from_static(b"signature"), }; let commit_cid = commit.to_cid()?; let commit_cbor = commit.to_cbor()?;
let mut bytes = Vec::new(); let mut writer = crate::car::CarWriter::new(crate::car::CarHeader::new_v1(vec![commit_cid]), &mut bytes); writer .write(commit_cid, &commit_cbor) .await .into_diagnostic()?; for (cid, node_bytes) in mst_nodes { writer.write(cid, &node_bytes).await.into_diagnostic()?; } let mut seen = std::collections::HashSet::new(); for (_, body) in records { let cid = crate::car::compute_cid(body); if seen.insert(cid) { writer.write(cid, body).await.into_diagnostic()?; } } writer.finish().await.into_diagnostic()?; Ok(BuiltCar { bytes: Bytes::from(bytes), mst_root, rev, }) }
struct Fixture { state: Arc<AppState>, did: Did, pds: PdsStub, /// one slot per pds, so a test can see whether a task still holds it admission: BackfillAdmission, strategy: BackfillStrategy, /// what [`Self::reopen`] opens the database with again #[cfg_attr(not(feature = "indexer_stream"), allow(dead_code))] config: Config, #[cfg_attr(not(feature = "indexer_stream"), allow(dead_code))] doc_base: Url, _tmp: tempfile::TempDir, }
// whatever a test does to the queue, its two keyspaces have to end in step impl Drop for Fixture { fn drop(&mut self) { if !std::thread::panicking() { self.state.db.indexer.assert_pending_dids_match(); } } }
/// records per backfill step in these tests: one, so every repo with more than one record /// goes through several steps, unless `HYDRANT_TEST_BACKFILL_CHUNK_RECORDS` says otherwise /// (0 writes each repo in one step) fn test_chunk_records() -> usize { std::env::var("HYDRANT_TEST_BACKFILL_CHUNK_RECORDS") .ok() .and_then(|records| records.parse().ok()) .unwrap_or(1) }
/// how big a car gets in memory before it spills in these tests, the default unless /// `HYDRANT_TEST_BACKFILL_SPILL_BYTES` says otherwise (0 spills every whole car) fn test_spill_bytes() -> usize { std::env::var("HYDRANT_TEST_BACKFILL_SPILL_BYTES") .ok() .and_then(|bytes| bytes.parse().ok()) .unwrap_or(Config::default().backfill_spill_bytes) }
impl Fixture { async fn new() -> miette::Result<Self> { Self::with_chunk_records(test_chunk_records()).await }
async fn with_chunk_records(records: usize) -> miette::Result<Self> { let pds = PdsStub::spawn().await; let endpoint = pds.base.clone(); Self::new_with(pds, endpoint, records, test_spill_bytes()).await }
/// a fixture whose backfills spill every whole car, at one record a step async fn spilling() -> miette::Result<Self> { let pds = PdsStub::spawn().await; let endpoint = pds.base.clone(); Self::new_with(pds, endpoint, 1, 0).await }
async fn new_with_pds(pds: PdsStub, pds_endpoint: Url) -> miette::Result<Self> { Self::new_with(pds, pds_endpoint, test_chunk_records(), test_spill_bytes()).await }
async fn new_with( pds: PdsStub, pds_endpoint: Url, chunk_records: usize, spill_bytes: usize, ) -> miette::Result<Self> { let tmp = tempfile::tempdir().into_diagnostic()?; let doc_base = spawn_doc_stub(did_doc(&pds_endpoint)).await; let config = Config { database_path: tmp.path().to_path_buf(), plc_urls: vec![Url::parse("http://public.example/").unwrap()], identity_cache_size: 64, backfill_chunk_records: chunk_records, backfill_spill_bytes: spill_bytes, ..Default::default() }; Ok(Self { state: Arc::new(Self::open(&config, &doc_base)?), did: Did::new_static(DID).into_diagnostic()?, pds, admission: BackfillAdmission::new(4, 1), strategy: BackfillStrategy::Full, config, doc_base, _tmp: tmp, }) }
fn open(config: &Config, doc_base: &Url) -> miette::Result<AppState> { let mut app_state = AppState::new(config)?; let public_client = reqwest::Client::builder() .no_proxy() .resolve( "public.example", std::net::SocketAddr::from(( std::net::Ipv4Addr::LOCALHOST, doc_base.port().unwrap(), )), ) .build() .into_diagnostic()?; app_state.resolver = crate::resolver::Resolver::with_plc_urls_and_client( config.plc_urls.clone(), 64, crate::net::PublicHttpClient::test(public_client), ); Ok(app_state) }
/// drops the database without a shutdown, the way a crash would, and opens it again #[cfg(feature = "indexer_stream")] fn reopen(&mut self) -> miette::Result<()> { // a database opens once at a time, so another one stands in while this one closes let spare = tempfile::tempdir().into_diagnostic()?; let stand_in = AppState::new(&Config { database_path: spare.path().to_path_buf(), ..Default::default() })?; let state = std::mem::replace(&mut self.state, Arc::new(stand_in)); drop(Arc::into_inner(state).expect("nothing else holds the app state")); self.state = Arc::new(Self::open(&self.config, &self.doc_base)?); Ok(()) }
fn seed( &self, repo: &RepoState<'static>, index_id: u64, records: &[(&str, &[u8])], ) -> miette::Result<()> { let mut batch = self.state.db.inner.batch(); batch.insert( &self.state.db.repos, keys::repo_key(&self.did), db::ser_repo_state(repo)?, ); batch.insert( &self.state.db.repo_metadata, keys::repo_metadata_key(&self.did), db::ser_repo_meta(&RepoMetadata::backfilling(index_id))?, ); self.state.db.indexer.stage_pending_insert( &mut batch, &self.did, &keys::pending_key(index_id), ); for &(rkey, body) in records { self.state.db.indexer.stage_head( &mut batch, keys::record_key(&self.did, COLLECTION, &DbRkey::new(rkey)), body, ); } batch.commit().into_diagnostic() }
/// mirrors the retry worker: a completed queue entry gives way to a fresh pending key. fn reindex_pending(&self, from: u64, to: u64) -> miette::Result<()> { let mut batch = self.state.db.inner.batch(); let indexer = &self.state.db.indexer; indexer.stage_pending_remove(&mut batch, &self.did, &keys::pending_key(from)); indexer.stage_pending_insert(&mut batch, &self.did, &keys::pending_key(to)); batch.insert( &self.state.db.repo_metadata, keys::repo_metadata_key(&self.did), db::ser_repo_meta(&RepoMetadata::backfilling(to))?, ); batch.commit().into_diagnostic() }
fn clear_resync(&self) -> miette::Result<()> { self.state .db .indexer .resync .remove(keys::repo_key(&self.did)) .into_diagnostic()?; Ok(()) }
/// the authoritative deletion primitive plus its `Deleted` tombstone: the same pair the /// firehose and list-repos paths apply under the repository lock, then the wipe's /// steps the way the owner shard runs them fn delete_authoritatively(&self, repo: &RepoState<'static>) -> miette::Result<()> { self.start_deletion(repo)?; FirehoseWorker::run_job(&self.state, &self.did); Ok(()) }
/// [`Self::delete_authoritatively`] up to its wipe's marker, which a shard runs from fn start_deletion(&self, repo: &RepoState<'static>) -> miette::Result<()> { let mut txn = DbTxn::new(&self.state.db); crate::ops::start_wipe(&mut txn, &self.state.db, &self.did, repo)?; crate::ops::transition_repo( &mut txn, &mut Vec::new(), &self.did, repo.clone(), RepoStatus::Deleted, )?; txn.commit() }
/// the bulk job the repo's marker names, if any fn marker(&self) -> miette::Result<Option<crate::types::BulkJobV1>> { db::read_bulk_job(&self.state.db, &self.did) }
/// starts a shard, which picks up every marker before anything else, and waits for it /// to run out of work async fn start_a_shard(&self) -> miette::Result<()> { let (tx, shard) = FirehoseWorker::spawn_one_shard(self.state.clone()); drop(tx); tokio::task::spawn_blocking(move || shard.join()) .await .into_diagnostic()? .map_err(|_| miette::miette!("indexer shard panicked")) }
async fn run(&self, pending_key: [u8; 8]) -> Result<TaskDisposition, BackfillError> { let (tx, shard) = FirehoseWorker::spawn_one_shard(self.state.clone()); let result = self.run_with(tx, pending_key).await; // the task held the shard's only sender, so the shard stops now tokio::task::spawn_blocking(move || shard.join()) .await .into_diagnostic()? .map_err(|_| miette::miette!("indexer shard panicked"))?; result }
/// `run`, handing the finished backfill to `tx`, which may have no shard behind it async fn run_with( &self, tx: IndexerTx, pending_key: [u8; 8], ) -> Result<TaskDisposition, BackfillError> { let (_, http) = public_client(self.pds.address, &self.state.throttler); let permit = Arc::new(Semaphore::new(1)).acquire_owned().await.unwrap(); did_task( &self.state, http, tx, &self.did, Slice::from(pending_key), self.admission.clone(), permit, false, self.strategy, ) .await }
/// a live commit creating `rkey` out of `car` fn live_commit( &self, car: &BuiltCar, rkey: &str, body: &[u8], ) -> miette::Result<StreamCommit<'static>> { let root = crate::car::parse_car(car.bytes.clone(), false)?.root; let cid = crate::car::compute_cid(body); Ok(StreamCommit { blobs: Vec::new(), blocks: car.bytes.clone(), commit: CidLink::from(root), ops: vec![RepoOp { action: RepoOpAction::Create, cid: Some(CidLink::from(cid)), path: CowStr::Owned(format!("{COLLECTION}/{rkey}").into()), prev: None, }], prev_data: None, rebase: false, repo: self.did.clone(), rev: car.rev.clone(), seq: 1, since: None, time: Datetime::try_from("2026-10-05T12:00:00Z".to_owned()).into_diagnostic()?, too_big: false, }) }
/// [`Self::live_commit`] for `c`, followed by an op whose rkey doesn't parse, so applying /// it fails halfway fn half_bad_commit( &self, car: &BuiltCar, c: &[u8], ) -> miette::Result<StreamCommit<'static>> { let mut commit = self.live_commit(car, "c", c)?; let mut bad = commit.ops[0].clone(); bad.path = CowStr::Owned(format!("{COLLECTION}/bad rkey").into()); commit.ops.push(bad); Ok(commit) }
/// hands `commit`, whose blocks are `car`, to a shard as a live event and waits for it async fn apply_live( &self, car: &BuiltCar, commit: StreamCommit<'static>, ) -> miette::Result<()> { let (tx, shard) = FirehoseWorker::spawn_one_shard(self.state.clone()); tx.send(live_event(car, commit, false)?) .await .map_err(|_| miette::miette!("the shard is gone"))?; drop(tx); tokio::task::spawn_blocking(move || shard.join()) .await .into_diagnostic()? .map_err(|_| miette::miette!("indexer shard panicked")) }
/// runs the task and `change` while it waits after its `after`th backfill step. the /// change gets the shard's sender, so it can hand the shard live events in between async fn run_pausing<F, Fut>( &self, pending_key: [u8; 8], after: usize, change: F, ) -> miette::Result<Result<TaskDisposition, BackfillError>> where F: FnOnce(IndexerTx) -> Fut, Fut: Future<Output = miette::Result<()>>, { let (tx, shard) = FirehoseWorker::spawn_one_shard(self.state.clone()); // the api hands an erase to the same shard self.state.indexer.lock().replace(tx.clone()); let (paused, mut waiting) = tokio::sync::mpsc::unbounded_channel(); let (resume, resumed) = std::sync::mpsc::channel::<()>(); let resumed = std::sync::Mutex::new(resumed); *self.state.after_chunk.lock() = Some(Arc::new(move |sent| { if sent == after { let _ = paused.send(()); let _ = resumed.lock().unwrap().recv(); } })); let run = async { let result = self.run_with(tx.clone(), pending_key).await; // lets go of `paused`, so a change waiting on a step that never came ends *self.state.after_chunk.lock() = None; result }; let change = async { if waiting.recv().await.is_none() { return Err(miette::miette!("the task never got to step {after}")); } let changed = change(tx.clone()).await; let _ = resume.send(()); changed }; let (result, changed) = tokio::join!(run, change); self.state.indexer.lock().take(); drop(tx); tokio::task::spawn_blocking(move || shard.join()) .await .into_diagnostic()? .map_err(|_| miette::miette!("indexer shard panicked"))?; changed?; Ok(result) }
/// [`Self::live_commit`], buffered the way the shard does while the repo is queued fn buffer(&self, car: &BuiltCar, rkey: &str, body: &[u8]) -> miette::Result<()> { self.buffer_commit(&self.live_commit(car, rkey, body)?) }
fn buffer_commit(&self, commit: &StreamCommit<'static>) -> miette::Result<()> { let mut txn = DbTxn::new(&self.state.db); crate::ops::persist_to_resync_buffer(&mut txn, &self.state.db, &self.did, commit)?; txn.commit() }
/// queues the repo under 7 and serves a car with record `a`, while the repo has moved on /// to a commit adding `b`. returns that commit's car and `b` async fn fetch_behind_a_live_commit(&self) -> miette::Result<(BuiltCar, Vec<u8>)> { self.seed(&rooted("3jzfcijpj2z2a")?, 7, &[])?; let a = post("a"); let b = post("b"); let fetched = build_car("3jzfcijpj2z2b", &[("a", a.as_slice())]).await?; let newer = build_car("3jzfcijpj2z2c", &[("a", a.as_slice()), ("b", b.as_slice())]).await?; self.pds.serve_car(fetched.bytes).await; Ok((newer, b)) }
/// [`Self::fetch_behind_a_live_commit`], after the shard buffered that commit during the /// fetch. returns its car async fn fetch_with_a_buffered_commit(&self) -> miette::Result<BuiltCar> { let (newer, b) = self.fetch_behind_a_live_commit().await?; self.buffer(&newer, "b", &b)?; Ok(newer) }
/// the repo is synced at `newer`, which added `b` to the fetched `a`, with nothing left /// queued or buffered fn assert_caught_up_to(&self, newer: &BuiltCar) -> miette::Result<()> { assert_eq!(self.record_rkeys()?, ["a", "b"]); let after = self.repo()?; assert_eq!( after.root.map(|root| root.rev), Some(crate::db::types::DbTid::from(&newer.rev)) ); assert_eq!(after.status, RepoStatus::Synced); assert!(self.pending_keys()?.is_empty()); assert_eq!(self.buffered(), 0); Ok(()) }
/// a channel with no shard behind it, so nobody answers the handoff fn unanswered(&self) -> IndexerTx { FirehoseWorker::new(self.state.clone(), 1).0 }
fn buffered(&self) -> usize { self.state .db .indexer .resync_buffer .prefix(keys::resync_buffer_prefix(&self.did)) .count() }
/// runs the task for `pending_key` and applies `change` while the task waits on getRepo, /// the way another writer can move the queue under a task that's already in flight async fn run_changing( &self, pending_key: [u8; 8], change: impl Future<Output = miette::Result<()>>, ) -> miette::Result<Result<TaskDisposition, BackfillError>> { self.run_changing_at(&self.pds.get_repo, pending_key, change) .await }
/// `run_changing` with the finished backfill handed to `tx`, a shard the test made, so the /// change can hand it work of its own async fn run_changing_on( &self, tx: IndexerTx, pending_key: [u8; 8], change: impl Future<Output = miette::Result<()>>, ) -> miette::Result<Result<TaskDisposition, BackfillError>> { let held = self.pds.get_repo.hold.lock().await; let arrived = self.pds.get_repo.arrived.notified(); tokio::pin!(arrived); arrived.as_mut().enable(); let (result, changed) = tokio::join!(self.run_with(tx, pending_key), async { arrived.await; let changed = change.await; drop(held); changed }); changed?; Ok(result) }
/// `run_changing`, with the change applied while the task waits at `gate` async fn run_changing_at( &self, gate: &Gate, pending_key: [u8; 8], change: impl Future<Output = miette::Result<()>>, ) -> miette::Result<Result<TaskDisposition, BackfillError>> { let held = gate.hold.lock().await; // registered before the task starts, because `notify_waiters` only wakes // waiters that already exist and keeps nothing for a later one, so a // request from an earlier run can't let this change in early let arrived = gate.arrived.notified(); tokio::pin!(arrived); arrived.as_mut().enable(); let (result, changed) = tokio::join!(self.run(pending_key), async { arrived.await; let changed = change.await; drop(held); changed }); changed?; Ok(result) }
fn put_repo(&self, repo: &RepoState<'_>) -> miette::Result<()> { let mut batch = self.state.db.inner.batch(); batch.insert( &self.state.db.repos, keys::repo_key(&self.did), db::ser_repo_state(repo)?, ); batch.commit().into_diagnostic() }
fn repo(&self) -> miette::Result<RepoState<'static>> { let bytes = self .state .db .repos .get(keys::repo_key(&self.did)) .into_diagnostic()? .expect("repository state row") .to_vec(); Ok(db::deser_repo_state(&bytes)?.into_static()) }
fn resync(&self) -> miette::Result<Option<ResyncState>> { self.state .db .indexer .resync .get(keys::repo_key(&self.did)) .into_diagnostic()? .map(|bytes| rmp_serde::from_slice::<ResyncState>(&bytes).into_diagnostic()) .transpose() }
fn pending_keys(&self) -> miette::Result<Vec<Vec<u8>>> { self.state .db .indexer .pending_iter() .map(|guard| Ok(guard.into_inner().into_diagnostic()?.0.to_vec())) .collect() }
fn record_rkeys(&self) -> miette::Result<Vec<String>> { let prefix = keys::record_prefix_did(&self.did); self.state .db .indexer .head_prefix(&prefix) .map(|guard| { let (key, _) = guard.into_inner().into_diagnostic()?; let (_, rkey) = keys::split_record_suffix(&key[prefix.len()..])?; Ok(rkey.to_string()) }) .collect() } }
/// `commit`, whose blocks are `car`, as the relay hands it to the indexer fn live_event( car: &BuiltCar, commit: StreamCommit<'static>, chain_break: bool, ) -> miette::Result<IndexerMessage> { Ok(IndexerMessage::Event(Box::new(IndexerEvent { seq: 1, firehose: Url::parse("wss://relay.example").into_diagnostic()?, ticket: None, row: None, data: IndexerEventData::Commit(IndexerCommitData { commit, chain_break, parsed_blocks: crate::car::parse_commit_car(car.bytes.clone(), false)?, }), }))) }
#[tokio::test] async fn unsafe_pds_endpoint_is_parked_until_fresh_identity_evidence() -> miette::Result<()> { let pds = PdsStub::spawn().await; let fixture = Fixture::new_with_pds(pds, Url::parse("http://127.0.0.1/").unwrap()).await?; fixture.seed(&RepoState::backfilling().into_static(), 13, &[])?;
assert_eq!( fixture.run(keys::pending_key(13)).await?, TaskDisposition::Finished, "unsafe endpoint work must park instead of entering retry backoff" ); assert!(fixture.pending_keys()?.is_empty()); assert!(fixture.resync()?.is_none()); assert_eq!( fixture.pds.requests(), 0, "unsafe endpoint must not connect" ); Ok(()) }
#[tokio::test] async fn car_whose_commit_names_another_did_is_rejected() -> miette::Result<()> { let fixture = Fixture::new().await?; fixture.seed(&RepoState::backfilling().into_static(), 9, &[])?; let body = post("a"); let other = "did:plc:bbbbbbbbbbbbbbbbbbbbbbbb"; let car = build_car_for(other, "3jzfcijpj2z2a", &[("a", body.as_slice())]).await?; fixture.pds.serve_car(car.bytes).await;
let err = fixture.run(keys::pending_key(9)).await.unwrap_err();
assert!(err.to_string().contains("commit is for"), "{err}"); assert!(fixture.record_rkeys()?.is_empty()); Ok(()) }
const WANTED: &str = "sh.tangled.repo";
fn only_collection(fixture: &Fixture, collection: &str) { let mut filter = crate::filter::FilterConfig::new(crate::filter::FilterMode::Filter); filter.collections = vec![collection.into()]; fixture.state.filter.store(Arc::new(filter)); }
fn stored(fixture: &Fixture, collection: &str, rkey: &str) -> miette::Result<Option<Vec<u8>>> { let key = keys::record_key(&fixture.did, collection, &DbRkey::new(rkey)); Ok(fixture .state .db .indexer .head(key) .into_diagnostic()? .map(|body| body.to_vec())) }
#[tokio::test] async fn filtered_full_backfill_stores_what_a_whole_car_would_for_the_wanted_collection() -> miette::Result<()> { let (core, site) = (tangled_repo("core"), tangled_repo("site")); let posts = (0..300) .map(|i| post(&format!("post {i}"))) .collect::<Vec<_>>(); let mut records = posts .iter() .enumerate() .map(|(i, body)| (format!("{COLLECTION}/3jzfcijpj{i:04}"), body.as_slice())) .collect::<Vec<_>>(); records.push((format!("{WANTED}/core"), core.as_slice())); records.push((format!("{WANTED}/site"), site.as_slice())); let car = build_car_keyed(DID, "3jzfcijpj2z2a", &records).await?;
let filtered = Fixture::new().await?; only_collection(&filtered, WANTED); filtered.seed(&RepoState::backfilling().into_static(), 9, &[])?; filtered.pds.serve_car(car.bytes.clone()).await; let whole = Fixture::new().await?; whole.seed(&RepoState::backfilling().into_static(), 9, &[])?; whole.pds.serve_car(car.bytes).await;
assert_eq!( filtered.run(keys::pending_key(9)).await?, TaskDisposition::Finished ); assert_eq!( whole.run(keys::pending_key(9)).await?, TaskDisposition::Finished );
assert_eq!(filtered.record_rkeys()?, vec!["core", "site"]); assert_eq!(stored(&filtered, WANTED, "core")?, Some(core.clone())); for rkey in ["core", "site"] { assert_eq!( stored(&filtered, WANTED, rkey)?, stored(&whole, WANTED, rkey)? ); } assert_eq!( filtered.repo()?.root.map(|root| root.data), whole.repo()?.root.map(|root| root.data) ); assert_eq!(filtered.pds.requests(), 1); Ok(()) }
#[tokio::test] async fn full_backfill_stores_every_record_a_whole_parse_finds() -> miette::Result<()> { let core = tangled_repo("core"); let posts = (0..500) .map(|i| post(&format!("post {i}"))) .collect::<Vec<_>>(); let mut records = posts .iter() .enumerate() .map(|(i, body)| (format!("{COLLECTION}/3jzfcijpj{i:04}"), body.as_slice())) .collect::<Vec<_>>(); records.push((format!("{WANTED}/core"), core.as_slice())); // the same body under two keys is one block in the car records.push((format!("{WANTED}/copy"), core.as_slice())); let car = build_car_keyed(DID, "3jzfcijpj2z2a", &records).await?;
let parsed = crate::car::parse_car(car.bytes.clone(), true)?; let want = crate::mst::leaves(parsed.blocks.map(), car.mst_root)? .into_iter() .map(|(key, cid)| (key.to_string(), parsed.blocks.get(&cid).unwrap().to_vec())) .collect::<std::collections::BTreeMap<_, _>>(); assert_eq!(want.len(), records.len());
let fixture = Fixture::new().await?; fixture.seed(&RepoState::backfilling().into_static(), 9, &[])?; fixture.pds.serve_car(car.bytes).await; assert_eq!( fixture.run(keys::pending_key(9)).await?, TaskDisposition::Finished );
let prefix = keys::record_prefix_did(&fixture.did); let stored = fixture .state .db .indexer .head_prefix(&prefix) .map(|guard| { let (key, body) = guard.into_inner().into_diagnostic()?; let (collection, rkey) = keys::split_record_suffix(&key[prefix.len()..])?; Ok((format!("{collection}/{rkey}"), body.to_vec())) }) .collect::<miette::Result<std::collections::BTreeMap<_, _>>>()?; assert_eq!(stored, want); Ok(()) }
/// `car` with the block under `cid` left out async fn car_without(car: &BuiltCar, cid: IpldCid) -> miette::Result<Bytes> { let parsed = crate::car::parse_car(car.bytes.clone(), false)?; let mut bytes = Vec::new(); let mut writer = crate::car::CarWriter::new(crate::car::CarHeader::new_v1(vec![parsed.root]), &mut bytes); for (block_cid, body) in parsed.blocks.map() { if *block_cid != cid { writer.write(*block_cid, body).await.into_diagnostic()?; } } writer.finish().await.into_diagnostic()?; Ok(Bytes::from(bytes)) }
#[tokio::test] async fn a_car_missing_an_mst_node_writes_nothing() -> miette::Result<()> { let posts = (0..300) .map(|i| post(&format!("post {i}"))) .collect::<Vec<_>>(); let records = posts .iter() .enumerate() .map(|(i, body)| (format!("{COLLECTION}/3jzfcijpj{i:04}"), body.as_slice())) .collect::<Vec<_>>(); let car = build_car_keyed(DID, "3jzfcijpj2z2a", &records).await?; // the root's last subtree, so the walk hands out leaves before it finds the gap let parsed = crate::car::parse_car(car.bytes.clone(), false)?; let root = parsed.blocks.get(&car.mst_root).expect("root node"); let subtree = crate::mst::decode_node(root)? .into_iter() .rev() .find_map(|entry| match entry { crate::mst::FlatEntry::Tree { cid } => Some(cid), crate::mst::FlatEntry::Leaf { .. } => None, }) .expect("300 records need more than one node");
let fixture = Fixture::new().await?; let old = post("old"); fixture.seed( &RepoState::backfilling().into_static(), 9, &[("old", old.as_slice())], )?; fixture .pds .serve_car(car_without(&car, subtree).await?) .await;
let err = fixture.run(keys::pending_key(9)).await.unwrap_err(); assert!(err.to_string().contains("missing from car"), "{err}"); assert_eq!(fixture.record_rkeys()?, ["old"]); assert_eq!(stored(&fixture, COLLECTION, "old")?, Some(old)); Ok(()) }
#[tokio::test] async fn a_car_with_a_key_that_doesnt_parse_writes_nothing() -> miette::Result<()> { let (a, bad) = (post("a"), post("bad")); // a fine mst key, but its collection isn't an nsid let records = [ (format!("{COLLECTION}/a"), a.as_slice()), ("notansid/b".to_string(), bad.as_slice()), ]; let car = build_car_keyed(DID, "3jzfcijpj2z2a", &records).await?; let fixture = Fixture::new().await?; let old = post("old"); fixture.seed( &RepoState::backfilling().into_static(), 9, &[("old", old.as_slice())], )?; fixture.pds.serve_car(car.bytes).await;
let err = fixture.run(keys::pending_key(9)).await.unwrap_err(); assert!(err.to_string().contains("invalid collection NSID"), "{err}"); assert_eq!(fixture.record_rkeys()?, ["old"]); Ok(()) }
#[tokio::test] async fn a_record_whose_block_the_car_left_out_is_skipped() -> miette::Result<()> { let (a, b) = (post("a"), post("b")); let car = build_car("3jzfcijpj2z2a", &[("a", a.as_slice()), ("b", b.as_slice())]).await?; let fixture = Fixture::new().await?; fixture.seed(&RepoState::backfilling().into_static(), 9, &[])?; let b_cid = crate::car::compute_cid(&b); fixture.pds.serve_car(car_without(&car, b_cid).await?).await;
assert_eq!( fixture.run(keys::pending_key(9)).await?, TaskDisposition::Finished ); assert_eq!(fixture.record_rkeys()?, ["a"]); Ok(()) }
#[tokio::test] async fn a_repo_without_a_signal_record_writes_nothing() -> miette::Result<()> { let fixture = Fixture::new().await?; let mut filter = crate::filter::FilterConfig::new(crate::filter::FilterMode::Filter); filter.signals = vec![WANTED.into()]; fixture.state.filter.store(Arc::new(filter)); let old = post("old"); fixture.seed( &RepoState::backfilling().into_static(), 9, &[("old", old.as_slice())], )?; let a = post("a"); let car = build_car("3jzfcijpj2z2a", &[("a", a.as_slice())]).await?; fixture.pds.serve_car(car.bytes).await;
assert_eq!( fixture.run(keys::pending_key(9)).await?, TaskDisposition::Finished ); assert_eq!(fixture.record_rkeys()?, ["old"]); assert_eq!(stored(&fixture, COLLECTION, "old")?, Some(old)); Ok(()) }
#[tokio::test] async fn a_filtered_backfill_keeps_and_counts_only_the_wanted_collection() -> miette::Result<()> { let (a, core, site) = (post("a"), tangled_repo("core"), tangled_repo("site")); let records = [ (format!("{COLLECTION}/a"), a.as_slice()), (format!("{WANTED}/core"), core.as_slice()), (format!("{WANTED}/site"), site.as_slice()), ]; let car = build_car_keyed(DID, "3jzfcijpj2z2a", &records).await?; let fixture = Fixture::new().await?; only_collection(&fixture, WANTED); let old = post("old"); fixture.seed( &RepoState::backfilling().into_static(), 9, &[("old", old.as_slice())], )?; let mut batch = fixture.state.db.inner.batch(); db::set_record_count(&mut batch, &fixture.state.db, &fixture.did, COLLECTION, 1); batch.commit().into_diagnostic()?; fixture.pds.serve_car(car.bytes).await;
assert_eq!( fixture.run(keys::pending_key(9)).await?, TaskDisposition::Finished ); // a stored record the filter no longer wants goes, like one the car doesn't have assert_eq!(fixture.record_rkeys()?, ["core", "site"]); let count = |collection| db::get_record_count(&fixture.state.db, &fixture.did, collection); assert_eq!(count(WANTED)?, 2); assert_eq!(count(COLLECTION)?, 0); Ok(()) }
#[cfg(feature = "indexer_stream")] /// the stream's record events as (collection/rkey, action), in stream order fn record_events( fixture: &Fixture, ) -> miette::Result<Vec<(String, crate::db::types::DbAction)>> { fixture .state .db .stream .event_range::<&[u8], _>(..) .map(|guard| { let value = guard.value().into_diagnostic()?; let event: crate::types::StoredEvent<'_> = rmp_serde::from_slice(&value).into_diagnostic()?; let key = format!("{}/{}", event.collection, event.rkey.to_smolstr()); Ok((key, event.action)) }) .collect() }
#[cfg(feature = "indexer_stream")] #[tokio::test] async fn a_backfill_over_stored_records_writes_only_what_changed() -> miette::Result<()> { let (a_old, a_new, b, c, d) = (post("a"), post("a2"), post("b"), post("c"), post("d")); let fixture = Fixture::new().await?; fixture.seed( &RepoState::backfilling().into_static(), 9, &[ ("a", a_old.as_slice()), ("b", b.as_slice()), ("c", c.as_slice()), ], )?; let car = build_car( "3jzfcijpj2z2a", &[ ("a", a_new.as_slice()), ("b", b.as_slice()), ("d", d.as_slice()), ], ) .await?; fixture.pds.serve_car(car.bytes).await;
assert_eq!( fixture.run(keys::pending_key(9)).await?, TaskDisposition::Finished ); assert_eq!(fixture.record_rkeys()?, ["a", "b", "d"]); assert_eq!(stored(&fixture, COLLECTION, "a")?, Some(a_new)); // in key order, so the delete of c sits between the other two assert_eq!( record_events(&fixture)?, [ ( format!("{COLLECTION}/a"), crate::db::types::DbAction::Update ), ( format!("{COLLECTION}/c"), crate::db::types::DbAction::Delete ), ( format!("{COLLECTION}/d"), crate::db::types::DbAction::Create ), ] ); assert_eq!( db::get_record_count(&fixture.state.db, &fixture.did, COLLECTION)?, 3 ); Ok(()) }
#[cfg(feature = "indexer_stream")] async fn car_of(records: &[&(String, Vec<u8>)], rev: &str) -> miette::Result<BuiltCar> { let keyed = records .iter() .map(|(key, body)| (key.clone(), body.as_slice())) .collect::<Vec<_>>(); build_car_keyed(DID, rev, &keyed).await }
#[cfg(feature = "indexer_stream")] #[tokio::test] async fn rebackfilling_over_prefix_related_collections_writes_only_what_changed() -> miette::Result<()> { // collections that prefix each other or differ right before the `/`, and both kinds // of rkey, which is where heads key order and mst order could part let collections = ["a.b.c", "a.b.c.d", "a.b.cd", "a.b-c.d"]; let rkeys = [ "3jzfcijpj2z2a", "3k2akgqbabc2x", "self", "a.b", "a-b", "~", "z:z", ]; let mut records = collections .iter() .flat_map(|collection| { rkeys.iter().map(move |rkey| { let body = serde_ipld_dagcbor::to_vec(&serde_json::json!({ "$type": collection, "rkey": rkey, })) .unwrap(); (format!("{collection}/{rkey}"), body) }) }) .collect::<Vec<_>>(); records.sort(); let every = records.iter().collect::<Vec<_>>(); // a third of them go in the last car, spread over every collection let (dropped, kept): (Vec<_>, Vec<_>) = every.iter().enumerate().partition(|(i, _)| i % 3 == 1); let kept = kept .into_iter() .map(|(_, record)| *record) .collect::<Vec<_>>(); let fixture = Fixture::new().await?; fixture.seed(&RepoState::backfilling().into_static(), 9, &[])?; let heads = || -> miette::Result<Vec<(Vec<u8>, Vec<u8>)>> { fixture .state .db .indexer .head_prefix(keys::record_prefix_did(&fixture.did)) .map(|guard| { let (key, value) = guard.into_inner().into_diagnostic()?; Ok((key.to_vec(), value.to_vec())) }) .collect() }; let counts = || -> miette::Result<Vec<u64>> { collections .iter() .map(|collection| db::get_record_count(&fixture.state.db, &fixture.did, collection)) .collect() }; let run = |from: u64, to: u64, car: BuiltCar| { let fixture = &fixture; async move { if from != to { fixture.reindex_pending(from, to)?; } fixture.pds.serve_car(car.bytes).await; assert_eq!( fixture.run(keys::pending_key(to)).await?, TaskDisposition::Finished ); Ok::<_, miette::Report>(()) } };
run(9, 9, car_of(&every, "3jzfcijpj2z2a").await?).await?; let (events, stored, counted) = (record_events(&fixture)?, heads()?, counts()?); assert_eq!(events.len(), records.len()); assert_eq!(stored.len(), records.len()); assert_eq!(counted, [rkeys.len() as u64; 4]);
// the same car again changes nothing run(9, 10, car_of(&every, "3jzfcijpj2z2a").await?).await?; assert_eq!(record_events(&fixture)?, events); assert_eq!(heads()?, stored); assert_eq!(counts()?, counted);
// a car without some of them deletes just those, in key order run(10, 11, car_of(&kept, "3jzfcijpj2z2b").await?).await?; let deletes = dropped .iter() .map(|(_, (key, _))| (key.clone(), crate::db::types::DbAction::Delete)) .collect::<Vec<_>>(); assert_eq!(record_events(&fixture)?[events.len()..], deletes); let kept_heads = stored .iter() .enumerate() .filter(|(i, _)| i % 3 != 1) .map(|(_, head)| head.clone()) .collect::<Vec<_>>(); assert_eq!(heads()?, kept_heads); assert_eq!(counts()?.iter().sum::<u64>(), kept.len() as u64); Ok(()) }
#[tokio::test] async fn a_full_backfill_holds_its_budget_until_its_last_step() -> miette::Result<()> { let body = post("a"); let car = build_car("3jzfcijpj2z2a", &[("a", body.as_slice())]).await?; let fixture = Fixture::new().await?; fixture.seed(&RepoState::backfilling().into_static(), 9, &[])?; fixture.pds.serve_car(car.bytes.clone()).await;
// the repo's shard, kept busy so the write waits behind it after the fetch is done let (tx, shard) = FirehoseWorker::spawn_one_shard(fixture.state.clone()); let (busy_tx, busy) = tokio::sync::oneshot::channel(); let (release, release_rx) = std::sync::mpsc::channel::<()>(); let release_rx = parking_lot::Mutex::new(release_rx); let did = fixture.did.clone(); let busy_shard = async { let work = move |_: &AppState, _: &mut crate::ingest::indexer::bulk::ShardState| { busy_tx.send(()).unwrap(); let _ = release_rx.lock().recv(); }; tx.inner .send(IndexerMessage::OnShard( crate::ingest::indexer::message::OnShard { did, work: Box::new(work), }, )) .await .map_err(|_| miette::miette!("the shard is gone")) }; let budget = fixture.state.backfill_budget.clone(); let check = async { busy.await.unwrap(); tokio::time::sleep(std::time::Duration::from_millis(300)).await; let held = budget.used(); release.send(()).unwrap(); held }; let (result, held) = tokio::join!( fixture.run_changing_on(tx.clone(), keys::pending_key(9), busy_shard), check ); drop(tx); tokio::task::spawn_blocking(move || shard.join()) .await .into_diagnostic()? .map_err(|_| miette::miette!("indexer shard panicked"))?;
assert_eq!(result??, TaskDisposition::Finished); // a spilled car holds its index and a step, not the car let peak = if fixture.config.backfill_spill_bytes < car.bytes.len() { crate::backfill::budget::spilled_peak_for(car.bytes.len()) } else { crate::backfill::budget::peak_for(car.bytes.len()) }; assert_eq!(held, peak); assert_eq!(budget.used(), 0); Ok(()) }
/// a repo with a record filed under a collection other than its `$type`, which a filtered /// car drops and the scan then finds missing async fn car_with_a_misfiled_record() -> miette::Result<(BuiltCar, Vec<u8>)> { let odd = post("filed under the wrong collection"); let core = tangled_repo("core"); let records = [ (format!("{WANTED}/core"), core.as_slice()), (format!("{WANTED}/odd"), odd.as_slice()), (format!("{COLLECTION}/3jzfcijpj0001"), odd.as_slice()), ]; let car = build_car_keyed(DID, "3jzfcijpj2z2a", &records).await?; Ok((car, odd)) }
/// a getBlocks answer holding `bodies` async fn blocks_car(bodies: &[&[u8]]) -> miette::Result<Bytes> { let cids = bodies .iter() .map(|body| crate::car::compute_cid(body)) .collect::<Vec<_>>(); let mut bytes = Vec::new(); let mut writer = crate::car::CarWriter::new(crate::car::CarHeader::new_v1(cids.clone()), &mut bytes); for (cid, body) in cids.iter().zip(bodies) { writer.write(*cid, body).await.into_diagnostic()?; } writer.finish().await.into_diagnostic()?; Ok(Bytes::from(bytes)) }
#[tokio::test] async fn get_blocks_brings_back_what_the_filtered_car_dropped() -> miette::Result<()> { let (car, odd) = car_with_a_misfiled_record().await?; let fixture = Fixture::new().await?; only_collection(&fixture, WANTED); fixture.seed(&RepoState::backfilling().into_static(), 9, &[])?; fixture.pds.serve_car(car.bytes).await; fixture .pds .serve_blocks(StatusCode::OK, blocks_car(&[&odd]).await?) .await;
// the refetch still counts against the pds, so nothing else takes its slot meanwhile let result = fixture .run_changing_at(&fixture.pds.get_blocks, keys::pending_key(9), async { let slot = fixture.admission.try_acquire(&fixture.pds.base).await; assert!(slot.is_none(), "the refetch let go of the pds admission"); Ok(()) }) .await?;
assert_eq!(result?, TaskDisposition::Finished); assert_eq!(fixture.record_rkeys()?, vec!["core", "odd"]); assert_eq!(stored(&fixture, WANTED, "odd")?, Some(odd)); assert_eq!( (fixture.pds.requests(), fixture.pds.block_requests()), (1, 1) ); let slot = fixture.admission.try_acquire(&fixture.pds.base).await; assert!(slot.is_some(), "the finished task kept the pds admission"); Ok(()) }
#[tokio::test] async fn a_ratelimited_get_blocks_backs_off_instead_of_fetching_the_whole_car() -> miette::Result<()> { let (car, _) = car_with_a_misfiled_record().await?; let fixture = Fixture::new().await?; only_collection(&fixture, WANTED); fixture.seed(&RepoState::backfilling().into_static(), 9, &[])?; fixture.pds.serve_car(car.bytes).await; fixture .pds .serve_blocks(StatusCode::TOO_MANY_REQUESTS, Bytes::new()) .await;
let err = fixture.run(keys::pending_key(9)).await.unwrap_err();
assert!(matches!(err, BackfillError::Ratelimited), "{err}"); assert_eq!( (fixture.pds.requests(), fixture.pds.block_requests()), (1, 1) ); assert!(fixture.record_rkeys()?.is_empty()); Ok(()) }
#[tokio::test] async fn a_repo_that_moved_on_before_the_refetch_is_retried() -> miette::Result<()> { let (car, _) = car_with_a_misfiled_record().await?; let core = tangled_repo("core"); let moved = build_car_keyed( DID, "3jzfcijpj2z2b", &[(format!("{WANTED}/core"), core.as_slice())], ) .await?; let fixture = Fixture::new().await?; only_collection(&fixture, WANTED); fixture.seed(&RepoState::backfilling().into_static(), 9, &[])?; fixture.pds.serve_car(car.bytes).await;
// the pds has no getBlocks, and by the time it says so the repo has a new commit let result = fixture .run_changing_at(&fixture.pds.get_blocks, keys::pending_key(9), async { fixture.pds.serve_car(moved.bytes.clone()).await; Ok(()) }) .await?;
let err = result.unwrap_err(); assert!(err.to_string().contains("moved on"), "{err}"); assert_eq!(fixture.pds.requests(), 2); assert!(fixture.record_rkeys()?.is_empty()); Ok(()) }
#[tokio::test] async fn a_repo_gone_before_the_refetch_is_retried_as_not_found() -> miette::Result<()> { let (car, _) = car_with_a_misfiled_record().await?; let fixture = Fixture::new().await?; only_collection(&fixture, WANTED); fixture.seed(&RepoState::backfilling().into_static(), 9, &[])?; fixture.pds.serve_car(car.bytes).await;
let result = fixture .run_changing_at(&fixture.pds.get_blocks, keys::pending_key(9), async { *fixture.pds.response.write().await = (StatusCode::BAD_REQUEST, Bytes::from_static(NOT_FOUND_BODY)); Ok(()) }) .await?;
assert!( matches!(result, Err(BackfillError::RepoNotFound)), "{result:?}" ); assert!(fixture.record_rkeys()?.is_empty()); Ok(()) }
#[tokio::test] async fn record_filed_under_another_type_comes_back_from_a_whole_car() -> miette::Result<()> { // the stub has no getBlocks, like the atcr hold, so the dropped record can only come // from a second, whole car let (car, odd) = car_with_a_misfiled_record().await?; let fixture = Fixture::new().await?; only_collection(&fixture, WANTED); fixture.seed(&RepoState::backfilling().into_static(), 9, &[])?; fixture.pds.serve_car(car.bytes).await;
assert_eq!( fixture.run(keys::pending_key(9)).await?, TaskDisposition::Finished );
assert_eq!(fixture.record_rkeys()?, vec!["core", "odd"]); assert_eq!(stored(&fixture, WANTED, "odd")?, Some(odd)); assert_eq!(fixture.pds.requests(), 2); Ok(()) }
#[tokio::test] async fn repo_not_found_keeps_a_rooted_snapshot_and_parks_a_retry() -> miette::Result<()> { let fixture = Fixture::new().await?; let seeded = rooted("3jzfcijpj2z2a")?; let body = post("a"); fixture.seed(&seeded, 7, &[("a", body.as_slice())])?;
let err = fixture.run(keys::pending_key(7)).await.unwrap_err();
assert!( matches!(err, BackfillError::RepoNotFound), "unexpected error: {err}" ); let after = fixture.repo()?; assert_eq!( after.root.as_ref().map(|root| root.data), seeded.root.as_ref().map(|root| root.data), "a non-authoritative fetch result must not clear the root" ); assert!( matches!(&after.status, RepoStatus::Error(_)), "expected a retryable error status, got {:?}", after.status ); assert!(after.active); assert_eq!(after.last_message_time, Some(101)); assert_eq!(fixture.record_rkeys()?, ["a"]); assert!( fixture .state .db .repo_metadata .get(keys::repo_metadata_key(&fixture.did)) .into_diagnostic()? .is_some(), "metadata must survive" ); assert!( fixture.pending_keys()?.is_empty(), "parked work must leave pending, or the worker spins" );
let Some(ResyncState::Error { kind, retry_count, next_retry, }) = fixture.resync()? else { panic!("expected a durable retryable error"); }; assert_eq!(kind, ResyncErrorKind::Generic); assert_eq!(retry_count, 1); assert!( next_retry > chrono::Utc::now().timestamp(), "the retry worker may only requeue a due error" ); Ok(()) }
#[cfg(feature = "indexer_stream")] /// what a relay's listRepos pass does to the repo when the relay lists it as taken down fn relay_lists_it_takendown(fixture: &Fixture) -> miette::Result<ReconcileOutcome> { let listing = RepoListing::inactive( fixture.did.clone(), crate::db::types::DbTid::from(&Tid::new("3jzfcijpj2z2b").into_diagnostic()?), ListedRepoStatus::Takendown, ); let mut txn = DbTxn::new(&fixture.state.db); let outcome = reconcile_listing(&fixture.state, &mut txn, &listing)?; txn.commit()?; Ok(outcome) }
#[cfg(feature = "indexer_stream")] fn account_events( events: &mut tokio::sync::broadcast::Receiver<crate::types::BroadcastEvent>, ) -> Vec<(bool, Option<String>)> { std::iter::from_fn(|| events.try_recv().ok()) .filter_map(|event| match event { crate::types::BroadcastEvent::Ephemeral(event) => event.account, _ => None, }) .map(|account| (account.active, account.status.map(|s| s.to_string()))) .collect() }
#[cfg(feature = "indexer_stream")] /// the pending key the repo's metadata names right now fn queued_key(fixture: &Fixture) -> miette::Result<[u8; 8]> { let bytes = fixture .state .db .repo_metadata .get(keys::repo_metadata_key(&fixture.did)) .into_diagnostic()? .expect("metadata row"); Ok(keys::pending_key(db::deser_repo_meta(&bytes)?.index_id)) }
#[cfg(feature = "indexer_stream")] #[tokio::test] async fn a_relay_takedown_does_not_beat_a_pds_that_still_serves_the_repo() -> miette::Result<()> { let fixture = Fixture::new().await?; fixture.seed(&RepoState::backfilling().into_static(), 7, &[])?; let body = post("a"); let car = build_car("3jzfcijpj2z2b", &[("a", body.as_slice())]).await?; fixture.pds.serve_car(car.bytes).await; assert_eq!( fixture.run(keys::pending_key(7)).await?, TaskDisposition::Finished ); let synced = fixture.repo()?; assert!(synced.active); assert_eq!(synced.status, RepoStatus::Synced); let mut events = fixture.state.db.stream.event_tx.subscribe();
// the first pass asks the pds, which serves the repo assert_eq!( relay_lists_it_takendown(&fixture)?, ReconcileOutcome::StateOnly ); let asked = fixture.repo()?; assert!(asked.active); assert_eq!(asked.status, RepoStatus::Synced); assert!(fixture.resync()?.is_none()); assert_eq!(fixture.pending_keys()?.len(), 1); assert_eq!( fixture.run(queued_key(&fixture)?).await?, TaskDisposition::Finished ); assert_eq!(fixture.pds.requests(), 2);
// later passes and a restart leave it synced without fetching it again for _ in 0..2 { assert_eq!( relay_lists_it_takendown(&fixture)?, ReconcileOutcome::Unchanged ); crate::backfill::manager::queue_gone_backfills(&fixture.state)?; assert!(fixture.pending_keys()?.is_empty()); assert!(fixture.resync()?.is_none()); let after = fixture.repo()?; assert!(after.active); assert_eq!(after.status, RepoStatus::Synced); } assert_eq!(fixture.pds.requests(), 2); assert_eq!(account_events(&mut events), []); assert_eq!(fixture.record_rkeys()?, ["a"]); Ok(()) }
#[cfg(feature = "indexer_stream")] #[tokio::test] async fn a_relay_takedown_the_pds_confirms_makes_the_repo_inactive() -> miette::Result<()> { let fixture = Fixture::new().await?; fixture.seed(&RepoState::backfilling().into_static(), 7, &[])?; let body = post("a"); let car = build_car("3jzfcijpj2z2b", &[("a", body.as_slice())]).await?; fixture.pds.serve_car(car.bytes).await; assert_eq!( fixture.run(keys::pending_key(7)).await?, TaskDisposition::Finished ); let mut events = fixture.state.db.stream.event_tx.subscribe();
assert_eq!( relay_lists_it_takendown(&fixture)?, ReconcileOutcome::StateOnly ); assert_eq!(account_events(&mut events), []); *fixture.pds.response.write().await = ( StatusCode::BAD_REQUEST, Bytes::from_static(br#"{"error":"RepoTakendown","message":"gone"}"#), ); assert_eq!( fixture.run(queued_key(&fixture)?).await?, TaskDisposition::Finished );
let after = fixture.repo()?; assert!(!after.active); assert_eq!(after.status, RepoStatus::Takendown); assert!(matches!( fixture.resync()?, Some(ResyncState::Gone { status: RepoStatus::Takendown }) )); assert!(fixture.pending_keys()?.is_empty()); assert_eq!( account_events(&mut events), [(false, Some("takendown".to_string()))] ); assert_eq!( relay_lists_it_takendown(&fixture)?, ReconcileOutcome::Unchanged ); assert_eq!(account_events(&mut events), []); Ok(()) }
#[cfg(feature = "indexer_stream")] #[tokio::test] async fn a_relay_listing_does_not_reactivate_a_repo_the_pds_refuses() -> miette::Result<()> { let fixture = Fixture::new().await?; fixture.seed(&RepoState::backfilling().into_static(), 7, &[])?; let body = post("a"); let car = build_car("3jzfcijpj2z2b", &[("a", body.as_slice())]).await?; fixture.pds.serve_car(car.bytes).await; assert_eq!( fixture.run(keys::pending_key(7)).await?, TaskDisposition::Finished ); *fixture.pds.response.write().await = ( StatusCode::BAD_REQUEST, Bytes::from_static(br#"{"error":"RepoDeactivated","message":"gone"}"#), ); fixture.reindex_pending(7, 8)?; assert_eq!( fixture.run(keys::pending_key(8)).await?, TaskDisposition::Finished ); let refused = fixture.repo()?; assert!(!refused.active); assert_eq!(refused.status, RepoStatus::Deactivated); let mut events = fixture.state.db.stream.event_tx.subscribe(); let relay_lists_it_active = || -> miette::Result<ReconcileOutcome> { let listing = RepoListing::active(fixture.did.clone(), crate::db::types::DbTid::from(&car.rev)); let mut txn = DbTxn::new(&fixture.state.db); let outcome = reconcile_listing(&fixture.state, &mut txn, &listing)?; txn.commit()?; Ok(outcome) };
// the listing only asks the pds, which still refuses assert_eq!(relay_lists_it_active()?, ReconcileOutcome::StateOnly); assert!(!fixture.repo()?.active); assert_eq!(account_events(&mut events), []); assert_eq!( fixture.run(queued_key(&fixture)?).await?, TaskDisposition::Finished ); assert_eq!(fixture.pds.requests(), 3); // the refused fetch says inactive again, like every gone retry does assert_eq!( account_events(&mut events), [(false, Some("deactivated".to_string()))] );
for _ in 0..2 { assert_eq!(relay_lists_it_active()?, ReconcileOutcome::Unchanged); let after = fixture.repo()?; assert!(!after.active); assert_eq!(after.status, RepoStatus::Deactivated); } assert_eq!(fixture.pds.requests(), 3); assert_eq!(account_events(&mut events), []); Ok(()) }
#[tokio::test] async fn backoff_grows_across_requeues_until_a_fetch_succeeds() -> miette::Result<()> { let fixture = Fixture::new().await?; fixture.seed(&RepoState::backfilling().into_static(), 9, &[])?; let metadata = || -> miette::Result<RepoMetadata> { let bytes = fixture .state .db .repo_metadata .get(keys::repo_metadata_key(&fixture.did)) .into_diagnostic()? .expect("metadata row"); db::deser_repo_meta(&bytes) };
for (failures, minutes) in [(1, 2), (2, 4), (3, 8), (4, 16), (5, 32), (6, 60), (7, 60)] { let before = chrono::Utc::now().timestamp(); let pending_key = keys::pending_key(metadata()?.index_id); assert!(matches!( fixture.run(pending_key).await, Err(BackfillError::RepoNotFound) ));
let Some(ResyncState::Error { retry_count, next_retry, .. }) = fixture.resync()? else { panic!("expected a retryable error after failure {failures}"); }; assert_eq!(retry_count, failures); assert_eq!(metadata()?.retry_count, failures); let delay = next_retry - before; let expected = minutes * 60; assert!( (expected * 9 / 10 - 1..=expected * 11 / 10 + 5).contains(&delay), "failure {failures} waits {delay}s, expected about {expected}s" ); assert_eq!(fixture.state.db.get_count_sync("resync"), 1); assert_eq!(fixture.state.db.get_count_sync("error_generic"), 1);
// what the retry worker does once the entry is due assert!(crate::backfill::manager::queue_resync_if( &fixture.state, &fixture.did, |_| true )?); assert!(fixture.resync()?.is_none()); assert_eq!(fixture.pending_keys()?.len(), 1); assert_eq!(fixture.state.db.get_count_sync("resync"), 0); assert_eq!(fixture.state.db.get_count_sync("error_generic"), 0); }
let body = post("a"); let car = build_car("3jzfcijpj2z2b", &[("a", body.as_slice())]).await?; fixture.pds.serve_car(car.bytes).await; let pending_key = keys::pending_key(metadata()?.index_id); assert_eq!(fixture.run(pending_key).await?, TaskDisposition::Finished); assert_eq!(metadata()?.retry_count, 0); assert!(fixture.resync()?.is_none()); assert!(fixture.pending_keys()?.is_empty()); Ok(()) }
#[tokio::test] async fn a_fetch_that_succeeds_leaves_the_refetch_allowance_as_it_was() -> miette::Result<()> { use crate::ingest::indexer::refetch::tests as refetch; let fixture = Fixture::new().await?; fixture.seed(&RepoState::backfilling().into_static(), 9, &[])?; let allowance = crate::types::Allowance { since: chrono::Utc::now().timestamp(), used: 3, }; let mut metadata = refetch::metadata_of(&fixture.state, &fixture.did); metadata.refetch = Some(allowance); refetch::put_metadata(&fixture.state, &fixture.did, &metadata);
let body = post("a"); let car = build_car("3jzfcijpj2z2b", &[("a", body.as_slice())]).await?; fixture.pds.serve_car(car.bytes).await; assert_eq!( fixture.run(keys::pending_key(metadata.index_id)).await?, TaskDisposition::Finished );
let after = refetch::metadata_of(&fixture.state, &fixture.did); assert_eq!(after.retry_count, 0); assert_eq!(after.refetch, Some(allowance)); Ok(()) }
#[tokio::test] async fn repo_not_found_on_a_never_rooted_repo_uses_the_same_retry_policy() -> miette::Result<()> { let fixture = Fixture::new().await?; fixture.seed(&RepoState::backfilling().into_static(), 9, &[])?;
let err = fixture.run(keys::pending_key(9)).await.unwrap_err();
assert!( matches!(err, BackfillError::RepoNotFound), "unexpected error: {err}" ); let after = fixture.repo()?; assert!(after.root.is_none(), "no root may be invented"); assert!( matches!(&after.status, RepoStatus::Error(_)), "expected a retryable error status, got {:?}", after.status ); assert!(fixture.record_rkeys()?.is_empty()); assert!(fixture.pending_keys()?.is_empty()); assert!(matches!( fixture.resync()?, Some(ResyncState::Error { kind: ResyncErrorKind::Generic, .. }) )); Ok(()) }
#[tokio::test] async fn a_key_whose_task_already_finished_is_not_run_again() -> miette::Result<()> { let fixture = Fixture::new().await?; let seeded = rooted("3jzfcijpj2z2a")?; let body = post("a"); fixture.seed(&seeded, 7, &[("a", body.as_slice())])?; // the task that owned key 7 already finished and removed it let mut batch = fixture.state.db.inner.batch(); fixture.state.db.indexer.stage_pending_remove( &mut batch, &fixture.did, &keys::pending_key(7), ); batch.commit().into_diagnostic()?;
assert_eq!( fixture.run(keys::pending_key(7)).await?, TaskDisposition::Finished );
assert_eq!(fixture.pds.requests(), 0); let after = fixture.repo()?; assert_eq!(after.status, seeded.status); assert_eq!( after.root.as_ref().map(|root| root.data), seeded.root.as_ref().map(|root| root.data) ); Ok(()) }
#[tokio::test] async fn repo_not_found_cannot_overwrite_a_replaced_pending_entry() -> miette::Result<()> { let fixture = Fixture::new().await?; let seeded = rooted("3jzfcijpj2z2a")?; let body = post("a"); fixture.seed(&seeded, 7, &[("a", body.as_slice())])?; let err = fixture .run_changing(keys::pending_key(7), async { fixture.reindex_pending(7, 11) }) .await? .unwrap_err();
assert!( matches!(err, BackfillError::RepoNotFound), "unexpected error: {err}" ); assert_eq!(fixture.pending_keys()?, [keys::pending_key(11).to_vec()]); assert!( fixture.resync()?.is_none(), "stale work must not park a retry against a newer queue entry" ); let after = fixture.repo()?; assert_eq!(after.status, RepoStatus::Synced); assert_eq!( after.root.as_ref().map(|root| root.data), seeded.root.as_ref().map(|root| root.data) ); assert_eq!(fixture.record_rkeys()?, ["a"]); Ok(()) }
#[tokio::test] async fn authoritative_deletion_is_not_resurrected_by_a_late_repo_not_found() -> miette::Result<()> { let fixture = Fixture::new().await?; let seeded = rooted("3jzfcijpj2z2a")?; let body = post("a"); fixture.seed(&seeded, 7, &[("a", body.as_slice())])?; let err = fixture .run_changing(keys::pending_key(7), async { fixture.delete_authoritatively(&seeded) }) .await? .unwrap_err();
assert!( matches!(err, BackfillError::RepoNotFound), "unexpected error: {err}" ); let after = fixture.repo()?; assert_eq!(after.status, RepoStatus::Deleted); assert!(!after.active, "the tombstone must stay inactive"); assert!( fixture.record_rkeys()?.is_empty(), "erased records must stay erased" ); assert!(fixture.pending_keys()?.is_empty()); assert!( fixture.resync()?.is_none(), "a deleted repo must not be re-queued" ); Ok(()) }
#[tokio::test] async fn authoritative_deletion_is_not_resurrected_by_a_late_successful_backfill() -> miette::Result<()> { let fixture = Fixture::new().await?; let seeded = rooted("3jzfcijpj2z2a")?; let body = post("a"); fixture.seed(&seeded, 7, &[("a", body.as_slice())])?; let newer = post("b"); let car = build_car( "3jzfcijpj2z2b", &[("a", body.as_slice()), ("b", newer.as_slice())], ) .await?; fixture.pds.serve_car(car.bytes).await;
let result = fixture .run_changing(keys::pending_key(7), async { fixture.delete_authoritatively(&seeded) }) .await?;
assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); let after = fixture.repo()?; assert_eq!(after.status, RepoStatus::Deleted); assert!(!after.active, "the tombstone must stay inactive"); assert!( fixture.record_rkeys()?.is_empty(), "erased records must stay erased" ); assert!(fixture.pending_keys()?.is_empty()); Ok(()) }
#[tokio::test] async fn an_untrack_during_the_fetch_is_not_undone() -> miette::Result<()> { let fixture = Fixture::new().await?; let seeded = rooted("3jzfcijpj2z2a")?; let body = post("a"); fixture.seed(&seeded, 7, &[("a", body.as_slice())])?; let newer = post("b"); let car = build_car( "3jzfcijpj2z2b", &[("a", body.as_slice()), ("b", newer.as_slice())], ) .await?; fixture.pds.serve_car(car.bytes).await;
let repos = crate::control::ReposControl::new(fixture.state.clone()); let result = fixture .run_changing(keys::pending_key(7), async { let untracked = repos.untrack([fixture.did.clone()]).await?; assert_eq!(untracked, std::slice::from_ref(&fixture.did)); Ok(()) }) .await?;
assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); let metadata = fixture .state .db .repo_metadata .get(keys::repo_metadata_key(&fixture.did)) .into_diagnostic()? .expect("repository metadata row"); assert!(!db::deser_repo_meta(&metadata)?.tracked); assert_eq!( fixture.record_rkeys()?, ["a"], "the fetched car must not land" ); assert_eq!( fixture.repo()?.root.map(|root| root.data), seeded.root.as_ref().map(|root| root.data) ); assert!(fixture.pending_keys()?.is_empty()); assert!(fixture.resync()?.is_none()); Ok(()) }
#[tokio::test] async fn an_exclude_during_the_fetch_takes_the_repo_out_of_the_pending_count() -> miette::Result<()> { let fixture = Fixture::new().await?; let repos = crate::control::ReposControl::new(fixture.state.clone()); assert_eq!(repos.track([fixture.did.clone()]).await?.len(), 1); assert_eq!(fixture.state.db.get_count("pending").await, 1); let pending_key: [u8; 8] = fixture.pending_keys()?[0] .as_slice() .try_into() .into_diagnostic()?; let a = post("a"); let car = build_car("3jzfcijpj2z2a", &[("a", a.as_slice())]).await?; fixture.pds.serve_car(car.bytes).await;
// after the task's own exclusion check, so it's the first step that finds it let result = fixture .run_changing(pending_key, async { fixture .state .db .filter .insert(crate::db::filter::exclude_key(fixture.did.as_str())?, []) .into_diagnostic() }) .await?;
assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); assert!(fixture.record_rkeys()?.is_empty()); assert!(fixture.pending_keys()?.is_empty()); assert_eq!(fixture.state.db.get_count("pending").await, 0); Ok(()) }
#[tokio::test] async fn a_deactivation_during_the_fetch_survives_it() -> miette::Result<()> { let fixture = Fixture::new().await?; let seeded = rooted("3jzfcijpj2z2a")?; let body = post("a"); fixture.seed(&seeded, 7, &[("a", body.as_slice())])?; let newer = post("b"); let car = build_car( "3jzfcijpj2z2b", &[("a", body.as_slice()), ("b", newer.as_slice())], ) .await?; fixture.pds.serve_car(car.bytes).await;
let result = fixture .run_changing(keys::pending_key(7), async { // what the relay writes for an account event that lands while getRepo streams let mut row = fixture.repo()?; row.active = false; row.status = RepoStatus::Deactivated; fixture.put_repo(&row) }) .await?;
assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); let after = fixture.repo()?; assert_eq!(after.status, RepoStatus::Deactivated); assert!(!after.active); assert_eq!( after.root.map(|root| root.rev), Some(crate::db::types::DbTid::from(&car.rev)) ); assert_eq!( after.pds.as_deref(), Some(fixture.pds.base.as_str()), "nothing else touched the identity, so the doc the backfill resolved lands" ); assert_eq!(fixture.record_rkeys()?, ["a", "b"]); Ok(()) }
#[tokio::test] async fn a_backfill_brings_back_a_repo_marked_inactive_before_it() -> miette::Result<()> { let fixture = Fixture::new().await?; // say a reactivation went by while hydrant was down let mut seeded = rooted("3jzfcijpj2z2a")?; seeded.active = false; seeded.status = RepoStatus::Deactivated; fixture.seed(&seeded, 7, &[])?; let body = post("a"); let car = build_car("3jzfcijpj2z2b", &[("a", body.as_slice())]).await?; fixture.pds.serve_car(car.bytes).await;
assert_eq!( fixture.run(keys::pending_key(7)).await?, TaskDisposition::Finished ); let after = fixture.repo()?; assert!(after.active); assert_eq!(after.status, RepoStatus::Synced); assert_eq!(fixture.record_rkeys()?, ["a"]); Ok(()) }
#[tokio::test] async fn a_status_written_during_the_fetch_beats_what_the_fetch_saw() -> miette::Result<()> { let fixture = Fixture::new().await?; let mut seeded = rooted("3jzfcijpj2z2a")?; seeded.active = false; seeded.status = RepoStatus::Deactivated; fixture.seed(&seeded, 7, &[])?; let body = post("a"); let car = build_car("3jzfcijpj2z2b", &[("a", body.as_slice())]).await?; fixture.pds.serve_car(car.bytes).await;
let result = fixture .run_changing(keys::pending_key(7), async { let mut row = fixture.repo()?; row.status = RepoStatus::Takendown; fixture.put_repo(&row) }) .await?;
assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); let after = fixture.repo()?; assert!(!after.active); assert_eq!(after.status, RepoStatus::Takendown); Ok(()) }
#[tokio::test] async fn a_commit_buffered_during_the_fetch_is_applied_before_the_repo_leaves_the_queue() -> miette::Result<()> { let fixture = Fixture::new().await?; let newer = fixture.fetch_with_a_buffered_commit().await?;
assert_eq!( fixture.run(keys::pending_key(7)).await?, TaskDisposition::Finished ); fixture.assert_caught_up_to(&newer) }
#[tokio::test] async fn a_commit_arriving_before_the_last_step_waits_for_the_drain() -> miette::Result<()> { let fixture = Fixture::new().await?; let (newer, b) = fixture.fetch_behind_a_live_commit().await?;
// the last step comes here instead of the shard, so a live commit can reach the shard // first let (handoff_tx, mut handoff_worker) = FirehoseWorker::new(fixture.state.clone(), 1); let mut handoff = handoff_worker .bulk_rxs .pop() .expect("single shard receiver"); let (shard_tx, shard) = FirehoseWorker::spawn_one_shard(fixture.state.clone()); let gone = |_| miette::miette!("the shard is gone"); let forward = async { let last = handoff.recv().await.expect("the task sends its last step"); let event = IndexerEvent { seq: 1, firehose: Url::parse("wss://relay.example").into_diagnostic()?, ticket: None, row: None, data: IndexerEventData::Commit(IndexerCommitData { commit: fixture.live_commit(&newer, "b", &b)?, chain_break: false, parsed_blocks: crate::car::parse_commit_car(newer.bytes.clone(), false)?, }), }; shard_tx .send(IndexerMessage::Event(Box::new(event))) .await .map_err(gone)?; // the shard serves its live messages in order, so once this answers the commit is in shard_tx.on_owner(&fixture.did, |_, _| ()).await?; assert_eq!(fixture.buffered(), 1, "applied before the drain"); assert!(fixture.record_rkeys()?.is_empty()); shard_tx .bulk .send(last) .await .map_err(|_| miette::miette!("the shard is gone")) }; let (result, forwarded) = tokio::join!(fixture.run_with(handoff_tx, keys::pending_key(7)), forward); forwarded?; drop(shard_tx); tokio::task::spawn_blocking(move || shard.join()) .await .into_diagnostic()? .map_err(|_| miette::miette!("indexer shard panicked"))?;
assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); fixture.assert_caught_up_to(&newer) }
#[tokio::test] async fn a_buffered_commit_the_car_already_covers_is_dropped() -> miette::Result<()> { let fixture = Fixture::new().await?; fixture.seed(&rooted("3jzfcijpj2z2a")?, 7, &[])?; let a = post("a"); let b = post("b"); // left over from an earlier failed backfill, and `b` was deleted again before the car let older = build_car("3jzfcijpj2z2b", &[("a", a.as_slice()), ("b", b.as_slice())]).await?; fixture.buffer(&older, "b", &b)?; let fetched = build_car("3jzfcijpj2z2c", &[("a", a.as_slice())]).await?; fixture.pds.serve_car(fetched.bytes).await;
assert_eq!( fixture.run(keys::pending_key(7)).await?, TaskDisposition::Finished ); assert_eq!(fixture.record_rkeys()?, ["a"]); let after = fixture.repo()?; assert_eq!( after.root.map(|root| root.rev), Some(crate::db::types::DbTid::from(&fetched.rev)) ); assert_eq!(after.status, RepoStatus::Synced); assert_eq!(fixture.buffered(), 0); Ok(()) }
#[tokio::test] async fn a_commit_the_relay_is_already_past_keeps_the_newer_root() -> miette::Result<()> { let fixture = Fixture::new().await?; fixture.seed(&rooted("3jzfcijpj2z2a")?, 7, &[])?; let a = post("a"); let b = post("b"); let fetched = build_car("3jzfcijpj2z2b", &[("a", a.as_slice())]).await?; fixture.pds.serve_car(fetched.bytes).await; assert_eq!( fixture.run(keys::pending_key(7)).await?, TaskDisposition::Finished ); // the relay validated this commit and the one after it before the shard got to it let live = build_car("3jzfcijpj2z2c", &[("a", a.as_slice()), ("b", b.as_slice())]).await?; let mut row = fixture.repo()?; row.root = rooted("3jzfcijpj2z2d")?.root; fixture .state .db .repos .insert(keys::repo_key(&fixture.did), db::ser_repo_state(&row)?) .into_diagnostic()?;
fixture .apply_live(&live, fixture.live_commit(&live, "b", &b)?) .await?;
assert_eq!(fixture.record_rkeys()?, ["a", "b"]); assert_eq!(fixture.repo()?.root, rooted("3jzfcijpj2z2d")?.root); Ok(()) }
/// queues the repo under 7 and serves a car at 2c that edited `a` and deleted `b` after a /// commit at 2b, which the relay validated before the fetch and which put `b` and the /// first `a` down. returns that commit's car and the commit async fn fetch_past_a_validated_commit( fixture: &Fixture, ) -> miette::Result<(BuiltCar, StreamCommit<'static>)> { fixture.seed(&rooted("3jzfcijpj2z2a")?, 7, &[])?; let a = post("a"); let b = post("b"); let older = build_car("3jzfcijpj2z2b", &[("a", a.as_slice()), ("b", b.as_slice())]).await?; let mut commit = fixture.live_commit(&older, "b", &b)?; commit.ops.push(RepoOp { action: RepoOpAction::Update, cid: Some(CidLink::from(crate::car::compute_cid(&a))), path: CowStr::Owned(format!("{COLLECTION}/a").into()), prev: None, }); let fetched = build_car("3jzfcijpj2z2c", &[("a", post("a, edited").as_slice())]).await?; fixture.pds.serve_car(fetched.bytes).await; Ok((older, commit)) }
fn assert_records_are_the_cars(fixture: &Fixture) -> miette::Result<()> { assert_eq!(stored(fixture, COLLECTION, "a")?, Some(post("a, edited"))); assert_eq!(fixture.record_rkeys()?, ["a"]); Ok(()) }
#[tokio::test] async fn a_live_commit_older_than_the_drained_car_leaves_its_records_alone() -> miette::Result<()> { let fixture = Fixture::new().await?; let (older, commit) = fetch_past_a_validated_commit(&fixture).await?; assert_eq!( fixture.run(keys::pending_key(7)).await?, TaskDisposition::Finished ); assert_records_are_the_cars(&fixture)?;
// the backfill's handoff got to the shard first, so this reaches a repo that isn't queued fixture.apply_live(&older, commit).await?; assert_records_are_the_cars(&fixture)?;
let c = post("c"); let newer = build_car( "3jzfcijpj2z2d", &[("a", post("a, edited").as_slice()), ("c", c.as_slice())], ) .await?; fixture .apply_live(&newer, fixture.live_commit(&newer, "c", &c)?) .await?; assert_eq!(fixture.record_rkeys()?, ["a", "c"]); Ok(()) }
#[tokio::test] async fn a_commit_older_than_the_car_buffered_before_its_last_step_is_not_drained() -> miette::Result<()> { let fixture = Fixture::new().await?; let (older, commit) = fetch_past_a_validated_commit(&fixture).await?;
// the last step comes here instead of the shard, so the older commit reaches the shard // first and gets buffered let (handoff_tx, mut handoff_worker) = FirehoseWorker::new(fixture.state.clone(), 1); let mut handoff = handoff_worker .bulk_rxs .pop() .expect("single shard receiver"); let (shard_tx, shard) = FirehoseWorker::spawn_one_shard(fixture.state.clone()); let gone = |_| miette::miette!("the shard is gone"); let forward = async { let last = handoff.recv().await.expect("the task sends its last step"); let event = IndexerEvent { seq: 1, firehose: Url::parse("wss://relay.example").into_diagnostic()?, ticket: None, row: None, data: IndexerEventData::Commit(IndexerCommitData { commit, chain_break: false, parsed_blocks: crate::car::parse_commit_car(older.bytes.clone(), false)?, }), }; shard_tx .send(IndexerMessage::Event(Box::new(event))) .await .map_err(gone)?; shard_tx .bulk .send(last) .await .map_err(|_| miette::miette!("the shard is gone")) }; let (result, forwarded) = tokio::join!(fixture.run_with(handoff_tx, keys::pending_key(7)), forward); forwarded?; drop(shard_tx); tokio::task::spawn_blocking(move || shard.join()) .await .into_diagnostic()? .map_err(|_| miette::miette!("indexer shard panicked"))?;
assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); assert_records_are_the_cars(&fixture)?; assert_eq!(fixture.buffered(), 0); assert!(fixture.pending_keys()?.is_empty()); Ok(()) }
#[tokio::test] async fn a_live_commit_to_a_repo_that_never_backfilled_applies() -> miette::Result<()> { let fixture = Fixture::new().await?; fixture.seed(&rooted("3jzfcijpj2z2a")?, 7, &[])?; let mut batch = fixture.state.db.inner.batch(); fixture.state.db.indexer.stage_pending_remove( &mut batch, &fixture.did, &keys::pending_key(7), ); batch.commit().into_diagnostic()?; let b = post("b"); let live = build_car("3jzfcijpj2z2b", &[("b", b.as_slice())]).await?;
fixture .apply_live(&live, fixture.live_commit(&live, "b", &b)?) .await?;
assert_eq!(fixture.record_rkeys()?, ["b"]); Ok(()) }
#[tokio::test] async fn a_buffered_commit_failing_halfway_writes_none_of_its_records() -> miette::Result<()> { let fixture = Fixture::new().await?; fixture.fetch_with_a_buffered_commit().await?; let c = post("c"); let bad = build_car("3jzfcijpj2z2d", &[("c", c.as_slice())]).await?; let commit = fixture.half_bad_commit(&bad, &c)?; let mut txn = DbTxn::new(&fixture.state.db); crate::ops::persist_to_resync_buffer(&mut txn, &fixture.state.db, &fixture.did, &commit)?; txn.commit()?;
let result = fixture.run(keys::pending_key(7)).await; assert!( matches!(result, Err(BackfillError::Generic(_))), "{result:?}" ); assert_eq!(fixture.record_rkeys()?, ["a", "b"]); assert_eq!(fixture.state.db.get_count_sync("records"), 2); Ok(()) }
#[tokio::test] async fn a_live_commit_failing_halfway_writes_none_of_its_records() -> miette::Result<()> { let fixture = Fixture::new().await?; fixture.seed(&rooted("3jzfcijpj2z2a")?, 7, &[])?; let a = post("a"); let c = post("c"); let fetched = build_car("3jzfcijpj2z2b", &[("a", a.as_slice())]).await?; fixture.pds.serve_car(fetched.bytes).await; assert_eq!( fixture.run(keys::pending_key(7)).await?, TaskDisposition::Finished ); let live = build_car("3jzfcijpj2z2c", &[("a", a.as_slice()), ("c", c.as_slice())]).await?;
fixture .apply_live(&live, fixture.half_bad_commit(&live, &c)?) .await?;
assert_eq!(fixture.record_rkeys()?, ["a"]); assert_eq!(fixture.state.db.get_count_sync("records"), 1); Ok(()) }
#[tokio::test] async fn an_orphaned_key_doesnt_write_its_car_over_live_records() -> miette::Result<()> { let fixture = Fixture::new().await?; // metadata names 8, which isn't queued, so the shard applied z live. 7 is an orphan let z = post("z"); fixture.seed(&rooted("3jzfcijpj2z2z")?, 7, &[("z", z.as_slice())])?; let mut batch = fixture.state.db.inner.batch(); batch.insert( &fixture.state.db.repo_metadata, keys::repo_metadata_key(&fixture.did), db::ser_repo_meta(&RepoMetadata::backfilling(8))?, ); batch.commit().into_diagnostic()?; let a = post("a"); let car = build_car("3jzfcijpj2z2b", &[("a", a.as_slice())]).await?; fixture.pds.serve_car(car.bytes).await;
assert_eq!( fixture.run(keys::pending_key(7)).await?, TaskDisposition::Finished ); assert_eq!(fixture.record_rkeys()?, ["z"]); assert!(fixture.pending_keys()?.is_empty()); Ok(()) }
#[tokio::test] async fn a_buffered_commit_that_cant_apply_costs_one_refetch() -> miette::Result<()> { let fixture = Fixture::new().await?; fixture.fetch_with_a_buffered_commit().await?; // a later live commit the drain can never apply, since its rkey doesn't parse let c = post("c"); let bad = build_car("3jzfcijpj2z2d", &[("c", c.as_slice())]).await?; fixture.buffer(&bad, "bad rkey", &c)?;
let first = fixture.run(keys::pending_key(7)).await; let Err(BackfillError::Generic(e)) = &first else { panic!("{first:?}"); }; // the shard says why the drain failed assert!(e.to_string().contains("bad rkey"), "{e}"); // the commit before it was drained anyway assert_eq!(fixture.record_rkeys()?, ["a", "b"]); assert_eq!(fixture.buffered(), 0); assert!(matches!( fixture.resync()?, Some(ResyncState::Error { retry_count: 1, .. }) ));
// the retry worker's requeue fixture.clear_resync()?; fixture.reindex_pending(7, 8)?; let again = fixture.run(keys::pending_key(8)).await; assert!(matches!(again, Ok(TaskDisposition::Finished)), "{again:?}"); assert_eq!(fixture.pds.requests(), 2); assert!(fixture.pending_keys()?.is_empty()); assert!(fixture.resync()?.is_none()); Ok(()) }
/// a commit the firehose skipped leaves the next one not following the car, and applied it /// would land on records it doesn't go with #[tokio::test] async fn a_buffered_commit_that_doesnt_follow_the_car_costs_one_refetch() -> miette::Result<()> { let fixture = Fixture::new().await?; let newer = fixture.fetch_with_a_buffered_commit().await?; let (a, b, d) = (post("a"), post("b"), post("d")); let after_gap = build_car( "3jzfcijpj2z2e", &[ ("a", a.as_slice()), ("b", b.as_slice()), ("d", d.as_slice()), ], ) .await?; let mut commit = fixture.live_commit(&after_gap, "d", &d)?; commit.since = Some(Tid::new("3jzfcijpj2z2d").into_diagnostic()?); fixture.buffer_commit(&commit)?;
let result = fixture.run(keys::pending_key(7)).await; let Err(BackfillError::Generic(e)) = &result else { panic!("{result:?}"); }; assert!(e.to_string().contains("doesn't follow"), "{e}"); // the commit before the gap still went in assert_eq!(fixture.record_rkeys()?, ["a", "b"]); assert_eq!( fixture.repo()?.root.map(|root| root.rev), Some(crate::db::types::DbTid::from(&newer.rev)) ); assert_eq!(fixture.buffered(), 0); assert!(matches!( fixture.resync()?, Some(ResyncState::Error { retry_count: 1, .. }) )); Ok(()) }
/// the relay puts every commit it forwards on the row, so one buffered during the fetch can /// already be the root when the drain gets to it, without its records #[tokio::test] async fn a_buffered_commit_the_relay_already_put_on_the_row_is_drained() -> miette::Result<()> { let fixture = Fixture::new().await?; let (newer, b) = fixture.fetch_behind_a_live_commit().await?; let mut commit = fixture.live_commit(&newer, "b", &b)?; commit.since = Some(Tid::new("3jzfcijpj2z2b").into_diagnostic()?); fixture.buffer_commit(&commit)?;
let result = fixture .run_changing(keys::pending_key(7), async { let mut row = fixture.repo()?; row.root = Some(crate::types::Commit { version: 3, rev: crate::db::types::DbTid::from(&newer.rev), data: newer.mst_root, prev: None, sig: Bytes::from_static(b"signature"), }); fixture.put_repo(&row) }) .await?;
assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); fixture.assert_caught_up_to(&newer) }
/// the pds is the authority on the repo, so a car older than what we had replaces the /// records, and the root goes back with them so the next commit shows the gap #[tokio::test] async fn a_car_behind_the_stored_root_takes_the_root_back_too() -> miette::Result<()> { let fixture = Fixture::new().await?; let (a, b) = (post("a"), post("b")); fixture.seed( &rooted("3jzfcijpj2z2c")?, 7, &[("a", a.as_slice()), ("b", b.as_slice())], )?; let behind = build_car("3jzfcijpj2z2b", &[("a", a.as_slice())]).await?; fixture.pds.serve_car(behind.bytes).await;
let result = fixture.run(keys::pending_key(7)).await; assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); assert_eq!(fixture.record_rkeys()?, ["a"]); assert_eq!( fixture.repo()?.root.map(|root| root.rev), Some(crate::db::types::DbTid::from(&behind.rev)) ); Ok(()) }
#[tokio::test] async fn a_backfill_the_shard_never_answers_is_retried_with_its_buffer_kept() -> miette::Result<()> { let fixture = Fixture::new().await?; fixture.fetch_with_a_buffered_commit().await?; let result = fixture .run_with(fixture.unanswered(), keys::pending_key(7)) .await;
let Err(BackfillError::Generic(e)) = &result else { panic!("{result:?}"); }; assert!(e.to_string().contains("shard is gone"), "{e}"); assert!(matches!( fixture.resync()?, Some(ResyncState::Error { retry_count: 1, .. }) )); assert!(fixture.pending_keys()?.is_empty()); assert_eq!(fixture.buffered(), 1); Ok(()) }
#[tokio::test] async fn a_backfill_stopped_before_its_drain_stays_queued() -> miette::Result<()> { let fixture = Fixture::new().await?; fixture.seed(&rooted("3jzfcijpj2z2a")?, 7, &[])?; let a = post("a"); let fetched = build_car("3jzfcijpj2z2b", &[("a", a.as_slice())]).await?; fixture.pds.serve_car(fetched.bytes).await; fixture.state.stop.trigger();
assert_eq!( fixture .run_with(fixture.unanswered(), keys::pending_key(7)) .await?, TaskDisposition::Finished ); assert_eq!(fixture.pending_keys()?, [keys::pending_key(7).to_vec()]); assert!(fixture.resync()?.is_none()); Ok(()) }
#[tokio::test] async fn a_newer_event_repeating_the_old_value_beats_what_the_fetch_saw() -> miette::Result<()> { let fixture = Fixture::new().await?; let mut seeded = rooted("3jzfcijpj2z2a")?; seeded.active = false; seeded.status = RepoStatus::Deactivated; seeded.last_account_time = Some(1_000); fixture.seed(&seeded, 7, &[])?; let body = post("a"); let car = build_car("3jzfcijpj2z2b", &[("a", body.as_slice())]).await?; fixture.pds.serve_car(car.bytes).await;
let result = fixture .run_changing(keys::pending_key(7), async { // what the relay writes for a newer deactivation that says what the row // already says let mut row = fixture.repo()?; row.last_account_time = Some(2_000); fixture.put_repo(&row) }) .await?;
assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); let after = fixture.repo()?; assert!(!after.active); assert_eq!(after.status, RepoStatus::Deactivated); assert_eq!(after.last_account_time, Some(2_000)); Ok(()) }
#[tokio::test] async fn an_identity_clock_moved_without_a_doc_keeps_the_backfills_doc() -> miette::Result<()> { let fixture = Fixture::new().await?; fixture.seed(&rooted("3jzfcijpj2z2a")?, 7, &[])?; let body = post("a"); let car = build_car("3jzfcijpj2z2b", &[("a", body.as_slice())]).await?; fixture.pds.serve_car(car.bytes).await;
let result = fixture .run_changing(keys::pending_key(7), async { // the relay handling an identity event it got no fresh doc for let mut row = fixture.repo()?; row.last_identity_time = Some(2_000); fixture.put_repo(&row) }) .await?;
assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); let after = fixture.repo()?; assert_eq!(after.pds.as_deref(), Some(fixture.pds.base.as_str())); assert_eq!(after.last_identity_time, Some(2_000)); Ok(()) }
#[tokio::test] async fn an_identity_written_during_the_fetch_outlives_the_backfills_doc() -> miette::Result<()> { let fixture = Fixture::new().await?; let seeded = rooted("3jzfcijpj2z2a")?; fixture.seed(&seeded, 7, &[])?; let body = post("a"); let car = build_car("3jzfcijpj2z2b", &[("a", body.as_slice())]).await?; fixture.pds.serve_car(car.bytes).await;
let result = fixture .run_changing(keys::pending_key(7), async { let mut row = fixture.repo()?; row.handle = Some(Handle::new_static("bob.test").into_diagnostic()?); fixture.put_repo(&row) }) .await?;
assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); let after = fixture.repo()?; assert_eq!(after.handle.as_ref().map(Handle::as_str), Some("bob.test")); assert_eq!(after.pds.as_deref(), seeded.pds.as_deref()); assert_eq!( after.root.map(|root| root.rev), Some(crate::db::types::DbTid::from(&car.rev)) ); assert_eq!(after.status, seeded.status); Ok(()) }
#[tokio::test] async fn a_failed_fetch_leaves_a_repo_that_went_inactive_inactive() -> miette::Result<()> { let fixture = Fixture::new().await?; let seeded = rooted("3jzfcijpj2z2a")?; fixture.seed(&seeded, 7, &[])?;
// the stub answers getRepo with RepoNotFound, which the task retries let result = fixture .run_changing(keys::pending_key(7), async { let mut row = fixture.repo()?; row.active = false; row.status = RepoStatus::Deactivated; fixture.put_repo(&row) }) .await?;
assert!( matches!(result, Err(BackfillError::RepoNotFound)), "{result:?}" ); let after = fixture.repo()?; assert_eq!(after.status, RepoStatus::Deactivated); assert!(!after.active); assert!(matches!( fixture.resync()?, Some(ResyncState::Error { retry_count: 1, .. }) )); assert!(fixture.pending_keys()?.is_empty()); Ok(()) }
#[tokio::test] async fn a_later_valid_car_replaces_the_error_after_requeue() -> miette::Result<()> { let fixture = Fixture::new().await?; let seeded = rooted("3jzfcijpj2z2a")?; let first = post("a"); fixture.seed(&seeded, 7, &[("a", first.as_slice())])?; assert!(matches!( fixture.run(keys::pending_key(7)).await.unwrap_err(), BackfillError::RepoNotFound )); assert_eq!(fixture.pds.requests(), 1);
fixture.clear_resync()?; fixture.reindex_pending(7, 21)?; let second = post("b"); let car = build_car( "3jzfcijpj2z2b", &[("a", first.as_slice()), ("b", second.as_slice())], ) .await?; fixture.pds.serve_car(car.bytes.clone()).await;
assert_eq!( fixture.run(keys::pending_key(21)).await.unwrap(), TaskDisposition::Finished );
let after = fixture.repo()?; assert_eq!( after.root.as_ref().map(|root| root.data), Some(car.mst_root) ); assert_eq!( after.root.as_ref().map(|root| root.rev.to_tid()), Some(car.rev.clone()) ); assert_eq!(fixture.record_rkeys()?, ["a", "b"]); assert!( fixture.resync()?.is_none(), "a successful import clears the error row" ); assert!(fixture.pending_keys()?.is_empty()); assert_eq!(fixture.pds.requests(), 2); Ok(()) }
// --- a backfill written in steps ------------------------------------------------------
/// three posts, one per step at one record a step, with the last one the last step async fn three_posts(fixture: &Fixture) -> miette::Result<BuiltCar> { fixture.seed(&rooted("3jzfcijpj2z2a")?, 7, &[])?; let (a, b, c) = (post("a"), post("b"), post("c")); let car = build_car( "3jzfcijpj2z2b", &[ ("a", a.as_slice()), ("b", b.as_slice()), ("c", c.as_slice()), ], ) .await?; fixture.pds.serve_car(car.bytes.clone()).await; Ok(car) }
fn untrack(fixture: &Fixture) -> impl Future<Output = miette::Result<Vec<Did>>> { let repos = crate::control::repos::ReposControl::new(fixture.state.clone()); let did = fixture.did.clone(); async move { repos.untrack([did]).await } }
#[tokio::test] async fn a_live_commit_during_the_steps_lands_after_the_last() -> miette::Result<()> { let fixture = Fixture::with_chunk_records(1).await?; three_posts(&fixture).await?; let d = post("d"); let newer = build_car( "3jzfcijpj2z2c", &[ ("a", post("a").as_slice()), ("b", post("b").as_slice()), ("c", post("c").as_slice()), ("d", d.as_slice()), ], ) .await?;
let (fixture, newer, d) = (&fixture, &newer, &d); let result = fixture .run_pausing(keys::pending_key(7), 1, |tx| async move { let commit = fixture.live_commit(newer, "d", d)?; tx.send(live_event(newer, commit, false)?) .await .map_err(|_| miette::miette!("the shard is gone")) }) .await?;
assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); assert_eq!(fixture.record_rkeys()?, ["a", "b", "c", "d"]); assert_eq!(fixture.buffered(), 0); assert!(fixture.pending_keys()?.is_empty()); Ok(()) }
#[tokio::test] async fn untrack_during_the_steps_is_refused_and_works_after() -> miette::Result<()> { let fixture = Fixture::with_chunk_records(1).await?; three_posts(&fixture).await?;
let result = fixture .run_pausing(keys::pending_key(7), 1, |_| async { assert!(untrack(&fixture).await?.is_empty()); Ok(()) }) .await?;
assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); assert_eq!(fixture.record_rkeys()?, ["a", "b", "c"]); assert_eq!(untrack(&fixture).await?, std::slice::from_ref(&fixture.did)); Ok(()) }
#[tokio::test] async fn resync_during_the_steps_does_nothing() -> miette::Result<()> { let fixture = Fixture::with_chunk_records(1).await?; three_posts(&fixture).await?; let repos = crate::control::repos::ReposControl::new(fixture.state.clone());
let result = fixture .run_pausing(keys::pending_key(7), 1, |_| async { assert!(repos.resync([fixture.did.clone()]).await?.is_empty()); assert_eq!(fixture.pending_keys()?, [keys::pending_key(7).to_vec()]); Ok(()) }) .await?;
assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); assert_eq!(fixture.record_rkeys()?, ["a", "b", "c"]); assert!(fixture.pending_keys()?.is_empty()); Ok(()) }
#[tokio::test] async fn a_chain_break_during_the_steps_queues_the_repo_again_after_them() -> miette::Result<()> { let fixture = Fixture::with_chunk_records(1).await?; three_posts(&fixture).await?; let d = post("d"); let newer = build_car("3jzfcijpj2z2c", &[("d", d.as_slice())]).await?;
let (fixture, newer, d) = (&fixture, &newer, &d); let result = fixture .run_pausing(keys::pending_key(7), 1, |tx| async move { let commit = fixture.live_commit(newer, "d", d)?; tx.send(live_event(newer, commit, true)?) .await .map_err(|_| miette::miette!("the shard is gone")) }) .await?;
assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); // the steps all landed instead of being thrown away assert_eq!(fixture.record_rkeys()?, ["a", "b", "c"]); // and the repo is queued again under a new key, with the commit kept for that backfill let pending = fixture.pending_keys()?; assert_eq!(pending.len(), 1); assert_ne!(pending[0], keys::pending_key(7)); assert_eq!(fixture.buffered(), 1); assert!(fixture.marker()?.is_none()); Ok(()) }
#[tokio::test] async fn an_erase_during_the_steps_leaves_nothing_behind() -> miette::Result<()> { let fixture = Fixture::with_chunk_records(1).await?; three_posts(&fixture).await?; let repos = crate::control::repos::ReposControl::new(fixture.state.clone());
let result = fixture .run_pausing(keys::pending_key(7), 1, |_| async { repos.erase_repo(&fixture.did, false).await?; Ok(()) }) .await?;
assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); assert!(fixture.record_rkeys()?.is_empty()); assert!(fixture.pending_keys()?.is_empty()); assert_eq!( db::get_record_count(&fixture.state.db, &fixture.did, COLLECTION)?, 0 ); assert!(fixture.marker()?.is_none()); Ok(()) }
#[tokio::test] async fn an_account_deletion_during_the_steps_leaves_nothing_behind() -> miette::Result<()> { let fixture = Fixture::with_chunk_records(1).await?; three_posts(&fixture).await?;
let fixture = &fixture; let result = fixture .run_pausing(keys::pending_key(7), 1, |tx| async move { let deleted = IndexerEvent { seq: 1, firehose: Url::parse("wss://relay.example").into_diagnostic()?, ticket: None, row: None, data: IndexerEventData::Account(crate::ingest::indexer::IndexerAccountData { account: crate::ingest::stream::Account { active: false, did: fixture.did.clone(), seq: 1, status: Some(crate::ingest::stream::types::AccountStatus::Deleted), time: Datetime::try_from("2026-10-05T12:00:00Z".to_owned()) .into_diagnostic()?, }, was_active: true, changed: true, }), }; tx.send(IndexerMessage::Event(Box::new(deleted))) .await .map_err(|_| miette::miette!("the shard is gone")) }) .await?;
assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); assert!(fixture.record_rkeys()?.is_empty()); assert!(fixture.pending_keys()?.is_empty()); assert!(fixture.marker()?.is_none()); Ok(()) }
#[tokio::test] async fn a_sparse_backfill_lets_its_pds_slot_go_before_its_steps() -> miette::Result<()> { let mut fixture = Fixture::with_chunk_records(1).await?; fixture.strategy = BackfillStrategy::SparseFilter; only_collection(&fixture, COLLECTION); fixture.seed(&RepoState::backfilling().into_static(), 7, &[])?; let (a, b) = (post("a"), post("b")); let car = build_car("3jzfcijpj2z2a", &[("a", a.as_slice()), ("b", b.as_slice())]).await?; fixture.pds.serve_sparse(&car).await?;
let pds = fixture.pds.base.clone(); let result = fixture .run_pausing(keys::pending_key(7), 1, |_| async { // one slot per pds, so this only gets it if the backfill gave it back let slot = fixture.admission.try_acquire(&pds).await; assert!(slot.is_some(), "the backfill still holds its pds slot"); Ok(()) }) .await?;
assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); assert_eq!(fixture.record_rkeys()?, ["a", "b"]); // it went sparse rather than falling back to a whole car assert_eq!(fixture.pds.requests(), 0); Ok(()) }
#[tokio::test] async fn an_exclude_during_the_steps_doesnt_stop_them() -> miette::Result<()> { let fixture = Fixture::with_chunk_records(1).await?; three_posts(&fixture).await?;
let result = fixture .run_pausing(keys::pending_key(7), 1, |_| async { fixture .state .db .filter .insert(crate::db::filter::exclude_key(fixture.did.as_str())?, []) .into_diagnostic() }) .await?;
assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); assert_eq!(fixture.record_rkeys()?, ["a", "b", "c"]); // the queue entry still goes, once the steps are done assert!(fixture.pending_keys()?.is_empty()); Ok(()) }
#[tokio::test] async fn a_repo_that_fits_in_a_step_goes_in_one() -> miette::Result<()> { let fixture = Fixture::with_chunk_records(1000).await?; three_posts(&fixture).await?; let steps = Arc::new(AtomicUsize::new(0)); let counted = steps.clone(); *fixture.state.after_chunk.lock() = Some(Arc::new(move |_| { counted.fetch_add(1, Ordering::SeqCst); }));
assert_eq!( fixture.run(keys::pending_key(7)).await?, TaskDisposition::Finished ); // the hook only runs between steps, and there were none assert_eq!(steps.load(Ordering::SeqCst), 0); assert_eq!(fixture.record_rkeys()?, ["a", "b", "c"]); Ok(()) }
#[tokio::test] async fn big_records_end_a_step_before_its_chunk_of_records() -> miette::Result<()> { let fixture = Fixture::with_chunk_records(1000).await?; fixture.seed(&RepoState::backfilling().into_static(), 7, &[])?; // a quarter of a step each, about as big as a record gets let big = |name: &str| post(&name.repeat(super::super::apply::STEP_BYTES / 4)); let records = ["a", "b", "c", "d", "e"].map(|name| (name, big(name))); let car = build_car( "3jzfcijpj2z2a", &records .each_ref() .map(|(name, body)| (*name, body.as_slice())), ) .await?; fixture.pds.serve_car(car.bytes).await; let steps = Arc::new(AtomicUsize::new(0)); let counted = steps.clone(); *fixture.state.after_chunk.lock() = Some(Arc::new(move |_| { counted.fetch_add(1, Ordering::SeqCst); }));
assert_eq!( fixture.run(keys::pending_key(7)).await?, TaskDisposition::Finished ); // four records pass the step's bytes, so the fifth goes in a step of its own. the hook // runs between steps only assert_eq!(steps.load(Ordering::SeqCst), 1); assert_eq!(fixture.record_rkeys()?, ["a", "b", "c", "d", "e"]); Ok(()) }
#[cfg(feature = "indexer_stream")] /// the did's records as key -> cid fn heads(fixture: &Fixture) -> miette::Result<std::collections::BTreeMap<String, IpldCid>> { let prefix = keys::record_prefix_did(&fixture.did); fixture .state .db .indexer .head_prefix(&prefix) .map(|guard| { let (key, value) = guard.into_inner().into_diagnostic()?; let (collection, rkey) = keys::split_record_suffix(&key[prefix.len()..])?; let cid = crate::db::ipld_cid_from_record_value(&value)?; Ok((format!("{collection}/{rkey}"), cid)) }) .collect() }
#[cfg(feature = "indexer_stream")] /// the did's history, without the death revs, which go by the clock fn history(fixture: &Fixture) -> miette::Result<Vec<(Vec<u8>, Vec<u8>)>> { let prefix = keys::record_prefix_did(&fixture.did); let mut end = prefix.clone(); *end.last_mut().unwrap() += 1; let mut entries = fixture .state .db .indexer .history_range(prefix..end) .map(|guard| { let (key, value) = guard.into_inner().into_diagnostic()?; Ok((key[..key.len() - 9].to_vec(), value.to_vec())) }) .collect::<miette::Result<Vec<_>>>()?; entries.sort(); Ok(entries) }
#[cfg(feature = "indexer_stream")] /// `start` with every record event in the stream played over it fn replayed( fixture: &Fixture, mut start: std::collections::BTreeMap<String, IpldCid>, ) -> miette::Result<std::collections::BTreeMap<String, IpldCid>> { for guard in fixture.state.db.stream.event_range::<&[u8], _>(..) { let value = guard.value().into_diagnostic()?; let event: crate::types::StoredEvent<'_> = rmp_serde::from_slice(&value).into_diagnostic()?; let key = format!("{}/{}", event.collection, event.rkey.to_smolstr()); match (event.action, event.data) { (crate::db::types::DbAction::Delete, _) => { start.remove(&key); } (_, crate::types::StoredData::Ptr(cid)) => { start.insert(key, cid); } (action, data) => panic!("{key}: {action:?} carries {data:?}"), } } Ok(start) }
#[cfg(feature = "indexer_stream")] #[tokio::test] async fn a_backfill_stopped_after_any_step_comes_out_the_same_once_it_runs_again() -> miette::Result<()> { let (a, a2, b, c, d, x) = ( post("a"), post("a2"), post("b"), post("c"), post("d"), post("x"), ); let (a, a2, b, c, d, x) = (&a[..], &a2[..], &b[..], &c[..], &d[..], &x[..]); let stored: &[(&str, &[u8])] = &[("a", a), ("b", b), ("x", x)]; // an update, two creates and a delete, so four steps at one record a step let car = build_car("3jzfcijpj2z2b", &[("a", a2), ("b", b), ("c", c), ("d", d)]).await?; let newer = build_car("3jzfcijpj2z2c", &[("b", b), ("d", d), ("e", a)]).await?; let cids = |records: &[(&str, &[u8])]| { records .iter() .map(|(rkey, body)| { let cid = crate::car::compute_cid(body); (format!("{COLLECTION}/{rkey}"), cid) }) .collect::<std::collections::BTreeMap<_, _>>() }; let start = cids(stored); let seeded = || async { let fixture = Fixture::with_chunk_records(1).await?; fixture.seed(&rooted("3jzfcijpj2z2a")?, 7, stored)?; miette::Result::<_>::Ok(fixture) };
let clean = seeded().await?; clean.pds.serve_car(car.bytes.clone()).await; assert_eq!( clean.run(keys::pending_key(7)).await?, TaskDisposition::Finished ); let want = cids(&[("a", a2), ("b", b), ("c", c), ("d", d)]); assert_eq!(heads(&clean)?, want);
for stopped_after in 1..4 { for redo in [&car, &newer] { let mut fixture = seeded().await?; fixture.pds.serve_car(car.bytes.clone()).await; let state = Arc::downgrade(&fixture.state); *fixture.state.after_chunk.lock() = Some(Arc::new(move |sent| { if sent == stopped_after && let Some(state) = state.upgrade() { state.stop.trigger(); } })); assert_eq!( fixture.run(keys::pending_key(7)).await?, TaskDisposition::Finished ); assert_eq!(fixture.pending_keys()?, [keys::pending_key(7).to_vec()]); *fixture.state.after_chunk.lock() = None;
fixture.reopen()?; // a spilled car was unlinked as soon as it was made, so the crash left nothing let mut spills = std::fs::read_dir(fixture.state.spills.dir()).into_diagnostic()?; assert!(spills.next().is_none()); fixture.pds.serve_car(redo.bytes.clone()).await; assert_eq!( fixture.run(keys::pending_key(7)).await?, TaskDisposition::Finished, "stopped after step {stopped_after}" );
let wanted = if std::ptr::eq(redo, &car) { want.clone() } else { cids(&[("b", b), ("d", d), ("e", a)]) }; assert_eq!( heads(&fixture)?, wanted, "stopped after step {stopped_after}" ); assert_eq!( db::get_record_count(&fixture.state.db, &fixture.did, COLLECTION)?, wanted.len() as u64 ); assert_eq!(replayed(&fixture, start.clone())?, wanted); if std::ptr::eq(redo, &car) { assert_eq!(history(&fixture)?, history(&clean)?); } assert!(fixture.pending_keys()?.is_empty()); assert!(fixture.resync()?.is_none()); } } Ok(()) }
// --- the drain after the last step, in steps of its own -------------------------------
/// queues the repo under 7 with a car holding `a`, and buffers `n` later commits that each /// add a post, the way the shard does while the repo is queued. returns the posts' rkeys and /// the car the pds has once they're all in async fn fetch_with_buffered_posts( fixture: &Fixture, n: usize, ) -> miette::Result<(Vec<String>, BuiltCar)> { use smol_str::ToSmolStr; fixture.seed(&rooted("3jzfcijpj2z2a")?, 7, &[])?; let a = post("a"); let fetched = build_car("3jzfcijpj2z2b", &[("a", a.as_slice())]).await?; fixture.pds.serve_car(fetched.bytes).await; let rev = |i: usize| { let base = crate::db::types::s32decode("3jzfcijpj2z2b"); crate::db::types::DbTid::new_from_bytes((base + 1 + i as u64).to_be_bytes()) .to_smolstr() .to_string() }; let rkeys = (0..n).map(|i| format!("p{i:03}")).collect::<Vec<_>>(); let bodies = rkeys.iter().map(|rkey| post(rkey)).collect::<Vec<_>>(); let mut records = vec![("a", a.as_slice())]; for (i, (rkey, body)) in rkeys.iter().zip(&bodies).enumerate() { records.push((rkey.as_str(), body.as_slice())); let car = build_car(&rev(i), &records).await?; fixture.buffer(&car, rkey, body)?; } let after_all = build_car(&rev(n), &records).await?; Ok((rkeys, after_all)) }
/// runs the task on its own shard with `hook` running on the shard after each drain step, /// given the step and a sender to the shard async fn run_draining( fixture: &Fixture, pending_key: [u8; 8], hook: impl Fn(usize, &IndexerTx) + Send + Sync + 'static, ) -> Result<TaskDisposition, BackfillError> { let (tx, shard) = FirehoseWorker::spawn_one_shard(fixture.state.clone()); let to_shard = tx.clone(); *fixture.state.after_drain_step.lock() = Some(Arc::new(move |step| hook(step, &to_shard))); let result = fixture.run_with(tx, pending_key).await; // the hook holds a sender, so the shard only stops once it's gone *fixture.state.after_drain_step.lock() = None; tokio::task::spawn_blocking(move || shard.join()) .await .into_diagnostic()? .map_err(|_| miette::miette!("indexer shard panicked"))?; result }
#[tokio::test] async fn a_big_drain_runs_in_steps_with_other_repos_served_between() -> miette::Result<()> { let fixture = Fixture::with_chunk_records(1).await?; let (rkeys, _) = fetch_with_buffered_posts(&fixture, 4).await?; let other = Did::new_static("did:plc:bbbbbbbbbbbbbbbbbbbbbbbb").into_diagnostic()?; let seen = Arc::new(parking_lot::Mutex::new(Vec::new())); let (state, recorded) = (Arc::downgrade(&fixture.state), seen.clone()); let queued_other = other.clone(); let result = run_draining(&fixture, keys::pending_key(7), move |step, tx| { let state = state.upgrade().unwrap(); let other_queued = state .db .repo_metadata .get(keys::repo_metadata_key(&queued_other)) .unwrap() .is_some(); recorded.lock().push((step, other_queued)); if step == 1 { tx.blocking_send(IndexerMessage::new_repo_for_test(queued_other.clone())) .unwrap(); } }) .await;
assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); // one commit a step, and the other repo's message went in right after the first assert_eq!(*seen.lock(), [(1, false), (2, true), (3, true), (4, true)]); let mut want = vec!["a".to_string()]; want.extend(rkeys); assert_eq!(fixture.record_rkeys()?, want); assert_eq!(fixture.buffered(), 0); assert_eq!( fixture.pending_keys()?.len(), 1, "only the other repo is queued" ); Ok(()) }
#[cfg(feature = "indexer_stream")] #[tokio::test] async fn a_commit_arriving_during_the_drain_lands_after_it_in_order() -> miette::Result<()> { let fixture = Fixture::with_chunk_records(1).await?; let (rkeys, after_all) = fetch_with_buffered_posts(&fixture, 3).await?; let q = post("q"); let later = { use smol_str::ToSmolStr; let rev = crate::db::types::s32decode(after_all.rev.as_str()) + 1; crate::db::types::DbTid::new_from_bytes(rev.to_be_bytes()).to_smolstr() }; let with_q = build_car(&later, &[("q", q.as_slice())]).await?; let commit = fixture.live_commit(&with_q, "q", &q)?; let event = std::sync::Mutex::new(Some(live_event(&with_q, commit, false)?)); let result = run_draining(&fixture, keys::pending_key(7), move |step, tx| { if step == 1 && let Some(event) = event.lock().unwrap().take() { tx.blocking_send(event).unwrap(); } }) .await;
assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); let mut want = vec![format!("{COLLECTION}/a")]; want.extend(rkeys.iter().map(|rkey| format!("{COLLECTION}/{rkey}"))); want.push(format!("{COLLECTION}/q")); let events = record_events(&fixture)? .into_iter() .map(|(key, _)| key) .collect::<Vec<_>>(); assert_eq!(events, want); assert_eq!(fixture.buffered(), 0); assert!(fixture.pending_keys()?.is_empty()); Ok(()) }
#[tokio::test] async fn untrack_during_the_drain_is_refused() -> miette::Result<()> { let fixture = Fixture::with_chunk_records(1).await?; fetch_with_buffered_posts(&fixture, 3).await?; let repos = crate::control::repos::ReposControl::new(fixture.state.clone()); let did = fixture.did.clone(); let answers = Arc::new(parking_lot::Mutex::new(Vec::new())); let recorded = answers.clone(); let handle = tokio::runtime::Handle::current(); let result = run_draining(&fixture, keys::pending_key(7), move |step, _| { if step == 1 { let untracked = handle.block_on(repos.untrack([did.clone()])).unwrap(); recorded.lock().push(untracked); } }) .await;
assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); assert_eq!(*answers.lock(), [Vec::<Did>::new()]); assert!(fixture.pending_keys()?.is_empty()); assert_eq!(untrack(&fixture).await?, std::slice::from_ref(&fixture.did)); Ok(()) }
#[tokio::test] async fn a_chain_break_during_the_drain_queues_the_repo_again_after_it() -> miette::Result<()> { let fixture = Fixture::with_chunk_records(1).await?; let (_, after_all) = fetch_with_buffered_posts(&fixture, 3).await?; let q = post("q"); let commit = fixture.live_commit(&after_all, "q", &q)?; let event = std::sync::Mutex::new(Some(live_event(&after_all, commit, true)?)); let result = run_draining(&fixture, keys::pending_key(7), move |step, tx| { if step == 1 && let Some(event) = event.lock().unwrap().take() { tx.blocking_send(event).unwrap(); } }) .await;
assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); // the first step drained one commit, then the drain stopped and the repo is queued // under a new key with the rest of the buffer, the chain break's commit included assert_eq!(fixture.record_rkeys()?, ["a", "p000"]); let pending = fixture.pending_keys()?; assert_eq!(pending.len(), 1); assert_ne!(pending[0], keys::pending_key(7)); assert_eq!(fixture.buffered(), 3); assert!(fixture.marker()?.is_none()); Ok(()) }
#[tokio::test] async fn a_drain_that_never_catches_up_takes_the_rest_in_one_step() -> miette::Result<()> { use crate::ingest::indexer::bulk::DRAIN_STEPS_BEFORE_ALL; let fixture = Fixture::with_chunk_records(1).await?; let (rkeys, _) = fetch_with_buffered_posts(&fixture, DRAIN_STEPS_BEFORE_ALL + 6).await?; let steps = Arc::new(AtomicUsize::new(0)); let counted = steps.clone(); let result = run_draining(&fixture, keys::pending_key(7), move |step, _| { counted.store(step, Ordering::SeqCst); }) .await;
assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); assert_eq!(steps.load(Ordering::SeqCst), DRAIN_STEPS_BEFORE_ALL + 1); assert_eq!(fixture.record_rkeys()?.len(), rkeys.len() + 1); assert_eq!(fixture.buffered(), 0); assert!(fixture.pending_keys()?.is_empty()); Ok(()) }
#[cfg(feature = "indexer_stream")] #[tokio::test] async fn a_drain_stopped_after_any_step_comes_out_the_same_once_it_runs_again() -> miette::Result<()> { const POSTS: usize = 4; let clean = Fixture::with_chunk_records(1).await?; let (_, after_all) = fetch_with_buffered_posts(&clean, POSTS).await?; assert_eq!( clean.run(keys::pending_key(7)).await?, TaskDisposition::Finished ); let want = heads(&clean)?; assert_eq!(want.len(), POSTS + 1);
for stopped_after in 1..POSTS { let mut fixture = Fixture::with_chunk_records(1).await?; fetch_with_buffered_posts(&fixture, POSTS).await?; let state = Arc::downgrade(&fixture.state); let result = run_draining(&fixture, keys::pending_key(7), move |step, _| { if step == stopped_after && let Some(state) = state.upgrade() { state.stop.trigger(); } }) .await; assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" ); // the queue entry and what the drain hadn't reached yet are still there assert_eq!(fixture.pending_keys()?, [keys::pending_key(7).to_vec()]); assert_eq!(fixture.buffered(), POSTS - stopped_after);
fixture.reopen()?; fixture.pds.serve_car(after_all.bytes.clone()).await; assert_eq!( fixture.run(keys::pending_key(7)).await?, TaskDisposition::Finished, "stopped after drain step {stopped_after}" ); assert_eq!( heads(&fixture)?, want, "stopped after drain step {stopped_after}" ); assert_eq!( replayed(&fixture, Default::default())?, want, "stopped after drain step {stopped_after}" ); assert_eq!(fixture.buffered(), 0); assert!(fixture.pending_keys()?.is_empty()); } Ok(()) }
// --- a whole car spilled to disk -------------------------------------------------------
#[tokio::test] async fn a_spilled_block_that_rots_on_disk_fails_its_step_and_retries() -> miette::Result<()> { let fixture = Fixture::spilling().await?; three_posts(&fixture).await?; let c = crate::car::compute_cid(&post("c")); fixture .state .spills .after_spill(move |spilled| spilled.corrupt(&c).unwrap());
let err = fixture.run(keys::pending_key(7)).await.unwrap_err(); assert!(err.to_string().contains("no longer hashes"), "{err}"); // the steps before the one with the rotten block went in, and that one didn't assert_eq!(fixture.record_rkeys()?, ["a", "b"]); assert!(fixture.pending_keys()?.is_empty()); assert!(matches!( fixture.resync()?, Some(ResyncState::Error { retry_count: 1, .. }) )); Ok(()) }
#[tokio::test] async fn a_car_with_no_disk_to_spill_to_writes_nothing_and_retries() -> miette::Result<()> { let fixture = Fixture::spilling().await?; three_posts(&fixture).await?; fixture.state.spills.set_free_bytes(0);
let err = fixture.run(keys::pending_key(7)).await.unwrap_err(); assert!(err.to_string().contains("bytes free"), "{err}"); assert!(fixture.record_rkeys()?.is_empty()); assert!(fixture.marker()?.is_none()); assert!(matches!( fixture.resync()?, Some(ResyncState::Error { retry_count: 1, .. }) )); let mut spills = std::fs::read_dir(fixture.state.spills.dir()).into_diagnostic()?; assert!(spills.next().is_none()); Ok(()) }
#[tokio::test] async fn a_spill_that_runs_out_of_disk_partway_writes_nothing_and_retries() -> miette::Result<()> { let fixture = Fixture::spilling().await?; three_posts(&fixture).await?; // room to start, but not for the car let spills = &fixture.state.spills; spills.set_free_bytes(crate::backfill::spill::Spills::HEADROOM + 10); spills.set_check_every(1);
let err = fixture.run(keys::pending_key(7)).await.unwrap_err(); assert!(err.to_string().contains("bytes free"), "{err}"); assert!(fixture.record_rkeys()?.is_empty()); assert!(fixture.marker()?.is_none()); assert!(matches!( fixture.resync()?, Some(ResyncState::Error { retry_count: 1, .. }) )); let mut spilled = std::fs::read_dir(spills.dir()).into_diagnostic()?; assert!(spilled.next().is_none()); Ok(()) }
#[tokio::test] async fn get_repo_of_a_repo_still_backfilling_answers_not_found() -> miette::Result<()> { let fixture = Fixture::with_chunk_records(1).await?; three_posts(&fixture).await?; let repos = crate::control::repos::ReposControl::new(fixture.state.clone());
let result = fixture .run_pausing(keys::pending_key(7), 1, |_| async { // its records are a mix of the old repo and the new one, which no root matches assert!(repos.get(&fixture.did).generate_car().await?.is_none()); Ok(()) }) .await?; assert!( matches!(result, Ok(TaskDisposition::Finished)), "{result:?}" );
use futures::TryStreamExt; let car = repos .get(&fixture.did) .generate_car() .await? .expect("a backfilled repo has a car") .map_ok(|chunk| chunk.to_vec()) .try_concat() .await .into_diagnostic()?; let parsed = crate::car::parse_car(car.into(), true)?; let root = parsed.blocks.get(&parsed.root).expect("the commit"); let data = AtpCommit::from_cbor(root)?.data; assert_eq!(crate::mst::leaves(parsed.blocks.map(), data)?.len(), 3);
// a resync entry alone doesn't hide a repo whose records match its root let resync = ResyncState::Error { kind: ResyncErrorKind::Generic, retry_count: 1, next_retry: 0, }; fixture .state .db .indexer .resync .insert( keys::repo_key(&fixture.did), rmp_serde::to_vec(&resync).into_diagnostic()?, ) .into_diagnostic()?; assert!(repos.get(&fixture.did).generate_car().await?.is_some()); Ok(()) }
// --- erases and wipes, a step at a time on the owner shard -----------------------------
/// the repo synced with posts `a`, `b` and `c` and counted, so a wipe or an erase at one /// record a step takes several fn three_stored_posts(fixture: &Fixture) -> miette::Result<RepoState<'static>> { let seeded = rooted("3jzfcijpj2z2a")?; let (a, b, c) = (post("a"), post("b"), post("c")); fixture.seed( &seeded, 7, &[ ("a", a.as_slice()), ("b", b.as_slice()), ("c", c.as_slice()), ], )?; let mut batch = fixture.state.db.inner.batch(); db::set_record_count(&mut batch, &fixture.state.db, &fixture.did, COLLECTION, 3); batch.commit().into_diagnostic()?; Ok(seeded) }
/// stops hydrant once the running erase or wipe has done `steps` steps #[cfg(feature = "indexer_stream")] fn stop_after_job_step(fixture: &Fixture, steps: usize) { let state = Arc::downgrade(&fixture.state); *fixture.state.after_job_step.lock() = Some(Arc::new(move |step| { if step == steps && let Some(state) = state.upgrade() { state.stop.trigger(); } })); }
impl Fixture { /// asks the api to erase the repo, on a shard that's gone once the answer is in #[cfg(feature = "indexer_stream")] async fn erase_on_a_shard( &self, ) -> miette::Result<miette::Result<crate::types::DeleteBodiesReport>> { let (tx, shard) = FirehoseWorker::spawn_one_shard(self.state.clone()); self.state.indexer.lock().replace(tx); let repos = crate::control::repos::ReposControl::new(self.state.clone()); let erased = repos.erase_repo(&self.did, false).await; self.state.indexer.lock().take(); tokio::task::spawn_blocking(move || shard.join()) .await .into_diagnostic()? .map_err(|_| miette::miette!("indexer shard panicked"))?; Ok(erased) }
/// the did's history, with the death revs #[cfg(feature = "indexer_stream")] fn wiped_history(&self) -> miette::Result<Vec<(Vec<u8>, Vec<u8>)>> { let prefix = keys::record_prefix_did(&self.did); let mut end = prefix.clone(); *end.last_mut().unwrap() += 1; self.state .db .indexer .history_range(prefix..end) .map(|guard| { let (key, value) = guard.into_inner().into_diagnostic()?; Ok((key.to_vec(), value.to_vec())) }) .collect() } }
#[cfg(feature = "indexer_stream")] #[tokio::test] async fn a_wipe_stopped_after_any_step_comes_out_the_same_once_it_runs_again() -> miette::Result<()> { // a head a step, and the third step also finds there are no more. none at all is a // crash right after the deletion's own txn for stopped_after in 0..=3 { let mut fixture = Fixture::with_chunk_records(1).await?; let seeded = three_stored_posts(&fixture)?; fixture.start_deletion(&seeded)?; let Some(crate::types::BulkJobV1::Wipe { death_rev }) = fixture.marker()? else { panic!("the deletion left no wipe marker"); }; if stopped_after > 0 { stop_after_job_step(&fixture, stopped_after); fixture.start_a_shard().await?; *fixture.state.after_job_step.lock() = None; }
fixture.reopen()?; fixture.start_a_shard().await?;
let want = ["a", "b", "c"] .map(|rkey| { let record = keys::record_key(&fixture.did, COLLECTION, &DbRkey::new(rkey)); (keys::history_key(&record, &death_rev), post(rkey)) }) .to_vec(); assert_eq!( fixture.wiped_history()?, want, "stopped after step {stopped_after}" ); assert!(fixture.record_rkeys()?.is_empty()); assert_eq!( db::get_record_count(&fixture.state.db, &fixture.did, COLLECTION)?, 0 ); assert!(fixture.marker()?.is_none()); assert!(fixture.pending_keys()?.is_empty()); assert_eq!(fixture.repo()?.status, RepoStatus::Deleted); } Ok(()) }
#[cfg(feature = "indexer_stream")] #[tokio::test] async fn an_erase_stopped_after_any_step_comes_out_the_same_once_it_runs_again() -> miette::Result<()> { let mut stopped_after = 0; loop { stopped_after += 1; let mut fixture = Fixture::with_chunk_records(1).await?; three_stored_posts(&fixture)?; let old = post("old"); let record = keys::record_key(&fixture.did, COLLECTION, &DbRkey::new("a")); let mut batch = fixture.state.db.inner.batch(); fixture.state.db.indexer.stage_history( &mut batch, keys::history_key(&record, &crate::db::types::DbTid::new_from_bytes([0; 8])), &old, ); batch.commit().into_diagnostic()?; fixture.buffer(&build_car("3jzfcijpj2z2b", &[]).await?, "d", &post("d"))?;
stop_after_job_step(&fixture, stopped_after); let finished = fixture.erase_on_a_shard().await?.is_ok(); *fixture.state.after_job_step.lock() = None; fixture.reopen()?; fixture.start_a_shard().await?;
let gone = format!("stopped after step {stopped_after}"); assert!(fixture.record_rkeys()?.is_empty(), "{gone}"); assert!(fixture.wiped_history()?.is_empty(), "{gone}"); assert_eq!(fixture.buffered(), 0, "{gone}"); assert!( fixture .state .db .repos .get(keys::repo_key(&fixture.did)) .into_diagnostic()? .is_none(), "{gone}" ); assert!(fixture.pending_keys()?.is_empty(), "{gone}"); assert!(fixture.marker()?.is_none(), "{gone}"); assert_eq!( db::get_record_count(&fixture.state.db, &fixture.did, COLLECTION)?, 0 ); if finished { assert!( stopped_after > 3, "every kind of entry takes a step of its own" ); break; } } Ok(()) }
#[tokio::test] async fn markers_whose_state_moved_on_are_dropped() -> miette::Result<()> { let stage = |fixture: &Fixture, job: crate::types::BulkJobV1| { let mut batch = fixture.state.db.inner.batch(); db::stage_bulk_job(&mut batch, &fixture.state.db, &fixture.did, &job)?; batch.commit().into_diagnostic() };
// a wipe for an account that's active again let fixture = Fixture::with_chunk_records(1).await?; three_stored_posts(&fixture)?; stage( &fixture, crate::types::BulkJobV1::Wipe { death_rev: crate::db::types::DbTid::new_from_bytes([0; 8]), }, )?; fixture.start_a_shard().await?; assert!(fixture.marker()?.is_none()); assert_eq!(fixture.record_rkeys()?, ["a", "b", "c"]);
// a backfill whose queue entry went away let fixture = Fixture::with_chunk_records(1).await?; three_stored_posts(&fixture)?; stage( &fixture, crate::types::BulkJobV1::Apply { pending_key: keys::pending_key(9).to_vec(), }, )?; fixture.start_a_shard().await?; assert!(fixture.marker()?.is_none()); assert_eq!(untrack(&fixture).await?, std::slice::from_ref(&fixture.did));
// an erase of a repo that was tracked again let fixture = Fixture::with_chunk_records(1).await?; three_stored_posts(&fixture)?; stage(&fixture, crate::types::BulkJobV1::Erase)?; fixture.start_a_shard().await?; assert!(fixture.marker()?.is_none()); assert_eq!(fixture.record_rkeys()?, ["a", "b", "c"]); Ok(()) }
#[cfg(feature = "indexer_stream")] #[tokio::test] async fn a_live_commit_during_a_wipe_or_an_erase_is_dropped_and_other_repos_go_between() -> miette::Result<()> { for erase in [false, true] { let fixture = Fixture::with_chunk_records(1).await?; let seeded = three_stored_posts(&fixture)?; let q = post("q"); let car = build_car("3jzfcijpj2z2b", &[("q", q.as_slice())]).await?; let event = live_event(&car, fixture.live_commit(&car, "q", &q)?, false)?; let other = Did::new_static("did:plc:bbbbbbbbbbbbbbbbbbbbbbbb").into_diagnostic()?;
let (tx, shard) = FirehoseWorker::spawn_one_shard(fixture.state.clone()); fixture.state.indexer.lock().replace(tx.clone()); let seen = Arc::new(parking_lot::Mutex::new(Vec::new())); let (state, recorded) = (Arc::downgrade(&fixture.state), seen.clone()); // the hook lets go of its sender after the first step, so the shard can stop let sends = parking_lot::Mutex::new(Some((tx.clone(), event, other.clone()))); *fixture.state.after_job_step.lock() = Some(Arc::new(move |step| { let state = state.upgrade().unwrap(); let other_queued = state .db .repo_metadata .get(keys::repo_metadata_key(&other)) .unwrap() .is_some(); recorded.lock().push((step, other_queued)); if let Some((tx, event, other)) = sends.lock().take() { tx.blocking_send(event).unwrap(); tx.blocking_send(IndexerMessage::new_repo_for_test(other)) .unwrap(); } })); if erase { let repos = crate::control::repos::ReposControl::new(fixture.state.clone()); repos.erase_repo(&fixture.did, false).await?; } else { fixture.start_deletion(&seeded)?; let did = fixture.did.clone(); tx.on_owner(&fixture.did, move |_, shard| shard.mark(did)) .await?; } fixture.state.indexer.lock().take(); drop(tx); tokio::task::spawn_blocking(move || shard.join()) .await .into_diagnostic()? .map_err(|_| miette::miette!("indexer shard panicked"))?; *fixture.state.after_job_step.lock() = None;
let seen = seen.lock().clone(); assert!(seen.len() > 1, "{seen:?}"); // the other repo's message went in right after the first step assert_eq!(seen[0], (1, false), "erase: {erase}"); assert!(seen[1..].iter().all(|&(_, queued)| queued), "{seen:?}"); assert!(fixture.record_rkeys()?.is_empty()); let q_event = format!("{COLLECTION}/q"); assert!( record_events(&fixture)? .iter() .all(|(key, _)| *key != q_event), "erase: {erase}" ); } Ok(()) }
#[cfg(feature = "indexer_stream")] #[tokio::test] async fn untrack_after_a_restart_partway_through_the_steps_waits_for_them_to_run_again() -> miette::Result<()> { let mut fixture = Fixture::with_chunk_records(1).await?; let car = three_posts(&fixture).await?; let state = Arc::downgrade(&fixture.state); *fixture.state.after_chunk.lock() = Some(Arc::new(move |sent| { if sent == 1 && let Some(state) = state.upgrade() { state.stop.trigger(); } })); assert_eq!( fixture.run(keys::pending_key(7)).await?, TaskDisposition::Finished ); *fixture.state.after_chunk.lock() = None;
fixture.reopen()?; fixture.start_a_shard().await?; assert!(untrack(&fixture).await?.is_empty()); assert_eq!(fixture.pending_keys()?, [keys::pending_key(7).to_vec()]);
fixture.pds.serve_car(car.bytes).await; assert_eq!( fixture.run(keys::pending_key(7)).await?, TaskDisposition::Finished ); assert_eq!(fixture.record_rkeys()?, ["a", "b", "c"]); assert!(fixture.marker()?.is_none()); assert_eq!(untrack(&fixture).await?, std::slice::from_ref(&fixture.did)); Ok(()) }}