From 45ebd14f517de2a26906262626f94cb51c655fea Mon Sep 17 00:00:00 2001 From: "@permadeath.com" Date: Mon, 26 Oct 2026 10:14:10 -0400 Subject: [PATCH] fix(index): read accounts let go for room back into a lost burn store The fill pass reads them too, until the burn store has read them whole, working out each tree from its registration when the store kept none. Co-Authored-By: Claude Opus 5.5 Change-Id: I775aeb43c542aded73ba78588379f6b3b3dac2f8 --- crates/vibescrobble-index/src/burn.rs | 44 ++++++++++++- crates/vibescrobble-index/src/fill.rs | 61 +++++++++++++++++-- crates/vibescrobble-index/src/operator.rs | 42 +++++++++++++ crates/vibescrobble-index/src/view.rs | 30 +++++++++ .../vibescrobble-index/tests/deploy/cold.rs | 1 - docs/state.md | 6 +- 6 files changed, 176 insertions(+), 8 deletions(-) diff --git a/crates/vibescrobble-index/src/burn.rs b/crates/vibescrobble-index/src/burn.rs index cb4dec5..5dd6e0d 100644 --- a/crates/vibescrobble-index/src/burn.rs +++ b/crates/vibescrobble-index/src/burn.rs @@ -73,7 +73,7 @@ use tracing::{debug, info, warn}; use crate::did::Did; use crate::feed::Scrobble; -use crate::view::{Event, View}; +use crate::view::{Agent, Event, View}; /// How many days a window row is kept before it is folded into dailies, /// unless `--burn-days` says otherwise. @@ -1497,6 +1497,40 @@ impl Lineage { chain.extend(ancestors.iter().map(|agent| agent.did.clone())); Some(Lineage { chain, operator }) } + + /// `did`'s, were the view to hold it now on the server `home` under + /// `operator`, with `lineage` from its registration: below the nearest + /// account of `lineage` held, when that is on the same server, as + /// [`View::attach`] hangs it. + pub fn under( + view: &View, + did: &str, + operator: Option<&str>, + home: &str, + lineage: &[String], + ) -> Self { + let parent = lineage + .iter() + .rev() + .find_map(|ancestor| view.agent(ancestor)) + .filter(|parent| parent.server == home && parent.did != did); + let mut ancestors: Vec<&Agent> = Vec::new(); + if let Some(parent) = parent { + ancestors.push(parent); + ancestors.extend(view.ancestors_of(&parent.did)); + } + let operator = operator + .map(str::to_owned) + .or_else(|| { + ancestors + .iter() + .find_map(|agent| agent.operator.as_ref().map(|human| human.did.clone())) + }) + .unwrap_or_else(|| did.to_owned()); + let mut chain = vec![did.to_owned()]; + chain.extend(ancestors.iter().map(|agent| agent.did.clone())); + Lineage { chain, operator } + } } /// The lineage of every author `store` counts: as `view` holds it, or as it @@ -1820,6 +1854,14 @@ impl Burn { .unwrap_or_default() } + /// Keeps `lineage` as the tree of `did`, which the view does not hold. + /// See [`Store::release`]. + pub fn release(&self, did: &str, lineage: &Lineage) { + if let Some(store) = self.store() { + store.release(did, lineage); + } + } + /// Lets go of the tree kept for `did`, taken off the board by its host. pub fn unrelease(&self, did: &str) { if let Some(store) = self.store() { diff --git a/crates/vibescrobble-index/src/fill.rs b/crates/vibescrobble-index/src/fill.rs index f4efaa6..cd5493a 100644 --- a/crates/vibescrobble-index/src/fill.rs +++ b/crates/vibescrobble-index/src/fill.rs @@ -24,6 +24,12 @@ //! the burn store's note back ([`crate::view::View::overran`]), since the //! account wrote more than that read reaches, and this reads it whole again. //! +//! An account the board let go for room is read here too, for its burn +//! alone, when the burn store has not read it whole and the ledgers hold a +//! week of it or the burn store its tree. A store that was lost, or let go +//! at a format change, keeps no tree, so the account's is read first +//! ([`crate::operator::lineage`]), a [`PACE`] after the last read. +//! //! A read follows the server's cursor, and the server is the account's own: //! one that names a next page forever would hold the fill on it. So a read //! takes at most [`PAGE`] records from a page and `--fill-pages` pages from @@ -85,7 +91,8 @@ pub struct Filled { pub lines: usize, /// Pages read. pub reads: usize, - /// Accounts left for the next pass: no server named, or a page not read. + /// Accounts left for the next pass: no server named, a page not read, or + /// let go for room and not drawn now. pub unread: usize, /// Accounts whose read was ended before the oldest week kept: past the /// page limit, at a cursor already followed, or at a page of nothing @@ -104,7 +111,7 @@ enum Read { Unread, } -/// Reads every held account not yet filled into the ledgers, `pace` apart +/// Reads every account not yet filled, held or let go for room, `pace` apart /// and at most `pages` pages each, back to the oldest week kept while /// `now`'s is the newest. pub async fn fill( @@ -123,8 +130,20 @@ pub async fn fill( view.agents_after(None, |agent| !shared.burn.filled(&agent.did)) .map(|agent| agent.did.clone()), ); + accounts.extend( + view.tallied_accounts() + .into_iter() + .chain(shared.burn.released()) + .filter(|did| { + !view.knows(did) + && !shared.burn.filled(did) + && !registry.is_gone(did) + && !shared.moderation.denies(did, None, None) + }), + ); accounts.into_iter().collect() }; + let kept: HashSet = shared.burn.released().into_iter().collect(); if accounts.is_empty() { return filled; } @@ -133,12 +152,38 @@ pub async fn fill( "counting each repository into the weekly ledgers and token burn" ); for did in accounts { + let held = shared.view.lock().await.knows(&did); + // Let go for room with no tree kept, as after the burn store was + // lost: who it rolls up to is read before its records. + let tree = if held || kept.contains(&did) { + None + } else { + tokio::time::sleep(pace).await; + let server = registry + .endpoint(&did) + .await + .and_then(|endpoint| ServerRef::parse(&endpoint).ok()); + let tree = match server { + Some(server) => crate::operator::lineage(shared, registry, &did, &server).await, + None => None, + }; + if tree.is_none() { + debug!( + did, + "let go for room, and not drawn now; left for the next pass" + ); + filled.unread += 1; + continue; + } + tree + }; let mut recount = Recount::default(); let read = account( shared, registry, indexer, &did, + held, pace, pages, now, @@ -161,6 +206,12 @@ pub async fn fill( view.mark_filled(&did); shared.burn.mark_filled(&did, recount); filled.accounts += 1; + } else if !held && !shared.moderation.denies(&did, None, None) { + if let Some(tree) = &tree { + shared.burn.release(&did, tree); + } + shared.burn.mark_filled(&did, recount); + filled.accounts += 1; } } info!( @@ -175,13 +226,15 @@ pub async fn fill( } /// Reads one account's records back to the oldest week kept, at most `pages` -/// pages of them. +/// pages of them. Its burn is counted while it is `held` as at the start, or +/// while it is let go for room and not denied. #[allow(clippy::too_many_arguments)] async fn account( shared: &Shared, registry: &Registry, indexer: &Indexer, did: &str, + held: bool, pace: Duration, pages: usize, now: OffsetDateTime, @@ -229,7 +282,7 @@ async fn account( if view.fill_line(scrobble) { filled.lines += 1; } - if view.knows(did) { + if view.knows(did) || (!held && !shared.moderation.denies(did, None, None)) { shared.burn.put_read(scrobble, recount); } } diff --git a/crates/vibescrobble-index/src/operator.rs b/crates/vibescrobble-index/src/operator.rs index 15cb018..f8b14c8 100644 --- a/crates/vibescrobble-index/src/operator.rs +++ b/crates/vibescrobble-index/src/operator.rs @@ -35,6 +35,7 @@ use tracing::{debug, info}; use crate::backfill::Overheard; +use crate::burn::Lineage; use crate::did::{hostname_is_at_or_below, AgentDid, DocumentExt}; use crate::http::HttpGet; use crate::identity::Registry; @@ -161,6 +162,47 @@ pub async fn acknowledged( } } +/// Who `did`, an account the view does not hold, would roll up to if +/// [`admit`] held it now, read from `server`. `None` for one it would not +/// draw, or whose profile could not be read. +pub async fn lineage( + shared: &Shared, + registry: &Registry, + did: &str, + server: &ServerRef, +) -> Option { + let pds = pds_host(Some(server.base_url())); + if shared.moderation.denies(did, None, pds.as_deref()) + || registry.is_bot(did, server).await != Some(true) + { + return None; + } + let handle = registry.handle(did).await; + if shared + .moderation + .denies(did, handle.as_deref(), pds.as_deref()) + { + return None; + } + let answer = answer(registry, did, Some(server)).await; + let operator = match &answer.operator { + Some(human) + if acknowledged(registry, &human.did, did, Some(server), &answer) + .await + .account + && !shared + .moderation + .denies_person(&human.did, human.handle.as_deref()) => + { + Some(human.did.as_str()) + } + _ => None, + }; + let home = crate::relay::server_for(server.base_url()).did; + let view = shared.view.lock().await; + Some(Lineage::under(&view, did, operator, &home, &answer.lineage)) +} + /// The host of `endpoint`, the server an account's document says its /// repository is on. pub(crate) fn pds_host(endpoint: Option<&str>) -> Option { diff --git a/crates/vibescrobble-index/src/view.rs b/crates/vibescrobble-index/src/view.rs index 5ceee77..aed2686 100644 --- a/crates/vibescrobble-index/src/view.rs +++ b/crates/vibescrobble-index/src/view.rs @@ -4205,4 +4205,34 @@ mod tests { } assert!(index.disown(PERSON).is_empty(), "once"); } + + /// The tree worked out for an account the view does not hold is the one + /// holding it gives: below the nearest held ancestor on its own server, + /// under its operator or else the nearest one above it. + #[test] + fn a_tree_worked_out_unheld_is_the_one_holding_it_gives() { + use crate::burn::Lineage; + let mut host = agent("did:web:host"); + host.operator = Some(human("did:plc:person")); + let mut elsewhere = agent("did:web:elsewhere"); + elsewhere.server = "did:web:other.example".to_owned(); + for (did, operator, lineage) in [ + ("did:web:session", None, vec!["did:web:host"]), + ("did:web:owned", Some("did:plc:other"), vec!["did:web:host"]), + ("did:web:stray", None, vec!["did:web:elsewhere"]), + ] { + let mut view = View::new(); + view.see_agent(host.clone()); + view.see_agent(elsewhere.clone()); + let lineage: Vec = lineage.into_iter().map(str::to_owned).collect(); + let unheld = Lineage::under(&view, did, operator, "did:web:agents.example", &lineage); + + let mut account = agent(did); + account.operator = operator.map(human); + view.register(did, operator.map(str::to_owned), lineage); + view.see_agent(account); + view.attach(did); + assert_eq!(Some(unheld), Lineage::of(&view, did), "{did}"); + } + } } diff --git a/crates/vibescrobble-index/tests/deploy/cold.rs b/crates/vibescrobble-index/tests/deploy/cold.rs index c41fe24..3cfaa46 100644 --- a/crates/vibescrobble-index/tests/deploy/cold.rs +++ b/crates/vibescrobble-index/tests/deploy/cold.rs @@ -109,7 +109,6 @@ async fn a_start_from_the_state_alone_comes_to_what_a_cold_start_serves() { /// The burn store let go, as one in an older format is at start-up, beside /// a board that has let an account go for room. #[tokio::test] -#[ignore = "issue 3myrrithsun2l: the burn of an account let go for room is not read back"] async fn a_burn_store_let_go_is_rebuilt_with_the_accounts_let_go_for_room() { let world = World::new(); let kept = DataDir::new("cold-burn-let-go"); diff --git a/docs/state.md b/docs/state.md index 057b0cd..3ba8362 100644 --- a/docs/state.md +++ b/docs/state.md @@ -70,8 +70,10 @@ it, each once, so a store that is new, lost or let go starts with the board's windows. Every record a read of a repository finds is counted, drawn or not. The file notes each account whose whole repository has been read into it, and an account it does not name is read whole by the same paced pass as -the ledgers, so a lost file is rebuilt from the repositories, all-time -dailies included: a day's daily is raised to what the repository holds for +the ledgers. So is one the board let go for room that the ledgers hold a +week of, or this file a tree of; with no tree kept, its tree is read from +its registration first. So a lost file is rebuilt from the repositories, +all-time dailies included: a day's daily is raised to what the repository holds for it, and never lowered. So is an account a read found had written more than its newest 32 while the index was away. A window whose record is gone from its repository is held only here. So is -- knotmirror 0.0.1