diff --git a/crates/vibescrobble-index/src/bin/vibescrobble-index.rs b/crates/vibescrobble-index/src/bin/vibescrobble-index.rs index 3f883fb..b907d5c 100644 --- a/crates/vibescrobble-index/src/bin/vibescrobble-index.rs +++ b/crates/vibescrobble-index/src/bin/vibescrobble-index.rs @@ -997,7 +997,7 @@ fn start_avatars( async fn released(shared: &Shared, announced: &[Event]) { let mut released = 0_usize; for event in announced { - if let Event::Released { did } = event { + if let Event::Released { did, .. } = event { shared.pulse.release(did).await; released += 1; } diff --git a/crates/vibescrobble-index/src/burn.rs b/crates/vibescrobble-index/src/burn.rs index bb71d78..438b371 100644 --- a/crates/vibescrobble-index/src/burn.rs +++ b/crates/vibescrobble-index/src/burn.rs @@ -48,7 +48,9 @@ //! gone from its repository is beyond reach. A denylisted account's rows are //! deleted. An account the view no longer holds, such as one that dropped //! its `bot` label, is left out at query time and is back when it is held -//! again. +//! again. One the view let go to keep the dormant accounts within +//! `--keep-dormant` is not: the store keeps who it rolled up to as it went +//! ([`Store::release`]), and counts it there until it is held again. use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::io::{BufRead, Write}; @@ -492,6 +494,16 @@ struct Saved { /// Accounts whose whole repository has been read into the store. #[serde(default)] filled: Vec, + /// Accounts the view let go for room, with the tree each had then. + #[serde(default)] + released: Vec, +} + +/// An account the view let go for room, and who it rolled up to. +#[derive(Debug, Serialize, Deserialize)] +struct LetGo { + author: Did, + lineage: Lineage, } /// One line of the journal. @@ -503,6 +515,8 @@ enum Entry { Forget(String), Filled(Did), Unfilled(Did), + Released(LetGo), + Held(Did), } #[derive(Debug, Default)] @@ -518,6 +532,9 @@ struct Inner { /// Accounts whose whole repository has been read in: see /// [`Store::filled`]. filled: BTreeSet, + /// Who each counted account the view let go for room rolled up to: + /// see [`Store::release`]. + released: BTreeMap, journal: Option, /// Lines written since the journal was last flushed to disk. unsynced: usize, @@ -553,6 +570,20 @@ impl Inner { self.windows.remove(&key).is_some() } + /// Whether anything is counted for `author`. + fn counts(&self, author: &Did) -> bool { + let first = |key: &Did| key == author; + self.windows + .range((author.clone(), String::new())..) + .next() + .is_some_and(|((key, _), _)| first(key)) + || self + .retired + .range((author.clone(), String::new(), String::new(), String::new())..) + .next() + .is_some_and(|((key, ..), _)| first(key)) + } + fn forget(&mut self, author: &str) -> bool { let before = self.windows.len() + self.retired.len(); self.windows.retain(|(held, _), _| held.as_str() != author); @@ -560,6 +591,7 @@ impl Inner { self.folded_rows .retain(|(held, _), _| held.as_str() != author); self.filled.retain(|held| held.as_str() != author); + self.released.retain(|held, _| held.as_str() != author); before != self.windows.len() + self.retired.len() } @@ -580,6 +612,12 @@ impl Inner { Entry::Unfilled(author) => { self.filled.remove(&author); } + Entry::Released(LetGo { author, lineage }) => { + self.released.insert(author, lineage); + } + Entry::Held(author) => { + self.released.remove(&author); + } } } @@ -742,6 +780,9 @@ impl Store { inner.retired.insert(daily.key(), daily); } inner.filled.extend(saved.filled); + for LetGo { author, lineage } in saved.released { + inner.released.insert(author, lineage); + } } } Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} @@ -825,6 +866,14 @@ impl Store { folded_windows: inner.folded_rows.values().cloned().collect(), retired: inner.retired.values().cloned().collect(), filled: inner.filled.iter().cloned().collect(), + released: inner + .released + .iter() + .map(|(author, lineage)| LetGo { + author: author.clone(), + lineage: lineage.clone(), + }) + .collect(), }; let beside = path.with_extension("tmp"); { @@ -942,6 +991,34 @@ impl Store { } } + /// Keeps who `did` rolled up to as the view let it go for room, so what + /// it is counted for stays in its tree's and its operator's totals. One + /// with nothing counted is nothing to keep. + pub fn release(&self, did: &str, lineage: &Lineage) { + let Some(author) = Did::key(did) else { + return; + }; + let mut inner = self.lock(); + if !inner.counts(&author) || inner.released.get(&author) == Some(lineage) { + return; + } + inner.released.insert(author.clone(), lineage.clone()); + debug!(did, "burn keeps the tree of an account let go for room"); + inner.write(&Entry::Released(LetGo { + author, + lineage: lineage.clone(), + })); + } + + /// Lets go of the tree kept for `did`, which the view holds again. + pub fn hold(&self, did: &str) { + let mut inner = self.lock(); + let Some((author, _)) = inner.released.remove_entry(did) else { + return; + }; + inner.write(&Entry::Held(author)); + } + /// What an announced event changes. pub fn observe(&self, event: &Event) { match event { @@ -949,6 +1026,11 @@ impl Store { self.put(scrobble); } Event::Unscrobbled { uri } => self.unsay(uri), + Event::Released { + did, + lineage: Some(lineage), + } => self.release(did, lineage), + Event::Provisioned(agent) => self.hold(&agent.did), _ => {} } } @@ -1140,39 +1222,44 @@ impl Row { /// Who an author rolls up to: itself, then its ancestors nearest first, /// which an agent's `usage` summary sums its subtree over, with the /// operator of the nearest one that has one. -struct Lineage { +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Lineage { chain: Vec, operator: String, } -fn lineages(view: &View, authors: impl Iterator) -> HashMap { - let mut out = HashMap::new(); - for author in authors { - if out.contains_key(&author) { - continue; - } - let Some(agent) = view.agent(author.as_str()) else { - continue; - }; - let ancestors = view.ancestors_of(author.as_str()); +impl Lineage { + /// `did`'s, as `view` holds it, or nothing when it is not held. + pub fn of(view: &View, did: &str) -> Option { + let agent = view.agent(did)?; + let ancestors = view.ancestors_of(did); let operator = std::iter::once(agent) .chain(ancestors.iter().copied()) .find_map(|agent| agent.operator.as_ref().map(|human| human.did.clone())) - .unwrap_or_else(|| author.to_string()); - let mut chain = vec![author.to_string()]; + .unwrap_or_else(|| did.to_owned()); + let mut chain = vec![did.to_owned()]; chain.extend(ancestors.iter().map(|agent| agent.did.clone())); - out.insert(author, Lineage { chain, operator }); + Some(Lineage { chain, operator }) } - out } -fn authors(store: &Store) -> HashSet { +/// The lineage of every author `store` counts: as `view` holds it, or as it +/// was when the view let the author go for room. +fn lineages(store: &Store, view: &View) -> HashMap { let inner = store.lock(); - inner + let authors: HashSet<&Did> = inner .windows .keys() - .map(|(author, _)| author.clone()) - .chain(inner.retired.keys().map(|key| key.0.clone())) + .map(|(author, _)| author) + .chain(inner.retired.keys().map(|key| &key.0)) + .collect(); + authors + .into_iter() + .filter_map(|author| { + let lineage = Lineage::of(view, author.as_str()) + .or_else(|| inner.released.get(author).cloned())?; + Some((author.clone(), lineage)) + }) .collect() } @@ -1189,9 +1276,10 @@ pub fn leaderboard(store: &Store, view: &View, board: &Board, now: OffsetDateTim pub struct Lineages(HashMap); impl Lineages { - /// Takes the lineage of every author `store` counts that `view` holds. + /// Takes the lineage of every author `store` counts that `view` holds, + /// or that it let go for room. pub fn of(store: &Store, view: &View) -> Self { - Self(lineages(view, authors(store).into_iter())) + Self(lineages(store, view)) } } @@ -1200,7 +1288,8 @@ pub fn rank(store: &Store, lineages: &Lineages, board: &Board, now: OffsetDateTi let lineages = &lineages.0; let mut rows: BTreeMap = BTreeMap::new(); store.parts(board.window, now, |part| { - // Hidden: an account the view does not hold is left out. + // Hidden: an account the view does not hold, and did not let go for + // room, is left out. let Some(lineage) = lineages.get(part.author) else { return; }; diff --git a/crates/vibescrobble-index/src/view.rs b/crates/vibescrobble-index/src/view.rs index 79a0921..86372dc 100644 --- a/crates/vibescrobble-index/src/view.rs +++ b/crates/vibescrobble-index/src/view.rs @@ -404,6 +404,10 @@ pub enum Event { Released { /// The account. did: String, + /// Who it rolled up to as it went, for the burn store to keep it in + /// its tree's totals. In this process only: not on the wire. + #[serde(skip)] + lineage: Option, }, /// How many accounts are held, shown or not. Announced when it changes. Held { @@ -1436,13 +1440,14 @@ impl View { } let mut events = Vec::with_capacity(going.len()); for (did, at) in going { + let lineage = crate::burn::Lineage::of(self, &did); if self.remove_account(&did).is_none() { continue; } if let Some(key) = Did::key(&did) { self.released.0.insert(key, at); } - events.push(Event::Released { did }); + events.push(Event::Released { did, lineage }); } events } @@ -1989,9 +1994,9 @@ impl View { Event::Scrobble(scrobble) => self.take_scrobble(scrobble, false, true), Event::Deleted { did, reaped: false } => self.forget(&did), Event::Deleted { did, reaped: true } => self.reap(&did), - Event::Released { did } => self + Event::Released { did, .. } => self .remove_account(&did) - .map(|_| Event::Released { did }) + .map(|_| Event::Released { did, lineage: None }) .into_iter() .collect(), Event::Disowned { did } => { @@ -3525,7 +3530,7 @@ mod tests { events .iter() .filter_map(|event| match event { - Event::Released { did } => Some(did.as_str()), + Event::Released { did, .. } => Some(did.as_str()), _ => None, }) .collect() @@ -3597,11 +3602,19 @@ mod tests { mirror.apply(Event::Scrobble(scrobble("at://a/1", "did:web:a.example"))); let did = "did:web:a.example".to_owned(); assert_eq!( - mirror.apply(Event::Released { did: did.clone() }), - [Event::Released { did: did.clone() }] + mirror.apply(Event::Released { + did: did.clone(), + lineage: None + }), + [Event::Released { + did: did.clone(), + lineage: None + }] ); assert_eq!((mirror.agents(), mirror.scrobbles()), (0, 0)); - assert!(mirror.apply(Event::Released { did }).is_empty()); + assert!(mirror + .apply(Event::Released { did, lineage: None }) + .is_empty()); } #[test] diff --git a/crates/vibescrobble-index/tests/burn_released.rs b/crates/vibescrobble-index/tests/burn_released.rs new file mode 100644 index 0000000..541f82a --- /dev/null +++ b/crates/vibescrobble-index/tests/burn_released.rs @@ -0,0 +1,187 @@ +//! An account let go to keep the dormant accounts within `--keep-dormant` is +//! still counted in its tree's and its operator's token burn, through a +//! restart, while an account the board forgets is still left out. + +use std::sync::Arc; + +use serde_json::{json, Value}; +use time::macros::datetime; +use vibescrobble_index::burn::{self, Board, Burn, Lineages, Store, Whose, RETENTION_DAYS}; +use vibescrobble_index::view::{Agent, Event, Human, Kept}; +use vibescrobble_index::Shared; + +const HOST: &str = "did:web:kiln.agents.localhost%3A3000"; +const SESSION: &str = "did:web:ember.agents.localhost%3A3000"; +const PERSON: &str = "did:plc:quernstone"; +const SCROBBLE: &str = "com.vibescrobble.scrobble"; + +/// When every process here runs: three hours after either account last spoke. +const NOW: time::OffsetDateTime = datetime!(2026-10-21 12:00 UTC); + +fn agent(did: &str, parent: Option<&str>) -> Agent { + let name = did.trim_start_matches("did:web:"); + let name = name.split('.').next().unwrap_or_default(); + Agent { + did: did.to_owned(), + agent_id: name.to_owned(), + handle: Some(format!("{name}.agents.localhost")), + server: "did:web:agents.localhost%3A3000".to_owned(), + operator: Some(Human { + did: PERSON.to_owned(), + handle: Some("quernstone.example".to_owned()), + }), + created_at: "2026-10-12T08:00:00Z".to_owned(), + parent: parent.map(str::to_owned), + agent_type: None, + registered_kind: None, + model: None, + status: None, + avatar: None, + banner: None, + display_name: None, + description: None, + active_at: None, + dormant: false, + idle: false, + } +} + +/// A line by `did` this morning, saying it spent `output` tokens. +async fn say(shared: &Shared, did: &str, minute: i64, output: u64) { + let uri = format!("at://{did}/{SCROBBLE}/3kiln{minute:04}"); + let at = (datetime!(2026-10-21 09:00 UTC) + time::Duration::minutes(minute)) + .format(&time::format_description::well_known::Rfc3339) + .expect("formats"); + let value = json!({ + "$type": SCROBBLE, + "text": "firing the kiln", + "emoji": "🧪", + "createdAt": at, + "model": "claude-opus-5-5", + "usage": { "input": 0, "output": output, "cacheRead": 0, "cacheWrite": 0 }, + }); + let scrobble = vibescrobble_index::feed::read_record(did, &uri, value).expect("a scrobble"); + let announced = shared.view.lock().await.see_scrobble(scrobble); + shared.announce(announced).await; +} + +fn scratch(name: &str) -> std::path::PathBuf { + let path = std::env::temp_dir().join(format!( + "vibescrobble-burn-released-{}-{name}", + std::process::id() + )); + for end in ["", ".journal"] { + let _ = std::fs::remove_file(format!("{}{end}", path.display())); + } + path +} + +/// A process on the burn store at `path`, with the board `kept`. +async fn start(path: &std::path::Path, kept: Option) -> Shared { + let store = Store::open_at(path, RETENTION_DAYS, NOW).expect("a burn store"); + let shared = Shared::new().with_burn(Burn::Local(Arc::new(store))); + if let Some(kept) = kept { + shared.view.lock().await.restore(kept, |_| true); + } + shared +} + +/// Writes down what a stop writes, and hands back the board. +async fn stop(shared: Shared) -> Kept { + shared.burn.store().expect("a local store").sync(); + shared.view.lock().await.keep() +} + +/// All-time `(windows, output)` under `whose`, from its summary's subtree. +async fn all_time(shared: &Shared, whose: Whose<'_>) -> (u64, u64) { + let store = shared.burn.store().expect("a local store"); + let view = shared.view.lock().await; + let summary = burn::sum(store, &Lineages::of(store, &view), whose, NOW).unwrap_or(Value::Null); + let all = &summary["subtree"]["all"]; + ( + all["scrobbles"].as_u64().unwrap_or(0), + all["output"].as_u64().unwrap_or(0), + ) +} + +/// The all-time leaderboard's rows by agent, by key. +async fn agents_ranked(shared: &Shared) -> Vec { + let store = shared.burn.store().expect("a local store"); + let view = shared.view.lock().await; + let board: Board = + serde_json::from_value(json!({ "by": "agent", "window": "all" })).expect("a board"); + let mut keys: Vec = burn::leaderboard(store, &view, &board, NOW)["rows"] + .as_array() + .expect("rows") + .iter() + .filter_map(|row| row["key"].as_str().map(str::to_owned)) + .collect(); + keys.sort(); + keys +} + +#[tokio::test] +async fn an_account_let_go_for_room_keeps_its_burn_in_its_tree() { + let path = scratch("burn.json"); + let shared = start(&path, None).await; + { + let mut view = shared.view.lock().await; + view.see_agent(agent(HOST, None)); + view.see_agent(agent(SESSION, Some(HOST))); + } + say(&shared, HOST, 0, 10).await; + say(&shared, SESSION, 1, 20).await; + let both = (2, 30); + assert_eq!(all_time(&shared, Whose::Operator(PERSON)).await, both); + + // Both go quiet; room for one dormant account lets the session go. + let events = { + let mut view = shared.view.lock().await; + view.set_prune(Some(std::time::Duration::from_secs(3600))); + view.set_keep_dormant(Some(1)); + view.sweep(NOW) + }; + let released: Vec<&str> = events + .iter() + .filter_map(|event| match event { + Event::Released { did, .. } => Some(did.as_str()), + _ => None, + }) + .collect(); + assert_eq!(released, [SESSION]); + shared.announce(events).await; + assert!(shared.view.lock().await.agent(SESSION).is_none()); + + assert_eq!( + all_time(&shared, Whose::Operator(PERSON)).await, + both, + "its operator's total keeps it" + ); + assert_eq!( + all_time(&shared, Whose::Agent(HOST)).await, + both, + "and so does its parent's" + ); + assert_eq!(agents_ranked(&shared).await, [SESSION, HOST]); + + // A restart keeps who it rolled up to. + let kept = stop(shared).await; + let restarted = start(&path, Some(kept)).await; + assert_eq!(all_time(&restarted, Whose::Operator(PERSON)).await, both); + assert_eq!(agents_ranked(&restarted).await, [SESSION, HOST]); + + // An account the board forgets is still left out. + let forgotten = restarted.view.lock().await.forget(HOST); + restarted.announce(forgotten).await; + assert_eq!(all_time(&restarted, Whose::Operator(PERSON)).await, (1, 20)); + assert_eq!(agents_ranked(&restarted).await, [SESSION]); + + // Held again, it is counted by the tree the board holds, through + // another restart. + let held = restarted.view.lock().await.see_agent(agent(SESSION, None)); + restarted.announce(held).await; + let kept = stop(restarted).await; + let again = start(&path, Some(kept)).await; + assert_eq!(all_time(&again, Whose::Operator(PERSON)).await, (1, 20)); + assert_eq!(all_time(&again, Whose::Agent(SESSION)).await, (1, 20)); +} diff --git a/docs/state.md b/docs/state.md index 504958e..ab62473 100644 --- a/docs/state.md +++ b/docs/state.md @@ -69,7 +69,9 @@ not. The file notes each account whose whole repository has been read into it, and an account it does not name is read whole by the same paced pass as the ledgers, so a lost file is rebuilt from the repositories. So is an account a read found had written more than its newest 32 while the index was -away. A window whose record is gone from its repository is held only here. +away. A window whose record is gone from its repository is held only here. So is +the tree, and the operator, of each counted account the board let go past +`--keep-dormant`: its burn stays in those totals until it is held again. `identities.jsonl` keeps every account it has resolved. A `did:web` account that has ended stops resolving, so its line is the only record of who wrote