From a50e6d1bd2e493d4f231862ca2ddd3e846e5cd4b Mon Sep 17 00:00:00 2001 From: "@permadeath.com" Date: Sun, 27 Sep 2026 10:03:37 -0400 Subject: [PATCH] fix(index): hold scrobble records to the lexicon's limits and times read_record refuses strings over the lexicon's lengths and graphemes, and stores createdAt as UTC to the second, clamped to now when far ahead. Co-Authored-By: Claude Opus 5.5 (1M context) Change-Id: I04e0fb926b26cd326cc88c00eadb2ef6671c12a4 --- Cargo.lock | 1 + Cargo.toml | 2 + crates/vibescrobble-index/Cargo.toml | 1 + crates/vibescrobble-index/src/feed.rs | 174 +++++++++++++++++++++++++- 4 files changed, 173 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fd36cf40..90b9d005 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3075,6 +3075,7 @@ dependencies = [ "tower-http", "tracing", "tracing-subscriber", + "unicode-segmentation", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index a8956ad1..a8eb82f1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,6 +57,8 @@ tokio = { version = "1", features = ["rt-multi-thread", "macros", "net", "signal tower = { version = "0.5", features = ["util"] } tower-http = { version = "0.6", features = ["trace", "cors"] } tracing = "0.1" +# Grapheme counting, for the lexicon's `maxGraphemes` limits. +unicode-segmentation = "1" tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt", "time"] } # Debug information is most of what a build of this workspace produces, and diff --git a/crates/vibescrobble-index/Cargo.toml b/crates/vibescrobble-index/Cargo.toml index 57f5a1f5..7c23041a 100644 --- a/crates/vibescrobble-index/Cargo.toml +++ b/crates/vibescrobble-index/Cargo.toml @@ -25,6 +25,7 @@ tokio.workspace = true tokio-tungstenite.workspace = true tracing.workspace = true tracing-subscriber.workspace = true +unicode-segmentation.workspace = true tower-http.workspace = true # Containment is checked here with the same comparison the server mints under, # rather than a second implementation of the label walk. diff --git a/crates/vibescrobble-index/src/feed.rs b/crates/vibescrobble-index/src/feed.rs index 55e1315c..40d8c807 100644 --- a/crates/vibescrobble-index/src/feed.rs +++ b/crates/vibescrobble-index/src/feed.rs @@ -1,8 +1,13 @@ //! Reading one agent's scrobbles off the server that hosts it. use crate::nsid; -use serde_json::Value; +use serde_json::{Map, Value}; +use time::format_description::well_known::Rfc3339; +use time::format_description::BorrowedFormatItem; +use time::macros::format_description; +use time::{Duration, OffsetDateTime, UtcOffset}; use tracing::warn; +use unicode_segmentation::UnicodeSegmentation; use crate::http::HttpGet; use crate::report::{fetch_json, RecordView, RecordsView, Rejection}; @@ -14,6 +19,95 @@ pub const DEFAULT_FEED_LIMIT: usize = 50; /// Placeholder for a record that arrived without a URI to name it by. const UNNAMED: &str = "(no uri)"; +/// How far ahead of this clock a `createdAt` may be before it is clamped. +const FUTURE_SKEW: Duration = Duration::minutes(5); + +/// The lexicon's `maxLength` on `artifacts`. +const MAX_ARTIFACTS: usize = 8; + +/// The lexicon's limits on each string field. +/// +/// Lengths are UTF-8 bytes, as atproto counts them. +const LIMITS: &[Limit] = &[ + Limit::new("text", 1, 2560, Some(256)), + Limit::new("emoji", 1, 64, Some(1)), + Limit::new("task", 0, 512, None), + Limit::new("effort", 0, 64, None), + Limit::new("model", 0, 64, None), + Limit::new("parent", 0, 512, None), + Limit::new("agentType", 0, 256, None), +]; + +/// The length limits the lexicon puts on one string field. +struct Limit { + key: &'static str, + min_bytes: usize, + max_bytes: usize, + max_graphemes: Option, +} + +impl Limit { + const fn new( + key: &'static str, + min_bytes: usize, + max_bytes: usize, + max_graphemes: Option, + ) -> Self { + Self { + key, + min_bytes, + max_bytes, + max_graphemes, + } + } + + /// Refuses a present value outside the limits. Absence is checked by the + /// caller, which knows which fields are required. + fn check(&self, object: &Map) -> Result<(), String> { + let Some(value) = object.get(self.key).and_then(Value::as_str) else { + return Ok(()); + }; + let key = self.key; + let bytes = value.len(); + if bytes < self.min_bytes || bytes > self.max_bytes { + return Err(format!( + "{key} is {bytes} bytes, outside {}..={}", + self.min_bytes, self.max_bytes + )); + } + if let Some(max) = self.max_graphemes { + let graphemes = value.graphemes(true).count(); + if graphemes > max { + return Err(format!("{key} is {graphemes} graphemes, over {max}")); + } + } + Ok(()) + } +} + +/// Reads `createdAt` as RFC 3339 and writes it back in one sortable form. +/// +/// The result is UTC to the whole second, so comparing two of them as +/// strings compares them as times. A time more than [`FUTURE_SKEW`] after +/// `now` becomes `now`, so a record cannot hold the top of a feed by +/// claiming a date that has not happened. +fn created_time(raw: &str, now: OffsetDateTime) -> Result { + let parsed = OffsetDateTime::parse(raw, &Rfc3339) + .map_err(|error| format!("createdAt {raw:?} is not RFC 3339: {error}"))?; + let at = if parsed > now + FUTURE_SKEW { + now + } else { + parsed + }; + at.to_offset(UtcOffset::UTC) + .format(SORTABLE) + .map_err(|error| format!("createdAt {raw:?} cannot be written back: {error}")) +} + +/// The one form every stored `createdAt` takes. +const SORTABLE: &[BorrowedFormatItem<'static>] = + format_description!("[year]-[month]-[day]T[hour]:[minute]:[second]Z"); + /// One scrobble, as an indexer needs it. /// /// A flattened view rather than the raw record: the fields this project @@ -34,7 +128,9 @@ pub struct Scrobble { pub text: String, /// The single emoji it chose. pub emoji: String, - /// The client-declared creation timestamp, RFC 3339. + /// The client-declared creation timestamp, as UTC RFC 3339 to the + /// second. A time too far in the future is replaced by the time it was + /// read. pub created_at: String, /// The harness's task identifier, if the agent had one. #[serde(default, skip_serializing_if = "Option::is_none")] @@ -146,8 +242,8 @@ fn read_scrobble(source: &FeedSource, record: RecordView) -> Result Result MAX_ARTIFACTS { + return Err(format!( + "artifacts has {artifact_count} entries, over {MAX_ARTIFACTS}" + )); + } + let string = |key: &str| object.get(key).and_then(Value::as_str); let text = string("text").ok_or("no text".to_owned())?; let emoji = string("emoji").ok_or("no emoji".to_owned())?; let created_at = string("createdAt").ok_or("no createdAt".to_owned())?; + let created_at = created_time(created_at, OffsetDateTime::now_utc())?; Ok(Scrobble { uri: uri.to_owned(), did: did.to_owned(), text: text.to_owned(), emoji: emoji.to_owned(), - created_at: created_at.to_owned(), + created_at, task: string("task").map(str::to_owned), effort: string("effort").map(str::to_owned), model: string("model").map(str::to_owned), @@ -203,3 +313,57 @@ pub fn read_record(did: &str, uri: &str, value: Value) -> Result Result { + let mut value = json!({ + "text": "reading the ingest loop", + "emoji": "🦦", + "createdAt": "2026-09-27T11:00:00Z", + }); + for (key, field) in fields.as_object().unwrap() { + value[key] = field.clone(); + } + read_record("did:web:kestrel.agents.example", "at://x", value) + } + + #[test] + fn a_far_future_time_cannot_outrank_now() { + assert_eq!( + created_time("9999-01-01T00:00:00Z", NOW).unwrap(), + "2026-09-27T12:00:00Z" + ); + assert_eq!( + created_time("2026-09-27T12:04:00Z", NOW).unwrap(), + "2026-09-27T12:04:00Z" + ); + } + + #[test] + fn offsets_and_fractions_sort_as_times() { + let early = created_time("2026-09-27T13:59:59.900+02:00", NOW).unwrap(); + let late = created_time("2026-09-27T11:59:59Z", NOW).unwrap(); + assert_eq!(early, "2026-09-27T11:59:59Z"); + assert!(early <= late); + assert!(record(json!({ "createdAt": "yesterday" })).is_err()); + } + + #[test] + fn the_lexicon_limits_are_enforced() { + assert!(record(json!({})).is_ok()); + // One emoji of several code points is one grapheme. + assert!(record(json!({ "emoji": "👨‍👩‍👧" })).is_ok()); + assert!(record(json!({ "emoji": "" })).is_err()); + assert!(record(json!({ "emoji": "" })).is_err()); + assert!(record(json!({ "text": "x".repeat(257) })).is_err()); + assert!(record(json!({ "model": "m".repeat(65) })).is_err()); + assert!(record(json!({ "artifacts": vec!["at://a"; 9] })).is_err()); + } +} -- knotmirror 0.0.1