diff --git a/.sqlx/query-5621466585209ebe477f2ae01e75c0b31f774619cfc7c087ebdc97dc4d875af6.json b/.sqlx/query-5621466585209ebe477f2ae01e75c0b31f774619cfc7c087ebdc97dc4d875af6.json new file mode 100644 index 0000000..db99a87 --- /dev/null +++ b/.sqlx/query-5621466585209ebe477f2ae01e75c0b31f774619cfc7c087ebdc97dc4d875af6.json @@ -0,0 +1,74 @@ +{ + "db_name": "SQLite", + "query": "SELECT uri AS \"uri!\", cid AS \"cid!\", name AS \"name!\", format, file_json AS \"file_json!\"\n FROM parts WHERE uri = ?", + "describe": { + "columns": [ + { + "name": "uri!", + "ordinal": 0, + "type_info": "Text", + "origin": { + "Table": { + "table": "parts", + "name": "uri" + } + } + }, + { + "name": "cid!", + "ordinal": 1, + "type_info": "Text", + "origin": { + "Table": { + "table": "parts", + "name": "cid" + } + } + }, + { + "name": "name!", + "ordinal": 2, + "type_info": "Text", + "origin": { + "Table": { + "table": "parts", + "name": "name" + } + } + }, + { + "name": "format", + "ordinal": 3, + "type_info": "Text", + "origin": { + "Table": { + "table": "parts", + "name": "format" + } + } + }, + { + "name": "file_json!", + "ordinal": 4, + "type_info": "Text", + "origin": { + "Table": { + "table": "parts", + "name": "file_json" + } + } + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + true, + false + ] + }, + "hash": "5621466585209ebe477f2ae01e75c0b31f774619cfc7c087ebdc97dc4d875af6" +} diff --git a/crates/polymodel-ldraw-core/src/geom.rs b/crates/polymodel-ldraw-core/src/geom.rs index 842fbd4..954e19e 100644 --- a/crates/polymodel-ldraw-core/src/geom.rs +++ b/crates/polymodel-ldraw-core/src/geom.rs @@ -292,6 +292,7 @@ pub(crate) fn parse_primitive(line: &ScannedLine) -> Option { } } +#[allow(clippy::too_many_arguments)] pub(crate) fn process_geometry<'src>( model: &mut ModelData<'src>, line: &ScannedLine<'src>, diff --git a/crates/polymodel-ldraw-core/src/lib.rs b/crates/polymodel-ldraw-core/src/lib.rs index 035f6e6..b6898bc 100644 --- a/crates/polymodel-ldraw-core/src/lib.rs +++ b/crates/polymodel-ldraw-core/src/lib.rs @@ -101,8 +101,10 @@ mod tests { #[test] fn line_limit_is_per_physical_line() { - let mut limits = LdrawLimits::default(); - limits.line_bytes = 3; + let limits = LdrawLimits { + line_bytes: 3, + ..LdrawLimits::default() + }; assert!(scan_lines(b"abc\ndef", &limits).is_ok()); assert!(scan_lines(b"abcd\n", &limits).is_err()); } diff --git a/crates/polymodel-ldraw-core/src/model.rs b/crates/polymodel-ldraw-core/src/model.rs index 3ec6091..c1f49f7 100644 --- a/crates/polymodel-ldraw-core/src/model.rs +++ b/crates/polymodel-ldraw-core/src/model.rs @@ -3,12 +3,11 @@ use crate::cache::{CacheKey, NormalizedPath}; use crate::geom::{Bounds3, ColourCode, GeometryRecord, Transform, deserialize_optional_bounds3}; use crate::mpd::OwnedVirtualFile; use crate::scanner::{LineType, Token, TokenKind}; -use crate::texmap::{TexmapAssociation, TexmapEvent, TexmapState, TextureDescriptor}; +use crate::texmap::{TexmapEvent, TexmapState, TextureDescriptor}; use crate::types::{Diagnostic, LimitCounters, SCHEMA_VERSION, SemanticArenas, Span}; use serde::{Deserialize, Serialize}; use std::borrow::Cow; use std::collections::BTreeMap; -use std::fmt; #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] pub struct ColourData { @@ -25,74 +24,19 @@ pub enum ColourEdge { Data(ColourData), } -#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ColourFinish { - Chrome, - Pearlescent, - Rubber, - MatteMetallic, - Metal, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ColourFabricKind { - Velvet, - Canvas, - String, - Fur, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ColourMaterialKind { - Glitter, - Speckle, - Fabric { variant: Option }, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct ColourMaterial<'src> { - /// The source spelling of the material kind, retained for compatibility. - #[serde(borrow)] - pub kind: Cow<'src, str>, - pub typed_kind: ColourMaterialKind, - pub value: Option, - pub fraction: Option, - pub vfraction: Option, - pub size: Option, - pub min_size: Option, - pub max_size: Option, - #[serde(borrow)] - pub extra: Vec>, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] pub struct Colour<'src> { pub code: ColourCode, #[serde(borrow)] pub name: Option>, pub data: ColourData, pub edge: ColourEdge, - #[serde(default)] - pub finishes: Vec, - #[serde(default)] - pub material: Option>, } -const FIRST_LOCAL_COLOUR_SLOT: u32 = 512; -const LAST_LOCAL_COLOUR_SLOT: u32 = 65_534; - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct ColourSlotError; - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] pub struct ColourTable<'src> { #[serde(borrow)] entries: BTreeMap>, - local_slots: BTreeMap, - next_local_slot: u32, } impl<'src> Default for ColourTable<'src> { @@ -105,8 +49,6 @@ impl<'src> ColourTable<'src> { pub fn new() -> Self { Self { entries: BTreeMap::new(), - local_slots: BTreeMap::new(), - next_local_slot: FIRST_LOCAL_COLOUR_SLOT, } } @@ -130,10 +72,6 @@ impl<'src> ColourTable<'src> { } } - pub fn local_slot(&self, code: impl Into) -> Option { - self.local_slots.get(&code.into()).copied() - } - pub fn len(&self) -> usize { self.entries.len() } @@ -142,16 +80,6 @@ impl<'src> ColourTable<'src> { self.entries.is_empty() } - pub(crate) fn merge_scope(&mut self, scope: ColourScope<'src>) { - self.entries.extend(scope.committed.into_iter().flatten()); - self.local_slots.extend(scope.local_slots); - self.next_local_slot = scope.next_local_slot; - } - - pub(crate) fn next_local_slot(&self) -> u32 { - self.next_local_slot - } - pub(crate) fn into_owned(self) -> ColourTable<'static> { ColourTable { entries: self @@ -165,382 +93,85 @@ impl<'src> ColourTable<'src> { name: colour.name.map(|name| Cow::Owned(name.into_owned())), data: colour.data, edge: colour.edge, - finishes: colour.finishes, - material: colour.material.map(|material| ColourMaterial { - kind: Cow::Owned(material.kind.into_owned()), - typed_kind: material.typed_kind, - value: material.value, - fraction: material.fraction, - vfraction: material.vfraction, - size: material.size, - min_size: material.min_size, - max_size: material.max_size, - extra: material - .extra - .into_iter() - .map(|value| Cow::Owned(value.into_owned())) - .collect(), - }), }, ) }) .collect(), - local_slots: self.local_slots, - next_local_slot: self.next_local_slot, } } } -/// File-local transactional colour definitions. -/// -/// Records are invisible until `commit`. Dropping or rolling back a scope -/// discards both definitions and provisional local slots, so failed parses do -/// not affect later virtual files or parses. -#[derive(Clone, Debug, PartialEq)] -pub struct ColourScope<'src> { - committed: Vec>>, - active: BTreeMap>, - local_slots: BTreeMap, - active_slot_start: u32, - next_local_slot: u32, -} - -impl<'src> ColourScope<'src> { - pub fn new() -> Self { - Self::with_first_local_slot(FIRST_LOCAL_COLOUR_SLOT) - } - - pub(crate) fn with_first_local_slot(first_local_slot: u32) -> Self { - Self { - committed: Vec::new(), - active: BTreeMap::new(), - local_slots: BTreeMap::new(), - active_slot_start: first_local_slot, - next_local_slot: first_local_slot, - } - } - - pub fn record(&mut self, colour: Colour<'src>) -> Result { - let code = colour.code; - if let Some(slot) = self.local_slots.get(&code).copied() { - self.active.insert(code, colour); - return Ok(slot); - } - let slot = self.allocate_local_slot()?; - self.local_slots.insert(code, slot); - self.active.insert(code, colour); - Ok(slot) - } - - pub fn commit(&mut self) { - if !self.active.is_empty() { - self.committed.push(std::mem::take(&mut self.active)); - } - self.active_slot_start = self.next_local_slot; - } - - pub fn rollback(&mut self) { - self.active.clear(); - self.local_slots - .retain(|_, slot| u32::from(*slot) < self.active_slot_start); - self.next_local_slot = self.active_slot_start; - } - - pub fn has_uncommitted(&self) -> bool { - !self.active.is_empty() - } - - pub fn get(&self, code: impl Into) -> Option<&Colour<'src>> { - if !self.active.is_empty() { - return None; - } - let code = code.into(); - self.committed.iter().rev().find_map(|map| map.get(&code)) - } - - pub fn find(&self, code: impl Into, find_edge: bool) -> Option<&ColourData> { - let mut code = code.into(); - for (map_index, map) in self.committed.iter().enumerate().rev() { - let Some(colour) = map.get(&code) else { - continue; - }; - if !find_edge { - return Some(&colour.data); - } - match &colour.edge { - ColourEdge::Data(data) => return Some(data), - ColourEdge::Code(edge_code) => { - if let Some(edge) = map.get(edge_code) { - return Some(&edge.data); - } - code = *edge_code; - for older_map in self.committed[..map_index].iter().rev() { - if let Some(edge) = older_map.get(&code) { - return Some(&edge.data); - } - } - return None; - } - } - } - None - } - - pub fn local_slot(&self, code: impl Into) -> Option { - self.local_slots.get(&code.into()).copied() - } - - fn allocate_local_slot(&mut self) -> Result { - if self.next_local_slot > LAST_LOCAL_COLOUR_SLOT { - return Err(ColourSlotError); - } - let slot = u16::try_from(self.next_local_slot).map_err(|_| ColourSlotError)?; - self.next_local_slot += 1; - Ok(slot) - } -} - -impl Default for ColourScope<'_> { - fn default() -> Self { - Self::new() - } -} - pub(crate) fn parse_colour<'src>(tokens: &[Token<'src>]) -> Option> { - if tokens.len() < 9 - || tokens.first()?.kind != TokenKind::Zero - || tokens.get(1)?.kind != TokenKind::Colour - || tokens.get(2)?.kind != TokenKind::Identifier - { - return None; - } - let mut index = 3; - if !matches_keyword(tokens.get(index), TokenKind::Code) { + if tokens.len() < 5 || tokens.get(2)?.kind != TokenKind::Identifier { return None; } - let code = parse_colour_code(tokens.get(index + 1)?.text)?; - index += 2; - if !matches_keyword(tokens.get(index), TokenKind::Value) { + if tokens[2].text.ends_with(';') { return None; } - let (r, g, b) = parse_rgb(tokens.get(index + 1)?.text)?; - index += 2; - if !matches_keyword(tokens.get(index), TokenKind::Edge) { + let fields = &tokens[3..]; + if !fields.len().is_multiple_of(2) { return None; } - let edge_token = tokens.get(index + 1)?.text; - let edge = parse_rgb(edge_token) - .map(|(r, g, b)| { - ColourEdge::Data(ColourData { - r, - g, - b, - alpha: None, - luminance: None, - }) - }) - .or_else(|| parse_colour_code(edge_token).map(ColourEdge::Code))?; - index += 2; - let mut data = ColourData { - r, - g, - b, - alpha: None, - luminance: None, - }; - let mut finishes = Vec::new(); - let mut material = None; - while index < tokens.len() { - match tokens[index].kind { - TokenKind::Alpha => { - if data.alpha.is_some() { - return None; - } - data.alpha = Some(parse_byte(tokens.get(index + 1)?.text)?); - index += 2; + let mut code = None; + let mut value = None; + let mut edge = None; + let mut alpha = None; + let mut luminance = None; + for pair in fields.chunks_exact(2) { + let key = pair[0].kind; + let value_token = &pair[1]; + match key { + TokenKind::Code if code.is_none() => code = parse_colour_code(value_token.text), + TokenKind::Value if value.is_none() => value = parse_rgb(value_token.text), + TokenKind::Edge if edge.is_none() => { + edge = parse_rgb(value_token.text) + .map(|(r, g, b)| { + ColourEdge::Data(ColourData { + r, + g, + b, + alpha: None, + luminance: None, + }) + }) + .or_else(|| parse_colour_code(value_token.text).map(ColourEdge::Code)); } - TokenKind::Luminance => { - if data.luminance.is_some() { - return None; - } - data.luminance = Some(parse_byte(tokens.get(index + 1)?.text)?); - index += 2; + TokenKind::Alpha if alpha.is_none() => alpha = value_token.text.parse().ok(), + TokenKind::Luminance if luminance.is_none() => { + luminance = value_token.text.parse().ok() } TokenKind::Chrome | TokenKind::Pearlescent | TokenKind::Rubber | TokenKind::MatteMetallic - | TokenKind::Metal => { - if !finishes.is_empty() || material.is_some() { - return None; - } - finishes.push(match tokens[index].kind { - TokenKind::Chrome => ColourFinish::Chrome, - TokenKind::Pearlescent => ColourFinish::Pearlescent, - TokenKind::Rubber => ColourFinish::Rubber, - TokenKind::MatteMetallic => ColourFinish::MatteMetallic, - TokenKind::Metal => ColourFinish::Metal, - _ => unreachable!(), - }); - index += 1; - } - TokenKind::Material => { - if !finishes.is_empty() || material.is_some() { - return None; - } - material = Some(parse_material(&tokens[index + 1..])?); - index = tokens.len(); - } + | TokenKind::Metal + | TokenKind::Material => return None, _ => return None, } } + Some(Colour { - code, + code: code?, name: Some(Cow::Borrowed(tokens[2].text)), - data, - edge, - finishes, - material, - }) -} - -fn matches_keyword(token: Option<&Token<'_>>, expected: TokenKind) -> bool { - token.is_some_and(|token| token.kind == expected) -} - -fn parse_material<'src>(tokens: &[Token<'src>]) -> Option> { - let kind_token = tokens.first()?; - let typed_kind = match kind_token.text { - "GLITTER" => ColourMaterialKind::Glitter, - "SPECKLE" => ColourMaterialKind::Speckle, - "FABRIC" => ColourMaterialKind::Fabric { variant: None }, - _ => return None, - }; - if matches!(typed_kind, ColourMaterialKind::Fabric { .. }) { - let variant = tokens.get(1).map(|token| match token.text { - "VELVET" => Some(ColourFabricKind::Velvet), - "CANVAS" => Some(ColourFabricKind::Canvas), - "STRING" => Some(ColourFabricKind::String), - "FUR" => Some(ColourFabricKind::Fur), - _ => None, - }); - if tokens.len() > 2 || variant.is_some_and(|variant| variant.is_none()) { - return None; - } - return Some(ColourMaterial { - kind: Cow::Borrowed(kind_token.text), - typed_kind: ColourMaterialKind::Fabric { - variant: variant.flatten(), - }, - value: None, - fraction: None, - vfraction: None, - size: None, - min_size: None, - max_size: None, - extra: Vec::new(), - }); - } - let mut value = None; - let mut fraction = None; - let mut vfraction = None; - let mut size = None; - let mut min_size = None; - let mut max_size = None; - let mut index = 1; - while index < tokens.len() { - let key = tokens[index].text; - if index + 1 >= tokens.len() { - return None; - } - let value_token = &tokens[index + 1]; - match key { - "VALUE" if value.is_none() => { - let (r, g, b) = parse_rgb(value_token.text)?; - value = Some(ColourData { - r, - g, - b, - alpha: None, - luminance: None, - }); - } - "ALPHA" if value.is_some() && value.as_ref()?.alpha.is_none() => { - value.as_mut()?.alpha = Some(parse_byte(value_token.text)?); - } - "LUMINANCE" if value.is_some() && value.as_ref()?.luminance.is_none() => { - value.as_mut()?.luminance = Some(parse_byte(value_token.text)?); - } - "FRACTION" if fraction.is_none() => fraction = Some(parse_fraction(value_token.text)?), - "VFRACTION" if vfraction.is_none() => { - vfraction = Some(parse_fraction(value_token.text)?) - } - "SIZE" if size.is_none() => size = Some(parse_positive(value_token.text)?), - "MINSIZE" if min_size.is_none() => min_size = Some(parse_positive(value_token.text)?), - "MAXSIZE" if max_size.is_none() => max_size = Some(parse_positive(value_token.text)?), - _ => return None, - } - index += 2; - } - let is_glitter = matches!(typed_kind, ColourMaterialKind::Glitter); - let is_speckle = matches!(typed_kind, ColourMaterialKind::Speckle); - if value.is_none() || fraction.is_none() { - return None; - } - if is_glitter { - if vfraction.is_none() || size.is_none() || min_size.is_some() || max_size.is_some() { - return None; - } - } else if is_speckle { - if vfraction.is_some() || size.is_some() || min_size.is_none() || max_size.is_none() { - return None; - } - } - if size.is_some() == (min_size.is_some() || max_size.is_some()) - || min_size.is_some() != max_size.is_some() - { - return None; - } - Some(ColourMaterial { - kind: Cow::Borrowed(kind_token.text), - typed_kind, - value, - fraction, - vfraction, - size, - min_size, - max_size, - extra: Vec::new(), + data: ColourData { + r: value?.0, + g: value?.1, + b: value?.2, + alpha, + luminance, + }, + edge: edge?, }) } fn parse_colour_code(text: &str) -> Option { - (!text.is_empty() && text.bytes().all(|byte| byte.is_ascii_digit())) - .then(|| text.parse::().ok()) - .flatten() - .map(ColourCode) -} - -fn parse_byte(text: &str) -> Option { - (!text.is_empty() && text.bytes().all(|byte| byte.is_ascii_digit())) - .then(|| text.parse::().ok()) - .flatten() - .and_then(|value| u8::try_from(value).ok()) -} - -fn parse_fraction(text: &str) -> Option { - let value = text.parse::().ok()?; - (value > 0.0 && value < 1.0 && value.is_finite()).then_some(value) -} - -fn parse_positive(text: &str) -> Option { - let value = text.parse::().ok()?; - (value > 0.0 && value.is_finite()).then_some(value) + let value = text.parse::().ok()?; + Some(ColourCode(value)) } fn parse_rgb(text: &str) -> Option<(u8, u8, u8)> { - let hex = text.strip_prefix('#').or_else(|| text.strip_prefix("0x"))?; + let hex = text.strip_prefix('#')?; if hex.len() != 6 { return None; } @@ -600,8 +231,6 @@ pub struct SemanticRecord<'src> { pub limits: Vec>, pub texmap_events: Vec>, #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub texmap_geometry: Vec, - #[serde(default, skip_serializing_if = "Vec::is_empty")] pub data_payloads: Vec, } impl<'src> SemanticRecord<'src> { @@ -632,7 +261,6 @@ impl<'src> SemanticRecord<'src> { .into_iter() .map(TexmapEvent::into_owned) .collect(), - texmap_geometry: self.texmap_geometry, data_payloads: self.data_payloads, } } @@ -642,20 +270,8 @@ pub struct InstanceRecord { pub instance_id: String, pub source_model: String, pub target_model: String, - #[serde(default)] - pub target_key: Option, pub transform: Transform, pub inverted: bool, - #[serde(default)] - pub bfc: Option, - #[serde(default)] - pub colour: Option, - #[serde(default)] - pub texmap_state: Option, - #[serde(default)] - pub texmap_reference: Option, - #[serde(default)] - pub texmap: Option, } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] @@ -683,23 +299,6 @@ pub struct CanonicalRecord<'src> { pub provenance_id: Cow<'src, str>, } -impl<'src> CanonicalRecord<'src> { - pub fn into_owned(self) -> CanonicalRecord<'static> { - CanonicalRecord { - schema_version: Cow::Owned(self.schema_version.into_owned()), - syntax: self - .syntax - .into_iter() - .map(SyntaxRecord::into_owned) - .collect(), - semantic: self.semantic.into_owned(), - scene: self.scene, - diagnostics: self.diagnostics, - provenance_id: Cow::Owned(self.provenance_id.into_owned()), - } - } -} - #[derive(Clone, Debug)] pub(crate) struct DataPayloadBuilder { pub(crate) filename: String, @@ -713,20 +312,7 @@ pub(crate) struct Include<'src> { pub transform: Transform, pub(crate) span: Span, pub(crate) inverted: bool, - pub(crate) bfc: BfcFrame, - pub(crate) colour: ColourCode, - pub(crate) texmap_state: TexmapState, - pub(crate) texmap: Option>, - pub(crate) texmap_association: Option, -} - -#[derive(Clone, Debug)] -pub(crate) enum ModelEvent<'src> { - Step, - Indices(u64), - Include(Cow<'src, str>), } - #[derive(Clone, Debug)] pub(crate) struct ModelData<'src> { pub(crate) file_id: u32, @@ -735,18 +321,12 @@ pub(crate) struct ModelData<'src> { pub(crate) bfc: BfcFrame, pub(crate) colour: ColourCode, pub(crate) steps: Vec>, - /// LDParse-compatible direct-mesh index offsets recorded at each STEP. - /// This is distinct from the root semantic step labels used by policy. - pub(crate) step_ends: Vec, - pub(crate) mesh_index_count: u64, - pub(crate) events: Vec>, pub(crate) texmap_state: TexmapState, pub(crate) texmap_descriptor: Option>, pub(crate) texmap_stack: Vec<(TexmapState, Option>, bool)>, pub(crate) texmap_next_span: Option, pub(crate) texmap_fallback_seen: bool, pub(crate) texmap_events: Vec>, - pub(crate) texmap_geometry: Vec, pub(crate) data_payload: Option, pub(crate) data_payloads: Vec, pub(crate) includes: Vec>, @@ -770,66 +350,6 @@ pub struct ParseResult<'src> { pub arenas: SemanticArenas, pub provenance_id: Cow<'src, str>, } - -/// An owned parse result. The retained semantic result and its reservation are -/// one object: projection borrows or clones data while this owner remains live. -pub struct OwnedParseResult { - pub result: ParseResult<'static>, -} - -impl fmt::Debug for OwnedParseResult { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("OwnedParseResult") - .field("schema_version", &SCHEMA_VERSION) - .field("reservation_id", &self.result.arenas.reservation_id()) - .finish() - } -} - -impl std::ops::Deref for OwnedParseResult { - type Target = ParseResult<'static>; - - fn deref(&self) -> &Self::Target { - &self.result - } -} - -impl OwnedParseResult { - pub(crate) fn from_parse_result(result: ParseResult<'_>) -> Self { - Self { - result: result.into_owned(), - } - } - - pub fn project(&self) -> CanonicalRecord<'static> { - self.result.project() - } - - pub fn ledger_reservation_id(&self) -> u64 { - self.result.arenas.reservation_id() - } - - /// Move both the canonical projection and its retained arena owner together. - pub fn into_projection(self) -> OwnedProjection { - OwnedProjection { - canonical: self.result.project(), - arenas: self.result.arenas, - } - } -} - -pub struct OwnedProjection { - pub canonical: CanonicalRecord<'static>, - pub arenas: SemanticArenas, -} - -impl OwnedProjection { - pub fn project(&self) -> &CanonicalRecord<'static> { - &self.canonical - } -} - #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct ModelSummary { pub id: String, @@ -842,8 +362,6 @@ pub struct ModelSummary { pub quads: u64, pub lines: u64, pub conditional_lines: u64, - /// LDParse-authoritative direct-mesh index offsets for this model's steps. - pub step_ends: Vec, pub geometry: Vec, pub bfc: BfcState, } @@ -859,25 +377,31 @@ impl<'src> ParseResult<'src> { } } - pub(crate) fn into_owned(self) -> ParseResult<'static> { - ParseResult { + pub fn into_project(self) -> CanonicalRecord<'src> { + CanonicalRecord { + schema_version: Cow::Borrowed(SCHEMA_VERSION), + syntax: self.syntax, + semantic: self.semantic, + scene: self.scene, + diagnostics: self.diagnostics, + provenance_id: self.provenance_id, + } + } + + pub fn into_owned_project(self) -> CanonicalRecord<'static> { + CanonicalRecord { + schema_version: Cow::Borrowed(SCHEMA_VERSION), syntax: self .syntax .into_iter() - .map(SyntaxRecord::into_owned) + .map(|record| record.into_owned()) .collect(), semantic: self.semantic.into_owned(), scene: self.scene, diagnostics: self.diagnostics, - counters: self.counters, - files: self.files, - models: self.models, - colours: self.colours.into_owned(), - arenas: self.arenas, provenance_id: Cow::Owned(self.provenance_id.into_owned()), } } - pub fn ledger_reservation_id(&self) -> u64 { self.arenas.reservation_id() } diff --git a/crates/polymodel-ldraw-core/src/parser.rs b/crates/polymodel-ldraw-core/src/parser.rs index 3fd594d..58b7014 100644 --- a/crates/polymodel-ldraw-core/src/parser.rs +++ b/crates/polymodel-ldraw-core/src/parser.rs @@ -3,20 +3,21 @@ use crate::bfc::{BfcState, bfc}; use crate::cache::{CacheKey, NormalizedPath}; use crate::geom::{parse_primitive, process_geometry}; use crate::model::{ - ColourScope, ColourTable, DataPayload, DataPayloadBuilder, ModelData, ModelEvent, ParseResult, + CanonicalRecord, ColourTable, DataPayload, DataPayloadBuilder, ModelData, ParseResult, SemanticRecord, SyntaxRecord, parse_colour, }; use crate::mpd::{OwnedVirtualFile, VirtualFile, split_mpd}; +use crate::resolve::ResolveRequest; use crate::scanner::{LineType, ScannedLine, TokenKind, scan_lines}; use crate::texmap::{TexmapState, TextureDescriptor, texmap}; use crate::traversal::traverse; use crate::types::{ - DEFAULT_COLOUR, DEFAULT_PROVENANCE, Diagnostic, DiagnosticCode, LdrawLimits, LimitCounters, - LimitKind, Materialization, ParseError, ParseOptions, ParserProfile, SemanticArenas, Span, - add_diag, limit_error, + BorrowedMaterialization, DEFAULT_COLOUR, DEFAULT_PROVENANCE, Diagnostic, DiagnosticCode, + LdrawLimits, LimitCounters, LimitKind, ParseError, ParseOptions, ParserProfile, SemanticArenas, + Span, add_diag, limit_error, }; use std::borrow::Cow; -use std::collections::BTreeSet; +use std::collections::{BTreeSet, HashSet}; pub struct LdrawParser; impl Default for LdrawParser { @@ -25,17 +26,79 @@ impl Default for LdrawParser { } } impl LdrawParser { - /// Parse immutable root bytes plus caller-admitted materializations. - /// - /// Discovery, fetching, precedence, and host policy are intentionally not - /// part of this core API. The adapter must provide every materialization it - /// wants the deterministic parser to consider. + pub async fn parse( + &self, + bytes: Vec, + mut options: ParseOptions<'_>, + resolver: &R, + ) -> Result, ParseError> { + let root_bytes = bytes; + let mut resolved = Vec::::new(); + let mut requested = HashSet::<(NormalizedPath, crate::types::RootId)>::new(); + loop { + let borrowed = resolved + .iter() + .map(|materialization| BorrowedMaterialization { + path: materialization.path.clone(), + root: materialization.root, + content_hash: materialization.content_hash.clone(), + bytes: &materialization.bytes, + available: materialization.available, + }) + .collect::>(); + let parsed = self.parse_bytes(&root_bytes, options.clone(), Some(&borrowed))?; + let mut next = None; + for model in &parsed.models { + for (include_index, include) in model.includes.iter().enumerate() { + let path = match NormalizedPath::new(include) { + Ok(path) => path, + Err(_) => continue, + }; + let root = options.resolved_root; + if !requested.insert((path.clone(), root)) { + continue; + } + next = Some(ResolveRequest { + source: model.cache_key.clone(), + path, + root, + depth: model.depth.saturating_add(1), + span: model.include_spans.get(include_index).copied(), + }); + break; + } + if next.is_some() { + break; + } + } + let Some(request) = next else { + return Ok(parsed.into_owned_project()); + }; + if options.fetch_count >= options.limits.fetches { + return Err(limit_error(LimitKind::Fetches, request.span)); + } + options.fetch_count = options + .fetch_count + .checked_add(1) + .ok_or(ParseError::Overflow("fetch count"))?; + let materialization = resolver.resolve(request).await?; + if !materialization.has_valid_hash() { + return Err(crate::resolve::ResolveError::Rejected { + path: materialization.path, + message: "resolver returned bytes with a mismatched content hash".into(), + } + .into()); + } + resolved.push(materialization); + } + } + pub fn parse_bytes<'src, 'a>( &self, bytes: &'src [u8], options: ParseOptions<'a>, - materializations: &[Materialization], - ) -> Result { + materializations: Option<&[BorrowedMaterialization<'src>]>, + ) -> Result, ParseError> { let mut counters = LimitCounters::default(); let input_len = u64::try_from(bytes.len()).map_err(|_| ParseError::Overflow("resource bytes"))?; @@ -45,14 +108,14 @@ impl LdrawParser { if options.fetch_count > options.limits.fetches { return Err(limit_error(LimitKind::Fetches, None)); } - if options.cancellation.checkpoint() { + if options.cancellation.cancelled { return Err(ParseError::Cancelled); } let budget = options.budget(bytes.len())?; let mut arenas = SemanticArenas::new(&options, budget)?; let scanned = scan_lines(bytes, &options.limits)?; for line in &scanned { - if options.cancellation.checkpoint() { + if options.cancellation.cancelled { return Err(ParseError::Cancelled); } let line_bytes = @@ -67,23 +130,10 @@ impl LdrawParser { && line.tokens.get(1).map(|token| token.kind) == Some(TokenKind::File) && line.tokens.len() >= 3 }); - if has_mpd_file { - for line in &scanned { - let is_file = line.line_type == Some(LineType::Zero) - && line.tokens.get(1).map(|token| token.kind) == Some(TokenKind::File) - && line.tokens.len() >= 3; - let is_nofile = line.line_type == Some(LineType::Zero) - && line.tokens.get(1).map(|token| token.kind) == Some(TokenKind::NoFile); - if is_file || is_nofile { - counters - .add(LimitKind::Commands, 1, &options.limits) - .map_err(|name| limit_error(name, Some(line.span)))?; - } - } - } else { + if !has_mpd_file { files[0].name = Cow::Owned(options.root_name.clone()); } - if options.cancellation.checkpoint() { + if options.cancellation.cancelled { return Err(ParseError::Cancelled); } let file_count = u64::try_from(files.len()) @@ -112,39 +162,19 @@ impl LdrawParser { let mut colours = ColourTable::new(); let mut texture_ids = BTreeSet::new(); for file in &files { - let mut scope = ColourScope::with_first_local_slot(colours.next_local_slot()); - let model_result = parse_model( + models.push(parse_model( file, &options, &mut counters, &mut diagnostics, &mut arenas, - &mut scope, + &mut colours, &mut texture_ids, - ); - let model = match model_result { - Ok(model) => model, - Err(error) => { - scope.rollback(); - return Err(error); - } - }; - scope.commit(); - colours.merge_scope(scope); - models.push(model); + )?); } - let source_model_count = models.len(); let mut materialized_keys = BTreeSet::new(); - for materialization in materializations { - if !materialization.has_valid_hash() { - return Err(ParseError::Diagnostic(Diagnostic { - code: DiagnosticCode::MaterializationHashMismatch, - severity: crate::types::Severity::Error, - message: format!("materialization hash mismatch for {}", materialization.path), - span: None, - })); - } + for materialization in materializations.into_iter().flatten() { if !materialization.available || materialization.bytes.is_empty() { continue; } @@ -166,7 +196,7 @@ impl LdrawParser { &options.limits, ) .map_err(|name| limit_error(name, None))?; - let resolved_scanned = scan_lines(&materialization.bytes, &options.limits)?; + let resolved_scanned = scan_lines(materialization.bytes, &options.limits)?; let file = VirtualFile { id: u32::try_from(files.len()) .map_err(|_| ParseError::Overflow("virtual file id"))?, @@ -184,24 +214,15 @@ impl LdrawParser { .add(LimitKind::Files, 1, &options.limits) .map_err(|name| limit_error(name, None))?; files.push(file.clone()); - let mut scope = ColourScope::with_first_local_slot(colours.next_local_slot()); - let mut model = match parse_model( + let mut model = parse_model( &file, &options, &mut counters, &mut diagnostics, &mut arenas, - &mut scope, + &mut colours, &mut texture_ids, - ) { - Ok(model) => model, - Err(error) => { - scope.rollback(); - return Err(error); - } - }; - scope.commit(); - colours.merge_scope(scope); + )?; model.key = CacheKey { canonical_path: materialization.path.clone(), resolved_root: materialization.root, @@ -209,14 +230,7 @@ impl LdrawParser { }; models.push(model); } - project_ldparse_step_ends(&mut models, source_model_count); - let (scene, summaries) = traverse( - &models, - &options, - &mut counters, - &mut diagnostics, - &mut arenas, - )?; + let (scene, summaries) = traverse(&models, &options, &mut counters, &mut diagnostics)?; let root = models.first(); let semantic = SemanticRecord { canonical_path: root.map(|m| Cow::Owned(m.path.to_string())), @@ -243,10 +257,9 @@ impl LdrawParser { .collect() }) .unwrap_or_default(), - texmap_geometry: root.map(|m| m.texmap_geometry.clone()).unwrap_or_default(), data_payloads: root.map(|m| m.data_payloads.clone()).unwrap_or_default(), }; - Ok(crate::OwnedParseResult::from_parse_result(ParseResult { + Ok(ParseResult { syntax, semantic, scene, @@ -257,7 +270,7 @@ impl LdrawParser { colours: colours.into_owned(), arenas, provenance_id: Cow::Borrowed(DEFAULT_PROVENANCE), - })) + }) } } @@ -267,10 +280,10 @@ fn parse_model<'src>( counters: &mut LimitCounters, diagnostics: &mut Vec, arenas: &mut SemanticArenas, - colours: &mut ColourScope<'src>, + colours: &mut ColourTable<'src>, texture_ids: &mut BTreeSet>, ) -> Result, ParseError> { - if options.cancellation.checkpoint() { + if options.cancellation.cancelled { return Err(ParseError::Cancelled); } let path = NormalizedPath::new(&file.name)?; @@ -293,16 +306,12 @@ fn parse_model<'src>( bfc: BfcFrame::default(), colour: DEFAULT_COLOUR, steps: Vec::new(), - step_ends: Vec::new(), - mesh_index_count: 0, - events: Vec::new(), texmap_state: TexmapState::Inactive, texmap_descriptor: None, texmap_stack: Vec::new(), texmap_next_span: None, texmap_fallback_seen: false, texmap_events: Vec::new(), - texmap_geometry: Vec::new(), data_payload: None, data_payloads: Vec::new(), includes: Vec::new(), @@ -317,7 +326,7 @@ fn parse_model<'src>( if line.blank { continue; } - if options.cancellation.checkpoint() { + if options.cancellation.cancelled { return Err(ParseError::Cancelled); } if counters @@ -382,111 +391,6 @@ fn parse_model<'src>( Ok(model) } -fn project_ldparse_step_ends(models: &mut [ModelData<'_>], source_model_count: usize) { - #[derive(Clone, Copy)] - struct ScanFrame { - source: usize, - next_event: usize, - } - - fn find_model(models: &[ModelData<'_>], limit: usize, name: &str) -> Option { - let normalized = NormalizedPath::new(name).ok()?; - models - .iter() - .take(limit) - .position(|model| model.path == normalized) - } - - let source_limit = source_model_count.min(models.len()); - for model in models.iter_mut() { - model.step_ends.clear(); - } - - // LDParse's callbacks are stateful pointers, not a tree-fold over model - // contents. Keep the parser's source cursor stack separate from the - // callback target: EOF always retargets callbacks to the root, while the - // saved include cursor is replayed against the now-cached child. - let mut callback_indices = vec![0_u64; models.len()]; - let mut cached = BTreeSet::new(); - let mut completed = vec![false; source_limit]; - let mut scan_stack = Vec::new(); - - for outer in 0..source_limit { - if completed[outer] { - continue; - } - cached.insert(outer); - let mut frame = ScanFrame { - source: outer, - next_event: 0, - }; - let mut callback_target = outer; - - loop { - if frame.next_event >= models[frame.source].events.len() { - completed[frame.source] = true; - // ModelBuilder::handleEOF calls recordTo(&target), where the - // EOF callback was retained on the root model. - callback_target = 0; - let Some(parent) = scan_stack.pop() else { - break; - }; - frame = parent; - continue; - } - - let event = models[frame.source].events[frame.next_event].clone(); - match event { - ModelEvent::Step => { - let offset = callback_indices[callback_target]; - models[callback_target].step_ends.push(offset); - frame.next_event += 1; - } - ModelEvent::Indices(count) => { - callback_indices[callback_target] = - callback_indices[callback_target].saturating_add(count); - frame.next_event += 1; - } - ModelEvent::Include(name) => { - let child = find_model(models, source_limit, name.as_ref()); - if let Some(child) = child { - if !cached.contains(&child) { - // Do not advance the include cursor. The real - // parser saves this line and reprocesses it after - // the child reaches EOF. - scan_stack.push(frame); - cached.insert(child); - frame = ScanFrame { - source: child, - next_event: 0, - }; - callback_target = child; - continue; - } - } - // A cached include is an ordinary callback and consumes - // its source line without changing the callback target. - frame.next_event += 1; - } - } - } - } - - // Caller-admitted materializations are parsed independently of the MPD - // stream and therefore retain their file-local callback target. - for index in source_limit..models.len() { - let mut indices = 0_u64; - let events = models[index].events.clone(); - for event in events { - match event { - ModelEvent::Step => models[index].step_ends.push(indices), - ModelEvent::Indices(count) => indices = indices.saturating_add(count), - ModelEvent::Include(_) => {} - } - } - } -} - fn finalize_data<'src>( model: &mut ModelData<'src>, arenas: &mut SemanticArenas, @@ -530,14 +434,9 @@ fn finalize_data<'src>( return Ok(()); } }; - if options.cancellation.checkpoint() { + if options.cancellation.cancelled { return Err(ParseError::Cancelled); } - let decoded_len = - u64::try_from(decoded.len()).map_err(|_| ParseError::Overflow("DATA resource bytes"))?; - counters - .add(LimitKind::ResourceBytes, decoded_len, &options.limits) - .map_err(|kind| limit_error(kind, Some(payload.span)))?; arenas.charge_data_bytes(options, decoded.len())?; model.data_payloads.push(DataPayload { filename: payload.filename, @@ -562,6 +461,7 @@ fn syntax_record<'src>(line: &ScannedLine<'src>) -> SyntaxRecord<'src> { } } +#[allow(clippy::too_many_arguments)] fn process_line<'src>( model: &mut ModelData<'src>, line: &ScannedLine<'src>, @@ -569,16 +469,15 @@ fn process_line<'src>( diagnostics: &mut Vec, counters: &mut LimitCounters, arenas: &mut SemanticArenas, - colours: &mut ColourScope<'src>, + colours: &mut ColourTable<'src>, texture_ids: &mut BTreeSet>, ) -> Result<(), ParseError> { let profile = options.profile; let limits = &options.limits; let primitive = parse_primitive(line); - let bang_colon = line.line_type == Some(LineType::Zero) - && line.tokens.get(1).map(|token| token.text) == Some("!:"); if let Some(payload) = model.data_payload.as_mut() { - let continuation = bang_colon && model.texmap_state == TexmapState::Inactive; + let continuation = line.line_type == Some(LineType::Zero) + && line.tokens.get(1).map(|token| token.text) == Some("!:"); if continuation { let encoded = line .tokens @@ -610,50 +509,29 @@ fn process_line<'src>( } else if matches!( line.line_type, Some(LineType::One | LineType::Two | LineType::Three | LineType::Four | LineType::Five) - ) { - if primitive.is_some() { - let descriptor = model.texmap_descriptor.clone(); - model.texmap_state = TexmapState::Active; - model.texmap_descriptor = descriptor.clone(); - let diagnostic_count = diagnostics.len(); - process_geometry( - model, - line, - primitive, - profile, - diagnostics, - counters, - limits, - arenas, - options, - )?; - if diagnostics.len() != diagnostic_count { - model.texmap_state = TexmapState::AwaitingNext; - model.texmap_descriptor = descriptor; - return Ok(()); - } - model.texmap_state = TexmapState::Inactive; - model.texmap_descriptor = None; - if let Some(descriptor) = descriptor { - let _ = texture_ids.insert(descriptor); - } - model.texmap_next_span = None; - return Ok(()); + ) && primitive.is_some() + { + let descriptor = model.texmap_descriptor.take(); + model.texmap_state = TexmapState::Active; + process_geometry( + model, + line, + primitive, + profile, + diagnostics, + counters, + limits, + arenas, + options, + )?; + model.texmap_state = TexmapState::Inactive; + if let Some(descriptor) = descriptor { + let _ = texture_ids.insert(descriptor); } + model.texmap_next_span = None; + return Ok(()); } } - if bang_colon { - return process_bang_colon( - model, - line, - profile, - diagnostics, - counters, - limits, - arenas, - options, - ); - } if line.line_type == Some(LineType::Zero) { let preserves_invert = line.tokens.get(1).map(|token| token.kind) == Some(TokenKind::Bfc) && line.tokens.get(2).map(|token| token.kind) == Some(TokenKind::InvertNext) @@ -685,112 +563,7 @@ fn process_line<'src>( ) } -fn process_bang_colon<'src>( - model: &mut ModelData<'src>, - line: &ScannedLine<'src>, - profile: ParserProfile, - diagnostics: &mut Vec, - counters: &mut LimitCounters, - limits: &LdrawLimits, - arenas: &mut SemanticArenas, - options: &ParseOptions<'_>, -) -> Result<(), ParseError> { - if !matches!( - model.texmap_state, - TexmapState::Active | TexmapState::Fallback - ) { - return add_diag( - profile, - diagnostics, - counters, - limits, - DiagnosticCode::TexmapBangColonOutOfScope, - line.span, - "bang-colon geometry requires an active TEXMAP frame", - true, - ); - } - let Some(embedded_type) = line.tokens.get(2).and_then(|token| match token.kind { - TokenKind::One => Some(LineType::One), - TokenKind::Two => Some(LineType::Two), - TokenKind::Three => Some(LineType::Three), - TokenKind::Four => Some(LineType::Four), - TokenKind::Five => Some(LineType::Five), - _ => None, - }) else { - return add_diag( - profile, - diagnostics, - counters, - limits, - DiagnosticCode::TexmapInvalidBangColonGeometry, - line.span, - "bang-colon requires an embedded line type 1 through 5", - true, - ); - }; - if matches!( - model.texmap_state, - TexmapState::Active | TexmapState::Fallback - ) && line.tokens.iter().skip(2).any(|token| token.text == "!:") - { - return add_diag( - profile, - diagnostics, - counters, - limits, - DiagnosticCode::TexmapNestedBangColon, - line.span, - "bang-colon geometry cannot be nested", - true, - ); - } - let embedded_line = ScannedLine { - line_type: Some(embedded_type), - raw: line.raw, - raw_tail: line.raw_tail, - tokens: line.tokens[2..].to_vec(), - span: line.span, - ending: line.ending.clone(), - blank: false, - }; - let embedded_primitive = crate::geom::parse_primitive(&embedded_line); - if embedded_primitive.is_none() - || embedded_primitive.as_ref().is_some_and(|primitive| { - matches!( - (embedded_type, primitive), - (LineType::One, crate::geom::ParsedPrimitive::Points(_)) - | ( - LineType::Two | LineType::Three | LineType::Four | LineType::Five, - crate::geom::ParsedPrimitive::Include(_) - ) - ) - }) - { - return add_diag( - profile, - diagnostics, - counters, - limits, - DiagnosticCode::TexmapInvalidBangColonGeometry, - line.span, - "malformed bang-colon geometry", - true, - ); - } - process_geometry( - model, - &embedded_line, - embedded_primitive, - profile, - diagnostics, - counters, - limits, - arenas, - options, - ) -} - +#[allow(clippy::too_many_arguments)] fn process_meta<'src>( model: &mut ModelData<'src>, line: &ScannedLine<'src>, @@ -798,7 +571,7 @@ fn process_meta<'src>( diagnostics: &mut Vec, counters: &mut LimitCounters, arenas: &mut SemanticArenas, - colours: &mut ColourScope<'src>, + colours: &mut ColourTable<'src>, texture_ids: &mut BTreeSet>, ) -> Result<(), ParseError> { let profile = options.profile; @@ -870,15 +643,13 @@ fn process_meta<'src>( return Ok(()); } match tokens[1].kind { - TokenKind::Step | TokenKind::Rotstep => { + TokenKind::Step => { let step = model .steps .len() .checked_add(1) .ok_or(ParseError::Overflow("step number"))?; model.steps.push(format!("step-{step:08}").into()); - model.step_ends.push(model.mesh_index_count); - model.events.push(ModelEvent::Step); model.bfc.invert_next = false; model.texmap_state = TexmapState::Inactive; model.texmap_descriptor = None; @@ -893,23 +664,11 @@ fn process_meta<'src>( } if let Some(colour) = parse_colour(tokens) { let code = colour.code; - if colours.local_slot(code).is_none() { + if colours.get(code).is_none() { arenas.charge_local_colour(options)?; } - if colours.record(colour).is_err() { - add_diag( - profile, - diagnostics, - counters, - limits, - DiagnosticCode::ColourSlotsLimit, - line.span, - "local colour slot table is exhausted", - true, - )?; - } else { - model.colour = code; - } + colours.record(colour); + model.colour = code; } else { add_diag( profile, diff --git a/crates/polymodel-ldraw-core/src/texmap.rs b/crates/polymodel-ldraw-core/src/texmap.rs index 55136d6..22d4521 100644 --- a/crates/polymodel-ldraw-core/src/texmap.rs +++ b/crates/polymodel-ldraw-core/src/texmap.rs @@ -139,6 +139,7 @@ use crate::types::{ }; use std::collections::BTreeSet; +#[allow(clippy::too_many_arguments)] pub(crate) fn texmap<'src>( model: &mut ModelData<'src>, line: &ScannedLine<'src>, @@ -231,7 +232,7 @@ pub(crate) fn texmap<'src>( .iter() .map(|token| { let value = token.text.parse::().ok()?; - value.is_finite().then(|| Cow::Borrowed(token.text)) + value.is_finite().then_some(Cow::Borrowed(token.text)) }) .collect::>>(); let Some(parameters) = parameters else { @@ -266,7 +267,7 @@ pub(crate) fn texmap<'src>( let glossmap = if has_gloss { line.tokens.get(png_index + 2).and_then(|token| { (!matches!(token.kind, TokenKind::Number | TokenKind::Garbage)) - .then(|| Cow::Borrowed(token.text)) + .then_some(Cow::Borrowed(token.text)) }) } else { None diff --git a/crates/polymodel-ldraw-core/src/types.rs b/crates/polymodel-ldraw-core/src/types.rs index adf7862..fcdc983 100644 --- a/crates/polymodel-ldraw-core/src/types.rs +++ b/crates/polymodel-ldraw-core/src/types.rs @@ -1,5 +1,4 @@ use crate::cache::NormalizedPath; -use crate::cache::{CacheKey, Sha256Hash}; use crate::geom::ColourCode; use miette::SourceSpan; use polymodel_renderer_ledger::{ @@ -8,10 +7,6 @@ use polymodel_renderer_ledger::{ }; use serde::{Deserialize, Serialize}; use std::fmt; -use std::sync::{ - Arc, - atomic::{AtomicBool, AtomicU64, Ordering}, -}; use thiserror::Error; pub const SCHEMA_VERSION: &str = "ldraw-canonical-v1"; @@ -30,18 +25,14 @@ pub const DEFAULT_TEXTURES: u64 = 1_024; pub const DEFAULT_FETCHES: u64 = 8; pub const DEFAULT_DIAGNOSTICS: u64 = 10_000; -#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Hash, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum ParserProfile { + #[default] Strict, Compatibility, Lossless, } -impl Default for ParserProfile { - fn default() -> Self { - Self::Strict - } -} #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] pub enum Severity { @@ -101,15 +92,10 @@ pub enum DiagnosticCode { TexmapFallbackOutOfScope, TexmapDuplicateFallback, TexmapInvalidEnd, - TexmapInvalidBangColonGeometry, - TexmapBangColonOutOfScope, - TexmapNestedBangColon, DataInvalid, DataTruncated, IncludeDepthLimit, GraphCycle, - AmbiguousMaterialization, - MaterializationHashMismatch, ResourceBytesLimit, CommandsLimit, InstancesLimit, @@ -155,15 +141,10 @@ impl fmt::Display for DiagnosticCode { Self::TexmapFallbackOutOfScope => "TEXMAP_FALLBACK_OUT_OF_SCOPE", Self::TexmapDuplicateFallback => "TEXMAP_DUPLICATE_FALLBACK", Self::TexmapInvalidEnd => "TEXMAP_INVALID_END", - Self::TexmapInvalidBangColonGeometry => "TEXMAP_INVALID_BANG_COLON_GEOMETRY", - Self::TexmapBangColonOutOfScope => "TEXMAP_BANG_COLON_OUT_OF_SCOPE", - Self::TexmapNestedBangColon => "TEXMAP_NESTED_BANG_COLON", Self::DataInvalid => "DATA_INVALID", Self::DataTruncated => "DATA_TRUNCATED", Self::IncludeDepthLimit => "INCLUDE_DEPTH_LIMIT", Self::GraphCycle => "GRAPH_CYCLE", - Self::AmbiguousMaterialization => "AMBIGUOUS_MATERIALIZATION", - Self::MaterializationHashMismatch => "MATERIALIZATION_HASH_MISMATCH", Self::ResourceBytesLimit => "RESOURCE_BYTES_LIMIT", Self::CommandsLimit => "COMMANDS_LIMIT", Self::InstancesLimit => "INSTANCES_LIMIT", @@ -220,6 +201,8 @@ pub enum LdrawError { Overflow(&'static str), #[error("parse cancelled")] Cancelled, + #[error(transparent)] + Resolver(#[from] crate::resolve::ResolveError), } pub type ParseError = LdrawError; @@ -336,115 +319,9 @@ impl LdrawLimits { } } -/// A target-safe live cancellation signal shared by a host and the parser. -/// -/// The parser checks this signal at every phase boundary; it never snapshots a -/// boolean before scanning. `AtomicBool` is available on native and WASM targets -/// and does not require a host runtime or an async executor. -#[derive(Clone, Debug)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub struct CancellationPolicy { - cancelled: Arc, - checkpoints: Arc, - /// `u64::MAX` is the disabled sentinel; every other value is a 1-based - /// checkpoint threshold, including zero for the next checkpoint. - cancel_after: Arc, -} - -impl Default for CancellationPolicy { - fn default() -> Self { - Self { - cancelled: Arc::new(AtomicBool::new(false)), - checkpoints: Arc::new(AtomicU64::new(0)), - cancel_after: Arc::new(AtomicU64::new(u64::MAX)), - } - } -} - -impl CancellationPolicy { - pub fn new() -> Self { - Self::default() - } - - pub fn cancel(&self) { - self.cancelled.store(true, Ordering::Release); - } - - /// Request cancellation after the given number of live checkpoints. - /// - /// This is intentionally a host-test and deterministic orchestration seam: - /// callers can cancel a deep traversal without another thread or a timing - /// dependency. A value of zero cancels at the next checkpoint; the default - /// policy has no checkpoint threshold. - pub fn cancel_after_checkpoints(&self, checkpoints: u64) { - let threshold = checkpoints.max(1); - self.cancel_after.store(threshold, Ordering::Release); - } - - pub(crate) fn checkpoint(&self) -> bool { - let count = self - .checkpoints - .fetch_add(1, Ordering::AcqRel) - .saturating_add(1); - let threshold = self.cancel_after.load(Ordering::Acquire); - self.is_cancelled() || (threshold != u64::MAX && count >= threshold) - } - - pub fn is_cancelled(&self) -> bool { - self.cancelled.load(Ordering::Acquire) - } -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct Materialization { - pub path: NormalizedPath, - pub root: RootId, - pub content_hash: Sha256Hash, - pub bytes: Vec, - pub available: bool, - pub provenance: Option, -} - -impl Materialization { - pub fn new( - path: NormalizedPath, - root: RootId, - bytes: Vec, - available: bool, - provenance: Option, - ) -> Self { - let content_hash = CacheKey::new(path.clone(), root, &bytes).content_hash; - Self { - path, - root, - content_hash, - bytes, - available, - provenance, - } - } - - pub fn unavailable(path: NormalizedPath, root: RootId, provenance: Option) -> Self { - Self::new(path, root, Vec::new(), false, provenance) - } - - pub fn key(&self) -> CacheKey { - CacheKey { - canonical_path: self.path.clone(), - resolved_root: self.root, - content_hash: self.content_hash.clone(), - } - } - - pub fn has_valid_hash(&self) -> bool { - CacheKey::new(self.path.clone(), self.root, &self.bytes).content_hash == self.content_hash - } -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct TargetSelection { - pub source: CacheKey, - pub span: Span, - pub target: CacheKey, + pub cancelled: bool, } #[derive(Clone)] @@ -458,10 +335,17 @@ pub struct ParseOptions<'a> { pub resolved_root: RootId, pub fetch_count: u64, pub cancellation: CancellationPolicy, - /// Exact resolver decisions, keyed by source model identity and include span. - /// An empty table requests the deterministic PM-75 precedence fallback, which - /// still rejects duplicate candidates at the same path/root. - pub target_selections: &'a [TargetSelection], +} + +/// A borrowed target table used by the compatibility `parse_bytes` entry point. +/// New callers should use [`crate::Materialization`] and [`crate::Resolver`]. +#[derive(Clone, Debug)] +pub struct BorrowedMaterialization<'a> { + pub path: NormalizedPath, + pub root: RootId, + pub content_hash: crate::cache::Sha256Hash, + pub bytes: &'a [u8], + pub available: bool, } /// Caller-owned storage for source buffers used during a parse. @@ -505,7 +389,6 @@ impl<'a> ParseOptions<'a> { resolved_root: RootId::UploadedManifest, fetch_count: 0, cancellation: CancellationPolicy::default(), - target_selections: &[], } } pub fn default_budget(input_len: usize) -> Result { @@ -585,6 +468,7 @@ pub(crate) fn limit_error(kind: LimitKind, span: Option) -> ParseError { }) } +#[allow(clippy::too_many_arguments)] pub(crate) fn add_diag( profile: ParserProfile, diagnostics: &mut Vec, @@ -673,13 +557,6 @@ impl SemanticArenas { self.child(options, 128) } - pub(crate) fn charge_geometry_record( - &mut self, - options: &ParseOptions<'_>, - ) -> Result<(), ParseError> { - self.child(options, 256) - } - pub(crate) fn charge_cache_entry( &mut self, options: &ParseOptions<'_>, diff --git a/crates/polymodel-mesh/src/parser.rs b/crates/polymodel-mesh/src/parser.rs index 22abd00..c59b4f8 100644 --- a/crates/polymodel-mesh/src/parser.rs +++ b/crates/polymodel-mesh/src/parser.rs @@ -103,6 +103,9 @@ pub fn load_mesh(source: &MeshSource<'_>) -> Result { MeshFormat::Obj => crate::obj::load_obj(source.primary), MeshFormat::Gltf => crate::gltf::load_gltf(source), MeshFormat::Threemf => crate::threemf::load_threemf(source.primary), + // LDraw is a compound scene, not a mesh. It must cross the renderer-owned + // scene adapter boundary rather than being guessed into another format. + MeshFormat::Ldraw => Err(MeshLoadError::UnsupportedFormat), } } @@ -172,6 +175,7 @@ impl MeshParser for DefaultMeshParser { MeshFormat::Stl | MeshFormat::Obj | MeshFormat::Gltf | MeshFormat::Threemf => { load_mesh(source) } + MeshFormat::Ldraw => Err(MeshLoadError::UnsupportedFormat), } } } diff --git a/crates/polymodel-renderer-ledger/src/lib.rs b/crates/polymodel-renderer-ledger/src/lib.rs index f8b34a6..feb3788 100644 --- a/crates/polymodel-renderer-ledger/src/lib.rs +++ b/crates/polymodel-renderer-ledger/src/lib.rs @@ -527,6 +527,7 @@ impl Drop for Reservation { } } +#[allow(clippy::result_large_err)] impl ReservationLedger { pub fn new() -> Self { Self { @@ -843,6 +844,7 @@ fn remove_apply(state: &mut State, entry: &Entry) { state.global_unified -= entry.bytes; } +#[allow(clippy::result_large_err)] fn check_policy( state: &State, owner: ReservationOwner, diff --git a/crates/polymodel-renderer-protocol/src/lib.rs b/crates/polymodel-renderer-protocol/src/lib.rs index f454705..3d7a0e8 100644 --- a/crates/polymodel-renderer-protocol/src/lib.rs +++ b/crates/polymodel-renderer-protocol/src/lib.rs @@ -11,6 +11,7 @@ use serde::{Deserialize, Serialize}; use sha2::Digest; +use std::collections::HashSet; mod load_state; @@ -153,6 +154,20 @@ pub struct EventEnvelope { } impl HandshakeState { + pub fn session_id(&self) -> Option { + match self { + Self::Negotiated(session) => Some(session.session_id), + _ => None, + } + } + + pub fn negotiated(&self) -> Option<&NegotiatedProtocol> { + match self { + Self::Negotiated(session) => Some(session), + _ => None, + } + } + pub fn accept_command( &mut self, envelope: &CommandEnvelope, @@ -162,11 +177,16 @@ impl HandshakeState { return Err(HandshakeError::NoVersionOverlap); } match (&*self, &envelope.payload) { - (Self::Created, CommandPayload::Hello(_)) => { + (Self::Created, CommandPayload::Hello(hello)) => { + hello.validate()?; *self = Self::Negotiating; Ok(None) } - (Self::Negotiating, CommandPayload::Accept(negotiated)) => { + (Self::Negotiating, CommandPayload::Accept(negotiated)) + if negotiated.version == PROTOCOL_VERSION + && negotiated.session_id != 0 + && negotiated.capabilities.windows(2).all(|pair| pair[0] != pair[1]) => + { *self = Self::Negotiated(negotiated.clone()); Ok(None) } @@ -195,10 +215,15 @@ impl HandshakeState { *self = Self::Negotiating; Ok(None) } - (Self::Negotiating, EventPayload::Negotiated(negotiated)) => { + (Self::Created | Self::Negotiating, EventPayload::Negotiated(negotiated)) => { *self = Self::Negotiated(negotiated.clone()); Ok(None) } + (Self::Negotiated(session), EventPayload::Negotiated(negotiated)) + if session == negotiated => + { + Ok(None) + } (Self::Negotiated(session), EventPayload::Event(event)) if session.session_id == envelope.session_id => { @@ -233,6 +258,8 @@ pub enum MeshFormat { /// 3D Manufacturing Format. Parsed client-side in the worker via `three-d-asset`'s /// `3mf` feature (`lib3mf`, pure Rust). Threemf, + /// LDraw source. The renderer adapter owns parsing and scene conversion. + Ldraw, } impl MeshFormat { @@ -444,6 +471,24 @@ pub struct CompoundLoadPlan { pub descriptors: Vec, } +pub const MAX_COMPOUND_DESCRIPTORS: usize = 16_384; +pub const MAX_COMPOUND_ENCODED_BYTES: usize = 8 * 1024 * 1024; +pub const MAX_COMPOUND_METADATA_BYTES: usize = 8 * 1024 * 1024; +pub const MAX_COMPOUND_TOTAL_BYTES: u64 = 128 * 1024 * 1024; +pub const MAX_COMPOUND_CIDS_PER_DESCRIPTOR: usize = 4_096; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CompoundPlanError { + Empty, + TooManyDescriptors, + EncodedBytesExceeded, + MetadataExceeded, + TotalBytesExceeded, + DuplicateKey, + PrimaryMissing, + InvalidDescriptor, +} + impl WorkerFetchDescriptor { pub fn validate(&self) -> Result<(), &'static str> { if self.key.is_empty() || self.key != self.path || self.route.is_empty() { @@ -458,8 +503,99 @@ impl WorkerFetchDescriptor { if self.sha256.len() != 32 || self.mime_type.is_empty() || self.target_cids.is_empty() { return Err("descriptor integrity metadata is incomplete"); } + if self.target_cids.len() > MAX_COMPOUND_CIDS_PER_DESCRIPTOR { + return Err("descriptor has too many target CIDs"); + } Ok(()) } + + fn metadata_bytes(&self) -> Option { + self.key + .len() + .checked_add(self.route.len())? + .checked_add(self.path.len())? + .checked_add(self.root.len())? + .checked_add(self.mime_type.len())? + .checked_add( + self.target_cids + .iter() + .try_fold(0usize, |total, cid| total.checked_add(cid.len()))?, + )? + .checked_add(self.sha256.len()) + } +} + +impl CompoundLoadPlan { + pub fn validate(&self) -> Result<(), CompoundPlanError> { + if self.descriptors.is_empty() { + return Err(CompoundPlanError::Empty); + } + if self.descriptors.len() > MAX_COMPOUND_DESCRIPTORS { + return Err(CompoundPlanError::TooManyDescriptors); + } + let mut keys = HashSet::with_capacity(self.descriptors.len()); + let mut metadata = self.primary.len(); + let mut total_bytes = 0u64; + for descriptor in &self.descriptors { + descriptor + .validate() + .map_err(|_| CompoundPlanError::InvalidDescriptor)?; + if !keys.insert(&descriptor.key) { + return Err(CompoundPlanError::DuplicateKey); + } + metadata = metadata + .checked_add( + descriptor + .metadata_bytes() + .ok_or(CompoundPlanError::MetadataExceeded)?, + ) + .ok_or(CompoundPlanError::MetadataExceeded)?; + total_bytes = total_bytes + .checked_add(descriptor.byte_length) + .ok_or(CompoundPlanError::TotalBytesExceeded)?; + if metadata > MAX_COMPOUND_METADATA_BYTES { + return Err(CompoundPlanError::MetadataExceeded); + } + if total_bytes > MAX_COMPOUND_TOTAL_BYTES { + return Err(CompoundPlanError::TotalBytesExceeded); + } + } + if !keys.contains(&self.primary) { + return Err(CompoundPlanError::PrimaryMissing); + } + Ok(()) + } + + pub fn encoded_size(&self) -> Result { + let encoded = + postcard::to_allocvec(self).map_err(|_| CompoundPlanError::EncodedBytesExceeded)?; + if encoded.len() > MAX_COMPOUND_ENCODED_BYTES { + return Err(CompoundPlanError::EncodedBytesExceeded); + } + Ok(encoded.len()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CompoundSceneSource { + pub primary: String, + pub descriptors: Vec<(WorkerFetchDescriptor, Vec)>, +} + +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum CompoundSceneError { + #[error("compound scene source is empty")] + Empty, +} + +/// Typed boundary between G's verified route fetch and PM-79's LDraw parser/scene adapter. +/// G owns acquisition and integrity only; PM-79 owns bytes-to-scene conversion and exact LDU +/// conversion. The source is returned intact so the mesh/worker boundary can consume it. +pub fn handoff_compound_scene(source: CompoundSceneSource) -> Result { + if source.descriptors.is_empty() || source.primary.is_empty() { + return Err(CompoundSceneError::Empty); + } + Ok(source) } /// Keyboard modifier flags accompanying a pointer state. @@ -527,19 +663,40 @@ pub enum RendererEvent { // --------------------------------------------------------------------------- /// Serialize a command in the mandatory versioned envelope. +/// +/// This compatibility helper is retained for callers that only have a command. New +/// session-bound transports should use [`serialize_command_for_session`]. pub fn serialize_command(cmd: &RendererCommand) -> Result, postcard::Error> { + serialize_command_for_session(cmd, 0) +} + +pub fn serialize_command_for_session( + cmd: &RendererCommand, + session_id: u64, +) -> Result, postcard::Error> { postcard::to_allocvec(&CommandEnvelope { version: PROTOCOL_VERSION, - session_id: 0, + session_id, payload: CommandPayload::Command(cmd.clone()), }) } /// Serialize an event in the mandatory versioned envelope. +/// +/// This compatibility helper is retained for worker events emitted before a +/// transport session is negotiated. Session-bound worker bridges use +/// [`serialize_event_for_session`]. pub fn serialize_event(evt: &RendererEvent) -> Result, postcard::Error> { + serialize_event_for_session(evt, 0) +} + +pub fn serialize_event_for_session( + evt: &RendererEvent, + session_id: u64, +) -> Result, postcard::Error> { postcard::to_allocvec(&EventEnvelope { version: PROTOCOL_VERSION, - session_id: 0, + session_id, payload: EventPayload::Event(evt.clone()), }) } diff --git a/crates/polymodel-renderer-protocol/src/load_state.rs b/crates/polymodel-renderer-protocol/src/load_state.rs index c86fc28..7fad031 100644 --- a/crates/polymodel-renderer-protocol/src/load_state.rs +++ b/crates/polymodel-renderer-protocol/src/load_state.rs @@ -206,7 +206,7 @@ impl LoadReducer { if let LoadEvent::StartLoad { identity, .. } = event { if self .current_identity - .map_or(true, |current| identity > current) + .is_none_or(|current| identity > current) { let mut disposed = self.take_resources(None); self.state = LoadState::Loading(identity); diff --git a/crates/polymodel-renderer-worker/Cargo.toml b/crates/polymodel-renderer-worker/Cargo.toml index 841f849..a6c7f34 100644 --- a/crates/polymodel-renderer-worker/Cargo.toml +++ b/crates/polymodel-renderer-worker/Cargo.toml @@ -35,6 +35,8 @@ web-sys = { version = "0.3", features = [ "Headers", "Blob", "BlobPropertyBag", + "AbortController", + "AbortSignal", ] } js-sys = "0.3" console_error_panic_hook = "0.1" diff --git a/crates/polymodel-renderer-worker/src/worker.rs b/crates/polymodel-renderer-worker/src/worker.rs index 6a20358..20b1bec 100644 --- a/crates/polymodel-renderer-worker/src/worker.rs +++ b/crates/polymodel-renderer-worker/src/worker.rs @@ -5,7 +5,6 @@ //! and the postcard message protocol bridge to the Dioxus main thread. use std::cell::RefCell; -use std::collections::HashMap; use std::rc::Rc; use std::sync::Arc; @@ -14,7 +13,7 @@ use wasm_bindgen::closure::Closure; use wasm_bindgen::prelude::*; use wasm_bindgen_futures::JsFuture; use wasm_bindgen_futures::spawn_local; -use web_sys::DedicatedWorkerGlobalScope; +use web_sys::{AbortController, DedicatedWorkerGlobalScope}; use polymodel_mesh::contract::ModelMesh; use polymodel_mesh::parser::load_mesh_with_companions; @@ -24,9 +23,10 @@ use polymodel_renderer_ledger::{ texture_charge, wasm_charge, }; use polymodel_renderer_protocol::{ - KeyModifiers, LoadError, LoadErrorCode, LoadEvent, LoadIdentity, LoadReducer, MeshFormat, - MeshStats, RendererCommand, RendererEvent, RendererNamespace, ResourceToken, - deserialize_command, serialize_event, + Capability, CommandPayload, EventEnvelope, EventPayload, HandshakeState, KeyModifiers, + LoadError, LoadErrorCode, LoadEvent, LoadIdentity, LoadReducer, MeshFormat, MeshStats, + NegotiatedProtocol, ProtocolHello, RendererCommand, RendererEvent, RendererNamespace, + ResourceToken, deserialize_command_envelope, serialize_event_envelope, }; use three_d_asset::{Geometry, Texture2D, TextureData}; @@ -106,6 +106,9 @@ struct WorkerState { canvas_reservation: Option, load_epoch: u32, scene_generation: u32, + compound_abort: Option, + handshake: HandshakeState, + local_hello: ProtocolHello, } impl WorkerState { @@ -127,19 +130,47 @@ impl WorkerState { canvas_reservation: None, load_epoch: 0, scene_generation: 0, + compound_abort: None, + handshake: HandshakeState::Created, + local_hello: ProtocolHello { + min_version: polymodel_renderer_protocol::PROTOCOL_VERSION, + max_version: polymodel_renderer_protocol::PROTOCOL_VERSION, + required: vec![Capability::CurrentFormats, Capability::Cancellation], + optional: vec![Capability::CompoundLdraw, Capability::Preview], + }, } } - fn post_event(&self, evt: &RendererEvent) { - match serialize_event(evt) { + fn post_envelope(&self, envelope: &EventEnvelope) { + match serialize_event_envelope(envelope) { Ok(bytes) => { let array = js_sys::Uint8Array::from(&bytes[..]); let _ = self.scope.post_message(&array); } - Err(e) => tracing::error!("failed to serialize event {evt:?}: {e}"), + Err(e) => tracing::error!("failed to serialize event envelope: {e}"), } } + fn post_event(&self, evt: &RendererEvent) { + let Some(session_id) = self.handshake.session_id() else { + tracing::warn!("suppressing renderer event before handshake"); + return; + }; + self.post_envelope(&EventEnvelope { + version: polymodel_renderer_protocol::PROTOCOL_VERSION, + session_id, + payload: EventPayload::Event(evt.clone()), + }); + } + + fn post_negotiated(&self, negotiated: &NegotiatedProtocol) { + self.post_envelope(&EventEnvelope { + version: polymodel_renderer_protocol::PROTOCOL_VERSION, + session_id: negotiated.session_id, + payload: EventPayload::Negotiated(negotiated.clone()), + }); + } + fn init_renderer(&mut self, canvas: web_sys::OffscreenCanvas) { tracing::info!("renderer worker: initializing WebGL2 renderer"); let owner = owner(self.load_epoch, self.scene_generation); @@ -1119,7 +1150,7 @@ async fn load_mesh_async( let expected_digest = expected_digest.clone(); async move { tracing::debug!(%url, verify_primary, "renderer worker: fetching mesh resource"); - let bytes = fetch_bytes(&url).await.map_err(|e| { + let bytes = fetch_bytes(&url, None).await.map_err(|e| { polymodel_mesh::contract::MeshLoadError::Decode(format!("fetch {url}: {e}")) })?; tracing::debug!( @@ -1171,42 +1202,54 @@ fn handle_compound_load( identity: LoadIdentity, plan: polymodel_renderer_protocol::CompoundLoadPlan, ) { + let abort = AbortController::new().ok(); + if let Some(abort) = &abort { + if let Ok(mut st) = state.try_borrow_mut() { + st.compound_abort = Some(abort.clone()); + } + } let state_clone = state.clone(); spawn_local(async move { - let result = load_compound_mesh(&state_clone, identity, plan).await; - finish_loaded_mesh(&state_clone, namespace, identity, result); + let result = load_compound_source(&state_clone, identity, plan, abort).await; + finish_compound_source(&state_clone, namespace, identity, result); }); } -async fn load_compound_mesh( +fn compound_is_current( + state: &Rc>, + identity: LoadIdentity, + abort: Option<&AbortController>, +) -> Result<(), String> { + if abort.is_some_and(|controller| controller.signal().aborted()) { + return Err("compound load cancelled".into()); + } + let st = state.borrow(); + if st.disposed || st.load_reducer.current_identity() != Some(identity) { + return Err("compound load superseded".into()); + } + Ok(()) +} + +async fn load_compound_source( state: &Rc>, identity: LoadIdentity, plan: polymodel_renderer_protocol::CompoundLoadPlan, -) -> Result { + abort: Option, +) -> Result { if plan.version != polymodel_renderer_protocol::PROTOCOL_VERSION { return Err("compound plan protocol version mismatch".into()); } - if plan.descriptors.is_empty() { - return Err("compound plan has no descriptors".into()); - } - let mut descriptors = HashMap::with_capacity(plan.descriptors.len()); - for descriptor in plan.descriptors { - descriptor.validate().map_err(str::to_owned)?; - if descriptors - .insert(descriptor.key.clone(), descriptor) - .is_some() - { - return Err("compound plan contains duplicate descriptor keys".into()); - } - } - let primary = descriptors - .get(&plan.primary) - .ok_or_else(|| "compound plan primary is missing".to_string())?; + plan.validate() + .map_err(|error| format!("compound plan rejected: {error:?}"))?; + plan.encoded_size() + .map_err(|error| format!("compound plan encoding rejected: {error:?}"))?; + compound_is_current(state, identity, abort.as_ref())?; + let descriptors = plan.descriptors; let owner = { let st = state.borrow(); owner(identity.load_epoch, st.scene_generation) }; - let total_bytes = descriptors.values().try_fold(0_u64, |total, descriptor| { + let total_bytes = descriptors.iter().try_fold(0_u64, |total, descriptor| { total .checked_add(descriptor.byte_length) .ok_or("compound plan byte bound overflow") @@ -1228,40 +1271,48 @@ async fn load_compound_mesh( ) .map_err(|error| format!("compound load admission: {error}"))? }; - let mut bytes_by_key = HashMap::with_capacity(descriptors.len()); - for (key, descriptor) in &descriptors { + let mut fetched = Vec::with_capacity(descriptors.len()); + for descriptor in descriptors { + compound_is_current(state, identity, abort.as_ref())?; let bytes = fetch_descriptor_bytes( &descriptor.route, descriptor.byte_length, &descriptor.sha256, + abort.as_ref(), ) .await?; - bytes_by_key.insert(key.clone(), bytes); - } - let primary_bytes = bytes_by_key - .remove(&primary.key) - .ok_or_else(|| "compound plan primary bytes are missing".to_string())?; - let mut companions = Vec::with_capacity(bytes_by_key.len()); - for (key, bytes) in bytes_by_key { - companions.push((key, bytes)); + fetched.push((descriptor, bytes)); } - let format = mesh_format_for_descriptor(primary); - let source = polymodel_mesh::parser::MeshSource { - format, - primary: &primary_bytes, - resources: polymodel_mesh::parser::MeshResources::new(&companions), + compound_is_current(state, identity, abort.as_ref())?; + let source = polymodel_renderer_protocol::CompoundSceneSource { + primary: plan.primary, + descriptors: fetched, }; - let model = polymodel_mesh::parser::load_mesh(&source).map_err(|error| error.to_string()); + let result = polymodel_renderer_protocol::handoff_compound_scene(source) + .map_err(|error| error.to_string()); drop(reservation); - model + result +} + +fn finish_compound_source( + state: &Rc>, + namespace: RendererNamespace, + identity: LoadIdentity, + result: Result, +) { + if let Ok(mut st) = state.try_borrow_mut() { + st.compound_abort = None; + } + finish_loaded_mesh(state, namespace, identity, result.map(|_| unreachable!())); } async fn fetch_descriptor_bytes( route: &str, expected_length: u64, expected_sha256: &[u8], + abort: Option<&AbortController>, ) -> Result, String> { - let bytes = fetch_bytes(route).await?; + let bytes = fetch_bytes(route, abort).await?; let expected_size = i64::try_from(expected_length) .map_err(|_| "compound descriptor byte length is too large".to_string())?; polymodel_renderer_protocol::verify_part_file(&bytes, expected_size, Some(expected_sha256)) @@ -1269,20 +1320,6 @@ async fn fetch_descriptor_bytes( Ok(bytes) } -fn mesh_format_for_descriptor( - descriptor: &polymodel_renderer_protocol::WorkerFetchDescriptor, -) -> MeshFormat { - if descriptor.mime_type.contains("stl") || descriptor.key.ends_with(".dat") { - MeshFormat::Stl - } else if descriptor.mime_type.contains("obj") || descriptor.key.ends_with(".obj") { - MeshFormat::Obj - } else if descriptor.mime_type.contains("3mf") || descriptor.key.ends_with(".3mf") { - MeshFormat::Threemf - } else { - MeshFormat::Gltf - } -} - fn finish_loaded_mesh( state: &Rc>, namespace: RendererNamespace, @@ -1478,12 +1515,17 @@ fn handle_render_preview_image( message, }, }; - match serialize_event(&evt) { + let envelope = EventEnvelope { + version: polymodel_renderer_protocol::PROTOCOL_VERSION, + session_id: 0, + payload: EventPayload::Event(evt), + }; + match serialize_event_envelope(&envelope) { Ok(bytes) => { let array = js_sys::Uint8Array::from(&bytes[..]); let _ = scope.post_message(&array); } - Err(e) => tracing::error!("failed to serialize preview event {evt:?}: {e}"), + Err(e) => tracing::error!("failed to serialize preview event: {e}"), } }); } @@ -1579,10 +1621,13 @@ async fn offscreen_canvas_png_bytes(canvas: &web_sys::OffscreenCanvas) -> Result Ok(js_sys::Uint8Array::new(&buffer).to_vec()) } -async fn fetch_bytes(url: &str) -> Result, String> { +async fn fetch_bytes(url: &str, abort: Option<&AbortController>) -> Result, String> { let opts = web_sys::RequestInit::new(); opts.set_method("GET"); opts.set_mode(web_sys::RequestMode::SameOrigin); + if let Some(abort) = abort { + opts.set_signal(Some(&abort.signal())); + } let request = web_sys::Request::new_with_str_and_init(url, &opts) .map_err(|e| format!("request: {:?}", e))?; @@ -1605,7 +1650,13 @@ async fn fetch_bytes(url: &str) -> Result, String> { .await .map_err(|e| format!("fetch: {:?}", e))? .dyn_into::() - .map_err(|_| "fetch: not a Response")?; + .map_err(|_| "fetch: not a Response".to_string())?; + if !response.ok() { + let status = response.status(); + let status_text = response.status_text(); + let bounded = status_text.chars().take(96).collect::(); + return Err(format!("fetch returned HTTP {status} {bounded}")); + } let array_buffer_promise = response .array_buffer() @@ -1631,6 +1682,26 @@ fn handle_command(state: &Rc>, cmd: RendererCommand) { if st.disposed { return; } + let Some(capabilities) = st + .handshake + .negotiated() + .map(|negotiated| negotiated.capabilities.clone()) + else { + tracing::warn!("rejecting scene command before negotiated session"); + return; + }; + let capability_allowed = match &cmd { + RendererCommand::StartCompoundLoad { .. } => + capabilities.contains(&Capability::CompoundLdraw), + RendererCommand::RenderPreviewImage { .. } => + capabilities.contains(&Capability::Preview), + RendererCommand::Cancel { .. } => capabilities.contains(&Capability::Cancellation), + _ => true, + }; + if !capability_allowed { + tracing::warn!("renderer command rejected by negotiated capabilities"); + return; + } match cmd { RendererCommand::StartLoad { @@ -1695,6 +1766,9 @@ fn handle_command(state: &Rc>, cmd: RendererCommand) { namespace, identity, } => { + if let Some(abort) = st.compound_abort.take() { + abort.abort(); + } let result = st.load_reducer.reduce(LoadEvent::Cancel { namespace, identity, @@ -1870,23 +1944,64 @@ pub fn entry() { } } - // String messages (handshake). + // String messages are only the loader readiness sentinel. if let Some(s) = data.as_string() { if s == "__worker_ready" { return; } } - // Postcard bytes. + // Every binary message is a versioned, session-gated command envelope. if let Ok(array) = data.dyn_into::() { let bytes = array.to_vec(); - match deserialize_command(&bytes) { - Ok(cmd) => { + let envelope = match deserialize_command_envelope(&bytes) { + Ok(envelope) => envelope, + Err(e) => { + tracing::error!("failed to deserialize command envelope: {e}"); + return; + } + }; + let mut st = match state_for_msg.try_borrow_mut() { + Ok(st) => st, + Err(_) => return, + }; + if matches!(envelope.payload, CommandPayload::Hello(_)) { + let CommandPayload::Hello(remote) = envelope.payload else { + return; + }; + let local = st.local_hello.clone(); + let session_id = js_sys::Math::random().mul_add(u64::MAX as f64, 1.0) as u64; + match polymodel_renderer_protocol::negotiate(&local, &remote, session_id.max(1)) { + Ok(negotiated) => { + st.handshake = HandshakeState::Negotiating; + st.post_envelope(&EventEnvelope { + version: polymodel_renderer_protocol::PROTOCOL_VERSION, + session_id: 0, + payload: EventPayload::Hello(local), + }); + st.post_negotiated(&negotiated); + // The main thread must echo this negotiated value as Accept. + } + Err(error) => { + st.handshake = HandshakeState::Failed; + st.post_envelope(&EventEnvelope { + version: polymodel_renderer_protocol::PROTOCOL_VERSION, + session_id: 0, + payload: EventPayload::Failed(format!("handshake failed: {error:?}")), + }); + } + } + return; + } + match st.handshake.accept_command(&envelope) { + Ok(Some(cmd)) => { + drop(st); tracing::debug!(command = ?cmd, "renderer worker: command received"); handle_command(&state_for_msg, cmd); } - Err(e) => { - tracing::error!("failed to deserialize command: {e}"); + Ok(None) => {} + Err(error) => { + tracing::warn!("rejected command before negotiated session: {error:?}") } } } diff --git a/migrations/010_remove_ldraw_resources.sql b/migrations/010_remove_ldraw_resources.sql index f8624a3..7504593 100644 --- a/migrations/010_remove_ldraw_resources.sql +++ b/migrations/010_remove_ldraw_resources.sql @@ -1,8 +1,4 @@ -- PM-80: retire the historical LDraw projection after the public/API surface was removed. --- Keep 009 in the migration chain so databases that applied it can upgrade safely. -DROP TABLE IF EXISTS ldraw_manifest_files; -DROP TABLE IF EXISTS ldraw_routes; -DROP TABLE IF EXISTS ldraw_verification; -DROP TABLE IF EXISTS ldraw_legacy_resources; -DROP TABLE IF EXISTS ldraw_official_snapshots; -DROP TABLE IF EXISTS ldraw_resources; +-- PM-80 retains the PM-76 resolver and route tables for compound resources. +-- This migration remains as a compatibility marker for databases that recorded +-- the earlier removal migration; it intentionally performs no destructive work. diff --git a/src/appview/downloads.rs b/src/appview/downloads.rs index ba973e4..adac58c 100644 --- a/src/appview/downloads.rs +++ b/src/appview/downloads.rs @@ -20,15 +20,16 @@ use axum::body::Body; use axum::extract::State; use axum::http::{StatusCode, header}; use axum::response::Response; -use futures::TryStreamExt; +use cid::Cid as ParsedCid; use futures::stream::{self, StreamExt}; use jacquard::identity::resolver::IdentityResolver; use jacquard_axum::ExtractXrpc; -use jacquard_common::types::string::Cid; +use jacquard_common::types::string::{Cid, Did}; use jacquard_common::xrpc::XrpcExt; use polymodel_api::com_atproto::sync::get_blob::GetBlob; use polymodel_api::space_polymodel::library::File; use polymodel_api::space_polymodel::library::get_part_file::GetPartFileRequest; +use sha2::{Digest, Sha256}; use super::error::{AppResult, db, internal, invalid_request, not_found}; use super::state::AppState; @@ -85,6 +86,63 @@ pub(super) async fn get_part_file( } validate_file_manifest(&file)?; + stream_projected_file(&state, did, row.name, file).await +} + +fn verify_chunk_cid(cid: &str, bytes: &[u8]) -> Result<(), String> { + let parsed = ParsedCid::try_from(cid).map_err(|error| format!("invalid chunk CID: {error}"))?; + let digest = parsed.hash().digest(); + if parsed.hash().code() != 0x12 { + return Err(format!( + "unsupported chunk CID hash code {}", + parsed.hash().code() + )); + } + let actual = Sha256::digest(bytes); + if digest != &actual[..] { + return Err("chunk CID digest mismatch".into()); + } + Ok(()) +} + +async fn collect_verified_blob( + response: jacquard_common::StreamingResponse, + expected_size: i64, + cid: &Cid, + pds: impl AsRef, +) -> AppResult> { + let (_, body) = response.into_parts(); + let pds = pds.as_ref(); + let mut stream = body.into_inner(); + let mut bytes = Vec::with_capacity(usize::try_from(expected_size.max(0)).unwrap_or_default()); + while let Some(chunk) = stream.next().await { + let chunk = + chunk.map_err(|error| internal(format!("blob fetch from {pds} failed: {error}")))?; + bytes.extend_from_slice(&chunk); + if bytes.len() as i64 > expected_size { + return Err(invalid_request("blob exceeded projected chunk size")); + } + } + if bytes.len() as i64 != expected_size { + return Err(invalid_request("blob did not match projected chunk size")); + } + verify_chunk_cid(cid.as_str(), &bytes) + .map_err(|error| invalid_request(format!("chunk integrity check failed: {error}")))?; + Ok(bytes) +} + +/// Stream a projected file through its owner's PDS. +/// +/// This is shared by the public Part download and authenticated compound +/// resource routes. The first blob is fetched before returning the response so +/// an initial PDS failure is reported as an HTTP error; later failures surface +/// as a body-stream error after headers are committed. +pub(super) async fn stream_projected_file( + state: &AppState, + did: Did, + name: String, + file: File, +) -> AppResult { let did_doc_response = state .resolver .resolve_did_doc(&did) @@ -115,7 +173,7 @@ pub(super) async fn get_part_file( .xrpc(pds_endpoint.borrow()) .download(&GetBlob { did: did.clone(), - cid: first_cid, + cid: first_cid.clone(), }) .await .map_err(|e| internal(format!("blob fetch from {pds_endpoint} failed: {e}")))?; @@ -125,51 +183,62 @@ pub(super) async fn get_part_file( first_response.status() ))); } - let (_, first_body) = first_response.into_parts(); - let first_stream = first_body.into_inner(); + let first_bytes = collect_verified_blob( + first_response, + ordered_chunks(&file)[0].size, + &first_cid, + pds_endpoint.as_str(), + ) + .await?; - let remaining_cids = ordered_cids[1..].to_vec(); + let remaining: Vec<(i64, Cid)> = ordered_chunks(&file)[1..] + .iter() + .zip(ordered_cids[1..].iter().cloned()) + .map(|(chunk, cid)| (chunk.size, cid)) + .collect(); let state_clone = state.clone(); let pds_clone = pds_endpoint.clone(); let did_clone = did.clone(); type BoxError = Box; - let first_stream = first_stream.map_err(|e| -> BoxError { Box::new(e) }); - - let rest_stream = stream::iter(remaining_cids) - .then(move |cid| { - let state = state_clone.clone(); - let pds = pds_clone.clone(); - let did = did_clone.clone(); - async move { - let response = state - .resolver - .xrpc(pds.borrow()) - .download(&GetBlob { - did: did.clone(), - cid, - }) - .await - .map_err(|e| -> BoxError { - Box::new(internal(format!("blob fetch from {pds} failed: {e}"))) - })?; - if !response.status().is_success() { - return Err(Box::new(internal(format!( - "blob fetch from {pds} failed with HTTP {}", - response.status() - ))) as BoxError); - } - let (_, body) = response.into_parts(); - let stream = body.into_inner().map_err(|e| -> BoxError { Box::new(e) }); - Ok::<_, BoxError>(stream) + let first_stream = + stream::once(async move { Ok::<_, BoxError>(bytes::Bytes::from(first_bytes)) }); + let rest_stream = stream::iter(remaining).then(move |(expected_size, cid)| { + let state = state_clone.clone(); + let pds = pds_clone.clone(); + let did = did_clone.clone(); + let pds_for_error = pds.clone(); + async move { + let response = state + .resolver + .xrpc(pds.borrow()) + .download(&GetBlob { + did: did.clone(), + cid: cid.clone(), + }) + .await + .map_err(|e| -> BoxError { + Box::new(internal(format!( + "blob fetch from {pds_for_error} failed: {e}" + ))) + })?; + if !response.status().is_success() { + return Err(Box::new(internal(format!( + "blob fetch from {pds} failed with HTTP {}", + response.status() + ))) as BoxError); } - }) - .try_flatten(); + let bytes = collect_verified_blob(response, expected_size, &cid, &pds) + .await + .map_err(|error| -> BoxError { Box::new(error) })?; + Ok::<_, BoxError>(bytes::Bytes::from(bytes)) + } + }); let body_stream = first_stream.chain(rest_stream); - let filename = download_filename(&row.name); + let filename = download_filename(&name); let content_type = file.mime_type.to_string(); Response::builder() .status(StatusCode::OK) @@ -185,7 +254,7 @@ pub(super) async fn get_part_file( /// Return a file's chunks sorted by offset. The streaming response yields them /// in this order so the client reassembles the original file. -fn ordered_chunks(file: &File) -> Vec<&polymodel_api::space_polymodel::library::Chunk> { +pub(super) fn ordered_chunks(file: &File) -> Vec<&polymodel_api::space_polymodel::library::Chunk> { let mut chunks: Vec<_> = file.chunks.iter().collect(); chunks.sort_by_key(|chunk| chunk.offset); chunks @@ -194,7 +263,7 @@ fn ordered_chunks(file: &File) -> Vec<&polymodel_api::space_polymodel::library:: /// Cheap manifest preflight: offset contiguity, chunk count, per-chunk sanity. /// Does **not** cap total file size — the PDS bounds each blob, and a total cap /// breaks the large-model case chunking exists for. -fn validate_file_manifest(file: &File) -> AppResult<()> { +pub(super) fn validate_file_manifest(file: &File) -> AppResult<()> { if file.size < 0 { return Err(invalid_request("part file size is negative")); } diff --git a/src/appview/ldraw.rs b/src/appview/ldraw.rs new file mode 100644 index 0000000..dcd47a3 --- /dev/null +++ b/src/appview/ldraw.rs @@ -0,0 +1,510 @@ +use axum::Json; +use axum::body::Body; +use axum::extract::{Query, State}; +use axum::http::{StatusCode, header}; +use axum::response::Response; +use chrono::Utc; +use futures::TryStreamExt; +use futures::stream::{self}; +use jacquard::client::Agent; +use jacquard::identity::resolver::IdentityResolver; +use jacquard_axum::oauth::ExtractOAuthSession; +use jacquard_common::types::string::AtUri; +use jacquard_common::xrpc::XrpcExt; +use rand::RngCore; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use super::error::{AppResult, db, internal, invalid_request, not_found, unauthorized}; +use super::state::AppState; +use super::writes::{ExtractSession, authenticated_did}; +use crate::ldraw::resolve::{SqliteResolver, serialize_source_identity}; +use crate::ldraw::verification::{VerificationRecord, VerificationState, claim, complete}; +use crate::ldraw::{CanonicalPath, LdrawResourceResolver}; +use polymodel_renderer_protocol::{ + CompoundLoadPlan, WorkerFetchDescriptor, MAX_COMPOUND_DESCRIPTORS, + MAX_COMPOUND_METADATA_BYTES, MAX_COMPOUND_TOTAL_BYTES, PROTOCOL_VERSION, +}; + +#[derive(Debug, Deserialize)] +pub(super) struct MintRequest { + pub project: AtUri, + pub path: String, +} + +#[derive(Debug, Serialize)] +pub(super) struct MintResponse { + pub token: String, + #[serde(rename = "expiresAt")] + pub expires_at: String, +} + +#[derive(Debug, Deserialize)] +pub(super) struct FetchQuery { + pub token: String, +} + +#[derive(Debug, Deserialize)] +pub(super) struct CompoundRequest { + pub project: AtUri, + pub path: String, +} + +#[derive(Debug, Serialize)] +pub(super) struct CompoundResponse { + pub plan: CompoundLoadPlan, +} + +pub(super) async fn compound_resources( + State(state): State, + ExtractOAuthSession(session): ExtractSession, + Json(request): Json, +) -> AppResult> { + let agent = Agent::from(session); + let principal = authenticated_did(&agent).await?; + let path = CanonicalPath::parse(&request.path) + .map_err(|diagnostic| invalid_request(format!("invalid LDraw path: {diagnostic:?}")))?; + let resolver = SqliteResolver::new(state.pool.clone()); + let primary = ensure_verified(&state, &resolver, Some(&principal), &request.project, &path) + .await + .map_err(resolve_error)?; + let mut resources = vec![primary]; + let mut seen = std::collections::HashSet::new(); + seen.insert(path.as_str().to_owned()); + let mut index = 0; + while index < resources.len() { + if resources.len() > MAX_COMPOUND_DESCRIPTORS { + return Err(invalid_request("compound resource descriptor bound exceeded")); + } + let current = &resources[index]; + let rows = sqlx::query!( + "SELECT canonical_path, ordinal FROM ldraw_manifest_files WHERE resource_uri = ? ORDER BY ordinal ASC", + current.resource.uri.as_ref(), + ) + .fetch_all(&state.pool) + .await + .map_err(|error| internal(format!("LDraw manifest enumeration failed: {error}")))?; + for row in rows { + let child_path = CanonicalPath::parse(&row.canonical_path) + .map_err(|_| not_found())?; + if seen.insert(child_path.as_str().to_owned()) { + let child = ensure_verified( + &state, + &resolver, + Some(&principal), + &request.project, + &child_path, + ) + .await + .map_err(resolve_error)?; + resources.push(child); + } + } + index += 1; + } + let mut descriptors = Vec::with_capacity(resources.len()); + let mut total_bytes = 0_u64; + let mut metadata_bytes = path.as_str().len(); + let mut route_tokens = Vec::with_capacity(resources.len()); + for resource in resources { + let token = issue_one_route(&state, &principal, &request.project, &resource).await?; + let descriptor = WorkerFetchDescriptor { + key: resource.path.to_string(), + route: format!("/xrpc/space.polymodel.library.getLdrawResource?token={token}"), + path: resource.path.to_string(), + root: resource.root.as_str().to_owned(), + byte_length: u64::try_from(resource.byte_length) + .map_err(|_| invalid_request("negative LDraw resource length"))?, + sha256: resource.sha256, + mime_type: resource.mime_type, + target_cids: resource.target.blob_cids, + }; + descriptor + .validate() + .map_err(|error| invalid_request(format!("invalid compound descriptor: {error}")))?; + total_bytes = total_bytes + .checked_add(descriptor.byte_length) + .ok_or_else(|| invalid_request("compound byte bound overflow"))?; + metadata_bytes = metadata_bytes + .checked_add(descriptor.key.len() + descriptor.route.len() + descriptor.root.len()) + .ok_or_else(|| invalid_request("compound metadata bound overflow"))?; + route_tokens.push(descriptor); + } + if total_bytes > MAX_COMPOUND_TOTAL_BYTES || metadata_bytes > MAX_COMPOUND_METADATA_BYTES { + return Err(invalid_request("compound plan bounds exceeded")); + } + let plan = CompoundLoadPlan { + version: PROTOCOL_VERSION, + primary: path.to_string(), + descriptors: route_tokens, + }; + plan.validate() + .map_err(|error| invalid_request(format!("invalid compound plan: {error:?}")))?; + plan.encoded_size() + .map_err(|error| invalid_request(format!("compound plan encoding failed: {error:?}")))?; + Ok(Json(plan)) +} + +async fn issue_one_route( + state: &AppState, + principal: &jacquard_common::types::string::Did, + project: &AtUri, + resource: &crate::ldraw::resolve::ResolvedResource, +) -> AppResult { + let mut token_bytes = [0_u8; 32]; + rand::rng().fill_bytes(&mut token_bytes); + let digest = Sha256::digest(token_bytes); + let digest_bytes: &[u8] = digest.as_ref(); + let now = Utc::now().timestamp_nanos_opt().unwrap_or_default(); + let expires_at = now + 300_000_000_000_i64; + db(sqlx::query!( + "INSERT INTO ldraw_routes (token_digest, principal_did, project_uri, resource_uri, source_identity, root, canonical_path, target_identity, expires_at, revoked_at, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?)", + digest_bytes, + principal.as_ref(), + project.as_ref(), + resource.resource.uri.as_ref(), + serialize_source_identity(&resource.cache_key.source), + resource.root.as_str(), + resource.path.as_str(), + serde_json::to_string(&resource.target.blob_cids).map_err(|error| internal(format!("target identity serialization failed: {error}")))?, + expires_at, + now, + ).execute(&state.pool).await)?; + Ok(encode_token(&token_bytes)) +} + +pub(super) async fn mint( + State(state): State, + ExtractOAuthSession(session): ExtractSession, + Json(request): Json, +) -> AppResult> { + let agent = Agent::from(session); + let principal = authenticated_did(&agent).await?; + let path = CanonicalPath::parse(&request.path) + .map_err(|diagnostic| invalid_request(format!("invalid LDraw path: {diagnostic:?}")))?; + let resolver = SqliteResolver::new(state.pool.clone()); + let resource = ensure_verified(&state, &resolver, Some(&principal), &request.project, &path) + .await + .map_err(resolve_error)?; + + let mut token_bytes = [0_u8; 32]; + rand::rng().fill_bytes(&mut token_bytes); + let digest = Sha256::digest(token_bytes); + let digest_bytes: &[u8] = digest.as_ref(); + let now = Utc::now().timestamp_nanos_opt().unwrap_or_default(); + let expires_at = now + 300_000_000_000_i64; + db(sqlx::query!( + "INSERT INTO ldraw_routes (token_digest, principal_did, project_uri, resource_uri, source_identity, root, canonical_path, target_identity, expires_at, revoked_at, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?)", + digest_bytes, + principal.as_ref(), + request.project.as_ref(), + resource.resource.uri.as_ref(), + serialize_source_identity(&resource.cache_key.source), + resource.root.as_str(), + resource.path.as_str(), + serde_json::to_string(&resource.target.blob_cids) + .map_err(|e| internal(format!("LDraw target identity serialization failed: {e}")))?, + expires_at, + now, + ).execute(&state.pool).await)?; + + Ok(Json(MintResponse { + token: encode_token(&token_bytes), + expires_at: chrono::DateTime::from_timestamp_nanos(expires_at).to_rfc3339(), + })) +} + +pub(super) async fn fetch( + State(state): State, + ExtractOAuthSession(session): ExtractSession, + Query(query): Query, +) -> AppResult { + let agent = Agent::from(session); + let principal = authenticated_did(&agent).await.map_err(|_| not_found())?; + let token = + decode_token(&query.token).ok_or_else(|| invalid_request("invalid LDraw route token"))?; + let digest = Sha256::digest(token); + let digest_bytes: &[u8] = digest.as_ref(); + let now = Utc::now().timestamp_nanos_opt().unwrap_or_default(); + let row = sqlx::query!( + "SELECT principal_did, project_uri, resource_uri, source_identity, root, canonical_path, target_identity FROM ldraw_routes WHERE token_digest = ? AND revoked_at IS NULL AND expires_at > ? AND principal_did = ?", + digest_bytes, + now, + principal.as_ref(), + ).fetch_optional(&state.pool).await.map_err(|_| not_found())?.ok_or_else(not_found)?; + let path = CanonicalPath::parse(&row.canonical_path).map_err(|_| not_found())?; + let project = AtUri::new_owned(&row.project_uri).map_err(|_| not_found())?; + let row_principal = row + .principal_did + .as_deref() + .and_then(|value| jacquard_common::types::string::Did::new_owned(value).ok()) + .ok_or_else(not_found)?; + if row_principal != principal { + return Err(not_found()); + } + let resolver = SqliteResolver::new(state.pool.clone()); + let resource = resolver + .resolve(Some(&principal), &project, &path) + .await + .map_err(resolve_error)?; + if resource.resource.uri.as_ref() != row.resource_uri + || serialize_source_identity(&resource.cache_key.source) != row.source_identity + || resource.root.as_str() != row.root + || serde_json::to_string(&resource.target.blob_cids).unwrap_or_default() + != row.target_identity + { + return Err(not_found()); + } + let mut tx = state.pool.begin().await.map_err(|_| not_found())?; + let consumed = sqlx::query!( + "UPDATE ldraw_routes SET revoked_at = ? WHERE token_digest = ? AND principal_did = ? AND revoked_at IS NULL AND expires_at > ?", + now, + digest_bytes, + principal.as_ref(), + now, + ) + .execute(&mut *tx) + .await + .map_err(|_| not_found())?; + if consumed.rows_affected() != 1 { + return Err(not_found()); + } + tx.commit().await.map_err(|_| not_found())?; + let bytes = fetch_verified_bytes(&state, &resource).await?; + Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, resource.mime_type.as_str()) + .header(header::CONTENT_LENGTH, bytes.len()) + .body(Body::from_stream(stream::once(async move { + Ok::<_, std::convert::Infallible>(bytes::Bytes::from(bytes)) + }))) + .map_err(|error| internal(format!("LDraw response build failed: {error}"))) +} + +async fn ensure_verified( + state: &AppState, + resolver: &SqliteResolver, + principal: Option<&jacquard_common::types::string::Did>, + project: &AtUri, + path: &CanonicalPath, +) -> Result { + let resource = resolver.candidate(principal, project, path).await?; + let source = serialize_source_identity(&resource.cache_key.source); + let row = sqlx::query!( + "SELECT state, attempt_token, lease_expires_at, expired_attempts, revoked_at FROM ldraw_verification WHERE resource_uri = ?", + resource.resource.uri.as_ref(), + ).fetch_optional(&state.pool).await.map_err(|_| crate::ldraw::ResolveError::Unavailable { diagnostic: crate::ldraw::resolve::AvailabilityDiagnostic::SourceUnavailable })?.ok_or(crate::ldraw::ResolveError::Unverified)?; + if row.state == "verified" { + return Ok(resource); + } + if row.state == "rejected" { + return Err(crate::ldraw::ResolveError::Rejected { + diagnostic: crate::ldraw::resolve::DiagnosticCode::InvalidManifest, + }); + } + let now = now_nanos(); + let mut token = [0_u8; 32]; + rand::rng().fill_bytes(&mut token); + let attempt = encode_token(&token); + let record = VerificationRecord { + state: match row.state.as_str() { + "unverified" => VerificationState::Unverified, + "verifying" => VerificationState::Verifying, + _ => return Err(crate::ldraw::ResolveError::SourceChanged), + }, + source_identity: source.clone(), + attempt_token: row.attempt_token.clone(), + lease_expires_at: row.lease_expires_at, + expired_attempts: row.expired_attempts.try_into().unwrap_or(u32::MAX), + diagnostic: None, + revoked_at: row.revoked_at, + }; + let lease = now + crate::ldraw::verification::LEASE_NANOS; + let claim = claim(&record, now, attempt.clone()); + match &claim { + crate::ldraw::verification::Transition::AlreadyVerifying => { + return Err(crate::ldraw::ResolveError::Verifying); + } + crate::ldraw::verification::Transition::Rejected(_reason) + if record.revoked_at.is_some() => + { + return Err(crate::ldraw::ResolveError::Rejected { + diagnostic: crate::ldraw::resolve::DiagnosticCode::InvalidManifest, + }); + } + crate::ldraw::verification::Transition::Rejected(reason) => { + let rejected = sqlx::query!( + "UPDATE ldraw_verification SET state = 'rejected', diagnostic = ?, attempt_token = NULL, lease_expires_at = NULL, updated_at = ? WHERE resource_uri = ? AND source_identity = ? AND state = 'verifying' AND attempt_token = ? AND (lease_expires_at IS NULL OR lease_expires_at <= ?)", + reason, now, resource.resource.uri.as_ref(), source, record.attempt_token, now, + ).execute(&state.pool).await.map_err(|_| crate::ldraw::ResolveError::Unavailable { diagnostic: crate::ldraw::resolve::AvailabilityDiagnostic::SourceUnavailable })?; + return if rejected.rows_affected() == 1 { + Err(crate::ldraw::ResolveError::Rejected { + diagnostic: crate::ldraw::resolve::DiagnosticCode::InvalidManifest, + }) + } else { + Err(crate::ldraw::ResolveError::SourceChanged) + }; + } + crate::ldraw::verification::Transition::Claim { .. } => {} + crate::ldraw::verification::Transition::Verified => { + return Ok(resource); + } + crate::ldraw::verification::Transition::Reset => { + return Err(crate::ldraw::ResolveError::SourceChanged); + } + } + let claimed = sqlx::query!( + "UPDATE ldraw_verification SET state = 'verifying', attempt_token = ?, lease_expires_at = ?, expired_attempts = expired_attempts + CASE WHEN state = 'verifying' AND (lease_expires_at IS NULL OR lease_expires_at <= ?) THEN 1 ELSE 0 END, updated_at = ? WHERE resource_uri = ? AND source_identity = ? AND expired_attempts < ? AND (state = 'unverified' OR (state = 'verifying' AND (lease_expires_at IS NULL OR lease_expires_at <= ?)))", + attempt, lease, now, now, resource.resource.uri.as_ref(), source, crate::ldraw::verification::MAX_EXPIRED_ATTEMPTS as i64, now, + ).execute(&state.pool).await.map_err(|_| crate::ldraw::ResolveError::Unavailable { diagnostic: crate::ldraw::resolve::AvailabilityDiagnostic::SourceUnavailable })?; + if claimed.rows_affected() != 1 { + return Err(crate::ldraw::ResolveError::Verifying); + } + let active_record = VerificationRecord { + state: VerificationState::Verifying, + source_identity: source.clone(), + attempt_token: Some(attempt.clone()), + lease_expires_at: Some(lease), + expired_attempts: record.expired_attempts + + u32::from(record.state == VerificationState::Verifying), + diagnostic: None, + revoked_at: record.revoked_at, + }; + let verification = fetch_verified_bytes(state, &resource).await; + match verification { + Ok(_) => { + complete(&active_record, &attempt, &source, now_nanos()) + .map_err(|_| crate::ldraw::ResolveError::SourceChanged)?; + let completed = sqlx::query!( + "UPDATE ldraw_verification SET state = 'verified', attempt_token = NULL, lease_expires_at = NULL, diagnostic = NULL, updated_at = ? WHERE resource_uri = ? AND source_identity = ? AND state = 'verifying' AND attempt_token = ? AND lease_expires_at > ?", + now_nanos(), resource.resource.uri.as_ref(), source, attempt, now_nanos(), + ).execute(&state.pool).await.map_err(|_| crate::ldraw::ResolveError::Unavailable { diagnostic: crate::ldraw::resolve::AvailabilityDiagnostic::SourceUnavailable })?; + if completed.rows_affected() == 1 { + Ok(resource) + } else { + Err(crate::ldraw::ResolveError::SourceChanged) + } + } + Err(_) => { + let rejection_now = now_nanos(); + let rejected = sqlx::query!( + "UPDATE ldraw_verification SET state = 'rejected', diagnostic = 'integrity-mismatch-v1', attempt_token = NULL, lease_expires_at = NULL, updated_at = ? WHERE resource_uri = ? AND source_identity = ? AND state = 'verifying' AND attempt_token = ? AND lease_expires_at > ?", + rejection_now, resource.resource.uri.as_ref(), source, attempt, rejection_now, + ).execute(&state.pool).await.map_err(|_| crate::ldraw::ResolveError::Unavailable { diagnostic: crate::ldraw::resolve::AvailabilityDiagnostic::SourceUnavailable })?; + if rejected.rows_affected() == 1 { + Err(crate::ldraw::ResolveError::Rejected { + diagnostic: crate::ldraw::resolve::DiagnosticCode::InvalidManifest, + }) + } else { + Err(crate::ldraw::ResolveError::SourceChanged) + } + } + } +} + +async fn fetch_verified_bytes( + state: &AppState, + resource: &crate::ldraw::resolve::ResolvedResource, +) -> AppResult> { + let did = + super::actor::resolve_actor(&state, &resource.source.resource_uri.authority().convert()) + .await?; + let did_doc_response = state + .resolver + .resolve_did_doc(&did) + .await + .map_err(|e| internal(format!("DID document resolution failed: {e}")))?; + let did_doc = did_doc_response + .parse() + .map_err(|e| internal(format!("DID document parse failed: {e}")))?; + let endpoint = did_doc + .pds_endpoint() + .ok_or_else(|| internal("DID document does not advertise a PDS endpoint"))? + .to_owned(); + let mut bytes = Vec::new(); + for value in &resource.source.blob_cids { + let cid = jacquard_common::types::string::Cid::new_owned(value.as_bytes()) + .map_err(|e| internal(format!("invalid LDraw blob CID: {e}")))?; + let response = state + .resolver + .xrpc(endpoint.borrow()) + .download(&polymodel_api::com_atproto::sync::get_blob::GetBlob { + did: did.clone(), + cid, + }) + .await + .map_err(|e| internal(format!("blob fetch failed: {e}")))?; + if !response.status().is_success() { + return Err(internal(format!( + "blob fetch failed with HTTP {}", + response.status() + ))); + } + let (_, body) = response.into_parts(); + let chunked = body + .into_inner() + .try_fold(Vec::new(), |mut bytes, chunk| async move { + bytes.extend_from_slice(&chunk); + Ok::<_, _>(bytes) + }) + .await + .map_err(|e| internal(format!("blob stream failed: {e}")))?; + bytes.extend_from_slice(&chunked); + } + if bytes.len() as i64 != resource.byte_length { + return Err(internal("LDraw byte length verification failed")); + } + if sha2::Sha256::digest(&bytes).as_slice() != resource.sha256.as_slice() { + return Err(internal("LDraw SHA-256 verification failed")); + } + Ok(bytes) +} + +fn now_nanos() -> i64 { + Utc::now().timestamp_nanos_opt().unwrap_or_default() +} + +fn resolve_error(error: crate::ldraw::ResolveError) -> super::error::AppError { + match error { + crate::ldraw::ResolveError::Unauthorized => { + unauthorized("LDraw resource is not authorized") + } + crate::ldraw::ResolveError::NotFound + | crate::ldraw::ResolveError::Revoked + | crate::ldraw::ResolveError::Rejected { .. } + | crate::ldraw::ResolveError::Unverified + | crate::ldraw::ResolveError::Verifying + | crate::ldraw::ResolveError::SourceChanged + | crate::ldraw::ResolveError::InvalidPath { .. } + | crate::ldraw::ResolveError::Collision { .. } + | crate::ldraw::ResolveError::Integrity { .. } => not_found(), + crate::ldraw::ResolveError::Unavailable { .. } => internal("LDraw resolver unavailable"), + } +} + +fn encode_token(token: &[u8; 32]) -> String { + token.iter().map(|byte| format!("{byte:02x}")).collect() +} + +fn decode_token(value: &str) -> Option> { + if value.len() != 64 || !value.is_ascii() { + return None; + } + (0..value.len()) + .step_by(2) + .map(|index| u8::from_str_radix(&value[index..index + 2], 16).ok()) + .collect() +} + +#[cfg(test)] +mod tests { + use super::{decode_token, encode_token}; + + #[test] + fn route_tokens_are_fixed_width_hex() { + let token = [7_u8; 32]; + let encoded = encode_token(&token); + assert_eq!(decode_token(&encoded), Some(token.to_vec())); + assert!(decode_token("short").is_none()); + } +} diff --git a/src/appview/mod.rs b/src/appview/mod.rs index 4a95f40..932871a 100644 --- a/src/appview/mod.rs +++ b/src/appview/mod.rs @@ -25,9 +25,11 @@ mod downloads; mod drafts; mod graph; mod library; +mod ldraw; mod proxy; mod writes; + #[cfg(test)] mod tests; @@ -123,6 +125,18 @@ pub fn router() -> Router { axum::routing::get(actor::get_session), ) .merge(GetPartFileRequest::into_router(downloads::get_part_file)) + .route( + "/app/ldraw/compound-resources", + axum::routing::post(ldraw::compound_resources), + ) + .route( + "/app/ldraw/resources", + axum::routing::post(ldraw::mint), + ) + .route( + "/xrpc/space.polymodel.library.getLdrawResource", + axum::routing::get(ldraw::fetch), + ) // PM-43 app-internal draft store + image upload. Not federated lexicons, // so these use a plain `/app/*` namespace (cookie-authenticated, // same-origin) rather than `/xrpc/space.polymodel.*`. diff --git a/src/appview/state.rs b/src/appview/state.rs index a46eb64..64f3015 100644 --- a/src/appview/state.rs +++ b/src/appview/state.rs @@ -13,8 +13,6 @@ //! the orphan rule); it is cheaply `Clone` (pool/resolver/oauth are //! `Arc`-backed). -use std::sync::Arc; - use axum::extract::FromRef; use axum_extra::extract::cookie::Key; use jacquard::client::BasicClient; @@ -22,8 +20,8 @@ use jacquard::identity::PublicResolver; use jacquard::oauth::client::OAuthClient; use jacquard_axum::oauth::{OAuthWebConfig, OAuthWebState}; use sqlx::SqlitePool; +use std::sync::Arc; use tokio::sync::Mutex; - use crate::oauth::{OAuthBootstrap, SqliteAuthStore}; /// State shared by all XRPC read handlers and the OAuth routes. diff --git a/src/ldraw/mod.rs b/src/ldraw/mod.rs new file mode 100644 index 0000000..cc9a429 --- /dev/null +++ b/src/ldraw/mod.rs @@ -0,0 +1,8 @@ +//! PM-76 LDraw resource identity, canonical paths, verification, and resolver API. + +pub mod paths; +pub mod resolve; +pub mod verification; + +pub use paths::{CanonicalPath, PathDiagnostic, RootId}; +pub use resolve::{CacheKey, LdrawResourceResolver, ResolveError, ResolvedResource}; diff --git a/src/ldraw/paths.rs b/src/ldraw/paths.rs new file mode 100644 index 0000000..f051b10 --- /dev/null +++ b/src/ldraw/paths.rs @@ -0,0 +1,181 @@ +use std::fmt; + +use unicode_normalization::UnicodeNormalization; + +/// A validated, NFC-preserving virtual filename. It is never interpreted as a URL. +#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct CanonicalPath(String); + +impl CanonicalPath { + pub fn parse(input: &str) -> Result { + if input.is_empty() || input.starts_with('/') || input.ends_with('/') { + return Err(PathDiagnostic::Slash); + } + if input.contains('%') { + return Err(PathDiagnostic::Percent); + } + if input.contains('\\') { + return Err(PathDiagnostic::Backslash); + } + if input.contains('?') || input.contains('#') { + return Err(PathDiagnostic::QueryOrFragment); + } + if input.chars().any(|c| c.is_control() || c == ':') { + return Err(PathDiagnostic::ControlOrColon); + } + if input.nfc().collect::() != input { + return Err(PathDiagnostic::NotNfc); + } + for segment in input.split('/') { + if segment.is_empty() || segment == "." || segment == ".." { + return Err(PathDiagnostic::Segment); + } + } + Ok(Self(input.to_owned())) + } + + pub fn as_str(&self) -> &str { + &self.0 + } + + pub fn segments(&self) -> impl Iterator { + self.0.split('/') + } +} + +impl fmt::Display for CanonicalPath { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub enum RootId { + Mpd, + Manifest, + Models, + Parts, + P, + OfficialParts, + OfficialP, +} + +impl RootId { + pub fn as_str(self) -> &'static str { + match self { + Self::Mpd => "mpd", + Self::Manifest => "manifest", + Self::Models => "models", + Self::Parts => "parts", + Self::P => "p", + Self::OfficialParts => "official-parts", + Self::OfficialP => "official-p", + } + } + + pub fn case_insensitive(self) -> bool { + !matches!(self, Self::Mpd | Self::Manifest) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum PathDiagnostic { + Slash, + Percent, + Backslash, + QueryOrFragment, + ControlOrColon, + NotNfc, + Segment, +} + +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct LookupEntry { + pub root: RootId, + pub path: CanonicalPath, + pub target: String, +} + +/// Resolve according to PM-75's fixed source precedence. +/// The maps contain already validated entries and only perform filename lookup. +pub fn resolve_precedence( + path: &CanonicalPath, + mpd: &[LookupEntry], + manifest: &[LookupEntry], + uploaded: &[LookupEntry], + official: &[LookupEntry], +) -> Option { + exact(path, mpd) + .or_else(|| exact(path, manifest)) + .or_else(|| roots(path, uploaded, &[RootId::Models, RootId::Parts, RootId::P])) + .or_else(|| roots(path, official, &[RootId::OfficialParts, RootId::OfficialP])) +} + +fn exact(path: &CanonicalPath, entries: &[LookupEntry]) -> Option { + entries.iter().find(|e| e.path == *path).cloned() +} + +fn roots(path: &CanonicalPath, entries: &[LookupEntry], roots: &[RootId]) -> Option { + roots.iter().find_map(|root| { + entries + .iter() + .filter(|e| e.root == *root) + .find(|e| { + e.path == *path + || (root.case_insensitive() + && e.path.as_str().eq_ignore_ascii_case(path.as_str())) + }) + .cloned() + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_unsafe_and_non_nfc_paths() { + for value in [ + "", + "/a", + "a/", + "a//b", + "a/../b", + "a/./b", + "a%2Fb", + "a\\b", + "a:b", + "a?b", + "a#b", + "a\u{0000}b", + ] { + assert!(CanonicalPath::parse(value).is_err(), "{value:?}"); + } + assert!(CanonicalPath::parse("cafe\u{301}").is_err()); + assert_eq!(CanonicalPath::parse("café").unwrap().as_str(), "café"); + } + + #[test] + fn precedence_and_ascii_case_policy_are_stable() { + let path = CanonicalPath::parse("parts/3001.dat").unwrap(); + let entry = |root, target: &str| LookupEntry { + root, + path: path.clone(), + target: target.into(), + }; + let found = + resolve_precedence(&path, &[], &[], &[entry(RootId::Models, "model")], &[]).unwrap(); + assert_eq!(found.target, "model"); + let upper = CanonicalPath::parse("PARTS/3001.DAT").unwrap(); + assert!( + resolve_precedence(&upper, &[], &[], &[entry(RootId::Models, "model")], &[]).is_some() + ); + let mpd = entry(RootId::Mpd, "mpd"); + assert_eq!( + resolve_precedence(&path, &[mpd], &[], &[], &[]) + .unwrap() + .target, + "mpd" + ); + } +} diff --git a/src/ldraw/resolve.rs b/src/ldraw/resolve.rs new file mode 100644 index 0000000..c698e2e --- /dev/null +++ b/src/ldraw/resolve.rs @@ -0,0 +1,320 @@ +use async_trait::async_trait; +use jacquard_common::types::string::{AtUri, Did}; +use sqlx::SqlitePool; +use std::collections::HashSet; + +use super::paths::{CanonicalPath, LookupEntry, PathDiagnostic, RootId, resolve_precedence}; + +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct SourceIdentity { + pub resource: AtUri, + pub content_hash: Vec, +} + +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct TargetIdentity { + pub blob_cids: Vec, +} + +/// The sole cache and route binding key shared across PM-76 and PM-80. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct CacheKey { + pub principal: Option, + pub project: AtUri, + pub source: SourceIdentity, + pub root: RootId, + pub path: CanonicalPath, + pub target: TargetIdentity, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ResourceIdentity { + pub uri: AtUri, + pub content_hash: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ByteSource { + pub resource_uri: AtUri, + pub blob_cids: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ResolvedResource { + pub cache_key: CacheKey, + pub resource: ResourceIdentity, + pub root: RootId, + pub path: CanonicalPath, + pub target: TargetIdentity, + pub mime_type: String, + pub byte_length: i64, + pub sha256: Vec, + pub source: ByteSource, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum DiagnosticCode { + InvalidManifest, + ExpiredAttempts, + Revoked, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum CollisionDiagnostic { + DuplicatePath, + FoldedPath, + OutOfRoot, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum IntegrityDiagnostic { + LengthMismatch, + DigestMismatch, + BlobCidMismatch, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum AvailabilityDiagnostic { + SourceUnavailable, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ResolveError { + NotFound, + Unauthorized, + Unverified, + Verifying, + Rejected { diagnostic: DiagnosticCode }, + Revoked, + SourceChanged, + InvalidPath { diagnostic: PathDiagnostic }, + Collision { diagnostic: CollisionDiagnostic }, + Integrity { diagnostic: IntegrityDiagnostic }, + Unavailable { diagnostic: AvailabilityDiagnostic }, +} + +#[async_trait] +pub trait LdrawResourceResolver: Send + Sync { + async fn resolve( + &self, + principal: Option<&Did>, + project: &AtUri, + path: &CanonicalPath, + ) -> Result; +} + +#[derive(Clone)] +pub struct SqliteResolver { + pool: SqlitePool, +} + +impl SqliteResolver { + pub fn new(pool: SqlitePool) -> Self { + Self { pool } + } + + /// Resolve the canonical candidate without requiring verification. This is + /// the only lookup path used by the lazy verifier and the verified resolver. + pub async fn candidate( + &self, + principal: Option<&Did>, + project: &AtUri, + path: &CanonicalPath, + ) -> Result { + let rows = sqlx::query!( + r#"SELECT resource_uri, owner_did, sha256, ordered_blob_cids, root, canonical_path, mime_type, byte_length + FROM ldraw_resources + WHERE project_uri = ? AND revoked_at IS NULL + AND (owner_did = ? OR owner_did = 'public')"#, + project.as_ref(), principal.map(Did::as_ref), + ).fetch_all(&self.pool).await.map_err(|_| ResolveError::Unavailable { diagnostic: AvailabilityDiagnostic::SourceUnavailable })?; + let mut entries = Vec::with_capacity(rows.len()); + let mut by_uri = std::collections::HashMap::new(); + for row in rows { + let Some(root) = root_id(&row.root) else { + continue; + }; + let Ok(candidate_path) = CanonicalPath::parse(&row.canonical_path) else { + continue; + }; + let Some(uri) = row.resource_uri.clone() else { + continue; + }; + entries.push(LookupEntry { + root, + path: candidate_path, + target: uri.clone(), + }); + by_uri.insert(uri, row); + } + let folded = entries + .iter() + .filter(|entry| entry.root.case_insensitive()) + .fold(HashSet::new(), |mut set, entry| { + set.insert((entry.root, entry.path.as_str().to_ascii_lowercase())); + set + }); + if folded.len() + != entries + .iter() + .filter(|entry| entry.root.case_insensitive()) + .count() + { + return Err(ResolveError::Collision { + diagnostic: CollisionDiagnostic::FoldedPath, + }); + } + let mut mpd = Vec::new(); + let mut manifest = Vec::new(); + let mut uploaded = Vec::new(); + let mut official = Vec::new(); + for entry in entries { + match entry.root { + RootId::Mpd => mpd.push(entry), + RootId::Manifest => manifest.push(entry), + RootId::Models | RootId::Parts | RootId::P => uploaded.push(entry), + RootId::OfficialParts | RootId::OfficialP => official.push(entry), + } + } + let chosen = resolve_precedence(path, &mpd, &manifest, &uploaded, &official) + .ok_or(ResolveError::NotFound)?; + let row = by_uri + .get(&chosen.target) + .ok_or(ResolveError::SourceChanged)?; + let resource = AtUri::new_owned(&chosen.target).map_err(|_| ResolveError::SourceChanged)?; + let blobs = parse_blob_cids(&row.ordered_blob_cids); + let target = TargetIdentity { + blob_cids: blobs.clone(), + }; + let content_hash = source_hash(&blobs, &row.sha256, row.byte_length); + let cache_key = CacheKey { + principal: principal.cloned(), + project: project.clone(), + source: SourceIdentity { + resource: resource.clone(), + content_hash: content_hash.clone(), + }, + root: chosen.root, + path: chosen.path.clone(), + target: target.clone(), + }; + Ok(ResolvedResource { + cache_key, + resource: ResourceIdentity { + uri: resource.clone(), + content_hash, + }, + root: chosen.root, + path: chosen.path, + target, + mime_type: row.mime_type.clone(), + byte_length: row.byte_length, + sha256: row.sha256.clone(), + source: ByteSource { + resource_uri: resource, + blob_cids: blobs, + }, + }) + } +} + +#[async_trait] +impl LdrawResourceResolver for SqliteResolver { + async fn resolve( + &self, + principal: Option<&Did>, + project: &AtUri, + path: &CanonicalPath, + ) -> Result { + let resource = self.candidate(principal, project, path).await?; + let verification = sqlx::query!( + "SELECT state, revoked_at FROM ldraw_verification WHERE resource_uri = ?", + resource.resource.uri.as_ref() + ) + .fetch_optional(&self.pool) + .await + .map_err(|_| ResolveError::Unavailable { + diagnostic: AvailabilityDiagnostic::SourceUnavailable, + })? + .ok_or(ResolveError::Unverified)?; + if verification.revoked_at.is_some() { + return Err(ResolveError::Revoked); + } + match verification.state.as_str() { + "verified" => Ok(resource), + "verifying" => Err(ResolveError::Verifying), + "rejected" => Err(ResolveError::Rejected { + diagnostic: DiagnosticCode::InvalidManifest, + }), + _ => Err(ResolveError::Unverified), + } + } +} + +/// Canonical serialization shared by projection, route bindings, and resolver. +pub fn serialize_source_identity(source: &SourceIdentity) -> String { + format!( + "{}|{}", + source.resource.as_ref(), + hex_bytes(&source.content_hash) + ) +} + +fn hex_bytes(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} + +fn parse_blob_cids(value: &str) -> Vec { + serde_json::from_str(value).unwrap_or_default() +} + +fn source_hash(cids: &[String], sha256: &[u8], size: i64) -> Vec { + let mut value = serde_json::to_vec(cids).unwrap_or_default(); + value.extend_from_slice(sha256); + value.extend_from_slice(&size.to_be_bytes()); + value +} + +fn root_id(value: &str) -> Option { + Some(match value { + "mpd" => RootId::Mpd, + "manifest" => RootId::Manifest, + "models" => RootId::Models, + "parts" => RootId::Parts, + "p" => RootId::P, + "official-parts" => RootId::OfficialParts, + "official-p" => RootId::OfficialP, + _ => return None, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_cache_dimension_changes_identity() { + let project = AtUri::new_owned("at://did:plc:a/space.polymodel.library.thing/p").unwrap(); + let path = CanonicalPath::parse("parts/3001.dat").unwrap(); + let did = Did::new_owned("did:plc:a").unwrap(); + let base = CacheKey { + principal: Some(did.clone()), + project: project.clone(), + source: SourceIdentity { + resource: project.clone(), + content_hash: vec![1], + }, + root: RootId::Parts, + path: path.clone(), + target: TargetIdentity { + blob_cids: vec!["bafk".into()], + }, + }; + let mut changed = base.clone(); + changed.principal = None; + assert_ne!(base, changed); + changed = base.clone(); + changed.path = CanonicalPath::parse("parts/3002.dat").unwrap(); + assert_ne!(base, changed); + } +} diff --git a/src/ldraw/verification.rs b/src/ldraw/verification.rs new file mode 100644 index 0000000..ff92b67 --- /dev/null +++ b/src/ldraw/verification.rs @@ -0,0 +1,167 @@ +use std::fmt; + +pub const LEASE_NANOS: i64 = 60_000_000_000; +pub const MAX_EXPIRED_ATTEMPTS: u32 = 3; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum VerificationState { + Unverified, + Verifying, + Verified, + Rejected, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct VerificationRecord { + pub state: VerificationState, + pub source_identity: String, + pub attempt_token: Option, + pub lease_expires_at: Option, + pub expired_attempts: u32, + pub diagnostic: Option, + pub revoked_at: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum Transition { + Claim { + token: String, + lease_expires_at: i64, + }, + AlreadyVerifying, + Verified, + Rejected(String), + Reset, +} + +#[derive(Debug, thiserror::Error, Eq, PartialEq)] +pub enum VerificationError { + #[error("verification attempt is not active")] + InvalidAttempt, + #[error("verification source changed")] + SourceChanged, + #[error("verification lease expired")] + Expired, + #[error("verification is revoked")] + Revoked, +} + +pub fn claim(record: &VerificationRecord, now: i64, token: impl Into) -> Transition { + if record.revoked_at.is_some() { + return Transition::Rejected("revoked".into()); + } + if record.state == VerificationState::Verifying { + if record.lease_expires_at.is_some_and(|expiry| expiry > now) { + return Transition::AlreadyVerifying; + } + if record.expired_attempts + 1 >= MAX_EXPIRED_ATTEMPTS { + return Transition::Rejected("verification-expired-attempt-limit-v1".into()); + } + } + Transition::Claim { + token: token.into(), + lease_expires_at: now + LEASE_NANOS, + } +} + +pub fn complete( + record: &VerificationRecord, + token: &str, + source_identity: &str, + now: i64, +) -> Result { + if record.revoked_at.is_some() { + return Err(VerificationError::Revoked); + } + if record.state != VerificationState::Verifying + || record.attempt_token.as_deref() != Some(token) + { + return Err(VerificationError::InvalidAttempt); + } + if record.source_identity != source_identity { + return Err(VerificationError::SourceChanged); + } + if record.lease_expires_at.is_none_or(|expiry| expiry <= now) { + return Err(VerificationError::Expired); + } + Ok(Transition::Verified) +} + +pub fn source_changed( + old: &VerificationRecord, + source_identity: impl Into, +) -> VerificationRecord { + VerificationRecord { + state: VerificationState::Unverified, + source_identity: source_identity.into(), + attempt_token: None, + lease_expires_at: None, + expired_attempts: 0, + diagnostic: None, + revoked_at: old.revoked_at, + } +} + +impl fmt::Display for VerificationState { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::Unverified => "unverified", + Self::Verifying => "verifying", + Self::Verified => "verified", + Self::Rejected => "rejected", + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn record() -> VerificationRecord { + VerificationRecord { + state: VerificationState::Unverified, + source_identity: "source-a".into(), + attempt_token: None, + lease_expires_at: None, + expired_attempts: 0, + diagnostic: None, + revoked_at: None, + } + } + + #[test] + fn claims_coalesce_and_completion_is_cas_guarded() { + let mut current = record(); + let Transition::Claim { + token, + lease_expires_at, + } = claim(¤t, 10, "one") + else { + panic!() + }; + current.state = VerificationState::Verifying; + current.attempt_token = Some(token.clone()); + current.lease_expires_at = Some(lease_expires_at); + assert_eq!(claim(¤t, 11, "two"), Transition::AlreadyVerifying); + assert_eq!( + complete(¤t, "wrong", "source-a", 11), + Err(VerificationError::InvalidAttempt) + ); + assert_eq!( + complete(¤t, &token, "source-a", 11), + Ok(Transition::Verified) + ); + } + + #[test] + fn source_change_resets_attempt_counter_and_lease() { + let mut current = record(); + current.state = VerificationState::Verified; + current.expired_attempts = 2; + current.attempt_token = Some("old".into()); + let reset = source_changed(¤t, "source-b"); + assert_eq!(reset.state, VerificationState::Unverified); + assert_eq!(reset.expired_attempts, 0); + assert!(reset.attempt_token.is_none()); + } +} diff --git a/src/main.rs b/src/main.rs index 527b2cd..52ef6bd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -26,6 +26,8 @@ mod viewer_route; // Server-only indexing infrastructure (Hydrant firehose + SQLite projection). #[cfg(any(feature = "server", test))] mod indexing; +#[cfg(any(feature = "server", test))] +mod ldraw; // Server-only XRPC read endpoints (appview query layer). #[cfg(any(feature = "server", test))] mod appview; diff --git a/src/publish.rs b/src/publish.rs index 4c8b3ba..5877c49 100644 --- a/src/publish.rs +++ b/src/publish.rs @@ -2504,6 +2504,7 @@ mod renderer_preview { MeshFormat::Obj => "model/obj", MeshFormat::Gltf => "model/gltf+json", MeshFormat::Threemf => "model/3mf", + MeshFormat::Ldraw => "application/octet-stream", } } diff --git a/src/viewer.rs b/src/viewer.rs index 586c6be..4af70d4 100644 --- a/src/viewer.rs +++ b/src/viewer.rs @@ -12,16 +12,21 @@ #![allow(dead_code)] +#[cfg(target_arch = "wasm32")] +use crate::client::PolymodelClient; use dioxus::prelude::*; #[cfg(not(target_arch = "wasm32"))] use polymodel_renderer_protocol::MeshStats; #[cfg(target_arch = "wasm32")] use polymodel_renderer_protocol::{ - KeyModifiers, LoadEvent, LoadIdentity, LoadReducer, LoadState, MeshStats, RendererCommand, - RendererEvent, RendererNamespace, serialize_command, + Capability, CommandEnvelope, CommandPayload, CompoundLoadPlan, EventPayload, HandshakeState, + KeyModifiers, LoadEvent, LoadIdentity, LoadReducer, MeshStats, ProtocolHello, RendererCommand, + RendererEvent, RendererNamespace, deserialize_event_envelope, serialize_command_envelope, }; use polymodel_renderer_protocol::{LoadErrorCode, MeshFormat}; +#[cfg(target_arch = "wasm32")] use std::cell::RefCell; +#[cfg(target_arch = "wasm32")] use std::rc::Rc; // --------------------------------------------------------------------------- @@ -47,6 +52,9 @@ pub struct ViewerFallbackImage { #[derive(Clone, Debug, PartialEq, Eq)] pub enum ViewerSourceFormat { Renderer(MeshFormat), + /// LDraw is fetched from an authenticated, server-issued compound plan; + /// it must not enter the generic mesh parser path. + CompoundLdraw, Step, } @@ -60,6 +68,16 @@ pub struct ViewerMeshSource { pub expected_digest: Option>, } +#[cfg(target_arch = "wasm32")] +#[derive(Clone, Debug, PartialEq, Eq)] +struct PendingCompoundLoad { + key: String, + plan: CompoundLoadPlan, +} + +#[cfg(not(target_arch = "wasm32"))] +type PendingCompoundLoad = (); + #[derive(Clone, Debug, PartialEq)] struct RenderSource { key: String, @@ -311,6 +329,7 @@ pub fn ViewerPage(initial_model: DemoModel) -> Element { let mut selected_model = use_signal(|| initial_model); let step_report = use_signal(StepConversionReport::idle); let step_render_source: Signal> = use_signal(|| None); + let compound_load: Signal> = use_signal(|| None); let active_step_object_url: Signal> = use_signal(|| None); let canvas_mounted = use_signal(|| false); let resize_count = use_signal(|| 0_u64); @@ -379,6 +398,7 @@ pub fn ViewerPage(initial_model: DemoModel) -> Element { ModelViewer { model: Some(model), render_source: step_render_source, + compound_load, fallback_to_demo: true, mesh_stats, status, @@ -399,6 +419,7 @@ pub fn AssetViewer( #[props(default)] source: Option, ) -> Element { let render_source: Signal> = use_signal(|| None); + let compound_load: Signal> = use_signal(|| None); let active_step_object_url: Signal> = use_signal(|| None); let step_report = use_signal(StepConversionReport::idle); let canvas_mounted = use_signal(|| false); @@ -416,6 +437,9 @@ pub fn AssetViewer( }); let fallback_to_demo = source.is_none() && initial_model.is_some(); + #[cfg(target_arch = "wasm32")] + let client = use_context::(); + let _mesh_load = use_resource(move || { let model = initial_model; let mut status = status; @@ -424,6 +448,10 @@ pub fn AssetViewer( let mut render_source = render_source; let active_step_object_url = active_step_object_url; let mut step_report = step_report; + #[cfg(target_arch = "wasm32")] + let compound_load = compound_load; + #[cfg(target_arch = "wasm32")] + let client = client.clone(); async move { let session_key = source .as_ref() @@ -444,6 +472,18 @@ pub fn AssetViewer( expected_digest: source.expected_digest, })); } + ViewerSourceFormat::CompoundLdraw => { + clear_step_render_source(render_source, active_step_object_url); + render_source.set(None); + #[cfg(target_arch = "wasm32")] + start_compound_plan_load( + source.key, + source.url, + client.clone(), + compound_load, + status, + ); + } ViewerSourceFormat::Step => { clear_step_render_source(render_source, active_step_object_url); step_report.set(StepConversionReport { @@ -476,6 +516,7 @@ pub fn AssetViewer( ModelViewer { model: initial_model, render_source, + compound_load, fallback_to_demo, mesh_stats, status, @@ -489,6 +530,52 @@ pub fn AssetViewer( } } +#[cfg(target_arch = "wasm32")] +#[derive(serde::Serialize)] +struct CompoundPlanRequest<'a> { + project: &'a str, + path: &'a str, +} + +#[cfg(target_arch = "wasm32")] +fn start_compound_plan_load( + key: String, + uri: String, + client: PolymodelClient, + mut pending: Signal>, + mut status: Signal, +) { + wasm_bindgen_futures::spawn_local(async move { + match client + .app_json::<_, CompoundLoadPlan>( + http::Method::POST, + "/app/ldraw/compound-resources", + Some(&CompoundPlanRequest { + project: &uri, + path: &key, + }), + ) + .await + { + Ok(plan) => { + if let Err(error) = plan + .validate() + .and_then(|_| plan.encoded_size().map(|_| ())) + { + status.set(ViewerStatus::MeshError(format!( + "compound plan rejected: {error:?}" + ))); + return; + } + pending.set(Some(PendingCompoundLoad { key, plan })); + } + Err(error) => status.set(ViewerStatus::MeshError(format!( + "compound plan acquisition failed: {error}" + ))), + } + }); +} + fn clear_step_render_source( mut step_render_source: Signal>, mut active_step_object_url: Signal>, @@ -723,6 +810,7 @@ fn worker_error_summary(data: &wasm_bindgen::JsValue) -> String { pub struct ModelViewerProps { model: Option, render_source: Signal>, + compound_load: Signal>, fallback_to_demo: bool, mesh_stats: Signal>, status: Signal, @@ -744,6 +832,7 @@ pub fn ModelViewer(props: ModelViewerProps) -> Element { let status = props.status; let model = props.model; let render_source = props.render_source; + let compound_load = props.compound_load; let fallback_to_demo = props.fallback_to_demo; let load_status = status.read().clone(); @@ -806,6 +895,7 @@ pub fn ModelViewer(props: ModelViewerProps) -> Element { WorkerRendererSession { model, render_source, + compound_load, fallback_to_demo, status, canvas_mounted, @@ -851,6 +941,7 @@ fn AssetPreview(summary: MeshSummary, stats: Option) -> Element { fn WorkerRendererSession( model: Option, render_source: Signal>, + compound_load: Signal>, fallback_to_demo: bool, status: Signal, canvas_mounted: Signal, @@ -863,6 +954,7 @@ fn WorkerRendererSession( WorkerSessionInner { model, render_source, + compound_load, fallback_to_demo, status, canvas_mounted, @@ -879,6 +971,7 @@ fn WorkerRendererSession( fn WorkerSessionInner( model: Option, render_source: Signal>, + compound_load: Signal>, fallback_to_demo: bool, status: Signal, canvas_mounted: Signal, @@ -890,6 +983,7 @@ fn WorkerSessionInner( let _ = ( model, render_source, + compound_load, fallback_to_demo, status, canvas_mounted, @@ -906,6 +1000,7 @@ fn WorkerSessionInner( fn WorkerSessionInner( model: Option, render_source: Signal>, + compound_load: Signal>, fallback_to_demo: bool, status: Signal, canvas_mounted: Signal, @@ -948,6 +1043,7 @@ fn WorkerSessionInner( // Transfer canvas + send LoadMesh/Resize when canvas is mounted. let model_val = model; + let compound_load_for_canvas = compound_load; let bridge_for_canvas = bridge.clone(); use_effect(move || { if !*worker_ready.read() { @@ -998,10 +1094,15 @@ fn WorkerSessionInner( }) .flatten() }); - if let Some(source) = source { + if let Some(compound) = compound_load_for_canvas.read().clone() { + bt.start_compound_load(compound); + if let Some(session) = bt.current_session.borrow().clone() { + status.set(ViewerStatus::Loading(session)); + mesh_stats.set(None); + } + } else if let Some(source) = source { bt.start_load(source); - let session = bt.current_session.borrow().clone(); - if let Some(session) = session { + if let Some(session) = bt.current_session.borrow().clone() { status.set(ViewerStatus::Loading(session)); mesh_stats.set(None); } @@ -1081,7 +1182,10 @@ struct WorkerBridgeState { load_reducer: Rc>, next_identity: LoadIdentity, current_session: Rc>>, - post_canvas_queue: Vec>, + handshake: Rc>, + session_id: Rc>>, + post_canvas_queue: Rc>>, + compound_started_key: Option, resize_cleanup: Option>, pointer_cleanup: Option>, pointer_listeners_attached: bool, @@ -1114,7 +1218,10 @@ impl WorkerBridgeState { ))), next_identity: LoadIdentity::new(0, 0), current_session: Rc::new(RefCell::new(None)), - post_canvas_queue: Vec::new(), + handshake: Rc::new(RefCell::new(HandshakeState::Created)), + session_id: Rc::new(RefCell::new(None)), + post_canvas_queue: Rc::new(RefCell::new(Vec::new())), + compound_started_key: None, resize_cleanup: None, pointer_cleanup: None, pointer_listeners_attached: false, @@ -1133,7 +1240,6 @@ impl WorkerBridgeState { mesh_stats: Signal>, mut worker_ready: Signal, ) { - use polymodel_renderer_protocol::deserialize_event; use wasm_bindgen::JsCast; let opts = web_sys::WorkerOptions::new(); @@ -1154,6 +1260,10 @@ impl WorkerBridgeState { let mut stats_msg = mesh_stats; let reducer_msg = self.load_reducer.clone(); let current_session_msg = self.current_session.clone(); + let handshake_msg = self.handshake.clone(); + let session_id_msg = self.session_id.clone(); + let queue_msg = self.post_canvas_queue.clone(); + let worker_for_message = worker.clone(); let onmessage = Closure::wrap(Box::new(move |event: web_sys::MessageEvent| { let data = event.data(); @@ -1162,114 +1272,190 @@ impl WorkerBridgeState { && s == "__worker_ready" { worker_ready.set(true); + let hello = CommandEnvelope { + version: polymodel_renderer_protocol::PROTOCOL_VERSION, + session_id: 0, + payload: CommandPayload::Hello(ProtocolHello { + min_version: polymodel_renderer_protocol::PROTOCOL_VERSION, + max_version: polymodel_renderer_protocol::PROTOCOL_VERSION, + required: vec![Capability::CurrentFormats, Capability::Cancellation], + optional: vec![Capability::CompoundLdraw, Capability::Preview], + }), + }; + match serialize_command_envelope(&hello) { + Ok(bytes) => { + let array = js_sys::Uint8Array::from(&bytes[..]); + if let Err(error) = worker_for_message.post_message(&array) { + tracing::error!("failed to send renderer protocol hello: {error:?}"); + } + } + Err(error) => { + tracing::error!("failed to serialize renderer protocol hello: {error}") + } + } return; } - // Postcard bytes (RendererEvent). + // Postcard bytes (versioned RendererEvent envelope). if let Ok(array) = data.dyn_into::() { let bytes = array.to_vec(); - match deserialize_event(&bytes) { - Ok(evt) => match evt { - RendererEvent::ContextAcquired => { - acquisitions_msg.set(1); - } - RendererEvent::RendererContextReady => {} - RendererEvent::LoadSucceeded { - namespace, - identity, - } => { - let result = - reducer_msg.borrow_mut().reduce(LoadEvent::LoadSucceeded { - namespace, - identity, - candidate: None, - }); - if matches!( - result.outcome, - polymodel_renderer_protocol::TransitionOutcome::Committed - ) && reducer_msg.borrow().current_identity() == Some(identity) - && let Some(session) = current_session_msg.borrow().clone() - { - status_msg.set(ViewerStatus::Installing(session)); + let envelope = match deserialize_event_envelope(&bytes) { + Ok(envelope) => envelope, + Err(e) => { + tracing::error!("failed to deserialize event envelope: {e}"); + return; + } + }; + if matches!(envelope.payload, EventPayload::Failed(_)) { + if let EventPayload::Failed(message) = envelope.payload { + tracing::error!("renderer handshake failed: {message}"); + failed_msg.set(true); + } + return; + } + let negotiated = match &envelope.payload { + EventPayload::Negotiated(protocol) => Some(protocol.clone()), + _ => None, + }; + let evt = match handshake_msg.borrow_mut().accept_event(&envelope) { + Ok(Some(evt)) => evt, + Ok(None) => { + if let Some(protocol) = negotiated { + *session_id_msg.borrow_mut() = Some(protocol.session_id); + let accept = CommandEnvelope { + version: polymodel_renderer_protocol::PROTOCOL_VERSION, + session_id: 0, + payload: CommandPayload::Accept(protocol), + }; + match serialize_command_envelope(&accept) { + Ok(bytes) => { + let array = js_sys::Uint8Array::from(&bytes[..]); + if let Err(error) = worker_for_message.post_message(&array) { + tracing::error!( + "failed to accept renderer handshake: {error:?}" + ); + } + } + Err(error) => { + tracing::error!("failed to serialize handshake accept: {error}") + } } - } - RendererEvent::InstallSucceeded { - namespace, - identity, - stats, - } => { - let result = - reducer_msg - .borrow_mut() - .reduce(LoadEvent::InstallSucceeded { - namespace, - identity, - scene: None, - }); - if matches!( - result.outcome, - polymodel_renderer_protocol::TransitionOutcome::Committed - ) && reducer_msg.borrow().current_identity() == Some(identity) - && let Some(session) = current_session_msg.borrow().clone() - { - stats_msg.set(Some(stats)); - status_msg.set(ViewerStatus::Ready(session)); + let queued = std::mem::take(&mut *queue_msg.borrow_mut()); + for command in queued { + let envelope = CommandEnvelope { + version: polymodel_renderer_protocol::PROTOCOL_VERSION, + session_id: protocol.session_id, + payload: CommandPayload::Command(command), + }; + if let Ok(bytes) = serialize_command_envelope(&envelope) { + let array = js_sys::Uint8Array::from(&bytes[..]); + let _ = worker_for_message.post_message(&array); + } } } - RendererEvent::LoadFailed { + return; + } + Err(error) => { + tracing::warn!("rejected renderer event envelope: {error:?}"); + return; + } + }; + match evt { + RendererEvent::ContextAcquired => { + acquisitions_msg.set(1); + } + RendererEvent::RendererContextReady => {} + RendererEvent::LoadSucceeded { + namespace, + identity, + } => { + let result = reducer_msg.borrow_mut().reduce(LoadEvent::LoadSucceeded { namespace, identity, - code, - detail, - } => { - let result = reducer_msg.borrow_mut().reduce(LoadEvent::LoadFailed { + candidate: None, + }); + if matches!( + result.outcome, + polymodel_renderer_protocol::TransitionOutcome::Committed + ) && reducer_msg.borrow().current_identity() == Some(identity) + && let Some(session) = current_session_msg.borrow().clone() + { + status_msg.set(ViewerStatus::Installing(session)); + } + } + RendererEvent::InstallSucceeded { + namespace, + identity, + stats, + } => { + let result = reducer_msg + .borrow_mut() + .reduce(LoadEvent::InstallSucceeded { namespace, identity, - error: polymodel_renderer_protocol::LoadError { - code, - detail: detail.clone(), - }, - candidate: None, + scene: None, }); - if matches!( - result.outcome, - polymodel_renderer_protocol::TransitionOutcome::Committed - ) && reducer_msg.borrow().current_identity() == Some(identity) - && let Some(session) = current_session_msg.borrow().clone() - { - status_msg.set(ViewerStatus::Failed(session, code, detail)); - } + if matches!( + result.outcome, + polymodel_renderer_protocol::TransitionOutcome::Committed + ) && reducer_msg.borrow().current_identity() == Some(identity) + && let Some(session) = current_session_msg.borrow().clone() + { + stats_msg.set(Some(stats)); + status_msg.set(ViewerStatus::Ready(session)); } - RendererEvent::LoadCancelled { + } + RendererEvent::LoadFailed { + namespace, + identity, + code, + detail, + } => { + let result = reducer_msg.borrow_mut().reduce(LoadEvent::LoadFailed { namespace, identity, - } => { - let result = reducer_msg.borrow_mut().reduce(LoadEvent::Cancel { - namespace, - identity, - candidate: None, - }); - if matches!( - result.outcome, - polymodel_renderer_protocol::TransitionOutcome::Committed - ) && reducer_msg.borrow().current_identity() == Some(identity) - && let Some(session) = current_session_msg.borrow().clone() - { - status_msg.set(ViewerStatus::Cancelled(session)); - } + error: polymodel_renderer_protocol::LoadError { + code, + detail: detail.clone(), + }, + candidate: None, + }); + if matches!( + result.outcome, + polymodel_renderer_protocol::TransitionOutcome::Committed + ) && reducer_msg.borrow().current_identity() == Some(identity) + && let Some(session) = current_session_msg.borrow().clone() + { + status_msg.set(ViewerStatus::Failed(session, code, detail)); } - RendererEvent::PreviewImageRendered { .. } - | RendererEvent::PreviewImageError { .. } => {} - RendererEvent::Error { fatal, message } => { - if fatal { - failed_msg.set(true); - status_msg - .set(ViewerStatus::RendererError(ViewerIssue::RendererInit)); - } - tracing::error!("renderer worker error: {message}"); + } + RendererEvent::LoadCancelled { + namespace, + identity, + } => { + let result = reducer_msg.borrow_mut().reduce(LoadEvent::Cancel { + namespace, + identity, + candidate: None, + }); + if matches!( + result.outcome, + polymodel_renderer_protocol::TransitionOutcome::Committed + ) && reducer_msg.borrow().current_identity() == Some(identity) + && let Some(session) = current_session_msg.borrow().clone() + { + status_msg.set(ViewerStatus::Cancelled(session)); } - }, - Err(e) => tracing::error!("failed to deserialize event: {e}"), + } + RendererEvent::PreviewImageRendered { .. } + | RendererEvent::PreviewImageError { .. } => {} + RendererEvent::Error { fatal, message } => { + if fatal { + failed_msg.set(true); + status_msg.set(ViewerStatus::RendererError(ViewerIssue::RendererInit)); + } + tracing::error!("renderer worker error: {message}"); + } } } }) as Box); @@ -1334,6 +1520,44 @@ impl WorkerBridgeState { next } + fn start_compound_load(&mut self, pending: PendingCompoundLoad) { + if self.compound_started_key.as_deref() == Some(pending.key.as_str()) { + return; + } + let compound_capability = { + let handshake = self.handshake.borrow(); + handshake.negotiated().is_some_and(|negotiated| { + negotiated.capabilities.contains(&Capability::CompoundLdraw) + }) + }; + if !compound_capability { + tracing::error!("renderer worker did not negotiate CompoundLdraw capability"); + return; + } + let identity = self.allocate_identity(); + let namespace = RendererNamespace::Interactive; + let result = self.load_reducer.borrow_mut().reduce(LoadEvent::StartLoad { + namespace, + identity, + }); + if !matches!( + result.outcome, + polymodel_renderer_protocol::TransitionOutcome::Committed + ) { + return; + } + self.compound_started_key = Some(pending.key.clone()); + *self.current_session.borrow_mut() = Some(SessionKey::new( + pending.key, + identity.scene_generation as u64, + )); + self.send_command(&RendererCommand::StartCompoundLoad { + namespace, + identity, + plan: pending.plan, + }); + } + fn start_load(&mut self, source: RenderSource) { let identity = self.allocate_identity(); let namespace = RendererNamespace::Interactive; @@ -1362,26 +1586,26 @@ impl WorkerBridgeState { } fn send_command(&mut self, cmd: &RendererCommand) { - if !self.canvas_transferred || !self.initial_resize_sent { - // Queue scene-dependent commands until canvas + initial resize are done. - if let Ok(bytes) = serialize_command(cmd) { - self.post_canvas_queue.push(bytes); - } + if !self.canvas_transferred + || !self.initial_resize_sent + || self.session_id.borrow().is_none() + { + self.post_canvas_queue.borrow_mut().push(cmd.clone()); return; } self.post_command(cmd); } fn flush_queue(&mut self) { - if !self.canvas_transferred || !self.initial_resize_sent { + if !self.canvas_transferred + || !self.initial_resize_sent + || self.session_id.borrow().is_none() + { return; } - let queue = std::mem::take(&mut self.post_canvas_queue); - for bytes in queue { - if let Some(worker) = &self.worker { - let array = js_sys::Uint8Array::from(&bytes[..]); - let _ = worker.post_message(&array); - } + let queue = std::mem::take(&mut *self.post_canvas_queue.borrow_mut()); + for command in queue { + self.post_command(&command); } } @@ -1402,12 +1626,20 @@ impl WorkerBridgeState { fn post_command(&self, cmd: &RendererCommand) { let Some(worker) = &self.worker else { return }; - match serialize_command(cmd) { + let Some(session_id) = *self.session_id.borrow() else { + return; + }; + let envelope = CommandEnvelope { + version: polymodel_renderer_protocol::PROTOCOL_VERSION, + session_id, + payload: CommandPayload::Command(cmd.clone()), + }; + match serialize_command_envelope(&envelope) { Ok(bytes) => { let array = js_sys::Uint8Array::from(&bytes[..]); let _ = worker.post_message(&array); } - Err(e) => tracing::error!("failed to serialize command: {e}"), + Err(e) => tracing::error!("failed to serialize command envelope: {e}"), } } diff --git a/tools/ldraw-compat-harness/src/main.rs b/tools/ldraw-compat-harness/src/main.rs index ea35e35..349cbe4 100644 --- a/tools/ldraw-compat-harness/src/main.rs +++ b/tools/ldraw-compat-harness/src/main.rs @@ -1,20 +1,19 @@ //! Native PM-77 corpus validator and D-phase LDParse oracle harness. use polymodel_ldraw_core::{ - AdapterRequest, CancellationPolicy, InProcessRustAdapter, ParserProfile as CoreProfile, RootId, + DiagnosticCode, InProcessRustAdapter, LdrawError, ParserProfile as CoreProfile, }; use polymodel_ldraw_testkit::{ - CorpusGateSet, GATE_NAMES, LDRAW_LIMITS, ORACLE_PATCH_IDENTITY, ORACLE_PROJECTION_RULE_VERSION, - ORACLE_SCHEMA_VERSION, ORACLE_WRAPPER_SCHEMA_VERSION, OracleColor, OracleDiagnostic, OracleError, - OracleProjection, - CANDIDATE_OBSERVATION_SCHEMA_VERSION, Profile, SCHEMA_VERSION, coverage_report, inventory, - named_gates, validate_inventory, + coverage_report, inventory, named_gates, serialize_canonical, validate_inventory, + CanonicalRecord, ExpectedOutcome, FixtureCase, CorpusGateSet, OracleDiagnostic, OracleError, + OracleModel, OracleProjection, Profile, SCHEMA_VERSION, ORACLE_SCHEMA_VERSION, GATE_NAMES, + LDRAW_LIMITS, }; use polymodel_renderer_ledger::{ReservationLedger, ReservationOwner}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use std::{ - env, fs, - io::Read, + env, + fs, path::{Path, PathBuf}, process::{Command, ExitCode, Output, Stdio}, thread, @@ -26,7 +25,7 @@ const LD_PARSE_REVISION: &str = "0fe78dcee22982f26cceced1b483340faa4d04e9"; const FUZZ_SEED: u64 = 0x504d3737_4435; const FUZZ_ITERATIONS: usize = 128; const MAX_MUTATED_BYTES: usize = 1024 * 1024; -const ORACLE_TIMEOUT: Duration = Duration::from_secs(10); +const ORACLE_TIMEOUT: Duration = Duration::from_secs(2); const FUZZ_BUDGET: Duration = Duration::from_secs(30); #[derive(Debug, Serialize)] @@ -44,115 +43,13 @@ struct Report<'a> { mutations: usize, } -#[derive(Debug, Serialize)] -struct Snapshot<'a> { - id: &'a str, - oracle_expected: Option, - oracle_error: Option, -} - -#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)] -#[serde(deny_unknown_fields)] -struct OraclePosition(Option, Option, Option); - -#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)] -#[serde(deny_unknown_fields)] -struct OracleMesh { - vertex_count: usize, - positions: Vec, - normal_count: usize, - normals: Vec, - indices: Vec, - bf_indices: Vec, - triangle_count: usize, - bf_triangle_count: usize, -} - -#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)] -#[serde(deny_unknown_fields)] -struct OracleTransform { - translation: OraclePosition, - matrix: Vec>, -} - -#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)] -#[serde(deny_unknown_fields)] -struct OracleChild { - index: usize, - bfc: String, - transform: OracleTransform, - child_name: Option, - child_path: Option, -} - -#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)] -#[serde(deny_unknown_fields)] -struct OracleModelDocument { - name: String, - model_index: u32, - path: String, - certify: String, - winding: String, - default_color: u64, - step_ends: Vec, - local_colors: Vec, - mesh: OracleMesh, - children: Vec, -} - -#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)] -#[serde(deny_unknown_fields)] -struct OracleToken { - kind: String, - kind_code: i32, - value: serde_json::Value, - line: i32, - column: i32, -} - -#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)] -#[serde(deny_unknown_fields)] -struct OracleTokenLine { - index: usize, - text: String, - tokens: Vec, -} - -#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)] -#[serde(deny_unknown_fields)] -struct OracleTokenizedModel { - model_index: usize, - name: String, - lines: Vec, -} - -#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)] -#[serde(deny_unknown_fields)] -struct OracleGeometry { - triangles: usize, - quads: usize, - lines: usize, - conditional_lines: usize, - includes: usize, - triangle_positions: Vec>, -} - #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] struct OracleDocument { schema_version: String, - oracle_revision: String, - wrapper_schema_version: String, - projection_rule_version: String, - patch_identity: String, fixture_path: String, status: String, - model_names: Vec, - root_model: Option, - models: Vec, - tokenized_models: Vec, - tokenized_geometry: Option, - resolved_geometry: Option, + models: Vec, diagnostics: Vec, error: Option, } @@ -162,12 +59,11 @@ fn main() -> ExitCode { let result = match command.as_str() { "validate" => validate(), "inventory" => report("inventory"), - "corpus" => corpus(), - "snapshot" => snapshot(), + "corpus" => report("corpus"), "fuzz" => fuzz(), "differential" => differential(), other => Err(format!( - "unknown command {other}; use validate, inventory, corpus, snapshot, fuzz, or differential" + "unknown command {other}; use validate, inventory, corpus, fuzz, or differential" )), }; match result { @@ -197,6 +93,16 @@ fn report(command: &'static str) -> Result { gates .validate_inventory(&fixtures) .map_err(|error| error.to_string())?; + for fixture in &fixtures { + if fixture.expected.outcome == ExpectedOutcome::Rejected { + validate_rejected(fixture)?; + continue; + } + let actual = actual_record(fixture)?; + for outcome in gates.evaluate(fixture, &actual) { + outcome.map_err(|error| error.to_string())?; + } + } let coverage = coverage_report(&fixtures); serde_json::to_string_pretty(&Report { schema_version: SCHEMA_VERSION, @@ -214,117 +120,6 @@ fn report(command: &'static str) -> Result { .map_err(|error| error.to_string()) } -fn corpus() -> Result { - let fixtures = inventory(); - validate_inventory(&fixtures)?; - let gates = CorpusGateSet::new(named_gates()).map_err(|error| error.to_string())?; - gates - .validate_inventory(&fixtures) - .map_err(|error| error.to_string())?; - let mut evaluated = 0usize; - for (index, fixture) in fixtures.iter().enumerate() { - let ledger = ReservationLedger::new(); - let parsed = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - if fixture.outcome == polymodel_ldraw_testkit::ExpectedOutcome::Cancelled { - let cancellation = CancellationPolicy::new(); - cancellation.cancel_after_checkpoints(1); - InProcessRustAdapter::parse(polymodel_ldraw_core::AdapterRequest { - fixture_id: fixture.id, - bytes: fixture.bytes, - profile: core_profile(fixture.profile), - provenance_id: fixture.id, - owner: ReservationOwner::preview(78, index as u64 + 1, 1), - ledger: &ledger, - semantic_budget: Some(16 * 1024 * 1024), - root_name: fixture.id, - root: RootId::UploadedManifest, - materializations: &[], - target_selections: &[], - cancellation, - }) - } else { - parse_fixture_with_root( - fixture, - ReservationOwner::preview(78, index as u64 + 1, 1), - &ledger, - ) - } - })) - .map_err(|_| format!("{} core parser panicked", fixture.id))?; - match (fixture.outcome, parsed) { - (polymodel_ldraw_testkit::ExpectedOutcome::Rejected, Err(error)) - | (polymodel_ldraw_testkit::ExpectedOutcome::Cancelled, Err(error)) => { - let results = gates.evaluate_rejection(fixture, &error); - if results.is_empty() { - return Err(format!("{} has no applicable rejection gates", fixture.id)); - } - for result in results { - result.map_err(|error| error.to_string())?; - evaluated += 1; - } - } - (polymodel_ldraw_testkit::ExpectedOutcome::Rejected, Ok(_)) - | (polymodel_ldraw_testkit::ExpectedOutcome::Cancelled, Ok(_)) => { - return Err(format!("{} unexpectedly accepted", fixture.id)); - } - (_, Ok(parsed)) => { - let actual = parsed.project(); - let results = gates.evaluate_with_files(fixture, &actual, &parsed.files); - if results.is_empty() { - return Err(format!("{} has no applicable acceptance gates", fixture.id)); - } - for result in results { - result.map_err(|error| error.to_string())?; - evaluated += 1; - } - drop(parsed); - } - (_, Err(error)) => return Err(format!("{} unexpectedly rejected: {error}", fixture.id)), - } - if ledger.snapshot().reservations != 0 { - return Err(format!("{} leaked parser reservations", fixture.id)); - } - } - Ok(serde_json::json!({ - "schema_version": SCHEMA_VERSION, - "command": "corpus", - "status": "pass", - "fixtures": fixtures.len(), - "gates": evaluated, - "profiles": 3, - "oracle_revision": LD_PARSE_REVISION, - "oracle_schema": ORACLE_SCHEMA_VERSION - }).to_string()) -} - -fn snapshot() -> Result { - let fixtures = inventory(); - let root = root_dir(); - let binary = oracle_binary(&root)?; - let mut snapshots = Vec::with_capacity(fixtures.len()); - for fixture in &fixtures { - let path = root - .join("crates/polymodel-ldraw-testkit") - .join(fixture.relative_path); - let (oracle_expected, oracle_error) = - match run_oracle( - &binary, - &path, - &root.join("crates/polymodel-ldraw-testkit/corpus"), - Duration::from_secs(10), - ) { - Ok(document) => (Some(projection(document, &path, fixture.bytes)?), None), - Err(error) => (None, Some(error)), - }; - snapshots.push(Snapshot { - id: fixture.id, - oracle_expected, - oracle_error, - }); - } - serde_json::to_string_pretty(&snapshots).map_err(|error| error.to_string()) -} - fn validate() -> Result { let fixtures = inventory(); validate_inventory(&fixtures)?; @@ -332,26 +127,19 @@ fn validate() -> Result { .map_err(|error| error.to_string())? .validate_inventory(&fixtures) .map_err(|error| error.to_string())?; - if LDRAW_LIMITS - .iter() - .any(|(name, limit)| name.is_empty() || *limit == 0) - { + if LDRAW_LIMITS.iter().any(|(name, limit)| name.is_empty() || *limit == 0) { return Err("invalid LDraw semantic limit".to_owned()); } for fixture in &fixtures { let encoded = serde_json::to_vec(&fixture.expected).map_err(|error| error.to_string())?; let second = serde_json::to_vec(&fixture.expected).map_err(|error| error.to_string())?; if encoded != second { - return Err(format!( - "non-deterministic expected serialization: {}", - fixture.id - )); + return Err(format!("non-deterministic expected serialization: {}", fixture.id)); } - if fixture.expected.oracle_expected.status.is_empty() { - return Err(format!( - "empty generated oracle projection status: {}", - fixture.id - )); + if let Some(projection) = &fixture.expected.oracle_projection { + if projection.status.is_empty() { + return Err(format!("empty authored oracle projection status: {}", fixture.id)); + } } } Ok(serde_json::json!({ @@ -385,20 +173,14 @@ fn run_checked(program: &str, args: &[&str], cwd: &Path) -> Result Result { - let nix_status = Command::new("nix") - .arg("--version") - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() - .map_err(|error| format!("could not start nix: {error}"))?; - if !nix_status.success() { + if !Command::new("nix").arg("--version").status().map_err(|error| format!("could not start nix: {error}"))?.success() { return Err("nix is required for the LDParse differential oracle".to_owned()); } let output = run_checked( "nix", &[ "build", - &format!("path:{}#ldraw-oracle", root.display()), + "path:/home/orual/Projects/epic-86#ldraw-oracle", "--no-link", "--print-out-paths", ], @@ -413,344 +195,77 @@ fn oracle_binary(root: &Path) -> Result { .to_owned(); let binary = Path::new(&store_path).join("bin/ldraw-oracle"); if !binary.is_file() { - return Err(format!( - "oracle binary is missing from successful Nix output: {}", - binary.display() - )); + return Err(format!("oracle binary is missing from successful Nix output: {}", binary.display())); } Ok(binary) } -fn run_oracle(binary: &Path, fixture: &Path, corpus: &Path, timeout: Duration) -> Result { - let fixture_name = fixture - .file_name() - .and_then(|name| name.to_str()) - .ok_or_else(|| format!("fixture path is not a UTF-8 filename: {}", fixture.display()))?; +fn run_oracle(binary: &Path, fixture: &Path, timeout: Duration) -> Result { let mut child = Command::new(binary) - .arg(fixture_name) - .current_dir(corpus) + .arg(fixture) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn() .map_err(|error| format!("could not start oracle: {error}"))?; - let mut stdout_pipe = child - .stdout - .take() - .ok_or_else(|| "oracle stdout pipe unavailable".to_owned())?; - let mut stderr_pipe = child - .stderr - .take() - .ok_or_else(|| "oracle stderr pipe unavailable".to_owned())?; - let stdout_reader = thread::spawn(move || { - let mut bytes = Vec::new(); - let _ = stdout_pipe.read_to_end(&mut bytes); - bytes - }); - let stderr_reader = thread::spawn(move || { - let mut bytes = Vec::new(); - let _ = stderr_pipe.read_to_end(&mut bytes); - bytes - }); let deadline = Instant::now() + timeout; let status = loop { - match child - .try_wait() - .map_err(|error| format!("oracle wait failed: {error}"))? - { + match child.try_wait().map_err(|error| format!("oracle wait failed: {error}"))? { Some(status) => break status, None if Instant::now() >= deadline => { let _ = child.kill(); let _ = child.wait(); - return Err(format!( - "oracle exceeded {}ms for {}", - timeout.as_millis(), - fixture.display() - )); + return Err(format!("oracle exceeded {}ms for {}", timeout.as_millis(), fixture.display())); } None => thread::sleep(Duration::from_millis(5)), } }; - let stdout = stdout_reader - .join() - .map_err(|_| "oracle stdout reader panicked".to_owned())?; - let stderr = stderr_reader - .join() - .map_err(|_| "oracle stderr reader panicked".to_owned())?; - if !status.success() { - return Err(format!( - "oracle exited {status}: {}", - String::from_utf8_lossy(&stderr) - )); + let output = child + .wait_with_output() + .map_err(|error| format!("could not collect oracle output: {error}"))?; + if !status.success() && !output.stdout.is_empty() { + return Err(format!("oracle exited {status}: {}", String::from_utf8_lossy(&output.stderr))); } - let stdout = String::from_utf8(stdout).map_err(|error| { - format!( - "oracle emitted non-UTF-8 stdout for {}: {error}", - fixture.display() - ) - })?; - let lines = stdout - .lines() - .filter(|line| !line.trim().is_empty()) - .collect::>(); - if lines.len() != 1 { - return Err(format!( - "oracle must emit exactly one JSON line for {}; got {}; stderr: {}", - fixture.display(), - lines.len(), - String::from_utf8_lossy(&stderr) - )); - } - let mut raw = serde_json::from_str::(lines[0]).map_err(|error| { - format!( - "invalid oracle JSON for {}: {error}; stderr: {}", - fixture.display(), - String::from_utf8_lossy(&stderr) - ) - })?; - let object = raw - .as_object_mut() - .ok_or_else(|| format!("oracle JSON for {} is not an object", fixture.display()))?; - match object.get("status").and_then(serde_json::Value::as_str) { - Some("ok") => { - object.entry("error").or_insert(serde_json::Value::Null); - if let Some(resolved) = object - .get_mut("resolved_geometry") - .and_then(serde_json::Value::as_object_mut) - { - resolved - .entry("triangle_positions") - .or_insert_with(|| serde_json::Value::Array(Vec::new())); - } - } - Some("error") => { - for field in ["model_names", "models", "tokenized_models"] { - object - .entry(field.to_owned()) - .or_insert_with(|| serde_json::Value::Array(Vec::new())); - } - if let Some(error) = object - .get_mut("error") - .and_then(serde_json::Value::as_object_mut) - { - error - .entry("diagnostics") - .or_insert_with(|| serde_json::Value::Array(Vec::new())); - } - } - _ => {} - } - let document = serde_json::from_value::(raw).map_err(|error| { - format!( - "invalid oracle JSON for {}: {error}; stderr: {}", - fixture.display(), - String::from_utf8_lossy(&stderr) - ) - })?; + let document: OracleDocument = serde_json::from_slice(&output.stdout) + .map_err(|error| format!("invalid oracle JSON for {}: {error}; stderr: {}", fixture.display(), String::from_utf8_lossy(&output.stderr)))?; validate_oracle_document(&document, fixture)?; Ok(document) } fn validate_oracle_document(document: &OracleDocument, fixture: &Path) -> Result<(), String> { if document.schema_version != ORACLE_SCHEMA_VERSION { - return Err(format!( - "{} has schema {}", - fixture.display(), - document.schema_version - )); - } - if document.oracle_revision != LD_PARSE_REVISION - || document.wrapper_schema_version != ORACLE_WRAPPER_SCHEMA_VERSION - || document.projection_rule_version != ORACLE_PROJECTION_RULE_VERSION - || document.patch_identity != ORACLE_PATCH_IDENTITY - { - return Err(format!( - "{} runtime oracle attestation does not match the checked-in contract", - fixture.display() - )); + return Err(format!("{} has schema {}", fixture.display(), document.schema_version)); } if document.fixture_path.is_empty() { return Err(format!("{} has empty fixture_path", fixture.display())); } match document.status.as_str() { - "ok" if document.error.is_none() && document.root_model.is_some() => {} - "error" if document.error.is_some() && document.root_model.is_none() => {} - status => { - return Err(format!( - "{} has invalid status/error shape: {status}", - fixture.display() - )); - } + "ok" if document.error.is_none() => {} + "error" if document.error.as_ref().is_some_and(|error| error.kind == "construct_null") => {} + status => return Err(format!("{} has invalid status/error shape: {status}", fixture.display())), } - if !polymodel_ldraw_testkit::unique_strings(document.model_names.clone()) { - return Err(format!( - "{} contains duplicate model names", - fixture.display() - )); - } - if let Some(root) = &document.root_model { - if document.model_names.first() != Some(&root.name) { - return Err(format!( - "{} root model is not first in model_names", - fixture.display() - )); + let mut previous = None; + for (index, model) in document.models.iter().enumerate() { + if previous == Some(model.name.as_str()) { + return Err(format!("{} contains duplicate model {}", fixture.display(), model.name)); } - if !matches!( - root.certify.as_str(), - "unknown" | "certified" | "uncertified" - ) { - return Err(format!( - "{} has invalid certification {}", - fixture.display(), - root.certify - )); + if model.model_index != index as u32 { + return Err(format!("{} has non-sequential model indices", fixture.display())); + } + previous = Some(model.name.as_str()); + if !matches!(model.certify.as_str(), "unknown" | "certified" | "uncertified") { + return Err(format!("{} has invalid certification {}", fixture.display(), model.certify)); } } Ok(()) } -fn project_model(model: OracleModelDocument) -> polymodel_ldraw_testkit::OracleModel { - polymodel_ldraw_testkit::OracleModel { - name: model.name, - model_index: model.model_index, - path: model.path, - certify: model.certify, - winding: model.winding, - default_color: model.default_color, - step_ends: model.step_ends.clone(), - local_colors: model.local_colors, - mesh: polymodel_ldraw_testkit::OracleMesh { - vertex_count: model.mesh.vertex_count, - positions: model - .mesh - .positions - .into_iter() - .map(|p| polymodel_ldraw_testkit::OraclePosition(p.0, p.1, p.2)) - .collect(), - normal_count: model.mesh.normal_count, - normals: model - .mesh - .normals - .into_iter() - .map(|p| polymodel_ldraw_testkit::OraclePosition(p.0, p.1, p.2)) - .collect(), - indices: model.mesh.indices, - bf_indices: model.mesh.bf_indices, - triangle_count: model.mesh.triangle_count, - bf_triangle_count: model.mesh.bf_triangle_count, - }, - children: model - .children - .into_iter() - .map(|child| polymodel_ldraw_testkit::OracleChild { - index: child.index, - bfc: child.bfc, - transform: polymodel_ldraw_testkit::OracleTransform { - translation: polymodel_ldraw_testkit::OraclePosition( - child.transform.translation.0, - child.transform.translation.1, - child.transform.translation.2, - ), - matrix: child.transform.matrix, - }, - child_name: child.child_name, - child_path: child.child_path, - }) - .collect(), - } -} - -fn projection( - document: OracleDocument, - fixture: &Path, - fixture_bytes: &[u8], -) -> Result { - let models = document - .models - .into_iter() - .map(project_model) - .collect::>(); - let root_model = document.root_model.map(project_model); - if document.status == "ok" && models.is_empty() { - return Err(format!( - "{} successful oracle document has no models", - fixture.display() - )); - } - let tokenized_models = document - .tokenized_models - .into_iter() - .map(|model| polymodel_ldraw_testkit::OracleTokenizedModel { - model_index: model.model_index, - name: model.name, - lines: model - .lines - .into_iter() - .map(|line| polymodel_ldraw_testkit::OracleTokenLine { - index: line.index, - text: line.text, - tokens: line - .tokens - .into_iter() - .map(|token| polymodel_ldraw_testkit::OracleToken { - kind: token.kind, - kind_code: token.kind_code, - value: token.value, - line: token.line, - column: token.column, - }) - .collect(), - }) - .collect(), - }) - .collect(); - let geometry = |geometry: Option| { - geometry.map(|value| polymodel_ldraw_testkit::OracleGeometry { - triangles: value.triangles, - quads: value.quads, - lines: value.lines, - conditional_lines: value.conditional_lines, - includes: value.includes, - triangle_positions: value - .triangle_positions - .into_iter() - .map(|points| { - points - .into_iter() - .map(|p| polymodel_ldraw_testkit::OraclePosition(p.0, p.1, p.2)) - .collect() - }) - .collect(), - }) - }; - let fixture_path = Path::new(&document.fixture_path) - .strip_prefix(root_dir().join("crates/polymodel-ldraw-testkit")) - .map(|path| format!("crates/polymodel-ldraw-testkit/{}", path.to_string_lossy().replace('\\', "/"))) - .unwrap_or_else(|_| { - format!( - "crates/polymodel-ldraw-testkit/corpus/{}", - document.fixture_path.replace('\\', "/") - ) - }); - Ok(OracleProjection { - schema_version: document.schema_version, - oracle_revision: document.oracle_revision, - wrapper_schema_version: document.wrapper_schema_version, - projection_rule_version: document.projection_rule_version, - patch_identity: document.patch_identity, - fixture_path, - fixture_sha256: polymodel_ldraw_testkit::hex_digest(fixture_bytes), +fn projection(document: OracleDocument) -> OracleProjection { + OracleProjection { status: document.status, - model_names: document.model_names, - root_model, - models, - tokenized_models, - tokenized_geometry: geometry(document.tokenized_geometry), - resolved_geometry: geometry(document.resolved_geometry), + models: document.models, diagnostics: document.diagnostics, error: document.error, - authority_rules: serde_json::from_str(include_str!("../../ldraw-oracle-generator/rules.json")) - .map_err(|error| format!("invalid checked-in authority rules: {error}"))?, - }) + } } fn core_profile(profile: Profile) -> CoreProfile { @@ -761,284 +276,51 @@ fn core_profile(profile: Profile) -> CoreProfile { } } -fn fixture_root_name(fixture: &polymodel_ldraw_testkit::FixtureCase) -> &str { - Path::new(fixture.relative_path) - .file_name() - .and_then(|name| name.to_str()) - .expect("fixture relative path must have a UTF-8 filename") -} - -fn parse_fixture_with_root<'a>( - fixture: &'a polymodel_ldraw_testkit::FixtureCase, - owner: ReservationOwner, - ledger: &'a ReservationLedger, -) -> Result { - InProcessRustAdapter::parse(AdapterRequest { - fixture_id: fixture.id, - bytes: fixture.bytes, - profile: core_profile(fixture.profile), - provenance_id: fixture.id, - owner, - ledger, - semantic_budget: Some(16 * 1024 * 1024), - root_name: fixture_root_name(fixture), - root: RootId::UploadedManifest, - materializations: &[], - target_selections: &[], - cancellation: CancellationPolicy::default(), - }) -} - -fn candidate_observation( - parsed: &polymodel_ldraw_core::OwnedParseResult, -) -> Result { - let result = &parsed.result; - Ok(serde_json::json!({ - "schema_version": CANDIDATE_OBSERVATION_SCHEMA_VERSION, - "status": "ok", - "syntax": result.syntax, - "semantic": { - "canonical_path": result.semantic.canonical_path, - "root_identity": result.semantic.root_identity, - "target_identity": result.semantic.target_identity, - "cache_identity": result.semantic.cache_identity, - "bfc_state": result.semantic.bfc_state, - "colour_state": result.semantic.colour_state, - "steps": result.semantic.steps, - "limits": result.semantic.limits, - "texmap_events": result.semantic.texmap_events, - "texmap_geometry": result.semantic.texmap_geometry, - "data_payloads": result.semantic.data_payloads, - }, - "scene": result.scene, - "diagnostics": result.diagnostics, - "models": result.models, - "files": result.files, - "counters": result.counters, - "non_comparable": [ - "oracle_expected.tokenized_models", - "oracle_expected.tokenized_geometry", - "oracle_expected.resolved_geometry.triangle_positions", - "oracle_expected.models[*].mesh.positions", - "oracle_expected.models[*].mesh.normals", - "oracle_expected.models[*].mesh.indices", - "oracle_expected.models[*].mesh.bf_indices", - ], - })) +fn actual_record(fixture: &FixtureCase) -> Result, String> { + let ledger = ReservationLedger::new(); + let parsed = InProcessRustAdapter::parse_fixture( + fixture.id, + fixture.bytes, + core_profile(fixture.expected.profile), + ReservationOwner::preview(1, 1, 1), + &ledger, + ) + .map_err(|error| format!("core parse failed for {}: {error}", fixture.id))?; + let record = parsed.project(); + if ledger.snapshot().reservations == 0 { + return Err(format!("core reservation was released before projection: {}", fixture.id)); + } + let _ = record; + Ok(parsed.into_owned_project()) } -fn compare_candidate_to_oracle( - fixture: &polymodel_ldraw_testkit::FixtureCase, - oracle: &OracleProjection, -) -> Result<(), String> { +fn validate_rejected(fixture: &FixtureCase) -> Result<(), String> { let ledger = ReservationLedger::new(); - let parsed = parse_fixture_with_root( - fixture, - ReservationOwner::preview(78, 0, 1), + let error = InProcessRustAdapter::parse_fixture( + fixture.id, + fixture.bytes, + core_profile(fixture.expected.profile), + ReservationOwner::preview(1, 1, 1), &ledger, ) - .map_err(|error| format!("{} candidate rejected while oracle status is {}: {error}", fixture.id, oracle.status))?; - let candidate = parsed.project(); - if oracle.status != "ok" { - if oracle.status != "error" - || oracle.error.is_none() - || oracle.root_model.is_some() - || oracle.resolved_geometry.is_some() - { - return Err(format!( - "{} oracle unavailable-surface shape is invalid: status={} error={} root_model={} resolved_geometry={}", - fixture.id, - oracle.status, - oracle.error.is_some(), - oracle.root_model.is_some(), - oracle.resolved_geometry.is_some() - )); - } - drop(parsed); - if ledger.snapshot().reservations != 0 { - return Err(format!("{} differential candidate leaked reservation", fixture.id)); - } - return Ok(()); - } - let resolved = oracle - .resolved_geometry - .as_ref() - .ok_or_else(|| format!("{} oracle success has no resolved geometry", fixture.id))?; - let oracle_triangles = resolved.triangles as u64; - if candidate.scene.triangles != oracle_triangles { - return Err(format!("{} triangle mismatch: oracle={} candidate={}", fixture.id, oracle_triangles, candidate.scene.triangles)); - } - if parsed.result.models.len() != oracle.models.len() { + .err() + .ok_or_else(|| format!("{} expected a rejected core parse", fixture.id))?; + let actual = match error { + LdrawError::Diagnostic(diagnostic) => diagnostic, + other => return Err(format!("{} rejected with unexpected error: {other}", fixture.id)), + }; + let expected = fixture + .expected + .canonical + .diagnostics + .first() + .ok_or_else(|| format!("{} has no authored rejection diagnostic", fixture.id))?; + if actual != *expected { return Err(format!( - "{} model count mismatch: oracle={} candidate={}", - fixture.id, - oracle.models.len(), - parsed.result.models.len() + "{} rejection diagnostic mismatch: actual {actual:?}, expected {expected:?}", + fixture.id )); } - let mut oracle_step_count = 0usize; - let mut consumed_candidate_indices = std::collections::BTreeSet::new(); - for (model_index, oracle_model) in oracle.models.iter().enumerate() { - if oracle_model.model_index as usize != model_index { - return Err(format!("{} oracle model order/index mismatch at {model_index}", fixture.id)); - } - let matching_candidate_indices = parsed - .result - .models - .iter() - .enumerate() - .filter_map(|(candidate_index, candidate)| { - if consumed_candidate_indices.contains(&candidate_index) { - return None; - } - let candidate_name = Path::new(&candidate.path) - .file_name() - .and_then(|name| name.to_str()) - .unwrap_or_default(); - let candidate_stem = Path::new(candidate_name) - .file_stem() - .and_then(|name| name.to_str()) - .unwrap_or_default(); - (candidate_name.eq_ignore_ascii_case(&oracle_model.name) - || (model_index == 0 - && oracle_model.name == format!("pm77-{candidate_stem}"))) - .then_some(candidate_index) - }) - .collect::>(); - let candidate_index = match matching_candidate_indices.as_slice() { - [] => { - return Err(format!( - "{} missing candidate model[{}] {}", - fixture.id, model_index, oracle_model.name - )); - } - [candidate_index] => *candidate_index, - indices => { - return Err(format!( - "{} ambiguous candidate model[{}] {} matched indices {:?}", - fixture.id, model_index, oracle_model.name, indices - )); - } - }; - consumed_candidate_indices.insert(candidate_index); - let candidate_model = &parsed.result.models[candidate_index]; - let candidate_name = Path::new(&candidate_model.path) - .file_name() - .and_then(|name| name.to_str()) - .unwrap_or_default(); - let candidate_stem = Path::new(candidate_name) - .file_stem() - .and_then(|name| name.to_str()) - .unwrap_or_default(); - let root_alias = model_index == 0 - && oracle_model.name == format!("pm77-{candidate_stem}"); - let oracle_path_name = Path::new(&oracle_model.path) - .file_name() - .and_then(|name| name.to_str()) - .unwrap_or_default(); - let mpd_path_alias = model_index == 0 - && oracle_path_name.eq_ignore_ascii_case(candidate_name); - if model_index == 0 && !mpd_path_alias && candidate_model.path != oracle_model.path { - return Err(format!( - "{} model[{}] path mismatch: oracle={} candidate={}", - fixture.id, model_index, oracle_model.path, candidate_model.path - )); - } - if candidate_name != oracle_model.name - && !candidate_name.eq_ignore_ascii_case(&oracle_model.name) - && !root_alias - { - return Err(format!( - "{} model[{}] name mismatch: oracle={} candidate={}", - fixture.id, model_index, oracle_model.name, candidate_name - )); - } - if candidate_model.step_ends != oracle_model.step_ends { - return Err(format!( - "{} step_ends mismatch for model[{}] {}: oracle={:?} candidate={:?}", - fixture.id, model_index, oracle_model.name, oracle_model.step_ends, candidate_model.step_ends - )); - } - let expected_certification = polymodel_ldraw_testkit::normalize_certification(&oracle_model.certify) - .ok_or_else(|| format!("{} invalid oracle certification at model[{}]", fixture.id, model_index))?; - let candidate_certification = match candidate_model.bfc { - polymodel_ldraw_core::BfcState::Unknown => "unknown", - polymodel_ldraw_core::BfcState::Certified => "certified", - polymodel_ldraw_core::BfcState::Uncertified => "uncertified", - }; - if candidate_certification != expected_certification { - return Err(format!( - "{} certification mismatch for model[{}]: oracle={} candidate={}", - fixture.id, model_index, expected_certification, candidate_certification - )); - } - let tokenized_model = oracle - .tokenized_models - .get(model_index) - .ok_or_else(|| format!("{} missing tokenized model[{}]", fixture.id, model_index))?; - let source_counts = tokenized_model.lines.iter().filter_map(|line| { - line.tokens.first().and_then(|token| match token.kind_code { - 1 => Some((1usize, 0, 0, 0, 0)), - 2 => Some((0, 1, 0, 0, 0)), - 3 => Some((0, 0, 1, 0, 0)), - 4 => Some((0, 0, 0, 1, 0)), - 5 => Some((0, 0, 0, 0, 1)), - _ => None, - }) - }).fold((0usize, 0, 0, 0, 0), |counts, line| { - ( - counts.0 + line.0, - counts.1 + line.1, - counts.2 + line.2, - counts.3 + line.3, - counts.4 + line.4, - ) - }); - if candidate_model.triangles as usize != source_counts.2 - || candidate_model.quads as usize != source_counts.3 - || candidate_model.lines as usize != source_counts.1 - || candidate_model.conditional_lines as usize != source_counts.4 - || candidate_model.includes.len() != source_counts.0 - { - return Err(format!( - "{} source geometry mismatch for model[{}]: oracle={:?} candidate=({},{},{},{},{})", - fixture.id, - model_index, - source_counts, - candidate_model.includes.len(), - candidate_model.lines, - candidate_model.triangles, - candidate_model.quads, - candidate_model.conditional_lines, - )); - } - let expected_children = source_counts.0; - if candidate_model.includes.len() != expected_children { - return Err(format!("{} child count mismatch for model[{}]: oracle={} candidate={}", fixture.id, model_index, expected_children, candidate_model.includes.len())); - } - oracle_step_count = oracle_step_count - .checked_add(oracle_model.step_ends.len()) - .ok_or_else(|| format!("{} oracle step count overflow", fixture.id))?; - } - if fixture.id == "external-ldr-tools" { - let root_step_count = oracle - .root_model - .as_ref() - .map(|model| model.step_ends.len()) - .unwrap_or_default(); - if root_step_count != 52 || oracle_step_count != 109 { - return Err(format!( - "{} expected root/submodel step_ends distribution 52/57, got root={} models_total={}", - fixture.id, - root_step_count, - oracle_step_count - )); - } - } - drop(parsed); - if ledger.snapshot().reservations != 0 { - return Err(format!("{} differential candidate leaked reservation", fixture.id)); - } Ok(()) } @@ -1047,36 +329,21 @@ fn differential() -> Result { validate_inventory(&fixtures)?; let root = root_dir(); let binary = oracle_binary(&root)?; - for (index, fixture) in fixtures.iter().enumerate() { - let path = root - .join("crates/polymodel-ldraw-testkit") - .join(fixture.relative_path); - let actual = run_oracle( - &binary, - &path, - &root.join("crates/polymodel-ldraw-testkit/corpus"), - ORACLE_TIMEOUT, - )?; - let actual_projection = projection(actual, &path, fixture.bytes)?; - let expected_oracle = &fixture.expected.oracle_expected; - polymodel_ldraw_testkit::compare_oracle_expected( - expected_oracle, + for fixture in &fixtures { + let path = root.join("crates/polymodel-ldraw-testkit").join(fixture.relative_path); + let actual = run_oracle(&binary, &path, ORACLE_TIMEOUT)?; + let actual_projection = projection(actual); + let expected_projection = fixture + .expected + .oracle_projection + .as_ref() + .ok_or_else(|| format!("missing authored oracle projection: {}", fixture.id))?; + polymodel_ldraw_testkit::compare_oracle_projection( + expected_projection, &actual_projection, fixture.id, ) - .map_err(|error| { - format!( - "{} (source: {}; oracle revision: {}; schema: {})", - error, fixture.provenance.id, LD_PARSE_REVISION, ORACLE_SCHEMA_VERSION - ) - })?; - if fixture.outcome == polymodel_ldraw_testkit::ExpectedOutcome::Accepted - || fixture.outcome == polymodel_ldraw_testkit::ExpectedOutcome::Preserved - || fixture.outcome == polymodel_ldraw_testkit::ExpectedOutcome::Limited - { - compare_candidate_to_oracle(fixture, &actual_projection) - .map_err(|error| format!("fixture {index}: {error}"))?; - } + .map_err(|error| format!("{} (source: {}; oracle revision: {}; schema: {})", error, fixture.provenance.id, LD_PARSE_REVISION, ORACLE_SCHEMA_VERSION))?; } serde_json::to_string_pretty(&Report { schema_version: SCHEMA_VERSION, @@ -1094,30 +361,6 @@ fn differential() -> Result { .map_err(|error| error.to_string()) } -fn mutation_operator(seed: u64, index: usize) -> &'static str { - match (seed - .wrapping_add(index as u64) - .wrapping_mul(6364136223846793005) - .wrapping_add(1) - % 4) as usize - { - 0 => "insert_meta", - 1 => "delete_bytes", - 2 => "replace_byte", - _ => "normalize_crlf", - } -} - -fn mutation_fixture_index(index: usize, fixture_count: usize) -> usize { - // Four complete rounds cover every fixture; the remaining mutations repeat - // the first four fixture slots with distinct indices and operators. - index % fixture_count -} - -fn mutation_index_is_authorized(index: usize, fixture_count: usize) -> bool { - index < FUZZ_ITERATIONS && fixture_count > 0 -} - fn mutate(seed: u64, index: usize, source: &[u8]) -> Vec { let mut state = seed.wrapping_add(index as u64); state = state.wrapping_mul(6364136223846793005).wrapping_add(1); @@ -1151,11 +394,62 @@ fn mutate(seed: u64, index: usize, source: &[u8]) -> Vec { output } +fn bounded_mutation_rejection(error: &LdrawError) -> bool { + match error { + LdrawError::Cancelled => true, + LdrawError::Diagnostic(diagnostic) => matches!( + diagnostic.code, + DiagnosticCode::InvalidUtf8 + | DiagnosticCode::PathOutOfRoot + | DiagnosticCode::PathForbiddenSyntax + | DiagnosticCode::PathEmpty + | DiagnosticCode::InvalidType1 + | DiagnosticCode::InvalidType2 + | DiagnosticCode::InvalidType3 + | DiagnosticCode::InvalidType4 + | DiagnosticCode::InvalidType5 + | DiagnosticCode::InvalidLineType + | DiagnosticCode::InvalidColour + | DiagnosticCode::MetaEmpty + | DiagnosticCode::TexmapNextType0 + | DiagnosticCode::ColourSlotsLimit + | DiagnosticCode::TexmapUnknownMetaStop + | DiagnosticCode::BfcInvalidDirective + | DiagnosticCode::BfcRecertifyUncertified + | DiagnosticCode::BfcInvalidCertifyArgument + | DiagnosticCode::BfcInvertNextOutsideCertified + | DiagnosticCode::BfcInvalidContext + | DiagnosticCode::TexmapInvalidArity + | DiagnosticCode::TexmapInvalidMethod + | DiagnosticCode::TexmapNextNested + | DiagnosticCode::TexmapInvalidValue + | DiagnosticCode::TexmapFallbackOutOfScope + | DiagnosticCode::TexmapDuplicateFallback + | DiagnosticCode::TexmapInvalidEnd + | DiagnosticCode::DataInvalid + | DiagnosticCode::DataTruncated + | DiagnosticCode::IncludeDepthLimit + | DiagnosticCode::GraphCycle + | DiagnosticCode::ResourceBytesLimit + | DiagnosticCode::CommandsLimit + | DiagnosticCode::InstancesLimit + | DiagnosticCode::TrianglesLimit + | DiagnosticCode::LinesLimit + | DiagnosticCode::TexturesLimit + | DiagnosticCode::FetchesLimit + | DiagnosticCode::DiagnosticsLimit + ), + LdrawError::LdrawDiagnostic(_) + | LdrawError::Reservation(_) + | LdrawError::Resolver(_) + | LdrawError::Overflow(_) => false, + } +} + fn fuzz() -> Result { let fixtures = inventory(); validate_inventory(&fixtures)?; - let root = root_dir(); - let binary = oracle_binary(&root)?; + let binary = oracle_binary(&root_dir())?; let temporary = env::temp_dir().join(format!("polymodel-pm77-fuzz-{}", std::process::id())); fs::create_dir_all(&temporary).map_err(|error| error.to_string())?; let started = Instant::now(); @@ -1164,13 +458,7 @@ fn fuzz() -> Result { if started.elapsed() >= FUZZ_BUDGET { return Err(format!("fuzz budget exceeded before mutation {index}")); } - if !mutation_index_is_authorized(index, fixtures.len()) { - return Err(format!( - "mutation {index} has no authorized fixture/profile/operator contract" - )); - } - let fixture_index = mutation_fixture_index(index, fixtures.len()); - let fixture = &fixtures[fixture_index]; + let fixture = &fixtures[index % fixtures.len()]; let bytes = mutate(FUZZ_SEED, index, fixture.bytes); if bytes.len() > MAX_MUTATED_BYTES { return Err(format!("mutation {index} exceeded 1 MiB")); @@ -1182,125 +470,72 @@ fn fuzz() -> Result { let digest = Sha256::digest(&bytes); let path = temporary.join(format!("{index}-{}.ldr", hex::encode(digest))); fs::write(&path, &bytes).map_err(|error| error.to_string())?; - let operator = mutation_operator(FUZZ_SEED, index); let ledger = ReservationLedger::new(); - let first = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let parsed = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { InProcessRustAdapter::parse_fixture( fixture.id, &bytes, - core_profile(fixture.profile), + core_profile(fixture.expected.profile), ReservationOwner::preview(1, index as u64 + 1, 1), &ledger, ) })) - .map_err(|_| { - format!( - "mutation {index} ({operator}) ({}) core parser panicked", - fixture.id - ) - })?; - let first_observation = match first { - Ok(parsed) => candidate_observation(&parsed).and_then(|observation| { - serde_json::to_vec(&observation) - }).map_err(|error| { - format!( - "mutation {index} ({operator}) ({}) candidate observation failed: {error}", - fixture.id - ) - }), - Err(error) => Err(format!("error:{error:?}")), - }; - if ledger.snapshot().reservations != 0 { - return Err(format!( - "mutation {index} ({operator}) ({}) leaked ledger reservations", - fixture.id - )); - } - let repeat_ledger = ReservationLedger::new(); - let repeat = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - InProcessRustAdapter::parse_fixture( - fixture.id, - &bytes, - core_profile(fixture.profile), - ReservationOwner::preview(1, index as u64 + 1, 1), - &repeat_ledger, - ) - })) - .map_err(|_| { - format!( - "mutation {index} ({operator}) ({}) repeated core parser panicked", - fixture.id - ) - })?; - let second_observation = match repeat { - Ok(parsed) => candidate_observation(&parsed).and_then(|observation| { - serde_json::to_vec(&observation) - }).map_err(|error| { - format!( - "mutation {index} ({operator}) ({}) repeated candidate observation failed: {error}", - fixture.id - ) - }), - Err(error) => Err(format!("error:{error:?}")), - }; - if repeat_ledger.snapshot().reservations != 0 { - return Err(format!( - "mutation {index} ({operator}) ({}) repeated parse leaked ledger reservations", - fixture.id - )); - } - if first_observation != second_observation { - return Err(format!( - "mutation {index} ({operator}) ({}) core outcome is not deterministic", - fixture.id - )); - } - let oracle_observation = |oracle: Result, repeated: bool| { - match oracle { - Ok(document) => serde_json::to_vec(&projection(document, &path, &bytes)?), - Err(error) => Ok(format!("error:{error}").into_bytes()), + .map_err(|_| format!("mutation {index} ({}) core parser panicked", fixture.id))?; + match parsed { + Ok(parsed) => { + let first = serialize_canonical(&parsed.project()) + .map_err(|error| format!("mutation {index} ({}) projection failed: {error}", fixture.id))?; + drop(parsed); + if ledger.snapshot().reservations != 0 { + return Err(format!("mutation {index} ({}) leaked ledger reservations", fixture.id)); + } + let repeat_ledger = ReservationLedger::new(); + let repeat = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + InProcessRustAdapter::parse_fixture( + fixture.id, + &bytes, + core_profile(fixture.expected.profile), + ReservationOwner::preview(1, index as u64 + 1, 1), + &repeat_ledger, + ) + })) + .map_err(|_| format!("mutation {index} ({}) repeated core parser panicked", fixture.id))? + .map_err(|error| format!("mutation {index} ({}) was nondeterministically rejected: {error}", fixture.id))?; + let second = serialize_canonical(&repeat.project()) + .map_err(|error| format!("mutation {index} ({}) repeated projection failed: {error}", fixture.id))?; + drop(repeat); + if repeat_ledger.snapshot().reservations != 0 { + return Err(format!("mutation {index} ({}) repeated parse leaked ledger reservations", fixture.id)); + } + if first != second { + return Err(format!("mutation {index} ({}) accepted parse is not deterministic", fixture.id)); + } + let oracle = run_oracle(&binary, &path, ORACLE_TIMEOUT) + .map_err(|error| format!("mutation {index} ({}) failed: {error}", fixture.id))?; + if oracle.status != "ok" { + return Err(format!("mutation {index} ({}) accepted core parse has oracle status {}", fixture.id, oracle.status)); + } + } + Err(error) if bounded_mutation_rejection(&error) => { + if index == 0 { + match error { + LdrawError::Diagnostic(diagnostic) + if diagnostic.code == DiagnosticCode::InvalidUtf8 => {} + other => { + return Err(format!( + "mutation 0 ({}) expected INVALID_UTF8, got {other}", + fixture.id + )); + } + } + } + if ledger.snapshot().reservations != 0 { + return Err(format!("mutation {index} ({}) rejected parse leaked ledger reservations", fixture.id)); + } + } + Err(error) => { + return Err(format!("mutation {index} ({}) core parse failed: {error}", fixture.id)); } - .map_err(|error| { - format!( - "mutation {index} ({operator}) ({}) {} oracle observation failed: {error}", - fixture.id, - if repeated { "repeated" } else { "initial" } - ) - }) - }; - let oracle_bytes = oracle_observation( - run_oracle( - &binary, - &path, - &root.join("crates/polymodel-ldraw-testkit/corpus"), - ORACLE_TIMEOUT, - ), - false, - )?; - let oracle_repeat_bytes = oracle_observation( - run_oracle( - &binary, - &path, - &root.join("crates/polymodel-ldraw-testkit/corpus"), - ORACLE_TIMEOUT, - ), - true, - )?; - if oracle_bytes != oracle_repeat_bytes { - let first_difference = oracle_bytes - .iter() - .zip(&oracle_repeat_bytes) - .position(|(left, right)| left != right) - .unwrap_or(oracle_bytes.len().min(oracle_repeat_bytes.len())); - return Err(format!( - "mutation {index} ({operator}) ({}) oracle outcome is not deterministic: first_difference={}, initial_len={}, repeat_len={}, initial_sha256={}, repeat_sha256={}", - fixture.id, - first_difference, - oracle_bytes.len(), - oracle_repeat_bytes.len(), - hex::encode(Sha256::digest(&oracle_bytes)), - hex::encode(Sha256::digest(&oracle_repeat_bytes)), - )); } } Ok(()) @@ -1325,10 +560,7 @@ fn fuzz() -> Result { mod hex { pub fn encode(bytes: impl AsRef<[u8]>) -> String { - bytes - .as_ref() - .iter() - .map(|byte| format!("{byte:02x}")) - .collect() + bytes.as_ref().iter().map(|byte| format!("{byte:02x}")).collect() } } +