Something went wrong. Try again.
STreaming ARchive format: simpler, stricter, smaller than CARs for a subset of use-cases.
star car archive atproto
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144//! High-level archive encoding.//!//! Both builders take the header commitment from the caller and write it out//! before the body, so the whole encode streams. Neither checks that the//! entries actually reproduce that commitment — a caller who passes the wrong//! one gets an archive that no reader will accept.//!//! TODO: offer a checking variant of both builders. Each can fold its own//! commitment as entries stream past and compare at `finish`: kind `L` needs//! only the 2 KiB LtHash state, and kind `C` needs only the MST node stack//! (one node per layer). Neither has to hold the archive or the tree.
use std::io::Write;
use star_repo::commit::Commit;use star_repo::space::SpaceMeta;use star_repo::{LtHash, compute_cid_dag_cbor};
use crate::Writer;use crate::error::Result;
/// Encode a kind `C` STAR-lite archive into `output` from a [`Commit`] and an/// iterator of `(key, record)` pairs.////// Keys must arrive in strict lex byte order; the writer rejects unsorted/// or duplicate keys. The header CID is taken from `commit.data`.////// `output` is any [`Write`]: a file, a `Vec<u8>`, an `os_pipe::PipeWriter`,/// a zstd encoder, etc. The writer flushes before returning.pub fn encode_archive<W, I, K>(output: W, commit: &Commit, entries: I) -> Result<()>where W: Write, I: IntoIterator<Item = (K, Vec<u8>)>, K: AsRef<str>,{ let mut builder = ArchiveBuilder::new(output, commit)?; for (k, v) in entries { builder.add_entry(k.as_ref(), &v)?; } builder.finish()}
/// Encode a kind `C` STAR-lite archive into `output` like [`encode_archive`],/// but with a push-style interface that works better for fallible content/// sources and sinks.pub struct ArchiveBuilder<W: Write> { writer: Writer<W>,}
impl<W: Write> ArchiveBuilder<W> { /// Start an archive write from the header + partial commit. pub fn new(output: W, commit: &Commit) -> Result<Self> { let partial = commit.to_partial_bytes()?; let mut writer = Writer::new(output); writer.write_header(commit.data, Some(&partial))?; Ok(Self { writer }) }
/// Append one record, in strict key-ascending order. pub fn add_entry(&mut self, key: &str, record: &[u8]) -> Result<()> { self.writer.write_entry(key, record) }
pub fn finish(self) -> Result<()> { self.writer.finish() }}
/// Encode a kind `L` (space repo) archive into `output`.////// `commitment` is supplied by the caller and written into the header before/// any entry, so the whole write streams — nothing is buffered. A producer/// that maintains its repo's LtHash incrementally already has this value;/// otherwise compute it with [`space_commitment`] in a prior pass.////// `meta` is optional. An archive with no meta object is still fully/// verifiable, since the commitment lives in the header either way.////// The commitment is taken on trust — see the module note.pub fn encode_space_archive<W, I, K>( output: W, commitment: [u8; 32], meta: Option<&SpaceMeta>, entries: I,) -> Result<()>where W: Write, I: IntoIterator<Item = (K, Vec<u8>)>, K: AsRef<str>,{ let mut builder = SpaceArchiveBuilder::new(output, commitment, meta)?; for (k, v) in entries { builder.add_entry(k.as_ref(), &v)?; } builder.finish()}
/// Compute the kind `L` commitment for a set of `(key, record)` pairs.////// Order-independent — the LtHash is a set hash — so this can be fed from any/// iteration order. The archive body itself must still be written in strict/// lex key order.pub fn space_commitment<'a, I>(entries: I) -> [u8; 32]where I: IntoIterator<Item = (&'a str, &'a [u8])>,{ let mut hash = LtHash::new(); for (key, record) in entries { hash.add_entry(key, &compute_cid_dag_cbor(record)); } hash.commitment()}
/// Push-style builder for a kind `L` archive.////// The header — including `commitment` — is written on construction, so/// entries stream straight through with no buffering.////// The commitment is taken on trust: the builder does not fold the entries to/// confirm it, exactly as [`ArchiveBuilder`] does not rebuild the MST to/// confirm `commit.data`. See the module note on adding checking variants.pub struct SpaceArchiveBuilder<W: Write> { writer: Writer<W>,}
impl<W: Write> SpaceArchiveBuilder<W> { /// Write the header and start the body. pub fn new(output: W, commitment: [u8; 32], meta: Option<&SpaceMeta>) -> Result<Self> { let encoded = meta.map(|m| m.to_bytes()).transpose()?; let mut writer = Writer::new(output); writer.write_space_header(commitment, encoded.as_deref())?; Ok(Self { writer }) }
/// Append one record, in strict key-ascending order. pub fn add_entry(&mut self, key: &str, record: &[u8]) -> Result<()> { self.writer.write_entry(key, record) }
pub fn finish(self) -> Result<()> { self.writer.finish() }}