Something went wrong. Try again.
A 5e storytelling engine with an LLM DM
Something went wrong. Try again.
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301use std::fmt;use std::fs;use std::io;use std::path::{Path, PathBuf};use std::time::Duration;
use flate2::read::GzDecoder;use serde_yaml_ng::{Mapping, Value};use sha2::{Digest, Sha256};use tar::Archive;
const SRD_VERSION: &str = "5.2.1";
const SRD_URL: &str = "https://media.dndbeyond.com/compendium-images/srd/5.2/SRD_CC_v5.2.1.pdf";const SRD_SHA256: &str = "8974902d109d6e63672d7c490bde9ccf052410503d9cfa768237154fbc5e3d87";const SRD_DESTINATION: &str = "rules/srd-5.2/sources/SRD_CC_v5.2.1.pdf";const SRD_META_PATH: &str = "sources/SRD_CC_v5.2.1.pdf";
const SRD_TEXT_REPO_URL: &str = "https://github.com/oldmanumby/dnd.srd.5.2.1";const SRD_TEXT_COMMIT: &str = "af537072cc95f362544c71ad14d56046a9aa065a";const SRD_TEXT_TARBALL_URL_PREFIX: &str = "https://codeload.github.com/oldmanumby/dnd.srd.5.2.1/tar.gz/";const SRD_TEXT_TARBALL_SHA256: &str = "b9bfc488f333b9713fd3b46a8597a1986f5123d9d57545a7df5104b1aa64e21d";const SRD_TEXT_DESTINATION: &str = "rules/srd-5.2/sources/dnd.srd.5.2.1";const SRD_TEXT_META_PATH: &str = "sources/dnd.srd.5.2.1/";
const META_DESTINATION: &str = "rules/srd-5.2/meta.yaml";
const DOWNLOAD_TIMEOUT: Duration = Duration::from_secs(60);
/// Where the SRD PDF comes from, where it lands, and the hash it must match.pub struct FetchConfig { pub url: String, pub destination: PathBuf, pub expected_sha256: String,}
impl FetchConfig { /// The config for SRD 5.2.1, the version this pipeline targets. pub fn srd_5_2_1() -> Self { Self { url: SRD_URL.to_string(), destination: PathBuf::from(SRD_DESTINATION), expected_sha256: SRD_SHA256.to_string(), } }}
/// Where the vendored SRD markdown source comes from, where it lands, and/// the pinned commit and tarball hash it must match.pub struct TextFetchConfig { pub url: String, pub destination: PathBuf, pub expected_tarball_sha256: String, pub commit: String, pub meta_path: PathBuf,}
impl TextFetchConfig { /// The config for the dnd.srd.5.2.1 markdown source, pinned to the /// commit this pipeline targets. pub fn srd_5_2_1_text() -> Self { Self { url: format!("{SRD_TEXT_TARBALL_URL_PREFIX}{SRD_TEXT_COMMIT}"), destination: PathBuf::from(SRD_TEXT_DESTINATION), expected_tarball_sha256: SRD_TEXT_TARBALL_SHA256.to_string(), commit: SRD_TEXT_COMMIT.to_string(), meta_path: PathBuf::from(META_DESTINATION), } }}
/// What `rules/srd-5.2/meta.yaml` records about the vendored SRD sources.pub struct SrdMeta { pub version: String, pub pdf_path: String, pub pdf_url: String, pub pdf_sha256: String, pub text_path: String, pub text_url: String, pub text_commit: String, pub text_tarball_sha256: String,}
impl SrdMeta { /// The meta record for SRD 5.2.1, the version this pipeline targets. pub fn srd_5_2_1() -> Self { Self { version: SRD_VERSION.to_string(), pdf_path: SRD_META_PATH.to_string(), pdf_url: SRD_URL.to_string(), pdf_sha256: SRD_SHA256.to_string(), text_path: SRD_TEXT_META_PATH.to_string(), text_url: SRD_TEXT_REPO_URL.to_string(), text_commit: SRD_TEXT_COMMIT.to_string(), text_tarball_sha256: SRD_TEXT_TARBALL_SHA256.to_string(), } }
/// Renders this record as the `meta.yaml` this pipeline reads and /// writes, starting from `existing` (the file's current contents, when /// there is one) and rewriting only the fields fetch owns: `version`, /// and each source entry's `path`, `url`, and pin. Every other key /// carries over untouched, including a text source's hand-maintained /// `patches` block: fetch does not know what patches exist and must /// never invent or drop that record. Source entries are matched by /// `path`; an entry `existing` doesn't already have gets a fresh one /// appended, in the same field order fetch has always written. With no /// `existing` file, this renders a fresh minimal `meta.yaml` from /// scratch. /// /// The text source is marked `filtered: true`: vendoring keeps only the /// numbered content directories and the top-level licensing files (see /// `should_vendor`), not the full pinned commit. pub fn render(&self, existing: Option<&str>) -> String { let mut top = existing .and_then(|text| serde_yaml_ng::from_str::<Value>(text).ok()) .and_then(|value| value.as_mapping().cloned()) .unwrap_or_default();
top.insert( Value::String("version".to_string()), Value::String(self.version.clone()), );
let mut sources = top .get("sources") .and_then(Value::as_sequence) .cloned() .unwrap_or_default();
let pdf_entry = owned_source_entry(&mut sources, &self.pdf_path); set_string(pdf_entry, "path", &self.pdf_path); set_string(pdf_entry, "url", &self.pdf_url); set_string(pdf_entry, "sha256", &self.pdf_sha256);
let text_entry = owned_source_entry(&mut sources, &self.text_path); set_string(text_entry, "path", &self.text_path); set_string(text_entry, "url", &self.text_url); set_string(text_entry, "commit", &self.text_commit); set_string(text_entry, "tarball_sha256", &self.text_tarball_sha256); text_entry.insert(Value::String("filtered".to_string()), Value::Bool(true));
top.insert( Value::String("sources".to_string()), Value::Sequence(sources), );
serde_yaml_ng::to_string(&Value::Mapping(top)) .expect("a mapping of strings and bools always serializes to YAML") }}
/// The entry in `sources` whose `path` field equals `path`, or a freshly/// appended empty entry when none matches. `SrdMeta::render` uses this to/// find the mapping it owns a few fields of without disturbing any entry,/// or any field of that entry, it doesn't recognize.fn owned_source_entry<'a>(sources: &'a mut Vec<Value>, path: &str) -> &'a mut Mapping { let position = sources.iter().position(|value| { value .as_mapping() .and_then(|mapping| mapping.get("path")) .and_then(Value::as_str) == Some(path) }); let index = position.unwrap_or_else(|| { sources.push(Value::Mapping(Mapping::new())); sources.len() - 1 }); sources[index] .as_mapping_mut() .expect("owned_source_entry only ever indexes an entry it confirmed is a mapping")}
fn set_string(mapping: &mut Mapping, key: &str, value: &str) { mapping.insert( Value::String(key.to_string()), Value::String(value.to_string()), );}
/// The PDF and markdown sources for one SRD version, and the meta record/// that documents their pins.pub struct SrdSources { pub pdf: FetchConfig, pub text: TextFetchConfig, pub meta: SrdMeta,}
impl SrdSources { /// The sources for SRD 5.2.1, the version this pipeline targets. pub fn srd_5_2_1() -> Self { Self { pdf: FetchConfig::srd_5_2_1(), text: TextFetchConfig::srd_5_2_1_text(), meta: SrdMeta::srd_5_2_1(), } }}
#[derive(Debug, PartialEq, Eq)]pub enum FetchOutcome { AlreadyPresent, Downloaded,}
#[derive(Debug)]pub enum FetchError { Download(String), Io(String), HashMismatch { expected: String, actual: String, }, UnsafeEntry(String), VendoredTreeExists { destination: String, meta_path: String, },}
impl fmt::Display for FetchError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { FetchError::Download(message) => write!(f, "download failed: {message}"), FetchError::Io(message) => write!(f, "file error: {message}"), FetchError::HashMismatch { expected, actual } => { write!(f, "hash mismatch: expected {expected}, got {actual}") } FetchError::UnsafeEntry(name) => { write!(f, "unsafe tarball entry: {name}") } FetchError::VendoredTreeExists { destination, meta_path, } => write!( f, "vendored tree already exists at '{destination}' and its recorded pin does not \ match. It may hold hand-applied patches recorded in the patches block of \ '{meta_path}'. Move '{destination}' aside on purpose, then fetch again." ), } }}
impl std::error::Error for FetchError {}
impl From<io::Error> for FetchError { fn from(error: io::Error) -> Self { FetchError::Io(error.to_string()) }}
/// Downloads the file at `config.url` to `config.destination`, verifying its/// SHA-256 against `config.expected_sha256`.////// If the destination already exists and its hash matches, the download is/// skipped. If it exists but the hash does not match, it is treated as/// corrupt and replaced.pub fn fetch(config: &FetchConfig) -> Result<FetchOutcome, FetchError> { if config.destination.exists() { let existing = fs::read(&config.destination)?; if sha256_hex(&existing) == config.expected_sha256 { return Ok(FetchOutcome::AlreadyPresent); } fs::remove_file(&config.destination)?; }
let bytes = download(&config.url)?; let actual_sha256 = sha256_hex(&bytes); if actual_sha256 != config.expected_sha256 { return Err(FetchError::HashMismatch { expected: config.expected_sha256.clone(), actual: actual_sha256, }); }
let parent = config.destination.parent().unwrap_or(Path::new("")); fs::create_dir_all(parent)?; fs::write(&config.destination, &bytes)?;
Ok(FetchOutcome::Downloaded)}
/// Fetches both the PDF and the markdown source for `sources`.pub fn fetch_all(sources: &SrdSources) -> Result<(FetchOutcome, FetchOutcome), FetchError> { let pdf_outcome = fetch(&sources.pdf)?; let text_outcome = fetch_text(&sources.text, &sources.meta)?; Ok((pdf_outcome, text_outcome))}
/// Downloads the tarball at `config.url`, verifies its SHA-256 against/// `config.expected_tarball_sha256`, and unpacks it into `config.destination`/// with the tarball's top-level directory stripped.////// If the destination already exists, this never removes it. When/// `config.meta_path` records `config.commit` as its pin, the tree is/// already up to date and the fetch is skipped. When the pin doesn't/// match, or can't be found at all, the tree may hold hand-applied patches/// that only exist on disk, and fetch has no way to tell an intentional/// edit from a stale download: it refuses and reports an error rather than/// guessing. Moving the tree aside is the caller's job. Only when the/// destination doesn't exist yet does fetch download, unpack, and rewrite/// `config.meta_path`.pub fn fetch_text(config: &TextFetchConfig, meta: &SrdMeta) -> Result<FetchOutcome, FetchError> { if config.destination.exists() { if meta_pin_matches(&config.meta_path, &config.commit) { return Ok(FetchOutcome::AlreadyPresent); } return Err(FetchError::VendoredTreeExists { destination: config.destination.display().to_string(), meta_path: config.meta_path.display().to_string(), }); }
let bytes = download(&config.url)?; let actual_sha256 = sha256_hex(&bytes); if actual_sha256 != config.expected_tarball_sha256 { return Err(FetchError::HashMismatch { expected: config.expected_tarball_sha256.clone(), actual: actual_sha256, }); }
unpack_stripped(&bytes, &config.destination)?;
let existing_meta = fs::read_to_string(&config.meta_path).ok(); let meta_parent = config.meta_path.parent().unwrap_or(Path::new("")); fs::create_dir_all(meta_parent)?; fs::write(&config.meta_path, meta.render(existing_meta.as_deref()))?;
Ok(FetchOutcome::Downloaded)}
/// Reports whether `meta_path`'s parsed `sources` list has an entry whose/// `commit` field equals `commit`. Parses the file as YAML rather than/// scanning its text, so a line that merely looks like the pin, such as/// prose inside a patch's `reason` field, can never be mistaken for it.fn meta_pin_matches(meta_path: &Path, commit: &str) -> bool { let Ok(contents) = fs::read_to_string(meta_path) else { return false; }; let Ok(value) = serde_yaml_ng::from_str::<Value>(&contents) else { return false; }; let Some(sources) = value .as_mapping() .and_then(|mapping| mapping.get("sources")) .and_then(Value::as_sequence) else { return false; }; sources.iter().any(|source| { source .as_mapping() .and_then(|mapping| mapping.get("commit")) .and_then(Value::as_str) == Some(commit) })}
/// Unpacks a gzipped tarball into `destination`, dropping each entry's/// top-level path component and keeping only entries `should_vendor`/// accepts.////// Assumes `destination` does not exist yet. `fetch_text` only ever calls/// this after confirming that, since overwriting an existing vendored tree/// can destroy hand-applied patches that have no record anywhere but the/// tree itself.fn unpack_stripped(tarball: &[u8], destination: &Path) -> Result<(), FetchError> { fs::create_dir_all(destination)?;
let mut archive = Archive::new(GzDecoder::new(tarball)); for entry in archive.entries()? { let mut entry = entry?; let entry_path = entry.path()?.into_owned(); if !is_safe_entry_path(&entry_path) { return Err(FetchError::UnsafeEntry(entry_path.display().to_string())); } let Some(relative_path) = strip_first_component(&entry_path) else { continue; }; if !should_vendor(&relative_path) { continue; } let entry_type = entry.header().entry_type(); if !is_safe_entry_type(entry_type) { return Err(FetchError::UnsafeEntry(format!( "{} is a {entry_type:?} entry, not a file or directory", entry_path.display() ))); } let target = destination.join(&relative_path); if entry_type.is_dir() { fs::create_dir_all(&target)?; } else { let file_parent = target.parent().unwrap_or(Path::new("")); fs::create_dir_all(file_parent)?; entry.unpack(&target)?; } } Ok(())}
/// Reports whether a tar entry's own path is safe to unpack: relative, and/// free of `..` components that could escape the destination directory once/// the top-level directory is stripped and the rest joined onto it.fn is_safe_entry_path(path: &Path) -> bool { !path.is_absolute() && !path .components() .any(|component| matches!(component, std::path::Component::ParentDir))}
/// Reports whether a tar entry's type is safe to unpack: a regular file or/// a directory, the only two shapes this vendored repo actually contains./// A symlink entry unpacked through `Entry::unpack` has its target used/// unvalidated, so a later entry whose path walks through that symlink can/// write outside `destination`; rejecting the symlink itself closes that/// off before any such entry is ever written.fn is_safe_entry_type(entry_type: tar::EntryType) -> bool { entry_type.is_file() || entry_type.is_dir()}
/// The upstream repo's top-level files that document the source and its/// licensing, rather than being part of the entry corpus itself.const VENDORED_TOP_LEVEL_FILES: [&str; 3] = ["Legal.md", "README.md", "structure.md"];
/// Reports whether a (top-level-stripped) path from the upstream repo/// belongs in the vendored corpus: a numbered content directory/// (`01_Playing_The_Game`, `07_Spells`, ...) or one of the top-level/// licensing files. Excludes everything else the upstream repo carries,/// such as `docs_original/` (their own PDF and pre-split text), compiled/// re-exports, `.github/`, and image assets.fn should_vendor(relative_path: &Path) -> bool { let mut components = relative_path.components(); let Some(first) = components.next() else { return false; }; let first = first.as_os_str().to_string_lossy(); if is_numbered_content_dir(&first) { return true; } components.next().is_none() && VENDORED_TOP_LEVEL_FILES.contains(&first.as_ref())}
/// Reports whether a path component looks like `NN_Some_Name`: two ASCII/// digits, an underscore, then anything.fn is_numbered_content_dir(component: &str) -> bool { let bytes = component.as_bytes(); bytes.len() > 2 && bytes[0].is_ascii_digit() && bytes[1].is_ascii_digit() && bytes[2] == b'_'}
/// Drops a path's first component, or `None` if that was its only component.fn strip_first_component(path: &Path) -> Option<PathBuf> { let mut components = path.components(); components.next()?; let rest: PathBuf = components.collect(); if rest.as_os_str().is_empty() { None } else { Some(rest) }}
fn download(url: &str) -> Result<Vec<u8>, FetchError> { let config = ureq::Agent::config_builder() .timeout_global(Some(DOWNLOAD_TIMEOUT)) .build(); let agent: ureq::Agent = config.into(); agent .get(url) .call() .and_then(|mut response| response.body_mut().read_to_vec()) .map_err(|error| FetchError::Download(error.to_string()))}
pub(crate) fn sha256_hex(bytes: &[u8]) -> String { let mut hasher = Sha256::new(); hasher.update(bytes); hasher .finalize() .iter() .map(|byte| format!("{byte:02x}")) .collect()}
#[cfg(test)]mod tests { use super::*; use crate::srd::testing::{append_entry, fixture_meta, serve_once, tiny_tar_gz}; use std::io::Write; use tempfile::TempDir;
/// Builds a tarball where a file entry (`07_Spells`) is later reused as /// a directory prefix (`07_Spells/b.md`), so unpacking it hits a real /// filesystem collision when it tries to create `07_Spells` as a /// directory. Both entries live under a numbered content directory name /// so `should_vendor` keeps them and the collision is actually reached. fn tar_gz_with_a_file_parent_collision(top_dir: &str) -> Vec<u8> { let mut builder = tar::Builder::new(Vec::new()); append_entry(&mut builder, &format!("{top_dir}/"), None); append_entry( &mut builder, &format!("{top_dir}/07_Spells"), Some(b"07_Spells is a file"), ); append_entry( &mut builder, &format!("{top_dir}/07_Spells/b.md"), Some(b"b"), ); let tar_bytes = builder.into_inner().unwrap();
let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); encoder.write_all(&tar_bytes).unwrap(); encoder.finish().unwrap() }
/// Writes `name` directly into the header's raw name field, bypassing /// `Header::set_path`'s validation. `tar::Header` refuses to build a /// `..`-laden or absolute path through its safe API (by design, for /// archives it writes); this simulates a malicious tarball that was not /// produced by this crate and carries one anyway. fn set_unchecked_entry_name(header: &mut tar::Header, name: &str) { let bytes = name.as_bytes(); let field = &mut header.as_old_mut().name; field.fill(0); field[..bytes.len()].copy_from_slice(bytes); }
fn append_entry_with_unchecked_path( builder: &mut tar::Builder<Vec<u8>>, name: &str, contents: &[u8], ) { let mut header = tar::Header::new_gnu(); set_unchecked_entry_name(&mut header, name); header.set_mode(0o644); header.set_size(contents.len() as u64); header.set_cksum(); builder.append(&header, contents).unwrap(); }
/// Builds a tarball with one entry whose name climbs out of `top_dir` /// with `..` components, the way a malicious upstream commit could try /// to write outside the unpack destination. fn tar_gz_with_a_parent_dir_traversal(top_dir: &str) -> Vec<u8> { let mut builder = tar::Builder::new(Vec::new()); append_entry(&mut builder, &format!("{top_dir}/"), None); append_entry_with_unchecked_path( &mut builder, &format!("{top_dir}/../../escape.txt"), b"escaped via parent dir", ); let tar_bytes = builder.into_inner().unwrap();
let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); encoder.write_all(&tar_bytes).unwrap(); encoder.finish().unwrap() }
/// Builds a tarball with one entry named as an absolute path, bypassing /// `top_dir` entirely. fn tar_gz_with_an_absolute_path_entry(top_dir: &str) -> Vec<u8> { let mut builder = tar::Builder::new(Vec::new()); append_entry(&mut builder, &format!("{top_dir}/"), None); append_entry_with_unchecked_path(&mut builder, "/etc/evil.txt", b"escaped via root"); let tar_bytes = builder.into_inner().unwrap();
let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); encoder.write_all(&tar_bytes).unwrap(); encoder.finish().unwrap() }
/// Builds a tarball with a symlink entry (`07_Spells/x`, targeting /// `symlink_target`) followed by a regular file entry whose path walks /// through that symlink (`07_Spells/x/evil.md`), the way a malicious /// upstream commit could try to write outside the unpack destination /// via a symlinked directory rather than a `..` path component. fn tar_gz_with_a_symlink_escape(top_dir: &str, symlink_target: &Path) -> Vec<u8> { let mut builder = tar::Builder::new(Vec::new()); append_entry(&mut builder, &format!("{top_dir}/"), None); append_entry(&mut builder, &format!("{top_dir}/07_Spells/"), None);
let mut symlink_header = tar::Header::new_gnu(); symlink_header .set_path(format!("{top_dir}/07_Spells/x")) .unwrap(); symlink_header.set_entry_type(tar::EntryType::Symlink); symlink_header.set_link_name(symlink_target).unwrap(); symlink_header.set_mode(0o777); symlink_header.set_size(0); symlink_header.set_cksum(); builder.append(&symlink_header, io::empty()).unwrap();
append_entry( &mut builder, &format!("{top_dir}/07_Spells/x/evil.md"), Some(b"evil"), ); let tar_bytes = builder.into_inner().unwrap();
let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); encoder.write_all(&tar_bytes).unwrap(); encoder.finish().unwrap() }
#[test] fn sha256_hex_matches_the_published_fips_180_vector_for_abc() { // Every other test in this module computes its expected hash by // calling `sha256_hex` itself, so a bug that truncates or // otherwise corrupts the hex encoding would pass all of them. // This pins one published test vector as a literal, so the // function's actual output is checked against a value nothing in // this file derived. assert_eq!( sha256_hex(b"abc"), "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" ); }
#[test] fn srd_5_2_1_targets_the_known_source() { let config = FetchConfig::srd_5_2_1();
assert_eq!(config.url, SRD_URL); assert_eq!(config.destination, PathBuf::from(SRD_DESTINATION)); assert_eq!(config.expected_sha256, SRD_SHA256); }
#[test] fn srd_5_2_1_text_targets_the_known_source() { let config = TextFetchConfig::srd_5_2_1_text();
assert_eq!( config.url, format!("{SRD_TEXT_TARBALL_URL_PREFIX}{SRD_TEXT_COMMIT}") ); assert_eq!(config.destination, PathBuf::from(SRD_TEXT_DESTINATION)); assert_eq!(config.expected_tarball_sha256, SRD_TEXT_TARBALL_SHA256); assert_eq!(config.commit, SRD_TEXT_COMMIT); assert_eq!(config.meta_path, PathBuf::from(META_DESTINATION)); }
#[test] fn srd_meta_srd_5_2_1_targets_the_known_sources() { let meta = SrdMeta::srd_5_2_1();
assert_eq!(meta.version, SRD_VERSION); assert_eq!(meta.pdf_path, SRD_META_PATH); assert_eq!(meta.pdf_url, SRD_URL); assert_eq!(meta.pdf_sha256, SRD_SHA256); assert_eq!(meta.text_path, SRD_TEXT_META_PATH); assert_eq!(meta.text_url, SRD_TEXT_REPO_URL); assert_eq!(meta.text_commit, SRD_TEXT_COMMIT); assert_eq!(meta.text_tarball_sha256, SRD_TEXT_TARBALL_SHA256); }
#[test] fn srd_meta_renders_the_documented_yaml_shape_with_no_existing_file() { let meta = fixture_meta();
let rendered = meta.render(None);
assert_eq!( rendered, format!( "version: 5.2.1\n\ sources:\n\ - path: sources/SRD_CC_v5.2.1.pdf\n\ \x20 url: https://example.test/srd.pdf\n\ \x20 sha256: {pdf_sha256}\n\ - path: sources/dnd.srd.5.2.1/\n\ \x20 url: https://example.test/dnd.srd.5.2.1\n\ \x20 commit: deadbeef\n\ \x20 tarball_sha256: {text_tarball_sha256}\n\ \x20 filtered: true\n", pdf_sha256 = "a".repeat(64), text_tarball_sha256 = "b".repeat(64), ) ); }
#[test] fn srd_meta_render_preserves_an_existing_patches_block_and_extra_fields() { let meta = fixture_meta(); let existing = "\ version: 5.2.0\n\ extra_top_level_field: keep-me\n\ sources:\n\ \x20 - path: sources/SRD_CC_v5.2.1.pdf\n\ \x20 url: https://old.example.test/srd.pdf\n\ \x20 sha256: oldpdfhash\n\ \x20 - path: sources/dnd.srd.5.2.1/\n\ \x20 url: https://example.test/dnd.srd.5.2.1\n\ \x20 commit: old-commit\n\ \x20 tarball_sha256: oldtarballhash\n\ \x20 filtered: true\n\ \x20 patches:\n\ \x20 - path: 11_Monsters/Monsters_Each/Winter_Wolf.md\n\ \x20 reason: file held only the H1 heading, no body\n\ \x20 repaired_from: 11_Monsters/Monsters_All.md\n";
let rendered = meta.render(Some(existing)); let value: Value = serde_yaml_ng::from_str(&rendered).unwrap(); let mapping = value.as_mapping().unwrap(); let sources = mapping.get("sources").unwrap().as_sequence().unwrap(); let text_entry = sources[1].as_mapping().unwrap(); let patches = text_entry.get("patches").unwrap().as_sequence().unwrap();
assert_eq!( mapping.get("extra_top_level_field").unwrap().as_str(), Some("keep-me") ); assert_eq!(mapping.get("version").unwrap().as_str(), Some("5.2.1")); assert_eq!(text_entry.get("commit").unwrap().as_str(), Some("deadbeef")); assert_eq!(patches.len(), 1); assert_eq!( patches[0] .as_mapping() .unwrap() .get("path") .unwrap() .as_str(), Some("11_Monsters/Monsters_Each/Winter_Wolf.md") ); }
#[test] fn is_numbered_content_dir_accepts_two_digit_prefixes() { assert!(is_numbered_content_dir("01_Playing_The_Game")); assert!(is_numbered_content_dir("12_Animals")); }
#[test] fn is_numbered_content_dir_rejects_non_numbered_names() { assert!(!is_numbered_content_dir("docs_original")); assert!(!is_numbered_content_dir(".github")); }
#[test] fn is_numbered_content_dir_rejects_a_name_with_no_room_for_a_prefix() { assert!(!is_numbered_content_dir("12")); }
#[test] fn should_vendor_rejects_an_empty_path() { assert!(!should_vendor(Path::new(""))); }
#[test] fn should_vendor_accepts_a_numbered_content_directory() { assert!(should_vendor(Path::new("07_Spells"))); }
#[test] fn should_vendor_accepts_a_file_nested_in_a_numbered_content_directory() { assert!(should_vendor(Path::new("07_Spells/Fireball.md"))); }
#[test] fn should_vendor_accepts_the_documented_top_level_licensing_files() { assert!(should_vendor(Path::new("Legal.md"))); assert!(should_vendor(Path::new("README.md"))); assert!(should_vendor(Path::new("structure.md"))); }
#[test] fn should_vendor_rejects_an_unlisted_top_level_file() { assert!(!should_vendor(Path::new("SRD-reForged.png"))); }
#[test] fn should_vendor_rejects_docs_original() { assert!(!should_vendor(Path::new( "docs_original/dnd_srd_5.2.1_cc.pdf" ))); }
#[test] fn should_vendor_rejects_dot_github() { assert!(!should_vendor(Path::new(".github/FUNDING.yml"))); }
#[test] fn fetch_downloads_and_verifies_the_hash() { let body = b"fixture pdf bytes".to_vec(); let expected = sha256_hex(&body); let (url, server) = serve_once(body.clone()); let pdf_dir = TempDir::new().unwrap(); let destination = pdf_dir.path().join("fixture.pdf"); let config = FetchConfig { url, destination: destination.clone(), expected_sha256: expected, };
let outcome = fetch(&config).unwrap();
assert_eq!(outcome, FetchOutcome::Downloaded); assert_eq!(fs::read(&destination).unwrap(), body); server.join().unwrap(); }
#[test] fn fetch_skips_download_when_the_existing_hash_matches() { let body = b"already present".to_vec(); let expected = sha256_hex(&body); let pdf_dir = TempDir::new().unwrap(); let destination = pdf_dir.path().join("fixture.pdf"); fs::write(&destination, &body).unwrap(); let never_dialed_url = "http://127.0.0.1:0".to_string(); let config = FetchConfig { url: never_dialed_url, destination, expected_sha256: expected, };
let outcome = fetch(&config).unwrap();
assert_eq!(outcome, FetchOutcome::AlreadyPresent); }
#[test] fn fetch_redownloads_a_corrupted_existing_file() { let body = b"good bytes".to_vec(); let expected = sha256_hex(&body); let (url, server) = serve_once(body.clone()); let pdf_dir = TempDir::new().unwrap(); let destination = pdf_dir.path().join("fixture.pdf"); fs::write(&destination, b"stale garbage").unwrap(); let config = FetchConfig { url, destination: destination.clone(), expected_sha256: expected, };
let outcome = fetch(&config).unwrap();
assert_eq!(outcome, FetchOutcome::Downloaded); assert_eq!(fs::read(&destination).unwrap(), body); server.join().unwrap(); }
#[test] fn fetch_rejects_a_hash_mismatch_and_leaves_no_file_behind() { let body = b"wrong bytes".to_vec(); let (url, server) = serve_once(body); let pdf_dir = TempDir::new().unwrap(); let destination = pdf_dir.path().join("fixture.pdf"); let config = FetchConfig { url, destination: destination.clone(), expected_sha256: "0".repeat(64), };
let error = fetch(&config).unwrap_err();
assert!(matches!(error, FetchError::HashMismatch { .. })); assert!(error.to_string().contains("expected")); assert!(error.to_string().contains("got")); assert!(!destination.exists()); server.join().unwrap(); }
#[test] fn fetch_reports_a_download_failure() { let pdf_dir = TempDir::new().unwrap(); let destination = pdf_dir.path().join("fixture.pdf"); let config = FetchConfig { url: "http://127.0.0.1:0/fixture.pdf".to_string(), destination, expected_sha256: "0".repeat(64), };
let error = fetch(&config).unwrap_err();
assert!(matches!(error, FetchError::Download(_))); assert!(error.to_string().contains("download failed")); }
#[test] fn fetch_reports_a_filesystem_error() { let body = b"any bytes".to_vec(); let expected = sha256_hex(&body); let (url, server) = serve_once(body); let dir = TempDir::new().unwrap(); let blocking_file = dir.path().join("blocker"); fs::write(&blocking_file, b"not a directory").unwrap(); let destination = blocking_file.join("nested").join("fixture.pdf"); let config = FetchConfig { url, destination, expected_sha256: expected, };
let error = fetch(&config).unwrap_err();
assert!(matches!(error, FetchError::Io(_))); assert!(error.to_string().contains("file error")); server.join().unwrap(); }
#[test] fn meta_pin_matches_the_parsed_commit_field() { let meta_dir = TempDir::new().unwrap(); let meta_path = meta_dir.path().join("meta.yaml"); fs::write( &meta_path, "sources:\n - path: sources/dnd.srd.5.2.1/\n commit: deadbeef\n", ) .unwrap();
assert!(meta_pin_matches(&meta_path, "deadbeef")); }
#[test] fn meta_pin_matches_is_false_when_the_file_is_missing() { let meta_dir = TempDir::new().unwrap();
assert!(!meta_pin_matches( &meta_dir.path().join("meta.yaml"), "deadbeef" )); }
#[test] fn meta_pin_matches_is_false_for_invalid_yaml() { let meta_dir = TempDir::new().unwrap(); let meta_path = meta_dir.path().join("meta.yaml"); fs::write(&meta_path, "not: [valid").unwrap();
assert!(!meta_pin_matches(&meta_path, "deadbeef")); }
#[test] fn meta_pin_matches_is_false_without_a_sources_list() { let meta_dir = TempDir::new().unwrap(); let meta_path = meta_dir.path().join("meta.yaml"); fs::write(&meta_path, "version: 5.2.1\n").unwrap();
assert!(!meta_pin_matches(&meta_path, "deadbeef")); }
#[test] fn meta_pin_matches_ignores_the_pin_text_inside_an_unrelated_value() { // A block scalar can contain a line that, read on its own, looks // exactly like a `commit:` pin line. A parser that walked the real // YAML structure instead of scanning lines of text must not // mistake that line for the actual `sources[].commit` field. let meta_dir = TempDir::new().unwrap(); let meta_path = meta_dir.path().join("meta.yaml"); fs::write( &meta_path, "sources:\n - path: sources/dnd.srd.5.2.1/\n commit: other-pin\n patches:\n - path: a.md\n reason: |\n commit: deadbeef\n", ) .unwrap();
assert!(!meta_pin_matches(&meta_path, "deadbeef")); }
#[test] fn fetch_text_skips_download_when_the_recorded_pin_matches() { let destination_dir = TempDir::new().unwrap(); let destination = destination_dir.path().join("dnd.srd.5.2.1"); fs::create_dir_all(&destination).unwrap(); let meta_dir = TempDir::new().unwrap(); let meta_path = meta_dir.path().join("meta.yaml"); fs::write( &meta_path, "sources:\n - path: sources/dnd.srd.5.2.1/\n commit: deadbeef\n", ) .unwrap(); let config = TextFetchConfig { url: "http://127.0.0.1:0".to_string(), destination, expected_tarball_sha256: "0".repeat(64), commit: "deadbeef".to_string(), meta_path, };
let outcome = fetch_text(&config, &fixture_meta()).unwrap();
assert_eq!(outcome, FetchOutcome::AlreadyPresent); }
#[test] fn fetch_text_downloads_and_unpacks_the_tarball_stripping_the_top_directory() { let body = tiny_tar_gz("dnd.srd.5.2.1-deadbeef"); let expected = sha256_hex(&body); let (url, server) = serve_once(body); let destination_dir = TempDir::new().unwrap(); let destination = destination_dir.path().join("dnd.srd.5.2.1"); let meta_dir = TempDir::new().unwrap(); let meta_path = meta_dir.path().join("meta.yaml"); let config = TextFetchConfig { url, destination: destination.clone(), expected_tarball_sha256: expected, commit: "deadbeef".to_string(), meta_path: meta_path.clone(), }; let meta = fixture_meta();
let outcome = fetch_text(&config, &meta).unwrap();
assert_eq!(outcome, FetchOutcome::Downloaded); assert_eq!( fs::read(destination.join("07_Spells/Fireball.md")).unwrap(), b"# Fireball\n" ); assert_eq!( fs::read(destination.join("Legal.md")).unwrap(), b"CC-BY-4.0\n" ); assert!(!destination.join(".github").exists()); assert!(!destination.join("docs_original").exists()); assert!(!destination.join("SRD-reForged.png").exists()); assert_eq!(fs::read_to_string(&meta_path).unwrap(), meta.render(None)); server.join().unwrap(); }
/// The names and contents of every file under `dir`, recursively, keyed /// by their path relative to `dir`. Lets a test snapshot a tree before /// an operation and assert nothing about it changed afterward. fn snapshot_tree(dir: &Path) -> std::collections::BTreeMap<PathBuf, Vec<u8>> { fn walk(dir: &Path, root: &Path, out: &mut std::collections::BTreeMap<PathBuf, Vec<u8>>) { for entry in fs::read_dir(dir).unwrap() { let entry = entry.unwrap(); let path = entry.path(); if path.is_dir() { walk(&path, root, out); } else { out.insert( path.strip_prefix(root).unwrap().to_path_buf(), fs::read(&path).unwrap(), ); } } } let mut out = std::collections::BTreeMap::new(); walk(dir, dir, &mut out); out }
#[test] fn fetch_text_refuses_an_existing_tree_with_a_mismatched_pin() { let destination_dir = TempDir::new().unwrap(); let destination = destination_dir.path().join("dnd.srd.5.2.1"); fs::create_dir_all(destination.join("07_Spells")).unwrap(); fs::write(destination.join("stale.md"), b"stale").unwrap(); fs::write(destination.join("07_Spells/stale-spell.md"), b"stale spell").unwrap(); let meta_dir = TempDir::new().unwrap(); let meta_path = meta_dir.path().join("meta.yaml"); let meta_contents = "sources:\n - path: sources/dnd.srd.5.2.1/\n commit: old-sha\n patches:\n - path: a.md\n"; fs::write(&meta_path, meta_contents).unwrap(); let tree_before = snapshot_tree(&destination); let config = TextFetchConfig { url: "http://127.0.0.1:0".to_string(), destination: destination.clone(), expected_tarball_sha256: "0".repeat(64), commit: "deadbeef".to_string(), meta_path: meta_path.clone(), };
let error = fetch_text(&config, &fixture_meta()).unwrap_err();
assert!(matches!(error, FetchError::VendoredTreeExists { .. })); assert!( error .to_string() .contains(&destination.display().to_string()) ); assert!(error.to_string().contains("patches")); assert_eq!(snapshot_tree(&destination), tree_before); assert_eq!(fs::read_to_string(&meta_path).unwrap(), meta_contents); }
#[test] fn fetch_text_refuses_an_existing_tree_with_no_recorded_pin() { let destination_dir = TempDir::new().unwrap(); let destination = destination_dir.path().join("dnd.srd.5.2.1"); fs::create_dir_all(&destination).unwrap(); let meta_dir = TempDir::new().unwrap(); let config = TextFetchConfig { url: "http://127.0.0.1:0".to_string(), destination: destination.clone(), expected_tarball_sha256: "0".repeat(64), commit: "deadbeef".to_string(), meta_path: meta_dir.path().join("meta.yaml"), };
let error = fetch_text(&config, &fixture_meta()).unwrap_err();
assert!(matches!(error, FetchError::VendoredTreeExists { .. })); assert!(destination.exists()); }
#[test] fn fetch_text_reports_a_filesystem_error_writing_meta() { let body = tiny_tar_gz("dnd.srd.5.2.1-deadbeef"); let expected = sha256_hex(&body); let (url, server) = serve_once(body); let blocker_dir = TempDir::new().unwrap(); let blocking_file = blocker_dir.path().join("blocker"); fs::write(&blocking_file, b"not a directory").unwrap(); let destination_dir = TempDir::new().unwrap(); let config = TextFetchConfig { url, destination: destination_dir.path().join("dnd.srd.5.2.1"), expected_tarball_sha256: expected, commit: "deadbeef".to_string(), meta_path: blocking_file.join("nested").join("meta.yaml"), };
let error = fetch_text(&config, &fixture_meta()).unwrap_err();
assert!(matches!(error, FetchError::Io(_))); server.join().unwrap(); }
#[test] fn fetch_text_reports_a_filesystem_error_unpacking_a_file_parent_collision() { let body = tar_gz_with_a_file_parent_collision("dnd.srd.5.2.1-deadbeef"); let expected = sha256_hex(&body); let (url, server) = serve_once(body); let destination_dir = TempDir::new().unwrap(); let meta_dir = TempDir::new().unwrap(); let config = TextFetchConfig { url, destination: destination_dir.path().join("dnd.srd.5.2.1"), expected_tarball_sha256: expected, commit: "deadbeef".to_string(), meta_path: meta_dir.path().join("meta.yaml"), };
let error = fetch_text(&config, &fixture_meta()).unwrap_err();
assert!(matches!(error, FetchError::Io(_))); server.join().unwrap(); }
#[test] fn fetch_text_rejects_a_parent_dir_traversal_entry() { let body = tar_gz_with_a_parent_dir_traversal("dnd.srd.5.2.1-deadbeef"); let expected = sha256_hex(&body); let (url, server) = serve_once(body); let destination_dir = TempDir::new().unwrap(); let destination = destination_dir.path().join("dnd.srd.5.2.1"); let escape_target = destination .parent() .unwrap() .parent() .unwrap() .join("escape.txt"); let meta_dir = TempDir::new().unwrap(); let config = TextFetchConfig { url, destination: destination.clone(), expected_tarball_sha256: expected, commit: "deadbeef".to_string(), meta_path: meta_dir.path().join("meta.yaml"), };
let error = fetch_text(&config, &fixture_meta()).unwrap_err();
assert!(matches!(error, FetchError::UnsafeEntry(_))); assert!(error.to_string().contains("unsafe tarball entry")); assert!(!escape_target.exists()); server.join().unwrap(); }
#[test] fn fetch_text_rejects_an_absolute_path_entry() { let body = tar_gz_with_an_absolute_path_entry("dnd.srd.5.2.1-deadbeef"); let expected = sha256_hex(&body); let (url, server) = serve_once(body); let destination_dir = TempDir::new().unwrap(); let meta_dir = TempDir::new().unwrap(); let config = TextFetchConfig { url, destination: destination_dir.path().join("dnd.srd.5.2.1"), expected_tarball_sha256: expected, commit: "deadbeef".to_string(), meta_path: meta_dir.path().join("meta.yaml"), };
let error = fetch_text(&config, &fixture_meta()).unwrap_err();
assert!(matches!(error, FetchError::UnsafeEntry(_))); assert!(!Path::new("/etc/evil.txt").exists()); server.join().unwrap(); }
#[test] fn fetch_text_rejects_a_symlink_entry() { // The symlink targets a directory under this test's own temp root, // rather than a fixed path like `/tmp`, so the assertion below // checks the actual destination of a real escape instead of a // path an unrelated file could already occupy, or a real escape // could land outside of. let escape_dir = TempDir::new().unwrap(); let body = tar_gz_with_a_symlink_escape("dnd.srd.5.2.1-deadbeef", escape_dir.path()); let expected = sha256_hex(&body); let (url, server) = serve_once(body); let destination_dir = TempDir::new().unwrap(); let meta_dir = TempDir::new().unwrap(); let config = TextFetchConfig { url, destination: destination_dir.path().join("dnd.srd.5.2.1"), expected_tarball_sha256: expected, commit: "deadbeef".to_string(), meta_path: meta_dir.path().join("meta.yaml"), };
let error = fetch_text(&config, &fixture_meta()).unwrap_err();
assert!(matches!(error, FetchError::UnsafeEntry(_))); assert!(error.to_string().contains("07_Spells/x")); assert!(error.to_string().contains("Symlink")); assert!(!escape_dir.path().join("evil.md").exists()); server.join().unwrap(); }
#[test] fn fetch_text_rejects_a_tarball_hash_mismatch() { let body = tiny_tar_gz("dnd.srd.5.2.1-deadbeef"); let (url, server) = serve_once(body); let destination_dir = TempDir::new().unwrap(); let destination = destination_dir.path().join("dnd.srd.5.2.1"); let meta_dir = TempDir::new().unwrap(); let config = TextFetchConfig { url, destination: destination.clone(), expected_tarball_sha256: "0".repeat(64), commit: "deadbeef".to_string(), meta_path: meta_dir.path().join("meta.yaml"), };
let error = fetch_text(&config, &fixture_meta()).unwrap_err();
assert!(matches!(error, FetchError::HashMismatch { .. })); assert!(!destination.exists()); server.join().unwrap(); }
#[test] fn fetch_text_reports_a_download_failure() { let destination_dir = TempDir::new().unwrap(); let meta_dir = TempDir::new().unwrap(); let config = TextFetchConfig { url: "http://127.0.0.1:0/dnd.srd.5.2.1.tar.gz".to_string(), destination: destination_dir.path().join("dnd.srd.5.2.1"), expected_tarball_sha256: "0".repeat(64), commit: "deadbeef".to_string(), meta_path: meta_dir.path().join("meta.yaml"), };
let error = fetch_text(&config, &fixture_meta()).unwrap_err();
assert!(matches!(error, FetchError::Download(_))); }
#[test] fn fetch_all_fetches_the_pdf_and_the_text_source() { let pdf_body = b"pdf bytes".to_vec(); let pdf_expected = sha256_hex(&pdf_body); let (pdf_url, pdf_server) = serve_once(pdf_body); let text_body = tiny_tar_gz("dnd.srd.5.2.1-deadbeef"); let text_expected = sha256_hex(&text_body); let (text_url, text_server) = serve_once(text_body); let pdf_dir = TempDir::new().unwrap(); let destination_dir = TempDir::new().unwrap(); let meta_dir = TempDir::new().unwrap(); let sources = SrdSources { pdf: FetchConfig { url: pdf_url, destination: pdf_dir.path().join("fixture.pdf"), expected_sha256: pdf_expected, }, text: TextFetchConfig { url: text_url, destination: destination_dir.path().join("dnd.srd.5.2.1"), expected_tarball_sha256: text_expected, commit: "deadbeef".to_string(), meta_path: meta_dir.path().join("meta.yaml"), }, meta: fixture_meta(), };
let (pdf_outcome, text_outcome) = fetch_all(&sources).unwrap();
assert_eq!(pdf_outcome, FetchOutcome::Downloaded); assert_eq!(text_outcome, FetchOutcome::Downloaded); pdf_server.join().unwrap(); text_server.join().unwrap(); }}