Something went wrong. Try again.
DASL — Data-Addressed Structures & Links — is a small set of simple, standard primitives for working with content-addressed, linked data. If
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521#!/usr/bin/env node
import { argv, exit } from 'node:process';import { dirname, basename, join } from 'node:path';import { rm, mkdir, cp, readFile, writeFile, readdir, access } from 'node:fs/promises';import { execFileSync } from 'node:child_process';import chokidar from 'chokidar';import chalk from 'chalk';import { JSDOM } from 'jsdom';import Turndown from 'turndown';import hljs from 'highlight.js/lib/core';import javascript from 'highlight.js/lib/languages/javascript';import json from 'highlight.js/lib/languages/json';hljs.registerLanguage('javascript', javascript);hljs.registerLanguage('json', json);
// Everything SISL produces lands here, and only what lands here gets deployed.// The directory is wiped on every build and is .gitignored.const buildDirName = 'build';
// SISL's databases. They feed the build but are never published.const dbFiles = new Set(['bibliography.json', 'people.json', 'maturities.json']);
// The home page. Hand-authored, but post-processed to label the spec list.const indexFile = 'index.html';
// Static files published as-is. Anything not matched here — spec sources, the// databases above, tooling, design files — stays in the repo.const staticFiles = [ /\.css$/, /\.png$/, 'dcbor42.html', // redirect stub kept for old links 'tiles-lexicon.json', 'polity.json', 'LICENSE.md',];const staticDirs = ['fonts'];
function isStaticFile (name) { return staticFiles.some((m) => (typeof m === 'string' ? m === name : m.test(name)));}
// SISL — Simple Implementation of a Specification Language// This is just a basic spec generator. Here's what it does:// - Treats every *.html in the root (bar the index and static files) as a spec source.// - Pulls metadata from those specs so they can reference one another.// - Watches bibliography.json for references that specs can use.// - Parses [[ref]] as a reference.// - Manages metadata and styling for specs.// - Creates anchors for definitions and the such// - Labels the spec list on the index with each spec's maturity.// - Writes the whole site, generated and static alike, to build/.
class SISL { constructor (dir) { this.baseDir = dir; this.buildDir = join(dir, buildDirName); this.turnDown = new Turndown({ bulletListMarker: '-', codeBlockStyle: 'fenced', headingStyle: 'atx', }); this.turnDown.addRule('meta-abstract', { filter: (n) => n.localName === 'table' && n.parentElement?.localName === 'header', replacement: (_, n) => n.querySelector('#abstract')?.textContent, }); this.turnDown.keep(['var', 'dfn']); this.turnDown.addRule('references', { filter: (n) => n.localName === 'dt' && /^ref-/.test(n.id), replacement: (cnt, n) => `<dfn id="${n.id}">${cnt}</dfn>`, }); this.turnDown.addRule('code', { filter: ['pre'], replacement: (cnt) => `\`\`\`\n${cnt.replace(/\n\s+\n?$/s, '')}\n\`\`\``, }); } async watch () { // For reasons that baffle me, chokidar's ignored option doesn't work correctly. So // we filter ourselves. Files under build/ are past `depth` and never show up here. const buildIfMatch = (path) => { const name = basename(path); if (!dbFiles.has(name) && !/\.(html|css)$/.test(name)) return; this.build(); }; chokidar .watch(this.baseDir, { depth: 0, ignoreInitial: true, }) .on('add', (path) => buildIfMatch(path)) .on('change', (path) => buildIfMatch(path)) .on('unlink', (path) => buildIfMatch(path)) .on('ready', () => this.build()) ; } async build () { // start from a clean build, so deleted specs don't linger await rm(this.buildDir, { recursive: true, force: true }); await mkdir(this.buildDir, { recursive: true }); // load data const bibliography = await loadJSON(this.baseDir, 'bibliography'); const people = await loadJSON(this.baseDir, 'people'); const maturities = await loadJSON(this.baseDir, 'maturities'); const ignoreRevs = await loadIgnoreRevs(this.baseDir); // list specs const specs = {}; const specList = (await readdir(this.baseDir)) .filter(f => /\.html$/.test(f) && f !== indexFile && !isStaticFile(f)); for (const s of specList) { const dom = new JSDOM(await readFile(join(this.baseDir, s), 'utf8')); const { window: { document: doc } } = dom; const meta = doc.querySelector('meta[name="authors"]'); const authors = meta?.getAttribute('content')?.split(/\s*,\s*/) || ['robin', 'bumblefudge']; if (meta) meta.remove(); // short_name: what the homepage shows when the full title is too long for a card. const snMeta = doc.querySelector('meta[name="short-name"]'); const shortName = snMeta?.getAttribute('content') || doc.title; if (snMeta) snMeta.remove(); const updated = lastUpdated(this.baseDir, basename(s, '.html'), ignoreRevs); specs[basename(s).replace(/\.html$/, '')] = { dom, doc, authors, shortName, updated }; } // extract metadata from all src and add to biblio Object.keys(specs).forEach(shortname => { bibliography[shortname] = this.htmlifyReference({ author: joinList(specs[shortname].authors.map(au => people[au].name)), title: specs[shortname].doc.title, date: specs[shortname].updated, url: `https://dasl.ing/${shortname}.html`, }); }); for (const shortname of Object.keys(specs)) { const { dom, doc, authors, updated } = specs[shortname]; const mat = maturities[shortname]; if (!mat) err(`No maturity set for ${shortname}`); const el = makeEl(doc); console.warn(`--- Processing ${shortname} "${doc.title}" (${doc.body.innerHTML.length}) ---`); // css const abstract = doc.querySelector('#abstract'); if (!abstract) err(`Missing abstract in ${doc.title}`); const head = doc.querySelector('head'); const cmt = doc.createComment(`
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! GENERATED SPEC — DO NOT EDIT. !!!!!!!!! Edit ${shortname}.html in the repository root instead.!!!!! !!!!!!!!! HEY YOU !!!!!!!!! YES ***YOU****! !!!!!!!!! !!!!!!!!! You're about to edit a generated document. !!!!!!!!! Don't do that! Why would you do that. !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
`); el('link', { rel: 'stylesheet', href: 'spec.css' }, [], head); el('link', { rel: 'preload', href: 'shared.css', as: 'style' }, [], head); head.append(doc.createTextNode('\n\n'), cmt, doc.createTextNode('\n\n')); el('link', { rel: 'icon', href: 'data:image/svg+xml,<svg xmlns=%22http://www.w3.org/2000/svg%22 viewBox=%220 0 100 100%22><rect x=%220%22 y=%220%22 width=%22100%22 height=%22100%22 fill=%22%2300ff75%22></rect></svg>' }, [], head); el('link', { rel: 'alternate', type: 'text/markdown', href: `https://dasl.ing/${shortname}.md` }, [], head); el('meta', { name: 'twitter:card', content: 'summary_large_image' }, [], head); el('meta', { name: 'twitter:title', property: 'og:title', content: `DASL: ${doc.title}` }, [], head); if (abstract) el('meta', { name: 'twitter:description', property: 'og:description', content: norm(abstract.textContent) }, [], head); el('meta', { name: 'twitter:image', property: 'og:image', content: `https://dasl.ing/${shortname}.png` }, [], head); el('meta', { name: 'twitter:image:alt', content: 'Very colourful stripes, so colourful it hurts' }, [], head); el('meta', { name: 'twitter:url', property: 'og:url', content: 'https://dasl.ing/' }, [], head); el('meta', { property: 'og:site_name', content: 'DASL' }, [], head); el('meta', { property: 'og:locale', content: 'en' }, [], head); el('meta', { name: 'theme-color', content: '#00ff75' }, [], head); // EXPERIMENT: politi.es integration // el('script', { src: 'https://politi.es/badge.js', defer: 'defer' }, [], head);
// main & header const main = doc.createElement('main'); const header = el('header', {}, [el('h1', {}, [doc.title])], main); const attribution = []; authors .map(au => people[au]) .forEach(({ site, name, email }, idx) => { if (idx) attribution.push(el('br')); attribution.push(el('a', { href: site }, [name]), ' <', el('a', { href: `mailto:${email}` }, [email]), '>'); }) el( 'table', {}, [ el('tr', {}, [ el('th', {}, ['last updated']), el('td', {}, [updated]), ]), // EXPERIMENT: politi.es integration // <polities-badge polity="did:polity:dasl-wg" deliverable="deliv-drisl"></polities-badge> el('tr', {}, [ el('th', {}, ['status']), // el('td', {}, [el('polities-badge', { // polity: 'dasl.ing', // deliverable: shortname, // })]), el('td', {}, [el('a', { href: 'governance.html', class: 'status', }, [maturityLabel(mat)])]), ]), el('tr', {}, [ el('th', {}, ['editors']), el('td', {}, attribution), ]), el('tr', {}, [ el('th', {}, ['abstract']), el('td', {}, [abstract]), ]), ], header ); main.append(...doc.body.childNodes); doc.body.append(main); // rainbow stripe, with the way home on the yellow band const stripe = el('nav', { class: 'stripe', 'aria-label': 'Site' }, [ el('a', { href: '/' }, [el('span', { 'aria-hidden': 'true' }, ['←']), 'DASL']), ...['green', 'purple', 'orange', 'red', 'blue'].map(c => el('span', { class: c, 'aria-hidden': 'true' })), ]); doc.body.prepend(stripe); // footer (outside <main>, so it stays out of the Markdown version) el('footer', { class: 'site-footer' }, [ el('p', {}, ['Made with 👾 by the IPFS Foundation.']), el('nav', { 'aria-label': 'Project' }, [ el('a', { href: '/#spec' }, ['← All specs']), el('a', { href: 'https://tangled.org/dasl.ing/dasl.ing' }, ['Source']), el('a', { href: 'https://tangled.org/dasl.ing/dasl.ing/issues' }, ['Issues']), ]), ], doc.body); // definitions & xrefs [...doc.querySelectorAll('dfn')].forEach(dfn => { const id = slugify(doc, dfn, 'dfn', true); dfn.setAttribute('id', id); }); [...doc.querySelectorAll('a:not([href])')].forEach(a => { const id = slugify(doc, a, 'dfn'); if (doc.getElementById(id)) { a.setAttribute('href', `#${id}`); a.className = 'dfn-ref'; } else { err(`Empty link "${a.textContent}" (#${id}) has no matching dfn.`); } }); // references const refs = {}; main.innerHTML = main.innerHTML.replace( /\[\[([\w-]+)\]\]/g, (_, ref) => { if (!bibliography[ref]) { err(`No "${ref}" entry in the bibliography.`); return `[[${ref}]]`; } refs[ref] = bibliography[ref]; return `[<a href="#ref-${ref}" class="ref">${ref}</a>]`; } ); if (Object.keys(refs).length) { const refSec = el('section', {}, [el('h2', {}, ['References'])], main); const dl = el('dl', {}, [], refSec); Object.keys(refs).sort().forEach(r => { el('dt', { id: `ref-${r}` }, [`[${r}]`], dl); const dd = el('dd', {}, [], dl); dd.innerHTML = refs[r]; }); } [...doc.querySelectorAll('pre')].forEach(pre => { // make them all be wrapped in code if (!(pre.firstElementChild?.localName === 'code')) { const c = doc.createElement('code'); c.append(...pre.childNodes); pre.append(c); hljs.highlightElement(c); } else { hljs.highlightElement(pre.querySelector('code')); } }); try { await access(join(this.baseDir, `${shortname}.png`)); } catch (e) { err(`No banner found for ${shortname}.`); } // save await writeFile(join(this.buildDir, `${shortname}.html`), dom.serialize()); // make some MD await writeFile(join(this.buildDir, `${shortname}.md`), this.turnDown.turndown(doc.body.querySelector('main'))); } await this.buildIndex(maturities, specs); await this.copyStatic(); } // The index lists the specs by hand, as a <dl> per group: a <dt> holding a link to // the spec and a <dd> with a one-line description. Everything else on each card is // data: the title is the spec's short-name, the subtitle is the rest of its full // title, and the tag comes from maturities.json. So nobody has to keep the index in // sync with the specs by hand. async buildIndex (maturities, specs) { const dom = new JSDOM(await readFile(join(this.baseDir, indexFile), 'utf8')); const { window: { document: doc } } = dom; const el = makeEl(doc); console.warn(`--- Processing ${indexFile} "${doc.title}" ---`); [...doc.querySelectorAll('#spec dl')].forEach(dl => { const cards = el('div', { class: 'cards' }); [...dl.querySelectorAll(':scope > dt')].forEach(dt => { const link = dt.querySelector('a[href$=".html"]'); if (!link) { err(`Spec list entry "${norm(dt.textContent)}" in ${indexFile} links to no spec.`); return; } const href = link.getAttribute('href'); const shortname = basename(href, '.html'); const spec = specs[shortname]; if (!spec) err(`${indexFile} lists ${shortname}, but there is no ${shortname}.html spec.`); const mat = maturities[shortname]; if (!mat) err(`No maturity set for ${shortname}, listed in ${indexFile}.`); const title = spec ? norm(spec.doc.title) : norm(link.textContent); const short = spec ? norm(spec.shortName) : title; const sub = cardSubtitle(title, short); const dd = dt.nextElementSibling?.localName === 'dd' ? dt.nextElementSibling : null; const head = el('div', { class: 'card-head' }, [el('a', { href }, [short])]); const tag = maturityTag(el, mat); if (tag) head.append(tag); const body = el('div', { class: 'card-body' }); if (sub) el('div', { class: 'card-sub' }, [sub], body); if (dd) el('p', {}, [...dd.childNodes], body); el('article', { class: 'card' }, [head, body], cards); }); dl.replaceWith(cards); }); await writeFile(join(this.buildDir, indexFile), dom.serialize()); } async copyStatic () { for (const entry of await readdir(this.baseDir, { withFileTypes: true })) { const { name } = entry; if (entry.isDirectory()) { if (!staticDirs.includes(name)) continue; await cp(join(this.baseDir, name), join(this.buildDir, name), { recursive: true }); } else if (isStaticFile(name)) { await cp(join(this.baseDir, name), join(this.buildDir, name)); } } } htmlifyReference ({ author, title, date, url }) { return `${esc(author)}. <a href="${esc(url)}"><cite>${esc(title)}</cite></a>. ${esc(date)}. URL: <a href="${esc(url)}">${esc(url)}</a>` }}
function err (str) { console.error(chalk.bold.bgRedBright.black(str));}function die (str) { err(str); exit(1);}
function makeEl (doc) { return (n, attr, kids, parent) => { const el = doc.createElement(n); if (attr) Object.keys(attr).forEach((k) => el.setAttribute(k, attr[k])); if (kids) { kids.forEach(k => { if (typeof k === 'string') k = doc.createTextNode(k); el.append(k); }); } if (parent) parent.append(el); return el; };}
function slugify (doc, el, pfx, unique) { if (el.hasAttribute('id')) return el.getAttribute('id'); let suf; const txt = (norm(el.textContent) || 'empty').toLowerCase().replace(/\W/g, '-').replace(/-{2,}/g, '-').replace(/^-|-$/g, ''); const idify = () => [pfx, txt, suf].filter(Boolean).join('-'); let id = idify(); if (unique) { while (doc.getElementById(id)) { if (!suf) suf = 0; suf++; id = idify(); } } return id;}
function today () { return new Date().toISOString().replace(/T.+/, '');}
// "Last updated" is the date the spec's text last changed, taken from git.//// Until SOURCE_CUTOVER, <name>.html was committed build output (rewritten on// every regen) and the text lived in <name>.src.html. So: use commits to// <name>.html after the cutover, and otherwise the last commit to// <name>.src.html before it. Commits listed in .dates-ignore-revs (one hash or// prefix per line, # for comments) are skipped, for repo-wide changes that// don't touch the specs' substance. Falls back to today when git or its history// isn't available (e.g. a shallow CI clone).const SOURCE_CUTOVER = '905f1b5'; // "simpler gen": specs move from *.src.html to *.html
function lastUpdated (baseDir, name, ignoreRevs) { const git = (...args) => execFileSync('git', args, { cwd: baseDir, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], }).trim(); const isDate = (d) => /^\d{4}-\d{2}-\d{2}$/.test(d); const ignored = (hash) => ignoreRevs.some((r) => hash.startsWith(r)); try { let cutover = null; try { cutover = git('rev-parse', '--verify', '-q', `${SOURCE_CUTOVER}^{commit}`); } catch (e) { // cutover not in this history; just use <name>.html } const range = cutover ? [`${cutover}..HEAD`] : []; const log = git('log', '--format=%H %cs', ...range, '--', `${name}.html`); for (const line of log.split('\n')) { const [hash, date] = line.split(' '); if (hash && !ignored(hash) && isDate(date)) return date; } if (cutover) { const legacy = git('log', '--format=%H %cs', `${cutover}~1`, '--', `${name}.src.html`); for (const line of legacy.split('\n')) { const [hash, date] = line.split(' '); if (hash && !ignored(hash) && isDate(date)) return date; } } } catch (e) { // fall through } return today();}
async function loadIgnoreRevs (baseDir) { try { return (await readFile(join(baseDir, '.dates-ignore-revs'), 'utf8')) .split('\n') .map((l) => l.replace(/#.*/, '').trim()) .filter(Boolean); } catch (e) { return []; }}
function esc (str) { return (str || '').replace(/&/g, '&').replace(/</g, '<').replace(/"/g, '"');}
function norm (str) { return (str || '').replace(/\s/g, ' ').replace(/^\s+|\s+$/g, '');}
const isWatch = argv[2] === '--watch';const sisl = new SISL(dirname(new URL(import.meta.url).toString().replace(/^file:\/\//, '')));if (isWatch) sisl.watch();else sisl.build();
async function loadJSON (baseDir, base) { try { return JSON.parse(await readFile(join(baseDir, `${base}.json`))); } catch (e) { die(e.message); }}
function joinList (authors) { if (authors.length === 1) return authors[0]; if (authors.length === 2) return authors.join(' & '); return authors.map((au, idx) => { if (idx === (authors.length - 1)) return `& ${au}`; return au; }).join(', ');}
function maturityName (mat) { if (mat === 'hns') return 'hot new stuff'; return mat;}
// "hot new stuff" → "Hot New Stuff", "stabilising" → "Stabilising"function maturityLabel (mat) { return maturityName(mat).replace(/\b\w/g, (c) => c.toUpperCase());}
// The homepage card subtitle: the part of the full title that isn't the short name,// when the title is "SHORT — Rest" or "Rest: SHORT". Otherwise, nothing.// "CID — Content Identifiers" / "CID" → "Content Identifiers"// "Tiles Protocol: Data Passing" / "Data Passing" → "Tiles Protocol"// "Governance of the DASL Project" / "Governance" → ""function cardSubtitle (title, short) { if (!short || title === short) return ''; for (const sep of [' — ', ': ']) { if (title.startsWith(short + sep)) return title.slice(short.length + sep.length); if (title.endsWith(sep + short)) return title.slice(0, -(sep.length + short.length)); } return '';}
// Maturity tags on the homepage cards. Informative documents aren't on the// maturity scale, so they get no tag.function maturityTag (el, mat) { if (!mat || mat === 'informative') return null; if (mat === 'hns') { return el('span', { class: 'tag hns' }, [ el('span', { 'aria-hidden': 'true' }, ['H', el('span', { class: 'boom' }, ['💥']), 't New Stuff']), el('span', { class: 'vh' }, ['Hot New Stuff']), ]); } return el('span', { class: `tag ${mat}` }, [maturityLabel(mat)]);}