From f96c2238bd19970bafd4befbef8e600d3b613082 Mon Sep 17 00:00:00 2001 From: Kieran Klukas Date: Wed, 12 Aug 2026 00:10:50 -0400 Subject: [PATCH] feat: shared sqlite catalog cache, crawl progress, cancellation --- README.md | 18 ++++- bunfig.toml | 2 + serve.ts | 92 ++++++++++++++++------ src/catalog.ts | 61 +++++++++++++++ src/client/app.css | 34 ++++++++ src/client/app.ts | 72 +++++++++++++---- src/client/bridge.ts | 40 +++++++++- src/client/crawl.test.ts | 110 ++++++++++++++++++++++++++ src/client/crawl.ts | 73 +++++++++++++++++ src/client/ctx.ts | 4 +- src/client/index.html | 6 ++ src/client/views/schedule.ts | 2 +- src/client/views/tree.ts | 12 ++- src/client/views/views.test.ts | 2 +- src/content.ts | 60 ++++++-------- src/server/store.test.ts | 138 +++++++++++++++++++++++++++++++++ src/server/store.ts | 132 +++++++++++++++++++++++++++++++ test/setup.ts | 11 +++ 18 files changed, 780 insertions(+), 89 deletions(-) create mode 100644 bunfig.toml create mode 100644 src/catalog.ts create mode 100644 src/client/crawl.test.ts create mode 100644 src/client/crawl.ts create mode 100644 src/server/store.test.ts create mode 100644 src/server/store.ts create mode 100644 test/setup.ts diff --git a/README.md b/README.md index ca659ac..d6f1c61 100644 --- a/README.md +++ b/README.md @@ -25,14 +25,20 @@ another tab, and click capture. src/requirements.ts Ellucian's 40-field Group as a tagged union src/merge.ts which course satisfies a requirement in both majors src/schedule.ts meeting times, seat counts, and date-aware conflicts + src/catalog.ts the one shape that is public rather than personal + src/server/store.ts SQLite cache of the section catalog + src/client/crawl.ts the fetch loop: cache-aware, paced, cancellable src/client/ the planner: no framework, one CSS file, mount/destroy views The split is by change rate. Auth bridging is stable and security-sensitive; the planner changes every time we learn something new about Colleague. The two halves share only `types.ts`. -Nothing is uploaded. There is no API and no database; a capture lives in the -student's own `localStorage`. +Two kinds of data, and only one of them is yours. Section times, seats and +instructors are identical for every student, so they are cached in SQLite on +the server and one student's crawl spares everyone else's. An evaluation is a +student record and never leaves their browser. There is no account system +because there is nothing here to attach to a person. ### what it refuses to guess @@ -52,8 +58,12 @@ range and every comparison uses it; a day-and-time check alone invents clashes and makes half the catalog look unschedulable. Loading sections fetches only courses that could still close an open -requirement, one call per course with a small delay. It is someone's registrar, -not a load test. +requirement, skips anything the shared cache already answers, paces itself, and +can be cancelled. It is someone's registrar, not a load test. + +Tests run with `test/setup.ts` preloaded, which pins `CATALOG_DB` to +`:memory:`. Without it a stray import of `serve.ts` would open the real +catalog database from a test. The canonical repo for this is hosted on tangled over at [`https://tangled.org/dunkirk.sh/the-cedarville-app`](https://tangled.org/dunkirk.sh/the-cedarville-app) diff --git a/bunfig.toml b/bunfig.toml new file mode 100644 index 0000000..8755352 --- /dev/null +++ b/bunfig.toml @@ -0,0 +1,2 @@ +[test] +preload = ["./test/setup.ts"] diff --git a/serve.ts b/serve.ts index 53112c8..83ed8be 100644 --- a/serve.ts +++ b/serve.ts @@ -1,43 +1,85 @@ /** - * Serves public/ and nothing else in anger. There is no database and no - * account system on purpose: the page holds a student's academic record, so - * the only copy lives in their own browser. + * Serves the planner and the shared section cache. * - * The one exception is POST /dev/capture, which writes the last capture to - * .data/ on this machine. That exists so an agent editing this repo can read - * a real Colleague response instead of inferring the schema — every bug found - * so far came from real data disagreeing with a guess. It is localhost-only, - * gitignored, and absent from any deployed build. + * The split that matters: the catalog is school-wide public data and lives in + * SQLite here, so one student's crawl spares everyone else's; a student's + * evaluation is their record and never leaves their browser. There is no + * account system because there is nothing here to attach to a person. + * + * POST /dev/capture is the exception, and it is localhost-only and + * gitignored: it drops the last capture on disk so an agent working on this + * repo can read a real Colleague response instead of guessing at the schema. */ import { mkdir } from "node:fs/promises"; +import type { TermCatalog } from "./src/catalog"; +import { CatalogStore } from "./src/server/store"; const PORT = 5173; const ROOT = "public"; -const DEV_DUMP = ".data/capture.json"; const dev = process.env.NODE_ENV !== "production"; -async function handle(request: Request): Promise { - const { pathname } = new URL(request.url); +await mkdir(".data", { recursive: true }); +const store = new CatalogStore(); + +const json = (body: unknown, status = 200) => + new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }); + +async function api(request: Request, pathname: string): Promise { + const term = /^\/catalog\/([^/]+)$/.exec(pathname)?.[1]; + + if (term && request.method === "GET") { + const wanted = new URL(request.url).searchParams.get("courses"); + return json(store.read(decodeURIComponent(term), wanted ? wanted.split(",") : undefined)); + } + + if (term && request.method === "PUT") { + const body = (await request.json()) as TermCatalog; + if (body.term !== decodeURIComponent(term)) return json({ error: "term mismatch" }, 400); + const written = store.write(body); + console.log(`cached ${written} courses for ${body.term}`); + return json({ written }); + } - if (dev && request.method === "POST" && pathname === "/dev/capture") { + if (pathname === "/catalog" && request.method === "GET") return json(store.stats()); + + if (dev && pathname === "/dev/capture" && request.method === "POST") { + // Named, because evaluations and the catalog are different artifacts and + // one file would mean the second dump silently ate the first. + const raw = new URL(request.url).searchParams.get("name") ?? "capture"; + const name = raw.replace(/[^a-z0-9-]/gi, "") || "capture"; const body = await request.text(); - await mkdir(".data", { recursive: true }); - await Bun.write(DEV_DUMP, body); - console.log(`wrote ${DEV_DUMP} (${(body.length / 1024).toFixed(0)}kb)`); + await Bun.write(`.data/${name}.json`, body); + console.log(`wrote .data/${name}.json (${(body.length / 1024).toFixed(0)}kb)`); return new Response(null, { status: 204 }); } - const path = decodeURIComponent(pathname === "/" ? "/index.html" : pathname); - // URL parsing folds away "..", but percent-encoded ones survive it. - if (path.includes("..")) return new Response("no", { status: 400 }); - - const file = Bun.file(ROOT + path); - return (await file.exists()) - ? new Response(file, { headers: { "cache-control": "no-store" } }) - : new Response("not found", { status: 404 }); + return null; } -Bun.serve({ port: PORT, fetch: handle }); +Bun.serve({ + port: PORT, + async fetch(request) { + const { pathname } = new URL(request.url); + + const handled = await api(request, pathname); + if (handled) return handled; -console.log(`planner on http://localhost:${PORT}${dev ? ` · captures land in ${DEV_DUMP}` : ""}`); + const path = decodeURIComponent(pathname === "/" ? "/index.html" : pathname); + // URL parsing folds away "..", but percent-encoded ones survive it. + if (path.includes("..")) return new Response("no", { status: 400 }); + + const file = Bun.file(ROOT + path); + return (await file.exists()) + ? new Response(file, { headers: { "cache-control": "no-store" } }) + : new Response("not found", { status: 404 }); + }, +}); + +for (const row of store.stats()) { + console.log(` cached ${row.term}: ${row.courses} courses, ${row.offered} offered`); +} +console.log(`planner on http://localhost:${PORT}`); diff --git a/src/catalog.ts b/src/catalog.ts new file mode 100644 index 0000000..f8f248b --- /dev/null +++ b/src/catalog.ts @@ -0,0 +1,61 @@ +/** + * The section catalog for one term. + * + * This is the one thing in the project that is *not* personal. Meeting times, + * seat counts and instructors are identical for every student at the school, + * which is why it can be cached and shared while a transcript never can. The + * boundary is worth stating plainly: catalog on the server, record in the + * browser. + */ + +import type { SectionsResponse } from "./types"; + +export interface TermCatalog { + term: string; + /** When these sections were fetched from Self-Service. */ + fetchedAt: string; + /** Raw responses keyed by course id, so a normalizer bug stays diagnosable. */ + sections: Record; + /** Course ids confirmed to have no section this term. Worth caching too. */ + notOffered: string[]; +} + +export const emptyCatalog = (term: string): TermCatalog => ({ + term, + fetchedAt: new Date().toISOString(), + sections: {}, + notOffered: [], +}); + +/** Course ids the catalog can already answer for, offered or not. */ +export function known(catalog: TermCatalog): Set { + return new Set([...Object.keys(catalog.sections), ...catalog.notOffered]); +} + +/** + * Later data wins, since the only reason to refetch a course is that the + * cached copy went stale. + */ +export function mergeCatalogs(base: TermCatalog, incoming: TermCatalog): TermCatalog { + const notOffered = new Set(base.notOffered); + for (const id of incoming.notOffered) notOffered.add(id); + + const sections = { ...base.sections, ...incoming.sections }; + // A course that now has sections is no longer "not offered". + for (const id of Object.keys(incoming.sections)) notOffered.delete(id); + + return { + term: base.term, + fetchedAt: incoming.fetchedAt > base.fetchedAt ? incoming.fetchedAt : base.fetchedAt, + sections, + notOffered: [...notOffered], + }; +} + +/** + * Seat counts move hourly during registration while meeting times do not, so + * age is only a reason to refetch availability, never the whole catalog. + */ +export function ageInHours(catalog: TermCatalog, now = Date.now()): number { + return (now - Date.parse(catalog.fetchedAt)) / 3_600_000; +} diff --git a/src/client/app.css b/src/client/app.css index 026376f..b2c0c19 100644 --- a/src/client/app.css +++ b/src/client/app.css @@ -340,3 +340,37 @@ label.section.clashes .when { .tag.full { color: var(--red); } + +#progress-bar { + display: flex; + align-items: center; + gap: 12px; + padding: 12px 0; + border-bottom: 1px solid var(--surface); +} + +progress { + flex: 1; + height: 6px; + border: 0; + border-radius: 3px; + background: var(--mantle); + appearance: none; +} + +progress::-webkit-progress-bar { + border-radius: 3px; + background: var(--mantle); +} + +progress::-webkit-progress-value { + border-radius: 3px; + background: var(--lavender); + transition: width 0.15s ease-out; +} + +#progress-text { + min-width: 260px; + color: var(--subtext); + font-size: 11px; +} diff --git a/src/client/app.ts b/src/client/app.ts index e116bd3..9e57606 100644 --- a/src/client/app.ts +++ b/src/client/app.ts @@ -7,9 +7,19 @@ * to send anything to, which is what keeps an academic record out of scope. */ -import type { Capture, SectionsCapture } from "../content"; +import type { TermCatalog } from "../catalog"; +import type { Capture } from "../content"; import { enumeratedCourseIds, normalize, openGroups, type ProgramTree } from "../requirements"; -import { capture, dumpForDev, installed, programs, sections, terms } from "./bridge"; +import { + capture, + dumpForDev, + fetchCached, + installed, + programs, + publishCached, + terms, +} from "./bridge"; +import { Cancelled, crawl, type Progress } from "./crawl"; import { $ } from "./dom"; import * as overlap from "./views/overlap"; import * as schedule from "./views/schedule"; @@ -22,7 +32,7 @@ type ViewName = keyof typeof VIEWS; let mounted: { destroy(): void } | null = null; let trees: ProgramTree[] = []; -let sectionData: SectionsCapture | undefined; +let sectionData: TermCatalog | undefined; const status = $("#status"); function say(text: string, bad = false) { @@ -59,7 +69,7 @@ $("#capture").addEventListener("click", async () => { try { const snapshot = await capture(select.value ? [select.value] : []); localStorage.setItem(STORE, JSON.stringify(snapshot)); - void dumpForDev(snapshot); + void dumpForDev("evaluations", snapshot); show(snapshot); say(`captured ${Object.keys(snapshot.evaluations).length} programs`); } catch (err) { @@ -83,6 +93,24 @@ function candidateCourseIds(): string[] { return [...ids]; } +let crawling: AbortController | null = null; + +function showProgress(p: Progress | null) { + const box = $("#progress-bar"); + box.hidden = p === null; + $("#cancel").hidden = p === null; + if (!p) return; + + const bar = $("#bar"); + bar.value = p.done; + bar.max = Math.max(p.total, 1); + $("#progress-text").textContent = + `${p.done}/${p.total}${p.current ? ` ${p.current}` : ""}` + + (p.cached ? ` · ${p.cached} already cached` : ""); +} + +$("#cancel").addEventListener("click", () => crawling?.abort()); + $("#load-sections").addEventListener("click", async () => { const button = $("#load-sections"); const term = $("#term").value; @@ -92,18 +120,36 @@ $("#load-sections").addEventListener("click", async () => { if (courseIds.length === 0) return say("capture your requirements first", true); button.disabled = true; - say(`fetching sections for ${courseIds.length} courses… this takes a minute`); + crawling = new AbortController(); + say("checking the shared cache…"); + try { - const data = await sections(courseIds, term); - sectionData = data; - localStorage.setItem(SECTIONS, JSON.stringify(data)); - void dumpForDev({ kind: "sections", ...data }); + // Anything another student already fetched costs this one nothing. + const have = (await fetchCached(term, courseIds)) ?? undefined; + const catalog = await crawl({ + courseIds, + term, + have, + signal: crawling.signal, + onProgress: showProgress, + }); + + sectionData = catalog; + localStorage.setItem(SECTIONS, JSON.stringify(catalog)); + void publishCached(catalog); + void dumpForDev("catalog", catalog); showView("schedule"); - const count = Object.keys(data.sections).length; - say(`${count} courses offered in ${term}, ${data.notOffered.length} not taught`); + + const offered = Object.keys(catalog.sections).length; + say(`${offered} courses offered in ${term}, ${catalog.notOffered.length} not taught`); } catch (err) { - say(err instanceof Error ? err.message : String(err), true); + say( + err instanceof Cancelled ? "cancelled" : String(err instanceof Error ? err.message : err), + err instanceof Cancelled ? false : true, + ); } finally { + showProgress(null); + crawling = null; button.disabled = false; } }); @@ -114,7 +160,7 @@ async function init() { const cachedSections = localStorage.getItem(SECTIONS); if (cachedSections) { try { - sectionData = JSON.parse(cachedSections) as SectionsCapture; + sectionData = JSON.parse(cachedSections) as TermCatalog; } catch { localStorage.removeItem(SECTIONS); } diff --git a/src/client/bridge.ts b/src/client/bridge.ts index 325e7aa..f7ed1a1 100644 --- a/src/client/bridge.ts +++ b/src/client/bridge.ts @@ -6,6 +6,7 @@ * extension gets a fresh random id each install and this could not address it. */ +import type { TermCatalog } from "../catalog"; import type { Capture, Reply, ReplyMap, Request } from "../content"; import type { ProgramSummary } from "../types"; @@ -43,8 +44,10 @@ function send(msg: Request & { type: K }): Promise send({ type: "ping" }); export const terms = () => send({ type: "terms" }); -export const sections = (courseIds: string[], term: string) => - send({ type: "sections", courseIds, term }); +export const sectionIds = (courseIds: string[], term: string) => + send({ type: "section-ids", courseIds, term }); +export const sections = (courseId: string, ids: string[]) => + send({ type: "sections", courseId, sectionIds: ids }); export const programs = (): Promise => send({ type: "programs" }); export const capture = (whatIf: string[] = []): Promise => send({ type: "capture", whatIf }); @@ -54,10 +57,10 @@ export const capture = (whatIf: string[] = []): Promise => * agent working on this code can read a real response instead of guessing at * the schema. Localhost only, gitignored, and a no-op anywhere else. */ -export async function dumpForDev(snapshot: unknown): Promise { +export async function dumpForDev(name: string, snapshot: unknown): Promise { if (location.hostname !== "localhost") return; try { - await fetch("/dev/capture", { + await fetch(`/dev/capture?name=${encodeURIComponent(name)}`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(snapshot), @@ -66,3 +69,32 @@ export async function dumpForDev(snapshot: unknown): Promise { // The dev server is optional; never let it break a capture. } } + +// ---- the shared catalog cache ----------------------------------------- + +/** + * Sections other students have already fetched. Public course data only; the + * server has never seen an evaluation and has nowhere to put one. + */ +export async function fetchCached(term: string, courseIds: string[]): Promise { + try { + const query = `?courses=${encodeURIComponent(courseIds.join(","))}`; + const res = await fetch(`/catalog/${encodeURIComponent(term)}${query}`); + return res.ok ? ((await res.json()) as TermCatalog) : null; + } catch { + // The cache is an optimisation; never let it block a crawl. + return null; + } +} + +export async function publishCached(catalog: TermCatalog): Promise { + try { + await fetch(`/catalog/${encodeURIComponent(catalog.term)}`, { + method: "PUT", + headers: { "content-type": "application/json" }, + body: JSON.stringify(catalog), + }); + } catch { + // Same: a failed contribution costs this student nothing. + } +} diff --git a/src/client/crawl.test.ts b/src/client/crawl.test.ts new file mode 100644 index 0000000..b9ff6e7 --- /dev/null +++ b/src/client/crawl.test.ts @@ -0,0 +1,110 @@ +/* + * The crawl loop, with the bridge stubbed. What matters here is not that it + * fetches but that it skips what the shared cache already knows, reports + * progress a human can read, and stops when told. + */ + +import { afterEach, describe, expect, mock, test } from "bun:test"; +import { ageInHours, emptyCatalog, known, mergeCatalogs, type TermCatalog } from "../catalog"; +import type { SectionsResponse } from "../types"; + +const sectionIds = mock(async (courseIds: string[], _term: string) => + courseIds.map((id) => ({ + courseId: id, + courseName: `SUBJ-${id}`, + // "gone" stands in for a course not taught this term. + sectionIds: id === "gone" ? [] : [`s-${id}`], + })), +); + +const sections = mock( + async (courseId: string, _ids: string[]) => + ({ + SectionsRetrieved: { Course: { Id: courseId }, TermsAndSections: [] }, + }) as unknown as SectionsResponse, +); + +mock.module("./bridge", () => ({ sectionIds, sections })); +const { Cancelled, crawl } = await import("./crawl"); + +afterEach(() => { + sectionIds.mockClear(); + sections.mockClear(); +}); + +const run = (courseIds: string[], have?: TermCatalog, extra = {}) => + crawl({ courseIds, term: "26/FA", have, delayMs: 0, ...extra }); + +describe("crawl", () => { + test("fetches every course and records the ones not taught", async () => { + const catalog = await run(["a", "b", "gone"]); + expect(Object.keys(catalog.sections).sort()).toEqual(["a", "b"]); + expect(catalog.notOffered).toEqual(["gone"]); + expect(sections).toHaveBeenCalledTimes(2); + }); + + test("skips what the shared cache already answers", async () => { + const have = mergeCatalogs(emptyCatalog("26/FA"), { + ...emptyCatalog("26/FA"), + sections: { a: {} as SectionsResponse }, + notOffered: ["gone"], + }); + + const catalog = await run(["a", "b", "gone"], have); + // Only "b" was unknown. + expect(sections).toHaveBeenCalledTimes(1); + expect(Object.keys(catalog.sections).sort()).toEqual(["a", "b"]); + expect(catalog.notOffered).toEqual(["gone"]); + }); + + test("reports progress that counts only the work left to do", async () => { + const have: TermCatalog = { ...emptyCatalog("26/FA"), notOffered: ["gone"] }; + const seen: string[] = []; + let last = { done: 0, total: 0, cached: 0 }; + + await run(["a", "b", "gone"], have, { + onProgress: (p: { done: number; total: number; current: string; cached: number }) => { + if (p.current) seen.push(p.current); + last = p; + }, + }); + + expect(last).toMatchObject({ done: 2, total: 2, cached: 1 }); + expect(seen).toContain("SUBJ-a"); + }); + + test("stops when cancelled and does not keep fetching", async () => { + const controller = new AbortController(); + const promise = run(["a", "b", "c"], undefined, { + signal: controller.signal, + onProgress: ({ done }: { done: number }) => { + if (done === 1) controller.abort(); + }, + }); + + await expect(promise).rejects.toBeInstanceOf(Cancelled); + expect(sections.mock.calls.length).toBeLessThan(3); + }); + + test("an empty course list is a no-op, not an error", async () => { + const catalog = await run([]); + expect(catalog.sections).toEqual({}); + expect(sections).not.toHaveBeenCalled(); + }); +}); + +describe("catalog merging", () => { + test("known() covers offered and not-offered alike", () => { + const catalog: TermCatalog = { + ...emptyCatalog("26/FA"), + sections: { a: {} as SectionsResponse }, + notOffered: ["b"], + }; + expect(known(catalog)).toEqual(new Set(["a", "b"])); + }); + + test("age is measured from the fetch stamp", () => { + const catalog = { ...emptyCatalog("26/FA"), fetchedAt: "2026-08-12T00:00:00.000Z" }; + expect(ageInHours(catalog, Date.parse("2026-08-12T06:00:00.000Z"))).toBe(6); + }); +}); diff --git a/src/client/crawl.ts b/src/client/crawl.ts new file mode 100644 index 0000000..e66f015 --- /dev/null +++ b/src/client/crawl.ts @@ -0,0 +1,73 @@ +/** + * Fetching a term's sections, politely and visibly. + * + * The loop lives here rather than in the extension because everything that + * makes it bearable is a user-experience decision: how fast to go, what to + * skip because the shared cache already has it, how to report progress, and + * how to stop halfway. The extension just fetches what it is told to. + */ + +import { emptyCatalog, known, mergeCatalogs, type TermCatalog } from "../catalog"; +import { sectionIds, sections } from "./bridge"; + +export interface Progress { + done: number; + total: number; + /** The course being fetched, for something concrete to read. */ + current: string; + /** Answered from the shared cache instead of Self-Service. */ + cached: number; +} + +export interface CrawlOptions { + courseIds: string[]; + term: string; + /** Already-known sections, typically from the server cache. */ + have?: TermCatalog; + onProgress?: (progress: Progress) => void; + signal?: AbortSignal; + /** Gap between requests. This is a registrar, not a load test. */ + delayMs?: number; +} + +export class Cancelled extends Error { + constructor() { + super("cancelled"); + } +} + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +export async function crawl(options: CrawlOptions): Promise { + const { courseIds, term, onProgress, signal, delayMs = 120 } = options; + const have = options.have ?? emptyCatalog(term); + const cachedIds = known(have); + + const fresh = emptyCatalog(term); + // One search covers every course, so it is cheap even for ones we cache. + const catalogue = await sectionIds(courseIds, term); + + const todo = catalogue.filter((c) => !cachedIds.has(c.courseId)); + const cached = catalogue.length - todo.length; + let done = 0; + + onProgress?.({ done, total: todo.length, current: "", cached }); + + for (const course of todo) { + if (signal?.aborted) throw new Cancelled(); + onProgress?.({ done, total: todo.length, current: course.courseName, cached }); + + if (course.sectionIds.length === 0) { + fresh.notOffered.push(course.courseId); + } else { + fresh.sections[course.courseId] = await sections(course.courseId, course.sectionIds); + // Only sleep after a real request; skipping costs nobody anything. + if (delayMs > 0) await sleep(delayMs); + } + + done++; + onProgress?.({ done, total: todo.length, current: course.courseName, cached }); + } + + return mergeCatalogs(have, fresh); +} diff --git a/src/client/ctx.ts b/src/client/ctx.ts index a6074f5..7bcfea4 100644 --- a/src/client/ctx.ts +++ b/src/client/ctx.ts @@ -1,9 +1,9 @@ /* What every view is handed. Grows as the app learns to fetch more. */ -import type { SectionsCapture } from "../content"; +import type { TermCatalog } from "../catalog"; import type { ProgramTree } from "../requirements"; export interface Ctx { trees: ProgramTree[]; - sections?: SectionsCapture; + sections?: TermCatalog; } diff --git a/src/client/index.html b/src/client/index.html index 81de419..b5b3f4c 100644 --- a/src/client/index.html +++ b/src/client/index.html @@ -24,9 +24,15 @@ + + +