Something went wrong. Try again.
A simple DAV sync application / cli
Something went wrong. Try again.
TypeScript
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387import { pathToFileURL } from "node:url";import { DavHttpError } from "@ordner.fyi/dav-client";import type { Outcome } from "../shared.ts";import type { LocalSide, TempFile } from "./local.ts";import { isBelow, mapLimit, nameOf, parentOf, rebase } from "./paths.ts";import { remoteLocked } from "./plan.ts";import { isOffline } from "./transient.ts";import type { RemoteSide } from "./remote.ts";import type { StateStore } from "./state.ts";import type { Action, LastSeen, SideMeta, Snapshot, States } from "./types.ts";
/** An action that could not go ahead because something changed since the scan. The next sync sees it. */export class Skip extends Error {}
export type { Outcome };
export interface ExecuteContext { folder: string; local: LocalSide; remote: RemoteSide; store: StateStore; /** The scanned states; kept current as actions change things. */ states: States; /** Names this run's trash directory. */ runTag: string; onAction?: (index: number, action: Action, outcome: Outcome) => void;}
const TRANSFER_CONCURRENCY = 4;
/** * Carries out a plan. Moves, deletes and new directories run one at a time, in * order; transfers run a few at a time. An action that fails is logged and the rest * go ahead: each settles its own last-seen record, so a failure leaves exactly that * entry's old record behind for the next sync to compare against again. */export async function execute(context: ExecuteContext, actions: Action[]): Promise<Outcome[]> { const outcomes: Outcome[] = new Array(actions.length); /** Set once the network is gone: nothing else is tried, everything waits for the next sync. */ let offline: string | null = null; const run = async (index: number) => { const action = actions[index]!; let outcome: Outcome; if (offline !== null) { outcomes[index] = { status: "skipped", detail: `not tried: ${offline}` }; context.onAction?.(index, action, outcomes[index]); return; } try { outcome = action.type === "hold" ? { status: "held", detail: null } : { status: "done", detail: (await perform(context, action)) ?? null }; } catch (error) { if (isOffline(error)) { offline = `the connection to the server failed (${errorCode(error)})`; outcome = { status: "skipped", detail: offline }; } else if (error instanceof Skip) outcome = { status: "skipped", detail: error.message }; else if (error instanceof DavHttpError && error.status === 412) outcome = { status: "skipped", detail: "changed remotely since the scan" }; else outcome = { status: "failed", detail: error instanceof Error ? error.message : String(error) }; } outcomes[index] = outcome; context.onAction?.(index, action, outcome); };
const isTransfer = (action: Action) => action.type === "upload" || action.type === "download" || action.type === "conflict"; let index = 0; while (index < actions.length) { if (!isTransfer(actions[index]!)) { await run(index++); continue; } const batch: number[] = []; while (index < actions.length && isTransfer(actions[index]!)) batch.push(index++); await mapLimit(batch, TRANSFER_CONCURRENCY, run); } return outcomes;}
async function perform(context: ExecuteContext, action: Action): Promise<string | void> { const { local, remote, states } = context; switch (action.type) { case "move": { if (action.on === "local") { await local.move(action.from, action.to); rekey(states.local, action.from, action.to, (moved) => pathToFileURL(local.fsPath(moved.path)).href); const moved = await local.stat(action.to); if (moved) states.local.set(action.to, { ...moved, sha256: states.local.get(action.to)?.sha256 ?? null, head: states.local.get(action.to)?.head ?? null }); } else { await remote.move(action.from, action.to, action.kind); rekey(states.remote, action.from, action.to, (moved) => remote.url(moved.path, moved.kind)); await refreshRemote(context, action.to); } return; }
case "mkdir": { if (action.on === "local") await local.mkdir(action.path); else await remote.mkdir(action.path); const [l, r] = await Promise.all([local.stat(action.path), remote.stat(action.path)]); if (!l || !r) throw new Error(`${action.path} is not there after creating it`); states.local.set(action.path, l); states.remote.set(action.path, r); settle(context, action.base, l, r, { sha256: null, head: null, size: 0 }); return; }
case "upload": { const now = await unchangedLocal(context, action.path); if (!now) throw new Skip("vanished locally since the scan"); const there = states.remote.get(action.path); if (there?.kind === "dir") throw new Error("a folder is in the way remotely"); const condition = there ? (there.etag ? { ifMatch: there.etag } : {}) : { ifNoneMatch: "*" as const }; const sent = await remote.upload(action.path, local.fsPath(action.path), now.size, condition); states.remote.set(action.path, sent.snapshot); // The stat from *before* the upload: if the file changed while it was read, // the next scan sees a difference and uploads again. settle(context, action.base, now, sent.snapshot, sent); return; }
case "download": { const was = await unchangedLocal(context, action.path); const there = states.remote.get(action.path); if (!there || there.kind !== "file") throw new Error("nothing to download"); if (was?.kind === "dir") throw new Error("a folder is in the way locally"); const incoming = await remote.download(action.path); const temp = await local.receive(incoming.body); await complete(context, temp, incoming, there); const remoteNow: Snapshot = { ...there, etag: incoming.etag ?? there.etag, sha256: temp.sha256, head: temp.head, size: temp.size }; states.remote.set(action.path, remoteNow); const localSha = was ? (was.sha256 ?? context.states.base.get(action.base ?? action.path)?.sha256 ?? null) : null; if (was && localSha === temp.sha256) { // Only the etag moved on; the bytes are the ones already here. await local.discard(temp); settle(context, action.base, was, remoteNow, temp); return "content unchanged; only the etag moved on"; } if (!sameStat(await local.stat(action.path), was)) { await local.discard(temp); throw new Skip("changed locally during the download"); } await local.place(temp, action.path, incoming.mtime ?? (there.mtime || null)); const placed = await local.stat(action.path); if (!placed) throw new Error("vanished right after it was written"); states.local.set(action.path, { ...placed, sha256: temp.sha256, head: temp.head }); settle(context, action.base, placed, remoteNow, temp); return; }
case "conflict": return resolveConflict(context, action.path, action.base);
case "delete": { if (action.on === "local") { // A folder goes to the trash whole, so even something created in it since the scan survives there. const present = action.kind === "dir" ? await local.stat(action.path) : await unchangedLocal(context, action.path); if (present) await local.trash(action.path, context.runTag); dropTree(states.local, action.path); } else { const there = states.remote.get(action.path); if (action.kind === "dir") await confirmRemoteTreeUnchanged(context, action.path); try { await remote.delete(action.path, action.kind, there?.etag ?? null); } catch (error) { if (!(error instanceof DavHttpError && error.status === 404)) throw error; } dropTree(states.remote, action.path); } context.store.transaction(() => action.kind === "dir" ? context.store.deleteBaseTree(context.folder, action.base) : context.store.deleteBase(context.folder, action.base), ); dropTree(states.base, action.base, action.kind === "file"); return action.on === "local" ? `moved to ${".davsync/trash/" + context.runTag}` : undefined; }
case "record": { const l = states.local.get(action.path); const r = states.remote.get(action.path); if (!l || !r) throw new Error("not present on both sides"); if (l.kind !== r.kind) throw new Error("a file on one side and a folder on the other"); const previous = action.base === null ? undefined : states.base.get(action.base); const sha256 = l.sha256 ?? r.sha256 ?? previous?.sha256 ?? null; const head = l.head ?? r.head ?? previous?.head ?? null; settle(context, action.base, l, r, { sha256, head, size: l.size }); return; }
case "forget": context.store.transaction(() => context.store.deleteBase(context.folder, action.base)); states.base.delete(action.base); return;
case "hold": return; }}
/** * Both sides have something at `path`. Equal content is simply adopted. Otherwise * the local version moves aside to a conflict name, the remote one takes its place, * and the local version is uploaded under the conflict name too, so both sides end * up with both. */async function resolveConflict(context: ExecuteContext, path: string, base: string | null): Promise<string> { const { local, remote, states } = context; const l = await unchangedLocal(context, path); const r = states.remote.get(path); if (!l || !r) throw new Skip("no longer on both sides"); if (l.kind !== "file" || r.kind !== "file") throw new Error("a file on one side and a folder on the other; rename one of them");
if (l.sha256 === null) Object.assign(l, await local.hash(l)); if (r.sha256 !== null && r.sha256 === l.sha256) { settle(context, base, l, r, { sha256: l.sha256, head: l.head, size: l.size }); return "same content on both sides"; }
const incoming = await remote.download(path); const temp = await local.receive(incoming.body); await complete(context, temp, incoming, r); const remoteNow: Snapshot = { ...r, etag: incoming.etag ?? r.etag, sha256: temp.sha256, head: temp.head, size: temp.size }; if (temp.sha256 === l.sha256) { await local.discard(temp); settle(context, base, l, remoteNow, temp); return "same content on both sides"; }
const aside = conflictName(path, (candidate) => states.local.has(candidate) || states.remote.has(candidate)); if (!sameStat(await local.stat(path), l)) { await local.discard(temp); throw new Skip("changed locally during the comparison"); } await local.move(path, aside); await local.place(temp, path, incoming.mtime ?? (r.mtime || null)); const placed = await local.stat(path); const movedAside = await local.stat(aside); if (!placed || !movedAside) throw new Error("a file vanished while resolving the conflict"); states.local.set(path, { ...placed, sha256: temp.sha256, head: temp.head }); states.remote.set(path, remoteNow); settle(context, base, placed, remoteNow, temp);
if (remoteLocked(states, aside)) { // The server will not take it: the local version stays here only, and the // next sync holds it as a change it cannot carry over. states.local.set(aside, { ...movedAside, sha256: l.sha256, head: l.head }); return `the local version was kept as ${aside}, here only: the server only lets you read this folder`; } const sent = await remote.upload(aside, local.fsPath(aside), movedAside.size, { ifNoneMatch: "*" }); states.local.set(aside, { ...movedAside, sha256: sent.sha256, head: sent.head }); states.remote.set(aside, sent.snapshot); settle(context, null, movedAside, sent.snapshot, sent); return `the local version was kept as ${aside}`;}
/** * Refuses a download that did not deliver what the server said it would: fewer * bytes than its Content-Length, or (for the version that was listed) than the * listing's size. A short read must never be stored, or recorded, as the file. */async function complete( context: ExecuteContext, temp: TempFile, incoming: { etag: string | null; length: number | null }, listed: Snapshot,): Promise<void> { const expected = incoming.length ?? (incoming.etag === null || incoming.etag === listed.etag ? listed.size : null); if (expected === null || temp.size === expected) return; await context.local.discard(temp); throw new Error(`the download was incomplete: ${temp.size} of ${expected} bytes arrived`);}
/** The most specific code an error or its causes carry, for the log. */function errorCode(error: unknown): string { for (let current = error, depth = 0; current && depth < 5; current = (current as { cause?: unknown }).cause, depth++) { const { code } = current as { code?: unknown }; if (typeof code === "string") return code; } return error instanceof Error ? error.message : String(error);}
/** "notes.txt" → "notes (conflict 2026-10-09 1530).txt", numbered if that is taken too. */export function conflictName(path: string, taken: (path: string) => boolean, now = new Date()): string { const name = nameOf(path); const dot = name.lastIndexOf("."); const [stem, extension] = dot > 0 ? [name.slice(0, dot), name.slice(dot)] : [name, ""]; const pad = (n: number) => String(n).padStart(2, "0"); const stamp = `${now.getFullYear()}-${pad(now.getMonth() + 1)}-${pad(now.getDate())} ${pad(now.getHours())}${pad(now.getMinutes())}`; const dir = parentOf(path); for (let n = 1; ; n++) { const candidate = `${dir ? dir + "/" : ""}${stem} (conflict ${stamp}${n > 1 ? ` ${n}` : ""})${extension}`; if (!taken(candidate)) return candidate; }}
/** Writes the last-seen record for `path`, replacing the one at `base`. */function settle( context: ExecuteContext, base: string | null, l: Snapshot, r: Snapshot, content: { sha256: string | null; head: string | null; size: number },): void { const entry: LastSeen = { path: l.path, kind: l.kind, size: l.kind === "dir" ? 0 : content.size, sha256: content.sha256, head: content.head, local: meta(l), remote: meta(r), syncedAt: Date.now(), }; context.store.transaction(() => { if (base !== null && base !== entry.path) context.store.deleteBase(context.folder, base); context.store.putBase(context.folder, entry); }); if (base !== null) context.states.base.delete(base); context.states.base.set(entry.path, entry);}
function meta(snapshot: Snapshot): SideMeta { return { mtime: snapshot.mtime, ctime: snapshot.ctime, etag: snapshot.etag, url: snapshot.url, id: snapshot.id };}
/** * The local entry as it is now, provided it is as the scan saw it (or absent, if the * scan saw nothing). Anything else means someone is working on it: leave it alone. */async function unchangedLocal(context: ExecuteContext, path: string): Promise<Snapshot | null> { const scanned = context.states.local.get(path) ?? null; const now = await context.local.stat(path); if (!sameStat(now, scanned)) throw new Skip("changed locally since the scan"); return scanned;}
function sameStat(a: Snapshot | null, b: Snapshot | null): boolean { if (a === null || b === null) return a === b; return a.kind === b.kind && (a.kind === "dir" || (a.size === b.size && a.mtime === b.mtime && a.ctime === b.ctime));}
/** After a remote move, the server may have given the moved entries new etags; take them in. */async function refreshRemote(context: ExecuteContext, path: string): Promise<void> { const { remote, states } = context; const updated = await remote.stat(path); const before = states.remote.get(path); if (updated) states.remote.set(path, { ...updated, sha256: before?.sha256 ?? null, head: before?.head ?? null }); if (updated?.kind !== "dir") return; const queue = [path]; while (queue.length > 0) { for (const member of await remote.members(queue.shift()!)) { const known = states.remote.get(member.path); if (member.kind === "dir") queue.push(member.path); // Only take what is recognisably the same entry; anything else is the next scan's business. if (known && known.kind === member.kind && known.size === member.size) { states.remote.set(member.path, { ...member, sha256: known.sha256, head: known.head }); } } }}
/** Refuses to delete a remote directory if anything below it is not as scanned. */async function confirmRemoteTreeUnchanged(context: ExecuteContext, path: string): Promise<void> { const queue = [path]; while (queue.length > 0) { for (const member of await context.remote.members(queue.shift()!)) { const known = context.states.remote.get(member.path); if (!known || known.kind !== member.kind || (member.kind === "file" && known.etag !== member.etag)) { throw new Skip(`${member.path} changed remotely since the scan`); } if (member.kind === "dir") queue.push(member.path); } }}
/** Re-keys `from` and everything below it to `to`. */function rekey(map: Map<string, Snapshot>, from: string, to: string, url: (moved: Snapshot) => string): void { for (const [path, snapshot] of [...map]) { if (path !== from && !isBelow(path, from)) continue; map.delete(path); const moved = { ...snapshot, path: rebase(path, from, to) }; map.set(moved.path, { ...moved, url: url(moved) }); }}
function dropTree<T>(map: Map<string, T>, path: string, onlySelf = false): void { map.delete(path); if (onlySelf) return; for (const key of [...map.keys()]) if (isBelow(key, path)) map.delete(key);}