Something went wrong. Try again.
A simple DAV sync application / cli
Something went wrong. Try again.
TypeScript
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447import { createReadStream } from "node:fs";import { Readable, Transform } from "node:stream";import { collectionMembers, DavClient, DavHttpError, type DavMemberResponse, DavProtocolError, decodeHref, type FetchLike } from "@ordner.fyi/dav-client";import type { PropertyValue, QName } from "@ordner.fyi/dav-core";import { DAV } from "@ordner.fyi/dav-core/wire";import { childElements, isElement, textContent } from "@ordner.fyi/xml";import { HEAD_SIZE } from "./detect.ts";import { ContentHasher, headOfBytes } from "./hash.ts";import { ancestors, isBelow, isSafeName, mapLimit, normalize } from "./paths.ts";import type { LastSeen, Snapshot } from "./types.ts";
/** Nextcloud / ownCloud. Asked for everywhere; servers without them just leave them out. */const OC = "http://owncloud.org/ns";
const PROPS: QName[] = [ { ns: DAV, name: "resourcetype" }, { ns: DAV, name: "getetag" }, { ns: DAV, name: "getcontentlength" }, { ns: DAV, name: "getlastmodified" }, { ns: DAV, name: "creationdate" }, { ns: OC, name: "fileid" }, { ns: OC, name: "checksums" }, // RFC 3744: what this account may do there, so nothing is tried that the server will refuse. { ns: DAV, name: "current-user-privilege-set" },];
const LIST_CONCURRENCY = 4;
export interface RemoteOptions { remoteUrl: string; username: string; password: string; fetch?: FetchLike;}
/** The remote half of a sync folder: a WebDAV collection, scanned and changed through DavClient. */export class RemoteSide { readonly #client: DavClient; /** The sync root's path on the server, as decoded segments. */ readonly #rootSegments: string[]; /** Collections known to exist, so parents are not created over and over. */ readonly #collections = new Set<string>([""]); /** Paths the last scan found this account may not change (see States.readOnly). */ readonly readOnly = new Set<string>(); /** * Paths as the server spells them, where that differs from the NFC path they * are compared by. One server may hold "ö" composed in one name and decomposed * in the next (mailbox.org does), and only its own spelling finds the file. */ readonly #spelling = new Map<string, string>();
constructor(options: RemoteOptions) { this.#client = new DavClient({ baseUrl: options.remoteUrl, ...(options.username ? { credentials: { user: options.username, password: options.password } } : {}), ...(options.fetch ? { fetch: options.fetch } : {}), userAgent: "davsync", timeoutMs: 60_000, }); this.#rootSegments = new URL(this.#client.url("")).pathname.split("/").filter(Boolean).map(safeDecode); }
/** * Every entry below the root, one PROPFIND (Depth: 1) per collection, since many * servers refuse Depth: infinity. With `trustCollectionEtags`, a collection whose * etag is as last seen is not listed at all: its last-seen subtree stands in for it. */ async scan(base: Map<string, LastSeen>, ignored: (path: string) => boolean, trustCollectionEtags: boolean): Promise<Map<string, Snapshot>> { const result = new Map<string, Snapshot>(); this.#collections.clear(); this.#collections.add(""); this.readOnly.clear(); this.#spelling.clear(); let level = [""]; while (level.length > 0) { const listings = await mapLimit(level, LIST_CONCURRENCY, (dir) => this.#list(dir)); const next: string[] = []; for (const listing of listings) { for (const snapshot of listing) { if (ignored(snapshot.path)) continue; result.set(snapshot.path, snapshot); if (snapshot.kind !== "dir") continue; this.#collections.add(snapshot.path); const seen = base.get(snapshot.path); if (trustCollectionEtags && seen?.kind === "dir" && snapshot.etag !== null && seen.remote.etag === snapshot.etag) { for (const [path, entry] of base) { if (isBelow(path, snapshot.path) && !ignored(path)) { result.set(path, fromLastSeen(entry)); this.#learnSpelling(path, entry.remote.url); if (entry.kind === "dir") this.#collections.add(path); } } } else { next.push(snapshot.path); } } } level = next; } return result; }
url(path: string, kind: Snapshot["kind"]): string { return this.#client.url(this.#target(path, kind)); }
/** * What a request for `path` addresses: the server's own spelling of it, or of * its nearest spelled ancestor plus the rest, for something not there yet. */ #spelled(path: string): string { if (path === "") return ""; const known = this.#spelling.get(path); if (known !== undefined) return known; const slash = path.lastIndexOf("/"); return slash === -1 ? path : `${this.#spelled(path.slice(0, slash))}/${path.slice(slash + 1)}`; }
#target(path: string, kind: Snapshot["kind"]): string { const spelled = this.#spelled(path); return kind === "dir" && spelled !== "" ? `${spelled}/` : spelled; }
/** Remembers how the server spelled `path`, from the URL it was last seen at. */ #learnSpelling(path: string, url: string): void { const segments = new URL(url).pathname.split("/").filter(Boolean).map(safeDecode).slice(this.#rootSegments.length); const spelled = segments.join("/"); if (spelled !== path && normalize(spelled) === path) this.#spelling.set(path, spelled); }
/** One entry, or null if there is nothing at `path`. */ async stat(path: string): Promise<Snapshot | null> { try { const [response] = await this.#client.propfind(this.#spelled(path), { depth: "0", props: PROPS }); return response ? this.#snapshot(response) : null; } catch (error) { if (error instanceof DavHttpError && error.status === 404) return null; throw error; } }
/** SHA-256 of the first 4 KiB, by Range request (or the start of a full response if ranges are ignored). */ async head(snapshot: Snapshot): Promise<string | null> { const content = await this.#client.get(this.#spelled(snapshot.path), { range: { start: 0, end: HEAD_SIZE - 1 } }); if (!content.body) return headOfBytes(new Uint8Array()); const chunks: Uint8Array[] = []; let size = 0; const reader = content.body.getReader(); while (size < HEAD_SIZE) { const { done, value } = await reader.read(); if (done) break; chunks.push(value); size += value.byteLength; } await reader.cancel().catch(() => {}); return headOfBytes(Buffer.concat(chunks)); }
/** * Starts a download. The body is read by whoever stores it; no timeout, because * files have no size limit. `length` is what the server declared, to check what * arrived against. */ async download(path: string): Promise<{ body: AsyncIterable<Uint8Array>; etag: string | null; mtime: number | null; length: number | null }> { const content = await this.#client.get(this.#spelled(path), { timeoutMs: 0 }); // fetch cancels a response's body once the Response itself is garbage // collected, even while someone holds the body stream: reading it then // yields nothing, silently. `content` keeps the Response reachable, so the // body is read through it, until it is done. async function* body(): AsyncGenerator<Uint8Array> { if (!content.body) return; for await (const chunk of content.body) yield chunk; } return { body: body(), etag: content.etag ?? null, mtime: content.lastModified?.getTime() ?? null, length: content.contentLength ?? null, }; }
/** * Streams a local file up, hashing what was actually sent. `ifMatch` guards an * overwrite (the remote file must still be what was scanned), `ifNoneMatch: "*"` * guards a creation (nothing may have appeared there since). */ async upload( path: string, file: string, size: number, condition: { ifMatch: string } | { ifNoneMatch: "*" } | {}, ): Promise<{ sha256: string; head: string; size: number; snapshot: Snapshot }> { await this.ensureParents(path); const hasher = new ContentHasher(); const tap = new Transform({ transform(chunk: Buffer, _encoding, done) { hasher.update(chunk); done(null, chunk); }, }); const body = Readable.toWeb(createReadStream(file).pipe(tap)) as ReadableStream<Uint8Array>; const result = await this.#client.put(this.#spelled(path), body, { ...condition, contentLength: size, timeoutMs: 0 }); const digest = hasher.digest(); // The PUT's ETag is the stored file's, when the server sends one. Other details // (fileid, mtime) need a PROPFIND either way. const snapshot = await this.stat(path); if (!snapshot) throw new Error(`${path} vanished right after it was uploaded`); if (result.etag !== undefined) snapshot.etag = result.etag; return { ...digest, snapshot: { ...snapshot, sha256: digest.sha256, head: digest.head } }; }
async move(from: string, to: string, kind: Snapshot["kind"]): Promise<void> { await this.ensureParents(to); const target = this.#target(to, kind); await this.#client.move(this.#target(from, kind), target, { overwrite: false }); // What was below `from` is now below `to`, spelled as the move spelled it. const fromSpelled = this.#spelled(from); for (const [path, spelled] of [...this.#spelling]) { if (path !== from && !isBelow(path, from)) continue; this.#spelling.delete(path); const moved = to + path.slice(from.length); const respelled = this.#spelled(to) + spelled.slice(fromSpelled.length); if (respelled !== moved) this.#spelling.set(moved, respelled); } for (const path of [...this.#collections]) { if (path === from || isBelow(path, from)) { this.#collections.delete(path); this.#collections.add(to + path.slice(from.length)); } } }
async delete(path: string, kind: Snapshot["kind"], ifMatch: string | null): Promise<void> { await this.#client.delete(this.#target(path, kind), ifMatch !== null && kind === "file" ? { ifMatch } : {}); for (const known of [...this.#collections]) if (known === path || isBelow(known, path)) this.#collections.delete(known); }
/** Lists a collection's direct members; used to confirm one is empty before deleting it. */ async members(path: string): Promise<Snapshot[]> { return this.#list(path); }
async mkdir(path: string): Promise<void> { await this.ensureParents(path); await this.#client.ensureCollection(this.#target(path, "dir")); this.#collections.add(path); }
async ensureParents(path: string): Promise<void> { for (const dir of ancestors(path)) { if (this.#collections.has(dir)) continue; await this.#client.ensureCollection(this.#target(dir, "dir")); this.#collections.add(dir); } }
async #list(dir: string): Promise<Snapshot[]> { const responses = await this.#client.propfind(this.#target(dir, "dir"), { depth: "1", props: PROPS }); const members: Snapshot[] = []; for (const response of responses) { if (response.status !== undefined && response.status >= 400) continue; const snapshot = this.#snapshot(response); if (!snapshot) continue; const privileges = privilegesOf(response.resource?.properties?.get(`{${DAV}}current-user-privilege-set`)); if (privileges && !(snapshot.kind === "dir" ? canChangeMembers(privileges) : canOverwrite(privileges))) this.readOnly.add(snapshot.path); if (snapshot.path !== dir) members.push(snapshot); } return members; }
/** * The entry a multistatus response describes, or null if it is not one we can * use. The href is the server's, so it is taken apart as it was sent: decoded a * segment at a time (a decoded "%2F" is part of a name, not a separator), and * refused outright if any segment is not a plain name. Otherwise ".." would put * a download outside the folder. */ #snapshot(response: DavMemberResponse): Snapshot | null { if (!response.resource) return null; // Usually an absolute path; some servers send a full URL. const raw = /^https?:/i.test(response.rawHref) ? new URL(response.rawHref).pathname : response.rawHref; const segments = raw.split("/").filter(Boolean).map(safeDecode); if (segments.length < this.#rootSegments.length || this.#rootSegments.some((segment, index) => segments[index] !== segment)) return null; const names = segments.slice(this.#rootSegments.length); if (!names.every(isSafeName)) return null; const spelled = names.join("/"); const path = normalize(spelled); if (spelled !== path) this.#spelling.set(path, spelled); return this.#snapshotAt(path, response.resource); }
#snapshotAt(path: string, resource: NonNullable<DavMemberResponse["resource"]>): Snapshot { const kind = resource.collection ? "dir" : "file"; const checksums = text(resource.properties?.get(`{${OC}}checksums`)); return { path, kind, size: kind === "dir" ? 0 : (resource.contentLength ?? 0), mtime: resource.lastModified?.getTime() ?? 0, ctime: resource.creationDate?.getTime() ?? null, etag: resource.etag ?? null, sha256: kind === "file" ? (/SHA256:([0-9a-f]{64})/i.exec(checksums ?? "")?.[1]?.toLowerCase() ?? null) : null, head: null, url: this.url(path, kind), id: text(resource.properties?.get(`{${OC}}fileid`))?.trim() || null, }; }}
/** The privilege names in a current-user-privilege-set, or null if the server did not say. */function privilegesOf(value: PropertyValue | undefined): Set<string> | null { if (!value) return null; // An empty set (no privileges at all) parses as empty text. if (value.kind === "text") return value.value.trim() === "" ? new Set() : null; if (value.kind !== "xml") return null; const names = new Set<string>(); for (const node of value.children) { if (!isElement(node) || node.name !== "privilege") continue; for (const privilege of childElements(node)) names.add(privilege.name); } return names;}
/** RFC 3744 §3: adding and removing members needs bind/unbind, which write and all include. */function canChangeMembers(privileges: Set<string>): boolean { return ["bind", "unbind", "write", "all"].some((name) => privileges.has(name));}
function canOverwrite(privileges: Set<string>): boolean { return ["write-content", "write", "all"].some((name) => privileges.has(name));}
function fromLastSeen(entry: LastSeen): Snapshot { return { path: entry.path, kind: entry.kind, size: entry.size, mtime: entry.remote.mtime, ctime: entry.remote.ctime, etag: entry.remote.etag, sha256: entry.sha256, head: entry.head, url: entry.remote.url, id: entry.remote.id, };}
function text(value: PropertyValue | undefined): string | undefined { if (value?.kind === "text") return value.value; if (value?.kind === "xml") return value.children.map((node) => textContent(node)).join(" "); return undefined;}
function safeDecode(path: string): string { try { return decodeURIComponent(path); } catch { return path; }}
/** A WebDAV server, as the setup window knows it before a folder is chosen. */export interface ServerOptions { serverUrl: string; username: string; password: string; fetch?: FetchLike;}
export interface CollectionListing { /** The listed collection below the server root ("" for the root itself). */ path: string; parent: string | null; folders: { name: string; label: string }[]; /** The server reports file ids (Nextcloud, ownCloud), whose collection etags change with their content. */ nextcloudLike: boolean;}
/** The sub-collections of `path` below the server root: what a folder browser shows. */export async function listCollections(server: ServerOptions, path: string): Promise<CollectionListing> { const client = serverClient(server); const segments = path.split("/").filter(Boolean); const target = segments.length ? `${segments.join("/")}/` : ""; try { const responses = await client.propfind(target, { depth: "1", props: [{ ns: DAV, name: "resourcetype" }, { ns: DAV, name: "displayname" }, { ns: OC, name: "fileid" }], }); const self = decodeHref(new URL(client.url(target)).pathname); const folders = collectionMembers(responses, self) .map((member) => ({ name: member.segment, label: member.label })) .sort((a, b) => a.label.localeCompare(b.label)); return { path: segments.join("/"), parent: segments.length ? segments.slice(0, -1).join("/") : null, folders, nextcloudLike: responses.some((response) => response.resource?.properties?.has(`{${OC}}fileid`)), }; } catch (error) { throw explain(error, server.serverUrl, segments.join("/")); }}
/** Creates `name` inside `path`; returns the new collection's path. */export async function makeCollection(server: ServerOptions, path: string, name: string): Promise<string> { const clean = name.trim(); if (!clean || clean === "." || clean === ".." || clean.includes("/")) throw new Error("A folder name cannot be empty or contain “/”"); const created = [...path.split("/").filter(Boolean), clean].join("/"); try { await serverClient(server).mkcol(`${created}/`); } catch (error) { if (error instanceof DavHttpError && error.status === 405) throw new Error(`There is already something called “${clean}” there`); throw explain(error, server.serverUrl, created); } return created;}
function serverClient(server: ServerOptions): DavClient { return new DavClient({ baseUrl: server.serverUrl, ...(server.username ? { credentials: { user: server.username, password: server.password } } : {}), ...(server.fetch ? { fetch: server.fetch } : {}), userAgent: "davsync", timeoutMs: 20_000, });}
/** Turns what went wrong talking to a server into something a person can act on. */export function explain(error: unknown, serverUrl: string, path: string): Error { if (error instanceof DavHttpError) { if (error.status === 401) return new Error("The server did not accept this user name and password."); if (error.status === 403) return new Error(`This account may not look into ${path ? `“${path}”` : "the server's root"}.`); if (error.status === 404) return new Error(`There is no folder ${path ? `“${path}”` : "at this address"} on the server.`); if (error.status === 405 || error.status === 501) return new Error("This address does not speak WebDAV (it does not answer PROPFIND)."); return new Error(`The server answered HTTP ${error.status}.`); } if (error instanceof DavProtocolError) return new Error("This address does not answer like a WebDAV server."); const cause = (error as { cause?: { code?: string; message?: string } })?.cause; let host = serverUrl; try { host = new URL(serverUrl).host; } catch {} if (error instanceof Error && (error.name === "TimeoutError" || error.name === "AbortError")) return new Error(`${host} did not answer in time.`); if (cause?.code || cause?.message) return new Error(`Could not reach ${host} (${cause.code ?? cause.message}).`); return error instanceof Error ? error : new Error(String(error));}