From ee08213a85e09889b9bd66beceecee92ac025801 Mon Sep 17 00:00:00 2001 From: Orual Date: Sat, 31 May 2025 10:06:36 -0400 Subject: [PATCH] bunch more lexicon stuff and buildout --- .gitignore | 2 + .zed/settings.json | 10 + Cargo.lock | 309 ++-- Cargo.toml | 3 +- crates/weaver-common/Cargo.toml | 2 + crates/weaver-common/src/agent.rs | 78 +- .../src/lexicons/app/bsky/feed/defs.rs | 10 + .../app/bsky/feed/get_feed_skeleton.rs | 3 + .../src/lexicons/app/bsky/feed/like.rs | 2 + .../src/lexicons/app/bsky/feed/repost.rs | 2 + .../bsky/notification/list_notifications.rs | 2 +- crates/weaver-common/src/lexicons/client.rs | 2 +- crates/weaver-common/src/lexicons/record.rs | 144 +- .../weaver-common/src/lexicons/sh/weaver.rs | 1 + .../src/lexicons/sh/weaver/actor/defs.rs | 11 + .../src/lexicons/sh/weaver/notebook.rs | 7 + .../src/lexicons/sh/weaver/notebook/book.rs | 4 +- .../lexicons/sh/weaver/notebook/chapter.rs | 24 + .../src/lexicons/sh/weaver/notebook/defs.rs | 9 + .../src/lexicons/sh/weaver/publish.rs | 10 + .../src/lexicons/sh/weaver/publish/blob.rs | 15 + .../src/lexicons/sh/weaver/publish/defs.rs | 2 + crates/weaver-common/src/lib.rs | 86 +- crates/weaver-renderer/Cargo.toml | 23 +- crates/weaver-renderer/src/atproto.rs | 18 + crates/weaver-renderer/src/base_html.rs | 693 +++++++++ crates/weaver-renderer/src/code_pretty.rs | 49 + crates/weaver-renderer/src/lib.rs | 323 +++++ crates/weaver-renderer/src/static_site.rs | 1245 +++++++++++++++++ crates/weaver-renderer/src/utils.rs | 244 ++++ crates/weaver-renderer/src/walker.rs | 113 ++ crates/weaver-server/Cargo.toml | 8 + crates/weaver-server/src/lib.rs | 1 + flake.lock | 12 +- flake.nix | 3 +- lexicons/sh/weaver/actor/defs.json | 14 + lexicons/sh/weaver/notebook/authors.json | 2 +- lexicons/sh/weaver/notebook/book.json | 9 +- lexicons/sh/weaver/notebook/chapter.json | 43 + lexicons/sh/weaver/notebook/defs.json | 12 + lexicons/sh/weaver/notebook/page.json | 36 + lexicons/sh/weaver/publish/blob.json | 23 + lexicons/sh/weaver/publish/defs.json | 5 + 43 files changed, 3419 insertions(+), 195 deletions(-) create mode 100644 crates/weaver-common/src/lexicons/sh/weaver/notebook/chapter.rs create mode 100644 crates/weaver-common/src/lexicons/sh/weaver/publish.rs create mode 100644 crates/weaver-common/src/lexicons/sh/weaver/publish/blob.rs create mode 100644 crates/weaver-common/src/lexicons/sh/weaver/publish/defs.rs create mode 100644 crates/weaver-renderer/src/atproto.rs create mode 100644 crates/weaver-renderer/src/base_html.rs create mode 100644 crates/weaver-renderer/src/code_pretty.rs create mode 100644 crates/weaver-renderer/src/static_site.rs create mode 100644 crates/weaver-renderer/src/utils.rs create mode 100644 crates/weaver-renderer/src/walker.rs create mode 100644 crates/weaver-server/src/lib.rs create mode 100644 lexicons/sh/weaver/notebook/chapter.json create mode 100644 lexicons/sh/weaver/notebook/page.json create mode 100644 lexicons/sh/weaver/publish/blob.json create mode 100644 lexicons/sh/weaver/publish/defs.json diff --git a/.gitignore b/.gitignore index 5e067b6..4b3a1fc 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,5 @@ .db/ **/.db/ + +**/.claude/settings.local.json diff --git a/.zed/settings.json b/.zed/settings.json index 5d2482d..9d30bab 100644 --- a/.zed/settings.json +++ b/.zed/settings.json @@ -11,6 +11,16 @@ "command": ["alejandra", "--quiet", "--"] // or ["nixfmt"] } } + }, + "rust-analyzer": { + "binary": { + "path": "/nix/store/qs17clvcja2b5wy5wmcqd5mqly6wypax-rust-default-1.89.0-nightly-2025-05-21/bin/rust-analyzer" + }, + "initialization_options": { + "cargo": { + "allFeatures": true + } + } } } } diff --git a/Cargo.lock b/Cargo.lock index 96540d8..69cbe1f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -121,12 +121,12 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "3.0.7" +version = "3.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" +checksum = "6680de5231bd6ee4c6191b8a1325daa282b415391ec9d3a37bd34f2060dc73fa" dependencies = [ "anstyle", - "once_cell", + "once_cell_polyfill", "windows-sys 0.59.0", ] @@ -185,9 +185,9 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "atrium-api" -version = "0.25.3" +version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7225f0ca3c78564b784828e3db3e92619cf6e786530c3468df73f49deebc0bd4" +checksum = "46355d3245edc7b3160b2a45fe55d09a6963ebd3eee0252feb6b72fb0eb71463" dependencies = [ "atrium-common", "atrium-xrpc", @@ -221,9 +221,9 @@ dependencies = [ [[package]] name = "atrium-identity" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84939a5e3a0a442467d6a000f586c092bb763a31c2d38ec1648f5179a720395a" +checksum = "c9e2d42bb4dbea038f4f5f45e3af2a89d61a9894a75f06aa550b74a60d2be380" dependencies = [ "atrium-api", "atrium-common", @@ -247,9 +247,9 @@ dependencies = [ [[package]] name = "atrium-oauth" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4f210da8f2d7199b15e6b02c0628175791ae5caaf506cfcafe45a20917f37c" +checksum = "ca22dc4eaf77fd9bf050b21192ac58cd654a437d28e000ec114ebd93a51d36f5" dependencies = [ "atrium-api", "atrium-common", @@ -421,6 +421,21 @@ dependencies = [ "serde", ] +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + [[package]] name = "bitflags" version = "1.3.2" @@ -469,6 +484,16 @@ dependencies = [ "alloc-stdlib", ] +[[package]] +name = "bstr" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "buf_redux" version = "0.8.4" @@ -517,9 +542,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.23" +version = "1.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f4ac86a9e5bc1e2b3449ab9d7d3a6a405e3d1bb28d7b9be8614f55846ae3766" +checksum = "16595d3be041c03b09d08d0858631facccee9221e579704070e6e9e4915d3bc7" dependencies = [ "shlex", ] @@ -573,9 +598,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.38" +version = "4.5.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed93b9805f8ba930df42c2590f05453d5ec36cbb85d018868a5b24d31f6ac000" +checksum = "fd60e63e9be68e5fb56422e397cf9baddded06dae1d2e523401542383bc72a9f" dependencies = [ "clap_builder", "clap_derive", @@ -583,9 +608,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.38" +version = "4.5.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "379026ff283facf611b0ea629334361c4211d1b12ee01024eec1591133b04120" +checksum = "89cc6392a1f72bbeb820d71f32108f61fdaf18bc526e1d23954168a67759ef51" dependencies = [ "anstream", "anstyle", @@ -682,9 +707,9 @@ dependencies = [ [[package]] name = "cordyceps" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0392f465ceba1713d30708f61c160ebf4dc1cf86bb166039d16b11ad4f3b5b6" +checksum = "688d7fbb8092b8de775ef2536f36c8c31f2bc4006ece2e8d8ad2d17d00ce0a2a" dependencies = [ "loom", "tracing", @@ -742,6 +767,16 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + [[package]] name = "crossbeam-epoch" version = "0.9.18" @@ -1085,6 +1120,27 @@ dependencies = [ "syn", ] +[[package]] +name = "dynosaur" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "277b2cb52d2df4acece06bb16bc0bb0a006970c7bf504eac2d310927a6f65890" +dependencies = [ + "dynosaur_derive", + "trait-variant", +] + +[[package]] +name = "dynosaur_derive" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a4102713839a8c01c77c165bc38ef2e83948f6397fa1e1dcfacec0f07b149d3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "ecdsa" version = "0.16.9" @@ -1231,6 +1287,16 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" +[[package]] +name = "fancy-regex" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b95f7c0680e4142284cf8b22c14a476e87d61b004a3a0861872b32ef7ead40a2" +dependencies = [ + "bit-set", + "regex", +] + [[package]] name = "fastrand" version = "2.3.0" @@ -1484,6 +1550,19 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +[[package]] +name = "globset" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54a1028dfc5f5df5da8a56a73e6c153c9a9708ec57232470703592a3f18e49f5" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata 0.4.9", + "regex-syntax 0.8.5", +] + [[package]] name = "group" version = "0.13.0" @@ -1566,6 +1645,15 @@ dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.3", +] + [[package]] name = "heck" version = "0.4.1" @@ -1719,11 +1807,10 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.5" +version = "0.27.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" +checksum = "03a01595e11bdcec50946522c32dde3fc6914743000a68b93000965f2f02406d" dependencies = [ - "futures-util", "http", "hyper", "hyper-util", @@ -1732,7 +1819,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", - "webpki-roots 0.26.11", + "webpki-roots", ] [[package]] @@ -1753,22 +1840,28 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.11" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497bbc33a26fdd4af9ed9c70d63f61cf56a938375fbb32df34db9b1cd6d643f2" +checksum = "b1c293b6b3d21eca78250dc7dbebd6b9210ec5530e038cbfe0661b5c47ab06e8" dependencies = [ + "base64 0.22.1", "bytes", "futures-channel", + "futures-core", "futures-util", "http", "http-body", "hyper", + "ipnet", "libc", + "percent-encoding", "pin-project-lite", "socket2", + "system-configuration", "tokio", "tower-service", "tracing", + "windows-registry", ] [[package]] @@ -1844,9 +1937,9 @@ checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" [[package]] name = "icu_properties" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2549ca8c7241c82f59c80ba2a6f415d931c5b58d24fb8412caa1a1f02c49139a" +checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" dependencies = [ "displaydoc", "icu_collections", @@ -1860,9 +1953,9 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8197e866e47b68f8f7d95249e172903bec06004b18b2937f1095d40a0c57de04" +checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" [[package]] name = "icu_provider" @@ -1908,6 +2001,22 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "ignore" +version = "0.4.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d89fd380afde86567dfba715db065673989d6253f42b88179abd3eae47bda4b" +dependencies = [ + "crossbeam-deque", + "globset", + "log", + "memchr", + "regex-automata 0.4.9", + "same-file", + "walkdir", + "winapi-util", +] + [[package]] name = "indexmap" version = "1.9.3" @@ -1978,6 +2087,16 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +[[package]] +name = "iri-string" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "is_ci" version = "1.2.0" @@ -2159,9 +2278,9 @@ dependencies = [ [[package]] name = "libsqlite3-sys" -version = "0.30.1" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +checksum = "947e6816f7825b2b45027c2c32e7085da9934defa535de4a6a46b10a4d5257fa" dependencies = [ "pkg-config", "vcpkg", @@ -2241,7 +2360,7 @@ checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] name = "markdown-weaver" version = "0.13.0" -source = "git+https://github.com/rsform/markdown-weaver#593f968ec3c4515c23273715eaeb33801342439b" +source = "git+https://github.com/rsform/markdown-weaver#5a93522bc6b69058a34e7eb46f89b39d1b6be006" dependencies = [ "bitflags 2.9.1", "getopts", @@ -2253,7 +2372,7 @@ dependencies = [ [[package]] name = "markdown-weaver-escape" version = "0.11.0" -source = "git+https://github.com/rsform/markdown-weaver#593f968ec3c4515c23273715eaeb33801342439b" +source = "git+https://github.com/rsform/markdown-weaver#5a93522bc6b69058a34e7eb46f89b39d1b6be006" [[package]] name = "matchers" @@ -2324,7 +2443,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c36f61da594ecad0ed986ceeb5061eba47a36fcf839576ce525c7e4ff08f3fa" dependencies = [ "merde_core", - "yaml-rust2", + "yaml-rust2 0.8.1", ] [[package]] @@ -2425,14 +2544,14 @@ dependencies = [ [[package]] name = "mio" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" +checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2657,13 +2776,19 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "once_cell_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" + [[package]] name = "onig" -version = "6.4.0" +version = "6.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f" +checksum = "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.9.1", "libc", "once_cell", "onig_sys", @@ -2671,9 +2796,9 @@ dependencies = [ [[package]] name = "onig_sys" -version = "69.8.1" +version = "69.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b829e3d7e9cc74c7e315ee8edb185bf4190da5acde74afd7fc59c35b1f086e7" +checksum = "c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc" dependencies = [ "cc", "pkg-config", @@ -2812,6 +2937,12 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "pathdiff" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" + [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -3244,9 +3375,9 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "reqwest" -version = "0.12.15" +version = "0.12.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d19c46a6fdd48bc4dab94b6103fccc55d34c67cc0ad04653aad4ea2a07cd7bbb" +checksum = "2bf597b113be201cb2269b4c39b39a804d01b99ee95a4278f0ed04e45cff1c71" dependencies = [ "async-compression", "base64 0.22.1", @@ -3272,25 +3403,23 @@ dependencies = [ "pin-project-lite", "quinn", "rustls 0.23.27", - "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", - "system-configuration", "tokio", "tokio-native-tls", "tokio-rustls", "tokio-util", "tower", + "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots 0.26.11", - "windows-registry", + "webpki-roots", ] [[package]] @@ -3451,15 +3580,6 @@ dependencies = [ "base64 0.13.1", ] -[[package]] -name = "rustls-pemfile" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" -dependencies = [ - "rustls-pki-types", -] - [[package]] name = "rustls-pki-types" version = "1.12.0" @@ -3483,9 +3603,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.20" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" +checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" [[package]] name = "ryu" @@ -3849,9 +3969,9 @@ checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" [[package]] name = "socket2" -version = "0.5.9" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f5fd57c80058a56cf5c777ab8a126398ece8e442983605d280a44ce79d0edef" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" dependencies = [ "libc", "windows-sys 0.52.0", @@ -3974,6 +4094,7 @@ checksum = "874dcfa363995604333cf947ae9f751ca3af4522c60886774c4963943b4746b1" dependencies = [ "bincode", "bitflags 1.3.2", + "fancy-regex", "flate2", "fnv", "once_cell", @@ -4151,7 +4272,7 @@ dependencies = [ "httpdate", "log", "rustls 0.20.9", - "rustls-pemfile 0.2.1", + "rustls-pemfile", "zeroize", ] @@ -4182,9 +4303,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.45.0" +version = "1.45.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2513ca694ef9ede0fb23fe71a4ee4107cb102b9dc1930f6d0fd77aae068ae165" +checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779" dependencies = [ "backtrace", "bytes", @@ -4358,12 +4479,14 @@ dependencies = [ "http-body-util", "http-range-header", "httpdate", + "iri-string", "mime", "mime_guess", "percent-encoding", "pin-project-lite", "tokio", "tokio-util", + "tower", "tower-layer", "tower-service", "tracing", @@ -4602,12 +4725,14 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" +checksum = "3cf4199d1e5d15ddd86a694e4d0dffa9c323ce759fea589f00fef9d81cc1931d" dependencies = [ "getrandom 0.3.3", + "js-sys", "serde", + "wasm-bindgen", ] [[package]] @@ -4818,12 +4943,14 @@ dependencies = [ "http", "jose-jwk", "markdown-weaver", + "markdown-weaver-escape", "merde", "miette", "minijinja", "multibase", "n0-future", "owo-colors", + "regex", "reqwest", "serde", "serde_bytes", @@ -4844,13 +4971,31 @@ dependencies = [ name = "weaver-renderer" version = "0.1.0" dependencies = [ + "async-trait", "atrium-api", + "bitflags 2.9.1", "compact_string", + "dashmap", + "dynosaur", "http", + "ignore", + "markdown-weaver", + "markdown-weaver-escape", + "miette", "n0-future", + "pathdiff", + "pin-project", + "pin-utils", + "regex", + "syntect", + "thiserror 2.0.12", + "tokio", + "tokio-util", + "unicode-normalization", "url", "weaver-common", "weaver-workspace-hack", + "yaml-rust2 0.10.2", ] [[package]] @@ -4948,15 +5093,6 @@ dependencies = [ "untrusted 0.9.0", ] -[[package]] -name = "webpki-roots" -version = "0.26.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" -dependencies = [ - "webpki-roots 1.0.0", -] - [[package]] name = "webpki-roots" version = "1.0.0" @@ -5038,15 +5174,15 @@ dependencies = [ [[package]] name = "windows-core" -version = "0.61.1" +version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46ec44dc15085cea82cf9c78f85a9114c463a369786585ad2882d1ff0b0acf40" +checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" dependencies = [ "windows-implement", "windows-interface", "windows-link", "windows-result", - "windows-strings 0.4.1", + "windows-strings 0.4.2", ] [[package]] @@ -5111,9 +5247,9 @@ dependencies = [ [[package]] name = "windows-result" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b895b5356fc36103d0f64dd1e94dfa7ac5633f1c9dd6e80fe9ec4adef69e09d" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" dependencies = [ "windows-link", ] @@ -5129,9 +5265,9 @@ dependencies = [ [[package]] name = "windows-strings" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a7ab927b2637c19b3dbe0965e75d8f2d30bdd697a1516191cad2ec4df8fb28a" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" dependencies = [ "windows-link", ] @@ -5407,7 +5543,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8902160c4e6f2fb145dbe9d6760a75e3c9522d8bf796ed7047c85919ac7115f8" dependencies = [ "arraydeque", - "hashlink", + "hashlink 0.8.4", +] + +[[package]] +name = "yaml-rust2" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18b783b2c2789414f8bb84ca3318fc9c2d7e7be1c22907d37839a58dedb369d3" +dependencies = [ + "arraydeque", + "encoding_rs", + "hashlink 0.10.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index d68c61d..1f6bd41 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,7 @@ minijinja-contrib = { version = "2.9.0", default-features = false } miette = { version = "7.6" } owo-colors = { version = "4.2.0" } thiserror = "2.0" -syntect = "5.2.0" +syntect = { version = "5.2.0", default-features = false } jane-eyre = "0.6.12" n0-future = "=0.1.3" tracing = { version = "0.1.41", default-features = false, features = ["std"] } @@ -38,6 +38,7 @@ lexicon_cid = { package = "cid", version = "0.10.1", features = [ "serde-codec", ] } markdown-weaver = { git = "https://github.com/rsform/markdown-weaver" } +markdown-weaver-escape = { git = "https://github.com/rsform/markdown-weaver" } esquema-codegen = { git = "https://github.com/fatfingers23/esquema.git", branch = "main" } atrium-lex = { git = "https://github.com/sugyan/atrium.git", rev = "f162f815a04b5ecb0421b390d521c883c41d5f75" } diff --git a/crates/weaver-common/Cargo.toml b/crates/weaver-common/Cargo.toml index 9e1393e..c03f46a 100644 --- a/crates/weaver-common/Cargo.toml +++ b/crates/weaver-common/Cargo.toml @@ -60,6 +60,8 @@ tower-service = "0.3.3" tower-layer = "0.3.3" multibase = "0.9.1" dirs = "6.0.0" +regex = "1.11.1" +markdown-weaver-escape = { workspace = true, features = ["std"] } diff --git a/crates/weaver-common/src/agent.rs b/crates/weaver-common/src/agent.rs index ab04295..c4ecc65 100644 --- a/crates/weaver-common/src/agent.rs +++ b/crates/weaver-common/src/agent.rs @@ -3,8 +3,8 @@ use crate::error::GenericXrpcError; use crate::resolver::HickoryDnsTxtResolver; use crate::sh::weaver::actor::defs::ProfileDataViewInnerRefs; use atrium_api::agent::{CloneWithProxy, Configure}; -use atrium_api::types::string::{Cid, Did, Handle, Nsid, RecordKey}; -use atrium_api::types::{Collection, Union, Unknown}; +use atrium_api::types::string::{Cid, Did, Handle, Nsid, RecordKey, Tid}; +use atrium_api::types::{BlobRef, Collection, LimitedU32, TryIntoUnknown, Union, Unknown}; use atrium_api::{agent::SessionManager, types::string::AtIdentifier}; use atrium_common::resolver::Resolver; use atrium_identity::did::{CommonDidResolver, CommonDidResolverConfig, DEFAULT_PLC_DIRECTORY_URL}; @@ -60,6 +60,7 @@ where AtprotoHandleResolver, >, >, + pub client: Arc, pub api: Service>, } @@ -86,6 +87,7 @@ where Self { session_manager, resolver: Arc::new(IdentityResolver::new(resolver_config)), + client: Arc::clone(&http_client), api, } } @@ -95,6 +97,10 @@ where pub async fn did(&self) -> Option { self.session_manager.did().await } + + pub fn pds(&self) -> String { + self.session_manager.base_uri() + } } impl WeaverAgent @@ -173,6 +179,33 @@ where } } } + pub async fn create_record( + &self, + collection: Nsid, + record: Unknown, + repo: AtIdentifier, + rkey: Option, + ) -> Result { + use crate::com::atproto::repo::create_record::*; + let result = self + .api + .com + .atproto + .repo + .create_record( + InputData { + collection, + record, + repo, + rkey, + swap_commit: None, + validate: None, + } + .into(), + ) + .await?; + Ok(result.data) + } pub async fn put_record( &self, @@ -249,6 +282,45 @@ where Ok(result.data) } + pub async fn publish_blob( + &self, + input: Vec, + ) -> Result< + ( + BlobRef, + crate::com::atproto::repo::create_record::OutputData, + ), + Error, + > { + use crate::sh::weaver::publish; + let blobref = self + .api + .com + .atproto + .repo + .upload_blob(input.into()) + .await? + .blob + .clone(); + let upload_record = publish::blob::RecordData { + upload: blobref.clone(), + }; + let at_identifier = AtIdentifier::try_from(self.did().await.expect("valid did")) + .expect("valid did should be valid at identifier"); + let result = self + .create_record( + Nsid::new(publish::Blob::NSID.to_string()) + .expect("sh.weaver.publish.blob should be a valid NSID"), + upload_record + .try_into_unknown() + .expect("this should just work"), + at_identifier, + None, + ) + .await?; + Ok((blobref, result)) + } + pub async fn upload_artifact( &self, content: String, @@ -313,9 +385,7 @@ where M: CloneWithProxy + SessionManager + Send + Sync, { /// Configures the atproto-proxy header to be applied on requests. - /// - /// Returns a new client service with the proxy header configured. pub fn api_with_proxy(&self, did: Did, service_type: impl AsRef) -> Service> { diff --git a/crates/weaver-common/src/lexicons/app/bsky/feed/defs.rs b/crates/weaver-common/src/lexicons/app/bsky/feed/defs.rs index e93b999..cf025c0 100644 --- a/crates/weaver-common/src/lexicons/app/bsky/feed/defs.rs +++ b/crates/weaver-common/src/lexicons/app/bsky/feed/defs.rs @@ -39,6 +39,9 @@ pub struct FeedViewPostData { pub reason: core::option::Option>, #[serde(skip_serializing_if = "core::option::Option::is_none")] pub reply: core::option::Option, + ///Unique identifier per request that may be passed back alongside interactions. + #[serde(skip_serializing_if = "core::option::Option::is_none")] + pub req_id: core::option::Option, } pub type FeedViewPost = atrium_api::types::Object; #[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq, Eq)] @@ -87,6 +90,9 @@ pub struct InteractionData { pub feed_context: core::option::Option, #[serde(skip_serializing_if = "core::option::Option::is_none")] pub item: core::option::Option, + ///Unique identifier per request that may be passed back alongside interactions. + #[serde(skip_serializing_if = "core::option::Option::is_none")] + pub req_id: core::option::Option, } pub type Interaction = atrium_api::types::Object; ///User liked the feed item @@ -142,7 +148,11 @@ pub type ReasonPin = atrium_api::types::Object; #[serde(rename_all = "camelCase")] pub struct ReasonRepostData { pub by: crate::app::bsky::actor::defs::ProfileViewBasic, + #[serde(skip_serializing_if = "core::option::Option::is_none")] + pub cid: core::option::Option, pub indexed_at: atrium_api::types::string::Datetime, + #[serde(skip_serializing_if = "core::option::Option::is_none")] + pub uri: core::option::Option, } pub type ReasonRepost = atrium_api::types::Object; #[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq, Eq)] diff --git a/crates/weaver-common/src/lexicons/app/bsky/feed/get_feed_skeleton.rs b/crates/weaver-common/src/lexicons/app/bsky/feed/get_feed_skeleton.rs index 4df33a4..560a6e3 100644 --- a/crates/weaver-common/src/lexicons/app/bsky/feed/get_feed_skeleton.rs +++ b/crates/weaver-common/src/lexicons/app/bsky/feed/get_feed_skeleton.rs @@ -18,6 +18,9 @@ pub struct OutputData { #[serde(skip_serializing_if = "core::option::Option::is_none")] pub cursor: core::option::Option, pub feed: Vec, + ///Unique identifier per request that may be passed back alongside interactions. + #[serde(skip_serializing_if = "core::option::Option::is_none")] + pub req_id: core::option::Option, } pub type Output = atrium_api::types::Object; #[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq, Eq)] diff --git a/crates/weaver-common/src/lexicons/app/bsky/feed/like.rs b/crates/weaver-common/src/lexicons/app/bsky/feed/like.rs index ff993ea..1edd3a9 100644 --- a/crates/weaver-common/src/lexicons/app/bsky/feed/like.rs +++ b/crates/weaver-common/src/lexicons/app/bsky/feed/like.rs @@ -6,6 +6,8 @@ use atrium_api::types::TryFromUnknown; pub struct RecordData { pub created_at: atrium_api::types::string::Datetime, pub subject: crate::com::atproto::repo::strong_ref::Main, + #[serde(skip_serializing_if = "core::option::Option::is_none")] + pub via: core::option::Option, } pub type Record = atrium_api::types::Object; impl From for RecordData { diff --git a/crates/weaver-common/src/lexicons/app/bsky/feed/repost.rs b/crates/weaver-common/src/lexicons/app/bsky/feed/repost.rs index 6950b46..55a4620 100644 --- a/crates/weaver-common/src/lexicons/app/bsky/feed/repost.rs +++ b/crates/weaver-common/src/lexicons/app/bsky/feed/repost.rs @@ -6,6 +6,8 @@ use atrium_api::types::TryFromUnknown; pub struct RecordData { pub created_at: atrium_api::types::string::Datetime, pub subject: crate::com::atproto::repo::strong_ref::Main, + #[serde(skip_serializing_if = "core::option::Option::is_none")] + pub via: core::option::Option, } pub type Record = atrium_api::types::Object; impl From for RecordData { diff --git a/crates/weaver-common/src/lexicons/app/bsky/notification/list_notifications.rs b/crates/weaver-common/src/lexicons/app/bsky/notification/list_notifications.rs index f231ca8..f08235e 100644 --- a/crates/weaver-common/src/lexicons/app/bsky/notification/list_notifications.rs +++ b/crates/weaver-common/src/lexicons/app/bsky/notification/list_notifications.rs @@ -46,7 +46,7 @@ pub struct NotificationData { pub is_read: bool, #[serde(skip_serializing_if = "core::option::Option::is_none")] pub labels: core::option::Option>, - ///Expected values are 'like', 'repost', 'follow', 'mention', 'reply', 'quote', 'starterpack-joined', 'verified', and 'unverified'. + ///The reason why this notification was delivered - e.g. your post was liked, or you received a new follower. pub reason: String, #[serde(skip_serializing_if = "core::option::Option::is_none")] pub reason_subject: core::option::Option, diff --git a/crates/weaver-common/src/lexicons/client.rs b/crates/weaver-common/src/lexicons/client.rs index d72df43..8f8fa6f 100644 --- a/crates/weaver-common/src/lexicons/client.rs +++ b/crates/weaver-common/src/lexicons/client.rs @@ -1150,7 +1150,7 @@ where _ => Err(atrium_xrpc::Error::UnexpectedResponseType), } } - ///Find posts matching search criteria, returning views of those posts. + ///Find posts matching search criteria, returning views of those posts. Note that this API endpoint may require authentication (eg, not public) for some service providers and implementations. pub async fn search_posts( &self, params: crate::app::bsky::feed::search_posts::Parameters, diff --git a/crates/weaver-common/src/lexicons/record.rs b/crates/weaver-common/src/lexicons/record.rs index e8b3126..bb7afc3 100644 --- a/crates/weaver-common/src/lexicons/record.rs +++ b/crates/weaver-common/src/lexicons/record.rs @@ -8,9 +8,7 @@ pub enum KnownRecord { #[serde(rename = "app.bsky.actor.status")] LexiconsAppBskyActorStatus(Box), #[serde(rename = "app.bsky.feed.generator")] - LexiconsAppBskyFeedGenerator( - Box, - ), + LexiconsAppBskyFeedGenerator(Box), #[serde(rename = "app.bsky.feed.like")] LexiconsAppBskyFeedLike(Box), #[serde(rename = "app.bsky.feed.post")] @@ -20,9 +18,7 @@ pub enum KnownRecord { #[serde(rename = "app.bsky.feed.repost")] LexiconsAppBskyFeedRepost(Box), #[serde(rename = "app.bsky.feed.threadgate")] - LexiconsAppBskyFeedThreadgate( - Box, - ), + LexiconsAppBskyFeedThreadgate(Box), #[serde(rename = "app.bsky.graph.block")] LexiconsAppBskyGraphBlock(Box), #[serde(rename = "app.bsky.graph.follow")] @@ -30,41 +26,23 @@ pub enum KnownRecord { #[serde(rename = "app.bsky.graph.list")] LexiconsAppBskyGraphList(Box), #[serde(rename = "app.bsky.graph.listblock")] - LexiconsAppBskyGraphListblock( - Box, - ), + LexiconsAppBskyGraphListblock(Box), #[serde(rename = "app.bsky.graph.listitem")] - LexiconsAppBskyGraphListitem( - Box, - ), + LexiconsAppBskyGraphListitem(Box), #[serde(rename = "app.bsky.graph.starterpack")] - LexiconsAppBskyGraphStarterpack( - Box, - ), + LexiconsAppBskyGraphStarterpack(Box), #[serde(rename = "app.bsky.graph.verification")] - LexiconsAppBskyGraphVerification( - Box, - ), + LexiconsAppBskyGraphVerification(Box), #[serde(rename = "app.bsky.labeler.service")] - LexiconsAppBskyLabelerService( - Box, - ), + LexiconsAppBskyLabelerService(Box), #[serde(rename = "chat.bsky.actor.declaration")] - LexiconsChatBskyActorDeclaration( - Box, - ), + LexiconsChatBskyActorDeclaration(Box), #[serde(rename = "com.atproto.lexicon.schema")] - LexiconsComAtprotoLexiconSchema( - Box, - ), + LexiconsComAtprotoLexiconSchema(Box), #[serde(rename = "sh.tangled.actor.profile")] - LexiconsShTangledActorProfile( - Box, - ), + LexiconsShTangledActorProfile(Box), #[serde(rename = "sh.weaver.actor.profile")] - LexiconsShWeaverActorProfile( - Box, - ), + LexiconsShWeaverActorProfile(Box), #[serde(rename = "sh.weaver.edit.cursor")] LexiconsShWeaverEditCursor(Box), #[serde(rename = "sh.weaver.edit.diff")] @@ -72,17 +50,15 @@ pub enum KnownRecord { #[serde(rename = "sh.weaver.edit.root")] LexiconsShWeaverEditRoot(Box), #[serde(rename = "sh.weaver.notebook.authors")] - LexiconsShWeaverNotebookAuthors( - Box, - ), + LexiconsShWeaverNotebookAuthors(Box), #[serde(rename = "sh.weaver.notebook.book")] - LexiconsShWeaverNotebookBook( - Box, - ), + LexiconsShWeaverNotebookBook(Box), + #[serde(rename = "sh.weaver.notebook.chapter")] + LexiconsShWeaverNotebookChapter(Box), #[serde(rename = "sh.weaver.notebook.entry")] - LexiconsShWeaverNotebookEntry( - Box, - ), + LexiconsShWeaverNotebookEntry(Box), + #[serde(rename = "sh.weaver.publish.blob")] + LexiconsShWeaverPublishBlob(Box), } impl From for KnownRecord { fn from(record: crate::lexicons::app::bsky::actor::profile::Record) -> Self { @@ -90,9 +66,7 @@ impl From for KnownRecord { } } impl From for KnownRecord { - fn from( - record_data: crate::lexicons::app::bsky::actor::profile::RecordData, - ) -> Self { + fn from(record_data: crate::lexicons::app::bsky::actor::profile::RecordData) -> Self { KnownRecord::LexiconsAppBskyActorProfile(Box::new(record_data.into())) } } @@ -112,9 +86,7 @@ impl From for KnownRecord { } } impl From for KnownRecord { - fn from( - record_data: crate::lexicons::app::bsky::feed::generator::RecordData, - ) -> Self { + fn from(record_data: crate::lexicons::app::bsky::feed::generator::RecordData) -> Self { KnownRecord::LexiconsAppBskyFeedGenerator(Box::new(record_data.into())) } } @@ -144,9 +116,7 @@ impl From for KnownRecord { } } impl From for KnownRecord { - fn from( - record_data: crate::lexicons::app::bsky::feed::postgate::RecordData, - ) -> Self { + fn from(record_data: crate::lexicons::app::bsky::feed::postgate::RecordData) -> Self { KnownRecord::LexiconsAppBskyFeedPostgate(Box::new(record_data.into())) } } @@ -166,9 +136,7 @@ impl From for KnownRecord } } impl From for KnownRecord { - fn from( - record_data: crate::lexicons::app::bsky::feed::threadgate::RecordData, - ) -> Self { + fn from(record_data: crate::lexicons::app::bsky::feed::threadgate::RecordData) -> Self { KnownRecord::LexiconsAppBskyFeedThreadgate(Box::new(record_data.into())) } } @@ -208,9 +176,7 @@ impl From for KnownRecord } } impl From for KnownRecord { - fn from( - record_data: crate::lexicons::app::bsky::graph::listblock::RecordData, - ) -> Self { + fn from(record_data: crate::lexicons::app::bsky::graph::listblock::RecordData) -> Self { KnownRecord::LexiconsAppBskyGraphListblock(Box::new(record_data.into())) } } @@ -220,9 +186,7 @@ impl From for KnownRecord { } } impl From for KnownRecord { - fn from( - record_data: crate::lexicons::app::bsky::graph::listitem::RecordData, - ) -> Self { + fn from(record_data: crate::lexicons::app::bsky::graph::listitem::RecordData) -> Self { KnownRecord::LexiconsAppBskyGraphListitem(Box::new(record_data.into())) } } @@ -232,9 +196,7 @@ impl From for KnownRecor } } impl From for KnownRecord { - fn from( - record_data: crate::lexicons::app::bsky::graph::starterpack::RecordData, - ) -> Self { + fn from(record_data: crate::lexicons::app::bsky::graph::starterpack::RecordData) -> Self { KnownRecord::LexiconsAppBskyGraphStarterpack(Box::new(record_data.into())) } } @@ -244,9 +206,7 @@ impl From for KnownReco } } impl From for KnownRecord { - fn from( - record_data: crate::lexicons::app::bsky::graph::verification::RecordData, - ) -> Self { + fn from(record_data: crate::lexicons::app::bsky::graph::verification::RecordData) -> Self { KnownRecord::LexiconsAppBskyGraphVerification(Box::new(record_data.into())) } } @@ -256,9 +216,7 @@ impl From for KnownRecord } } impl From for KnownRecord { - fn from( - record_data: crate::lexicons::app::bsky::labeler::service::RecordData, - ) -> Self { + fn from(record_data: crate::lexicons::app::bsky::labeler::service::RecordData) -> Self { KnownRecord::LexiconsAppBskyLabelerService(Box::new(record_data.into())) } } @@ -268,9 +226,7 @@ impl From for KnownReco } } impl From for KnownRecord { - fn from( - record_data: crate::lexicons::chat::bsky::actor::declaration::RecordData, - ) -> Self { + fn from(record_data: crate::lexicons::chat::bsky::actor::declaration::RecordData) -> Self { KnownRecord::LexiconsChatBskyActorDeclaration(Box::new(record_data.into())) } } @@ -280,9 +236,7 @@ impl From for KnownRecor } } impl From for KnownRecord { - fn from( - record_data: crate::lexicons::com::atproto::lexicon::schema::RecordData, - ) -> Self { + fn from(record_data: crate::lexicons::com::atproto::lexicon::schema::RecordData) -> Self { KnownRecord::LexiconsComAtprotoLexiconSchema(Box::new(record_data.into())) } } @@ -292,9 +246,7 @@ impl From for KnownRecord } } impl From for KnownRecord { - fn from( - record_data: crate::lexicons::sh::tangled::actor::profile::RecordData, - ) -> Self { + fn from(record_data: crate::lexicons::sh::tangled::actor::profile::RecordData) -> Self { KnownRecord::LexiconsShTangledActorProfile(Box::new(record_data.into())) } } @@ -304,9 +256,7 @@ impl From for KnownRecord { } } impl From for KnownRecord { - fn from( - record_data: crate::lexicons::sh::weaver::actor::profile::RecordData, - ) -> Self { + fn from(record_data: crate::lexicons::sh::weaver::actor::profile::RecordData) -> Self { KnownRecord::LexiconsShWeaverActorProfile(Box::new(record_data.into())) } } @@ -346,9 +296,7 @@ impl From for KnownRecor } } impl From for KnownRecord { - fn from( - record_data: crate::lexicons::sh::weaver::notebook::authors::RecordData, - ) -> Self { + fn from(record_data: crate::lexicons::sh::weaver::notebook::authors::RecordData) -> Self { KnownRecord::LexiconsShWeaverNotebookAuthors(Box::new(record_data.into())) } } @@ -358,24 +306,40 @@ impl From for KnownRecord { } } impl From for KnownRecord { - fn from( - record_data: crate::lexicons::sh::weaver::notebook::book::RecordData, - ) -> Self { + fn from(record_data: crate::lexicons::sh::weaver::notebook::book::RecordData) -> Self { KnownRecord::LexiconsShWeaverNotebookBook(Box::new(record_data.into())) } } +impl From for KnownRecord { + fn from(record: crate::lexicons::sh::weaver::notebook::chapter::Record) -> Self { + KnownRecord::LexiconsShWeaverNotebookChapter(Box::new(record)) + } +} +impl From for KnownRecord { + fn from(record_data: crate::lexicons::sh::weaver::notebook::chapter::RecordData) -> Self { + KnownRecord::LexiconsShWeaverNotebookChapter(Box::new(record_data.into())) + } +} impl From for KnownRecord { fn from(record: crate::lexicons::sh::weaver::notebook::entry::Record) -> Self { KnownRecord::LexiconsShWeaverNotebookEntry(Box::new(record)) } } impl From for KnownRecord { - fn from( - record_data: crate::lexicons::sh::weaver::notebook::entry::RecordData, - ) -> Self { + fn from(record_data: crate::lexicons::sh::weaver::notebook::entry::RecordData) -> Self { KnownRecord::LexiconsShWeaverNotebookEntry(Box::new(record_data.into())) } } +impl From for KnownRecord { + fn from(record: crate::lexicons::sh::weaver::publish::blob::Record) -> Self { + KnownRecord::LexiconsShWeaverPublishBlob(Box::new(record)) + } +} +impl From for KnownRecord { + fn from(record_data: crate::lexicons::sh::weaver::publish::blob::RecordData) -> Self { + KnownRecord::LexiconsShWeaverPublishBlob(Box::new(record_data.into())) + } +} impl Into for KnownRecord { fn into(self) -> atrium_api::types::Unknown { atrium_api::types::TryIntoUnknown::try_into_unknown(&self).unwrap() diff --git a/crates/weaver-common/src/lexicons/sh/weaver.rs b/crates/weaver-common/src/lexicons/sh/weaver.rs index 6c98d69..cc5fb7a 100644 --- a/crates/weaver-common/src/lexicons/sh/weaver.rs +++ b/crates/weaver-common/src/lexicons/sh/weaver.rs @@ -4,3 +4,4 @@ pub mod actor; pub mod edit; pub mod embed; pub mod notebook; +pub mod publish; diff --git a/crates/weaver-common/src/lexicons/sh/weaver/actor/defs.rs b/crates/weaver-common/src/lexicons/sh/weaver/actor/defs.rs index 2330ffb..d95f82e 100644 --- a/crates/weaver-common/src/lexicons/sh/weaver/actor/defs.rs +++ b/crates/weaver-common/src/lexicons/sh/weaver/actor/defs.rs @@ -1,5 +1,16 @@ // @generated - This file is generated by esquema-codegen (forked from atrium-codegen). DO NOT EDIT. //!Definitions for the `sh.weaver.actor.defs` namespace. +///A single author in a Weaver notebook. +#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct AuthorData { + pub did: atrium_api::types::string::Did, + #[serde(skip_serializing_if = "core::option::Option::is_none")] + pub display_name: core::option::Option, + #[serde(skip_serializing_if = "core::option::Option::is_none")] + pub handle: core::option::Option, +} +pub type Author = atrium_api::types::Object; #[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq, Eq)] #[serde(rename_all = "camelCase")] pub struct ProfileDataViewData { diff --git a/crates/weaver-common/src/lexicons/sh/weaver/notebook.rs b/crates/weaver-common/src/lexicons/sh/weaver/notebook.rs index 11a5de2..7a5632c 100644 --- a/crates/weaver-common/src/lexicons/sh/weaver/notebook.rs +++ b/crates/weaver-common/src/lexicons/sh/weaver/notebook.rs @@ -2,6 +2,7 @@ //!Definitions for the `sh.weaver.notebook` namespace. pub mod authors; pub mod book; +pub mod chapter; pub mod defs; pub mod entry; #[derive(Debug)] @@ -17,6 +18,12 @@ impl atrium_api::types::Collection for Book { type Record = book::Record; } #[derive(Debug)] +pub struct Chapter; +impl atrium_api::types::Collection for Chapter { + const NSID: &'static str = "sh.weaver.notebook.chapter"; + type Record = chapter::Record; +} +#[derive(Debug)] pub struct Entry; impl atrium_api::types::Collection for Entry { const NSID: &'static str = "sh.weaver.notebook.entry"; diff --git a/crates/weaver-common/src/lexicons/sh/weaver/notebook/book.rs b/crates/weaver-common/src/lexicons/sh/weaver/notebook/book.rs index aa2bcea..355369c 100644 --- a/crates/weaver-common/src/lexicons/sh/weaver/notebook/book.rs +++ b/crates/weaver-common/src/lexicons/sh/weaver/notebook/book.rs @@ -4,11 +4,11 @@ use atrium_api::types::TryFromUnknown; #[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq, Eq)] #[serde(rename_all = "camelCase")] pub struct RecordData { - pub authors: crate::sh::weaver::notebook::defs::AuthorListView, + pub authors: Vec, ///Client-declared timestamp when this was originally created. #[serde(skip_serializing_if = "core::option::Option::is_none")] pub created_at: core::option::Option, - pub entry_list: Vec, + pub entry_list: Vec, #[serde(skip_serializing_if = "core::option::Option::is_none")] pub tags: core::option::Option, #[serde(skip_serializing_if = "core::option::Option::is_none")] diff --git a/crates/weaver-common/src/lexicons/sh/weaver/notebook/chapter.rs b/crates/weaver-common/src/lexicons/sh/weaver/notebook/chapter.rs new file mode 100644 index 0000000..15add65 --- /dev/null +++ b/crates/weaver-common/src/lexicons/sh/weaver/notebook/chapter.rs @@ -0,0 +1,24 @@ +// @generated - This file is generated by esquema-codegen (forked from atrium-codegen). DO NOT EDIT. +//!Definitions for the `sh.weaver.notebook.chapter` namespace. +use atrium_api::types::TryFromUnknown; +#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct RecordData { + pub authors: Vec, + ///Client-declared timestamp when this was originally created. + #[serde(skip_serializing_if = "core::option::Option::is_none")] + pub created_at: core::option::Option, + pub entry_list: Vec, + ///The notebook this chapter belongs to. + pub notebook: crate::com::atproto::repo::strong_ref::Main, + #[serde(skip_serializing_if = "core::option::Option::is_none")] + pub tags: core::option::Option, + #[serde(skip_serializing_if = "core::option::Option::is_none")] + pub title: core::option::Option, +} +pub type Record = atrium_api::types::Object; +impl From for RecordData { + fn from(value: atrium_api::types::Unknown) -> Self { + Self::try_from_unknown(value).unwrap() + } +} diff --git a/crates/weaver-common/src/lexicons/sh/weaver/notebook/defs.rs b/crates/weaver-common/src/lexicons/sh/weaver/notebook/defs.rs index d5eb121..827a8aa 100644 --- a/crates/weaver-common/src/lexicons/sh/weaver/notebook/defs.rs +++ b/crates/weaver-common/src/lexicons/sh/weaver/notebook/defs.rs @@ -30,6 +30,15 @@ pub struct BookEntryViewData { pub prev: core::option::Option, } pub type BookEntryView = atrium_api::types::Object; +///The format of the content. This is used to determine how to render the content. +#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct ContentFormatData { + ///The format of the content. This is used to determine how to render the content. + #[serde(skip_serializing_if = "core::option::Option::is_none")] + pub markdown: core::option::Option, +} +pub type ContentFormat = atrium_api::types::Object; #[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq, Eq)] #[serde(rename_all = "camelCase")] pub struct EntryViewData { diff --git a/crates/weaver-common/src/lexicons/sh/weaver/publish.rs b/crates/weaver-common/src/lexicons/sh/weaver/publish.rs new file mode 100644 index 0000000..5bd2f08 --- /dev/null +++ b/crates/weaver-common/src/lexicons/sh/weaver/publish.rs @@ -0,0 +1,10 @@ +// @generated - This file is generated by esquema-codegen (forked from atrium-codegen). DO NOT EDIT. +//!Definitions for the `sh.weaver.publish` namespace. +pub mod blob; +pub mod defs; +#[derive(Debug)] +pub struct Blob; +impl atrium_api::types::Collection for Blob { + const NSID: &'static str = "sh.weaver.publish.blob"; + type Record = blob::Record; +} diff --git a/crates/weaver-common/src/lexicons/sh/weaver/publish/blob.rs b/crates/weaver-common/src/lexicons/sh/weaver/publish/blob.rs new file mode 100644 index 0000000..e4e3940 --- /dev/null +++ b/crates/weaver-common/src/lexicons/sh/weaver/publish/blob.rs @@ -0,0 +1,15 @@ +// @generated - This file is generated by esquema-codegen (forked from atrium-codegen). DO NOT EDIT. +//!Definitions for the `sh.weaver.publish.blob` namespace. +use atrium_api::types::TryFromUnknown; +#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct RecordData { + ///Reference to the uploaded file + pub upload: atrium_api::types::BlobRef, +} +pub type Record = atrium_api::types::Object; +impl From for RecordData { + fn from(value: atrium_api::types::Unknown) -> Self { + Self::try_from_unknown(value).unwrap() + } +} diff --git a/crates/weaver-common/src/lexicons/sh/weaver/publish/defs.rs b/crates/weaver-common/src/lexicons/sh/weaver/publish/defs.rs new file mode 100644 index 0000000..fece57b --- /dev/null +++ b/crates/weaver-common/src/lexicons/sh/weaver/publish/defs.rs @@ -0,0 +1,2 @@ +// @generated - This file is generated by esquema-codegen (forked from atrium-codegen). DO NOT EDIT. +//!Definitions for the `sh.weaver.publish.defs` namespace. diff --git a/crates/weaver-common/src/lib.rs b/crates/weaver-common/src/lib.rs index d173e9a..53ed80a 100644 --- a/crates/weaver-common/src/lib.rs +++ b/crates/weaver-common/src/lib.rs @@ -8,8 +8,12 @@ pub mod lexicons; pub mod oauth; pub mod resolver; pub mod xrpc_server; -use atrium_api::types::{BlobRef, TypedBlobRef, string::Did}; +use std::sync::OnceLock; + +pub use atrium_api::types::*; pub use lexicons::*; +use regex::Regex; +use string::Did; pub use crate::error::{Error, IoError, ParseError, SerDeError}; @@ -56,3 +60,83 @@ pub fn avatar_cdn_url(did: &Did, blob_ref: &BlobRef) -> String { mime_type.strip_prefix("image/").unwrap_or(mime_type) ) } + +pub fn blob_url(did: &Did, pds: &str, blob_ref: &BlobRef) -> String { + let cid = match blob_ref { + BlobRef::Typed(TypedBlobRef::Blob(b)) => atrium_api::types::string::Cid::new(b.r#ref.0) + .as_ref() + .to_string(), + + BlobRef::Untyped(r) => r.cid.clone(), + }; + format!( + "https://{}/xrpc/com.atproto.repo.getBlob?did={}&cid={}", + pds, + did.as_str(), + cid, + ) +} + +pub fn match_identifier(maybe_identifier: &str) -> Option<&str> { + static RE_HANDLE: OnceLock = OnceLock::new(); + static RE_DID: OnceLock = OnceLock::new(); + if maybe_identifier.len() > 253 { + None + } else if !RE_DID.get_or_init(|| Regex::new(r"^did:[a-z]+:[a-zA-Z0-9._:%-]*[a-zA-Z0-9._-]$").unwrap()) + .is_match(&maybe_identifier) && !RE_HANDLE + .get_or_init(|| Regex::new(r"^([a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?$").unwrap()) + .is_match(&maybe_identifier) + { + None + } else { + Some(maybe_identifier) + } +} + +pub fn match_nsid(maybe_nsid: &str) -> Option<&str> { + static RE_NSID: OnceLock = OnceLock::new(); + if maybe_nsid.len() > 317 { + None + } else if !RE_NSID + .get_or_init(|| Regex::new(r"^[a-zA-Z]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(\.[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)+(\.[a-zA-Z]([a-zA-Z0-9]{0,62}[a-zA-Z0-9])?)$").unwrap()) + .is_match(&maybe_nsid) + { + None + } else { + Some(maybe_nsid) + } +} + +/// Convert an ATURI to a HTTP URL +/// Currently has some failure modes and should restrict the NSIDs to a known subset +pub fn aturi_to_http<'s>(aturi: &'s str, appview: &'s str) -> Option> { + use markdown_weaver::CowStr; + + if aturi.starts_with("at://") { + let rest = aturi.strip_prefix("at:://").unwrap(); + let mut split = rest.splitn(2, '/'); + let maybe_identifier = split.next()?; + let maybe_nsid = split.next()?; + let maybe_rkey = split.next()?; + + // https://atproto.com/specs/handle#handle-identifier-syntax + let identifier = match_identifier(maybe_identifier)?; + + let nsid = if let Some(nsid) = match_nsid(maybe_nsid) { + // Last part of the nsid is generally the middle component of the URL + // TODO: check for bsky ones specifically, because those are the ones where this is valid + nsid.rsplitn(1, '.').next()? + } else { + return None; + }; + Some(CowStr::Boxed( + format!( + "https://{}/profile/{}/{}/{}", + appview, identifier, nsid, maybe_rkey + ) + .into_boxed_str(), + )) + } else { + Some(CowStr::Borrowed(aturi)) + } +} diff --git a/crates/weaver-renderer/Cargo.toml b/crates/weaver-renderer/Cargo.toml index 6133bd9..06ccb5b 100644 --- a/crates/weaver-renderer/Cargo.toml +++ b/crates/weaver-renderer/Cargo.toml @@ -6,12 +6,31 @@ license.workspace = true publish = false [dependencies] -n0-future = { workspace = true } +n0-future.workspace = true atrium-api = { version = "0.25.3", default-features = false } weaver-common = { path = "../weaver-common" } - +markdown-weaver = { workspace = true } weaver-workspace-hack = { version = "0.1", path = "../weaver-workspace-hack" } compact_string = "0.1.0" http = "1.3.1" url = "2.5.4" +syntect = { workspace = true, default-features = false, features = ["default-fancy"]} +markdown-weaver-escape = { workspace = true, features = ["std"] } +thiserror.workspace = true +miette.workspace = true +pathdiff = "0.2.3" +unicode-normalization = "0.1.24" +yaml-rust2 = { version = "0.10.2" } +bitflags = "2.9.1" +ignore = "0.4.23" +dashmap = "6.1.0" +regex = "1.11.1" +pin-utils = "0.1.0" +pin-project = "1.1.10" +dynosaur = "0.2.0" +async-trait = "0.1.88" + +[target.'cfg(not(all(target_family = "wasm", target_os = "unknown")))'.dependencies] +tokio = { version = "1.28", features = ["rt", "time"] } +tokio-util = { version = "0.7.14", features = ["rt"] } diff --git a/crates/weaver-renderer/src/atproto.rs b/crates/weaver-renderer/src/atproto.rs new file mode 100644 index 0000000..a279bdf --- /dev/null +++ b/crates/weaver-renderer/src/atproto.rs @@ -0,0 +1,18 @@ +//! Atproto renderer +//! +//! This mode of the renderer renders either an entire notebook or entries in it to files suitable for inclusion +//! in a single-page app and uploads them to your Atproto PDS +//! It can be accessed via the appview at {your-handle}.weaver.sh/{notebook-name}. +//! +//! It can also be edited there. +//! +//! Link altering logic: +//! - Option 1: leave (non-embed) links the same as in the markdown, have the CSM deal with them via some means +//! such as adding "data-did" and "data-cid" attributes to the `` tag containing the DID and CID +//! Pushes toward having the SPA/Appview do a bit more, but makes this step MUCH simpler +//! - In this scenario, the rendering step can happen upon access (and then be cached) in the appview +//! - More flexible in some ways, less in others +//! - Option 2: alter links to point to other rendered blobs. Requires a certain amount of work to handle +//! scenarios with a complex mesh of internal links, as the CID is altered by the editing of the link. +//! Such cycles are handled in the simplest way, by rendering an absolute url which will make a call to the appview. +//! diff --git a/crates/weaver-renderer/src/base_html.rs b/crates/weaver-renderer/src/base_html.rs new file mode 100644 index 0000000..b52354f --- /dev/null +++ b/crates/weaver-renderer/src/base_html.rs @@ -0,0 +1,693 @@ +// use std::{string::String, vec::Vec}; +// #[cfg(all(feature = "std", not(feature = "hashbrown")))] +// use std::collections::HashMap; + +// #[cfg(feature = "hashbrown")] +// use hashbrown::HashMap; +// +use std::collections::HashMap; +//#[cfg(feature = "std")] +use markdown_weaver_escape::IoWriter; +use markdown_weaver_escape::{ + FmtWriter, StrWrite, escape_href, escape_html, escape_html_body_text, +}; + +use markdown_weaver::{ + Alignment, BlockQuoteKind, CodeBlockKind, CowStr, Event, Event::*, LinkType, Tag, TagEnd, +}; + +pub enum TableState { + Head, + Body, +} + +struct HtmlWriter<'a, I, W> { + /// Iterator supplying events. + iter: I, + + /// Writer to write to. + writer: W, + + /// Whether or not the last write wrote a newline. + end_newline: bool, + + /// Whether if inside a metadata block (text should not be written) + in_non_writing_block: bool, + + table_state: TableState, + table_alignments: Vec, + table_cell_index: usize, + numbers: HashMap, usize>, +} + +impl<'a, I, W> HtmlWriter<'a, I, W> +where + I: Iterator>, + W: StrWrite, +{ + fn new(iter: I, writer: W) -> Self { + Self { + iter, + writer, + end_newline: true, + in_non_writing_block: false, + table_state: TableState::Head, + table_alignments: vec![], + table_cell_index: 0, + numbers: HashMap::new(), + } + } + + /// Writes a new line. + #[inline] + fn write_newline(&mut self) -> Result<(), W::Error> { + self.end_newline = true; + self.writer.write_str("\n") + } + + /// Writes a buffer, and tracks whether or not a newline was written. + #[inline] + fn write(&mut self, s: &str) -> Result<(), W::Error> { + self.writer.write_str(s)?; + + if !s.is_empty() { + self.end_newline = s.ends_with('\n'); + } + Ok(()) + } + + fn run(mut self) -> Result<(), W::Error> { + while let Some(event) = self.iter.next() { + match event { + Start(tag) => { + self.start_tag(tag)?; + } + End(tag) => { + self.end_tag(tag)?; + } + Text(text) => { + if !self.in_non_writing_block { + escape_html_body_text(&mut self.writer, &text)?; + self.end_newline = text.ends_with('\n'); + } + } + Code(text) => { + self.write("")?; + escape_html_body_text(&mut self.writer, &text)?; + self.write("")?; + } + InlineMath(text) => { + self.write(r#""#)?; + escape_html(&mut self.writer, &text)?; + self.write("")?; + } + DisplayMath(text) => { + self.write(r#""#)?; + escape_html(&mut self.writer, &text)?; + self.write("")?; + } + Html(html) | InlineHtml(html) => { + self.write(&html)?; + } + SoftBreak => { + self.write_newline()?; + } + HardBreak => { + self.write("
\n")?; + } + Rule => { + if self.end_newline { + self.write("
\n")?; + } else { + self.write("\n
\n")?; + } + } + FootnoteReference(name) => { + let len = self.numbers.len() + 1; + self.write("")?; + let number = *self.numbers.entry(name).or_insert(len); + write!(&mut self.writer, "{}", number)?; + self.write("")?; + } + TaskListMarker(true) => { + self.write("\n")?; + } + TaskListMarker(false) => { + self.write("\n")?; + } + WeaverBlock(_text) => {} + } + } + Ok(()) + } + + /// Writes the start of an HTML tag. + fn start_tag(&mut self, tag: Tag<'a>) -> Result<(), W::Error> { + match tag { + Tag::HtmlBlock => Ok(()), + Tag::Paragraph => { + if self.end_newline { + self.write("

") + } else { + self.write("\n

") + } + } + Tag::Heading { + level, + id, + classes, + attrs, + } => { + if self.end_newline { + self.write("<")?; + } else { + self.write("\n<")?; + } + write!(&mut self.writer, "{}", level)?; + if let Some(id) = id { + self.write(" id=\"")?; + escape_html(&mut self.writer, &id)?; + self.write("\"")?; + } + let mut classes = classes.iter(); + if let Some(class) = classes.next() { + self.write(" class=\"")?; + escape_html(&mut self.writer, class)?; + for class in classes { + self.write(" ")?; + escape_html(&mut self.writer, class)?; + } + self.write("\"")?; + } + for (attr, value) in attrs { + self.write(" ")?; + escape_html(&mut self.writer, &attr)?; + if let Some(val) = value { + self.write("=\"")?; + escape_html(&mut self.writer, &val)?; + self.write("\"")?; + } else { + self.write("=\"\"")?; + } + } + self.write(">") + } + Tag::Table(alignments) => { + self.table_alignments = alignments; + self.write("") + } + Tag::TableHead => { + self.table_state = TableState::Head; + self.table_cell_index = 0; + self.write("") + } + Tag::TableRow => { + self.table_cell_index = 0; + self.write("") + } + Tag::TableCell => { + match self.table_state { + TableState::Head => { + self.write(" { + self.write(" self.write(" style=\"text-align: left\">"), + Some(&Alignment::Center) => self.write(" style=\"text-align: center\">"), + Some(&Alignment::Right) => self.write(" style=\"text-align: right\">"), + _ => self.write(">"), + } + } + Tag::BlockQuote(kind) => { + let class_str = match kind { + None => "", + Some(kind) => match kind { + BlockQuoteKind::Note => " class=\"markdown-alert-note\"", + BlockQuoteKind::Tip => " class=\"markdown-alert-tip\"", + BlockQuoteKind::Important => " class=\"markdown-alert-important\"", + BlockQuoteKind::Warning => " class=\"markdown-alert-warning\"", + BlockQuoteKind::Caution => " class=\"markdown-alert-caution\"", + }, + }; + if self.end_newline { + self.write(&format!("\n", class_str)) + } else { + self.write(&format!("\n\n", class_str)) + } + } + Tag::CodeBlock(info) => { + if !self.end_newline { + self.write_newline()?; + } + match info { + CodeBlockKind::Fenced(info) => { + let lang = info.split(' ').next().unwrap(); + if lang.is_empty() { + self.write("
")
+                        } else {
+                            self.write("
")
+                        }
+                    }
+                    CodeBlockKind::Indented => self.write("
"),
+                }
+            }
+            Tag::List(Some(1)) => {
+                if self.end_newline {
+                    self.write("
    \n")
+                } else {
+                    self.write("\n
      \n")
+                }
+            }
+            Tag::List(Some(start)) => {
+                if self.end_newline {
+                    self.write("
        \n")
+            }
+            Tag::List(None) => {
+                if self.end_newline {
+                    self.write("