diff --git a/lib/mst/tree.ex b/lib/mst/tree.ex index 084a1ab..0e5c11a 100644 --- a/lib/mst/tree.ex +++ b/lib/mst/tree.ex @@ -204,7 +204,9 @@ defmodule MST.Tree do def to_list(%__MODULE__{root: nil}), do: {:ok, []} def to_list(%__MODULE__{root: root, store: store}) do - walk(store, root, []) + with {:ok, reversed} <- walk(store, root, []) do + {:ok, Enum.reverse(reversed)} + end end @doc """ @@ -228,16 +230,18 @@ defmodule MST.Tree do def stream(%__MODULE__{root: root, store: store}) do Stream.resource( - fn -> [root] end, + fn -> [{:visit, root}] end, fn [] -> {:halt, []} - [cid | rest] -> + [{:yield, k, v} | rest] -> + {[{k, v}], rest} + + [{:visit, cid} | rest] -> node = fetch_node!(store, cid) full_keys = Node.keys(node) - {yields, next_stack} = node_to_stream_items(node, full_keys, rest) - {yields, next_stack} + {[], in_order_items(node, full_keys) ++ rest} end, fn _ -> :ok end ) @@ -573,7 +577,8 @@ defmodule MST.Tree do @spec walk_entries(store(), [Entry.t()], [binary()], [{binary(), CID.t()}]) :: {:ok, [{binary(), CID.t()}]} | tree_error() - defp walk_entries(_store, [], [], acc), do: {:ok, Enum.reverse(acc)} + # Accumulates pairs in reverse order; to_list/1 reverses once at the top. + defp walk_entries(_store, [], [], acc), do: {:ok, acc} defp walk_entries(store, [entry | rest_e], [key | rest_k], acc) do acc = [{key, entry.value} | acc] @@ -587,28 +592,22 @@ defmodule MST.Tree do # Private — stream helpers # --------------------------------------------------------------------------- - # Turn a node into a stream item list (yields) and the updated DFS stack. - # We push right subtrees and yield leaf entries in left-to-right order. - @spec node_to_stream_items(Node.t(), [binary()], list()) :: {[{binary(), CID.t()}], list()} - defp node_to_stream_items(node, full_keys, rest_stack) do - # Build a plan: [{:visit, cid} | {:yield, key, val}] in order - left_visits = if node.left, do: [{:visit, node.left}], else: [] + # Expand a node into an in-order sequence of {:visit, cid} and {:yield, k, v} + # items. When prepended to a DFS stack, this produces correct left-to-right + # traversal: left subtree is visited first, then each entry interleaved with + # its right subtree. + @spec in_order_items(Node.t(), [binary()]) :: list() + defp in_order_items(node, full_keys) do + left_items = if node.left, do: [{:visit, node.left}], else: [] entry_items = Enum.zip(node.entries, full_keys) |> Enum.flat_map(fn {e, k} -> - right_visits = if e.right, do: [{:visit, e.right}], else: [] - [{:yield, k, e.value} | right_visits] + right_items = if e.right, do: [{:visit, e.right}], else: [] + [{:yield, k, e.value} | right_items] end) - plan = left_visits ++ entry_items - - # Collect yields and build the new stack (visits go onto the front). - # We want left-to-right order: process plan in reverse to prepend onto stack. - Enum.reduce(Enum.reverse(plan), {[], rest_stack}, fn - {:yield, k, v}, {yields, stack} -> {[{k, v} | yields], stack} - {:visit, cid}, {yields, stack} -> {yields, [cid | stack]} - end) + left_items ++ entry_items end # --------------------------------------------------------------------------- diff --git a/test/fixtures/interop/commit_proof.json b/test/fixtures/interop/commit_proof.json new file mode 100644 index 0000000..7b35fe8 --- /dev/null +++ b/test/fixtures/interop/commit_proof.json @@ -0,0 +1,96 @@ +[ + { + "comment": "two deep split", + "leafValue": "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454", + "keys": [ + "A0/501344", + "B1/293486", + "C0/535043", + "E0/922708", + "F1/415452", + "G0/714257" + ], + "adds": ["D2/915466"], + "dels": [], + "rootBeforeCommit": "bafyreibthlzzn3rwvmomwf4dz6utt7yeh5eyn6qwbumvjfv35gwanh7ovq", + "rootAfterCommit": "bafyreidb6bxxylhmlzs4a6ruhcunv3fd32o6i5phlzkmjk6arletj2ua6a", + "blocksInProof": [ + "bafyreidb6bxxylhmlzs4a6ruhcunv3fd32o6i5phlzkmjk6arletj2ua6a", + "bafyreifjsxnultnc3tbvnrawqpmrk6d76ymcstwcr5e3hn6u472nasb2xq", + "bafyreibzch5k5j5xkg6dcwmur2p6jqwavyjhdtvifr6g2gnccwhixibzsi", + "bafyreiamcu5ud3j4ovclrgq2sdyev5oajsmpnl2fdu5ffgpfint64n2jme", + "bafyreidxvw3sbdg4t5b2mbtozitnyu7kjien2zcrtgdj4ssgmjb72mzawe" + ] + }, + { + "comment": "two deep leafless split", + "leafValue": "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454", + "keys": ["A0/501344", "B0/436099", "D0/360671", "E0/922708"], + "adds": ["C2/953910"], + "dels": [], + "rootBeforeCommit": "bafyreid7jnvjg7mr4akmyf7rtaz47duex2l47rz36nvs4i7yjnpuhfmehe", + "rootAfterCommit": "bafyreih2ry5gae5r4m47unhhuw4w2qjdhe6oprw3w2uico2tzbflwi74eu", + "blocksInProof": [ + "bafyreih2ry5gae5r4m47unhhuw4w2qjdhe6oprw3w2uico2tzbflwi74eu", + "bafyreiag5ata5gtynbpef26l4kus2uz4nshuo526h275oljwlm5dwsvhqm", + "bafyreiaybgpm7ahyiy7fko7c4czjokhzajvimot6lfi6mxqzw2bzwoddn4", + "bafyreiheqxxydll4b4zlyemmegb7q3chs7aacczuotpxkqils6bufnsyse", + "bafyreigkijiuasyl5x4f2j3kxzou2vsdyc3vockx63r6bvgoip4ybhj2sa" + ] + }, + { + "comment": "add on edge with neighbor two layers down", + "leafValue": "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454", + "keys": ["A0/501344", "B2/303249", "C0/535043"], + "adds": ["D2/915466"], + "dels": [], + "rootBeforeCommit": "bafyreifoy7ierkqljk37wozudqhqjuuahjnubqvd3qprx5ocwcfrx5v3hm", + "rootAfterCommit": "bafyreid2i3nxmsvv3ifb53nlkjh3qaymygrrxuno6z22gctzdme5lbptky", + "blocksInProof": [ + "bafyreid2i3nxmsvv3ifb53nlkjh3qaymygrrxuno6z22gctzdme5lbptky", + "bafyreiagiwrefvm27hvgryirykp7reqcpz56v6txzksgbargjlibtpsqwu", + "bafyreiewdvzcopoza6bdntvhmvdfqeolql6sckkiu75jpvfnwwnfi57jia" + ] + }, + { + "comment": "merge and split in multi-op commit", + "leafValue": "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454", + "keys": ["A0/501344", "B2/303249", "D2/915466", "E0/922708"], + "adds": ["C2/953910"], + "dels": ["B2/303249", "D2/915466"], + "rootBeforeCommit": "bafyreielnllkafudlseizljjx32rkkivlgxziqayhctgbxncw2srrox7ny", + "rootAfterCommit": "bafyreih6464tr7ue67qgllhiekgfmwiz45zuthrv72gwi2tjpuu5dbxt3a", + "blocksInProof": [ + "bafyreih6464tr7ue67qgllhiekgfmwiz45zuthrv72gwi2tjpuu5dbxt3a", + "bafyreihexby6fnhajsjzzqkmegqpqt2lrr3rpesyl6kt3t3xppid7tuvfy", + "bafyreiciix65xuk62hu6ew6jdy3m2swqstvnuhuwcwffidk3nduf7eaoh4", + "bafyreieneexkszoung4zc5jzkjukjbbxm74ukz6mylydj7q2v42zqp6vmy", + "bafyreidxvw3sbdg4t5b2mbtozitnyu7kjien2zcrtgdj4ssgmjb72mzawe" + ] + }, + { + "comment": "complex multi-op commit", + "leafValue": "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454", + "keys": [ + "B0/436099", + "C2/953910", + "D0/360671", + "E2/413113", + "F0/606463", + "H0/740256" + ], + "adds": ["A2/239654", "G2/536869"], + "dels": ["C2/953910"], + "rootBeforeCommit": "bafyreiej4jqggfhidabjfrjgogdwed5eglhnboepxscbwfrss4uclnrrmi", + "rootAfterCommit": "bafyreifykpu67c4w4ynkx4lvjfjwxdofax6gx7j2wxrl6ewt3yslezcb6i", + "blocksInProof": [ + "bafyreifykpu67c4w4ynkx4lvjfjwxdofax6gx7j2wxrl6ewt3yslezcb6i", + "bafyreig5pe2hdnhfbqleo6yyipkw3tdiju7tlm4sqp7btsiicxe4tex5de", + "bafyreievjgro75jk6ma3xwuqvalsydtzgvbbaduhazbvajvslaf3l6kcxu", + "bafyreieax6243224jnbout6ynursux2dvt6fabonofdu47dxupkxvmflvu", + "bafyreie44qmlnwlyeh6ubb2eocfko6st7gmbarplmcci6c7ilx24vh4iym", + "bafyreihlhqn4quwcgbum5g4wzkini2c42j7zi5dsjdgkzm55jxyvebndue", + "bafyreiggcbzkb2wgenvyfhkh2nggf7pohb7uzjm6bs7hixhjxw2xpmnq6u" + ] + } +] diff --git a/test/fixtures/interop/common_prefix.json b/test/fixtures/interop/common_prefix.json new file mode 100644 index 0000000..2c31287 --- /dev/null +++ b/test/fixtures/interop/common_prefix.json @@ -0,0 +1,15 @@ +[ + {"left": "", "right": "", "len": 0}, + {"left": "abc", "right": "abc", "len": 3}, + {"left": "", "right": "abc", "len": 0}, + {"left": "abc", "right": "", "len": 0}, + {"left": "ab", "right": "abc", "len": 2}, + {"left": "abc", "right": "ab", "len": 2}, + {"left": "abcde", "right": "abc", "len": 3}, + {"left": "abc", "right": "abcde", "len": 3}, + {"left": "abcde", "right": "abc1", "len": 3}, + {"left": "abcde", "right": "abb", "len": 2}, + {"left": "abcde", "right": "qbb", "len": 0}, + {"left": "abc", "right": "abc\u0000", "len": 3}, + {"left": "abc\u0000", "right": "abc", "len": 3} +] diff --git a/test/fixtures/interop/example_keys.txt b/test/fixtures/interop/example_keys.txt new file mode 100644 index 0000000..d1d1904 --- /dev/null +++ b/test/fixtures/interop/example_keys.txt @@ -0,0 +1,81 @@ +A0/501344 +A1/700567 +A2/239654 +A3/570745 +A4/231700 +A5/343219 +B0/436099 +B1/293486 +B2/303249 +B3/690557 +B4/522003 +B5/528640 +C0/535043 +C1/970596 +C2/953910 +C3/016643 +C4/687126 +C5/136391 +D0/360671 +D1/637976 +D2/915466 +D3/722333 +D4/816246 +D5/611412 +E0/922708 +E1/710014 +E2/413113 +E3/226890 +E4/349347 +E5/574759 +F0/606463 +F1/415452 +F2/410478 +F3/000172 +F4/438093 +F5/131765 +G0/714257 +G1/254594 +G2/536869 +G3/188348 +G4/627086 +G5/436727 +H0/740256 +H1/113887 +H2/783135 +H3/911996 +H4/413212 +H5/205035 +I0/123247 +I1/186251 +I2/455864 +I3/874267 +I4/700662 +I5/355687 +J0/651505 +J1/747356 +J2/880562 +J3/337247 +J4/333302 +J5/802321 +K0/513509 +K1/512199 +K2/998695 +K3/030175 +K4/843537 +K5/621841 +L0/110539 +L1/902119 +L2/433601 +L3/578589 +L4/179159 +L5/411430 +M0/233209 +M1/807305 +M2/593452 +M3/412948 +M4/230935 +M5/340624 +N0/719700 +N1/322330 +N2/554 diff --git a/test/fixtures/interop/key_heights.json b/test/fixtures/interop/key_heights.json new file mode 100644 index 0000000..113dd51 --- /dev/null +++ b/test/fixtures/interop/key_heights.json @@ -0,0 +1,11 @@ +[ + {"key": "", "height": 0}, + {"key": "asdf", "height": 0}, + {"key": "blue", "height": 1}, + {"key": "2653ae71", "height": 0}, + {"key": "88bfafc7", "height": 2}, + {"key": "2a92d355", "height": 4}, + {"key": "884976f5", "height": 6}, + {"key": "app.bsky.feed.post/454397e440ec", "height": 4}, + {"key": "app.bsky.feed.post/9adeb165882c", "height": 8} +] diff --git a/test/mst/height_test.exs b/test/mst/height_test.exs index 8884255..ec06672 100644 --- a/test/mst/height_test.exs +++ b/test/mst/height_test.exs @@ -3,6 +3,10 @@ defmodule MST.HeightTest do doctest MST.Height + @interop_fixtures Path.join([__DIR__, "..", "fixtures", "interop", "key_heights.json"]) + |> File.read!() + |> Jason.decode!() + describe "for_key/1" do # Spec examples from https://atproto.com/specs/repository#mst-structure test "spec example: depth 0" do @@ -50,4 +54,13 @@ defmodule MST.HeightTest do assert d0 != d1 end end + + describe "key_heights.json interop" do + test "all fixture entries match expected height" do + for %{"key" => key, "height" => expected} <- @interop_fixtures do + assert MST.Height.for_key(key) == expected, + "for_key(#{inspect(key)}) expected #{expected}, got #{MST.Height.for_key(key)}" + end + end + end end diff --git a/test/mst/interop_test.exs b/test/mst/interop_test.exs new file mode 100644 index 0000000..d3d2891 --- /dev/null +++ b/test/mst/interop_test.exs @@ -0,0 +1,315 @@ +defmodule MST.InteropTest do + @moduledoc """ + Interoperability tests using fixtures from atproto-interop-tests and + jacquard's additional edge-case vectors. + + Fixture sources: + - https://github.com/bluesky-social/atproto-interop-tests/tree/main/mst + - https://github.com/orual/jacquard (tests/fixtures/) + + Covers: + - 156 real-world-shaped keys: full insert, selective delete, insertion-order + determinism. + - 5 commit-proof CID-exact scenarios stressing height-gap splits, leafless + splits, edge inserts, and merge-then-split sequences. + - The "rsky" 2-key tree regression from rsky's + `handle_new_layers_that_are_two_higher_than_existing` test. + - Trees spanning heights 0–8 to exercise multi-level intermediate empty + nodes. + """ + + use ExUnit.Case, async: true + + alias DASL.CID + alias MST.Tree + + @fixture_dir Path.join([__DIR__, "..", "fixtures", "interop"]) + + # 156 keys of the form "X{level}/{number}" where each key's MST height + # matches the digit in the name (generated by atproto-interop-tests/mst/gen_keys.py). + @example_keys Path.join(@fixture_dir, "example_keys.txt") + |> File.read!() + |> String.split("\n", trim: true) + + # Stable key→CID mapping reused across example-key tests so that insertion + # order doesn't affect which value a key maps to. + @example_kv Map.new(Enum.with_index(@example_keys), fn {k, i} -> + {k, CID.compute(<>, :raw)} + end) + + # 5 commit-proof scenarios; each specifies an initial key set, a batch of + # adds/deletes, and the expected root CIDs before and after. + @commit_proof Path.join(@fixture_dir, "commit_proof.json") + |> File.read!() + |> Jason.decode!() + + defp new_tree, do: Tree.new(MST.Store.Memory.new()) + + # --------------------------------------------------------------------------- + # Example keys — insert / delete / determinism + # --------------------------------------------------------------------------- + + describe "example keys" do + @tag :slow + test "insert all keys and retrieve each" do + tree = + Enum.reduce(@example_keys, new_tree(), fn key, acc -> + {:ok, t} = Tree.put(acc, key, @example_kv[key]) + t + end) + + for key <- @example_keys do + assert {:ok, @example_kv[key]} == Tree.get(tree, key), + "key not found after insert: #{key}" + end + + assert {:ok, count} = Tree.length(tree) + assert count == length(@example_keys) + end + + @tag :slow + test "delete every other key; correct half remains" do + tree = + Enum.reduce(@example_keys, new_tree(), fn key, acc -> + {:ok, t} = Tree.put(acc, key, @example_kv[key]) + t + end) + + indexed = Enum.with_index(@example_keys) + {evens, odds} = Enum.split_with(indexed, fn {_, i} -> rem(i, 2) == 0 end) + + tree = + Enum.reduce(evens, tree, fn {key, _}, acc -> + {:ok, t} = Tree.delete(acc, key) + t + end) + + for {key, _} <- evens do + assert {:error, :not_found} == Tree.get(tree, key), + "deleted key still present: #{key}" + end + + for {key, _} <- odds do + assert {:ok, @example_kv[key]} == Tree.get(tree, key), + "surviving key not found: #{key}" + end + + assert {:ok, remaining} = Tree.length(tree) + assert remaining == length(odds) + end + + @tag :slow + test "root CID is identical regardless of insertion order" do + forward = + Enum.reduce(@example_keys, new_tree(), fn key, acc -> + {:ok, t} = Tree.put(acc, key, @example_kv[key]) + t + end) + + reverse = + Enum.reduce(Enum.reverse(@example_keys), new_tree(), fn key, acc -> + {:ok, t} = Tree.put(acc, key, @example_kv[key]) + t + end) + + assert forward.root == reverse.root, + "insertion order changed root CID" + end + + @tag :slow + test "delete all keys produces empty tree" do + tree = + Enum.reduce(@example_keys, new_tree(), fn key, acc -> + {:ok, t} = Tree.put(acc, key, @example_kv[key]) + t + end) + + empty = + Enum.reduce(@example_keys, tree, fn key, acc -> + {:ok, t} = Tree.delete(acc, key) + t + end) + + assert {:ok, []} = Tree.to_list(empty) + assert empty.root == nil + end + end + + # --------------------------------------------------------------------------- + # Commit-proof fixtures — CID-exact spec vectors + # --------------------------------------------------------------------------- + + describe "commit proof fixtures" do + # Each fixture drives a scenario that would catch specific structural bugs: + # + # "two deep split" — height-2 insert between height-1 nodes, + # requires two levels of intermediate empties. + # "two deep leafless split" — height-2 insert with no height-1 nodes + # anywhere near the split point. + # "add on edge with neighbor two layers down" + # — new height-2 key adjacent to a subtree + # whose highest key is 2 levels lower. + # "merge and split in multi-op" — simultaneous adds and deletes; the + # merge path (delete) and split path + # (insert) both execute. + # "complex multi-op commit" — larger batch with both creates and + # deletes across multiple height levels. + + for fixture <- @commit_proof do + @fixture fixture + test @fixture["comment"] do + run_commit_proof_fixture(@fixture) + end + end + end + + # --------------------------------------------------------------------------- + # rsky edge case — 2-key tree with a known root CID + # --------------------------------------------------------------------------- + + describe "rsky simple case" do + # Regression from rsky's `handle_new_layers_that_are_two_higher_than_existing`. + # Two height-0 keys (same collection prefix) that differ only in their last + # few chars. The expected root CID is taken from the reference TypeScript + # implementation. + test "two height-0 keys produce the known root CID" do + {:ok, leaf} = CID.new("bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454") + + {:ok, tree} = Tree.put(new_tree(), "com.example.record/3jqfcqzm3ft2j", leaf) + {:ok, tree} = Tree.put(tree, "com.example.record/3jqfcqzm3fz2j", leaf) + + assert CID.encode(tree.root) == + "bafyreidfcktqnfmykz2ps3dbul35pepleq7kvv526g47xahuz3rqtptmky" + end + end + + # --------------------------------------------------------------------------- + # Multi-height trees — spec keys spanning heights 0–8 + # --------------------------------------------------------------------------- + + describe "keys spanning multiple heights" do + # Keys taken directly from the atproto spec and key_heights.json; each has + # a well-known height. Together they force intermediate empty nodes at every + # level from 0 up to the maximum height. + @spec_keys [ + {"2653ae71", 0}, + {"blue", 1}, + {"88bfafc7", 2}, + {"2a92d355", 4}, + {"884976f5", 6}, + {"app.bsky.feed.post/9adeb165882c", 8} + ] + + test "insert keys at heights 0/1/2/4/6/8 — all retrievable" do + val = CID.compute("v", :raw) + + tree = + Enum.reduce(@spec_keys, new_tree(), fn {key, _}, acc -> + {:ok, t} = Tree.put(acc, key, val) + t + end) + + for {key, _} <- @spec_keys do + assert {:ok, ^val} = Tree.get(tree, key), "not found: #{key}" + end + + assert {:ok, n} = Tree.length(tree) + assert n == length(@spec_keys) + end + + test "delete the height-4 key; other heights unaffected" do + val = CID.compute("v", :raw) + + tree = + Enum.reduce(@spec_keys, new_tree(), fn {key, _}, acc -> + {:ok, t} = Tree.put(acc, key, val) + t + end) + + {del_key, _} = Enum.find(@spec_keys, fn {_, h} -> h == 4 end) + {:ok, tree} = Tree.delete(tree, del_key) + + assert {:error, :not_found} = Tree.get(tree, del_key) + + for {key, _} <- @spec_keys, key != del_key do + assert {:ok, ^val} = Tree.get(tree, key), "not found after delete: #{key}" + end + end + + test "root CID is stable regardless of insertion order" do + val = CID.compute("v", :raw) + keys = Enum.map(@spec_keys, &elem(&1, 0)) + + forward = + Enum.reduce(keys, new_tree(), fn key, acc -> + {:ok, t} = Tree.put(acc, key, val) + t + end) + + reverse = + Enum.reduce(Enum.reverse(keys), new_tree(), fn key, acc -> + {:ok, t} = Tree.put(acc, key, val) + t + end) + + assert forward.root == reverse.root + end + + test "full delete cycle returns to empty" do + val = CID.compute("v", :raw) + keys = Enum.map(@spec_keys, &elem(&1, 0)) + + tree = + Enum.reduce(keys, new_tree(), fn key, acc -> + {:ok, t} = Tree.put(acc, key, val) + t + end) + + empty = + Enum.reduce(keys, tree, fn key, acc -> + {:ok, t} = Tree.delete(acc, key) + t + end) + + assert {:ok, []} = Tree.to_list(empty) + assert empty.root == nil + end + end + + # --------------------------------------------------------------------------- + # Helpers + # --------------------------------------------------------------------------- + + defp run_commit_proof_fixture(fixture) do + {:ok, leaf} = CID.new(fixture["leafValue"]) + + before_tree = + Enum.reduce(fixture["keys"], new_tree(), fn key, acc -> + {:ok, t} = Tree.put(acc, key, leaf) + t + end) + + assert CID.encode(before_tree.root) == fixture["rootBeforeCommit"], + ~s(root before commit: expected #{fixture["rootBeforeCommit"]}, ) <> + ~s(got #{CID.encode(before_tree.root)}) + + after_tree = + before_tree + |> then(fn t -> + Enum.reduce(fixture["adds"], t, fn key, acc -> + {:ok, t2} = Tree.put(acc, key, leaf) + t2 + end) + end) + |> then(fn t -> + Enum.reduce(fixture["dels"], t, fn key, acc -> + {:ok, t2} = Tree.delete(acc, key) + t2 + end) + end) + + assert CID.encode(after_tree.root) == fixture["rootAfterCommit"], + ~s(root after commit: expected #{fixture["rootAfterCommit"]}, ) <> + ~s(got #{CID.encode(after_tree.root)}) + end +end diff --git a/test/mst/node_test.exs b/test/mst/node_test.exs index 73916c8..13374f4 100644 --- a/test/mst/node_test.exs +++ b/test/mst/node_test.exs @@ -12,6 +12,10 @@ defmodule MST.NodeTest do @cid_b CID.compute("value_b", :raw) @cid_c CID.compute("value_c", :raw) + @prefix_interop Path.join([__DIR__, "..", "fixtures", "interop", "common_prefix.json"]) + |> File.read!() + |> Jason.decode!() + describe "empty/0" do test "returns an empty node" do assert %Node{left: nil, entries: []} = Node.empty() @@ -180,6 +184,23 @@ defmodule MST.NodeTest do end end + describe "common_prefix.json interop" do + # The common prefix length between two keys is what determines prefix_len + # in the second of any two adjacent compress_entries inputs. We test via + # compress_entries since common_prefix_length/2 is private. + test "all fixture pairs produce correct prefix_len" do + cid = CID.compute("test", :raw) + + for %{"left" => left, "right" => right, "len" => expected} <- @prefix_interop do + [_first, second] = Node.compress_entries([{left, cid, nil}, {right, cid, nil}]) + + assert second.prefix_len == expected, + "common_prefix(#{inspect(left)}, #{inspect(right)}) " <> + "expected #{expected}, got #{second.prefix_len}" + end + end + end + describe "decode/1 error cases" do test "returns error for non-CBOR bytes" do assert {:error, :decode, _} = Node.decode(<<0xFF, 0xFF, 0xFF>>) diff --git a/test/mst/stress_test.exs b/test/mst/stress_test.exs new file mode 100644 index 0000000..e8e2050 --- /dev/null +++ b/test/mst/stress_test.exs @@ -0,0 +1,285 @@ +defmodule MST.StressTest do + @moduledoc """ + Stress tests for MST structural correctness under long random mutation sequences. + + Inspired by jacquard's `large_proof_tests.rs`, which applies hundreds of + random create/update/delete operations and validates commit proofs after each + batch. We can't validate firehose commits (that layer doesn't exist here), but + we capture the same invariant: after every mutation the tree's contents must + exactly match a shadow map maintained in parallel. + + Three test dimensions: + - **small**: 100 ops verified after every single operation — catches bugs in + individual transitions. + - **large**: 300 ops verified after every batch of 10 — catches accumulated + drift. + - **high-height**: 50 ops over a pool that includes height 8 keys — stresses + intermediate empty-node creation and destruction. + + Each test ends with a determinism check: rebuild the same key-value set into + a fresh tree and assert the root CID matches the evolved tree. + """ + + use ExUnit.Case, async: true + + alias DASL.CID + alias MST.Tree + + @fixture_dir Path.join([__DIR__, "..", "fixtures", "interop"]) + + # 81 real-world-shaped keys at heights 0–5 from the interop fixture. + @key_pool @fixture_dir + |> Path.join("example_keys.txt") + |> File.read!() + |> String.split("\n", trim: true) + + # Additional keys at heights 2, 4, 6, 8 — not in the example_keys file — + # to ensure multi-level intermediate empty nodes are exercised. + @high_height_keys [ + "88bfafc7", + "2a92d355", + "884976f5", + "app.bsky.feed.post/9adeb165882c" + ] + + @full_pool @key_pool ++ @high_height_keys + + # Keys used for the focused high-height test. Heights: 0, 0, 1, 2, 4, 4, 6, 8. + @height_spanning_keys [ + "2653ae71", + "asdf", + "blue", + "88bfafc7", + "2a92d355", + "app.bsky.feed.post/454397e440ec", + "884976f5", + "app.bsky.feed.post/9adeb165882c" + ] + + defp new_tree, do: Tree.new(MST.Store.Memory.new()) + + # --------------------------------------------------------------------------- + # PRNG helpers — explicit state threading for reproducibility + # --------------------------------------------------------------------------- + + # Returns {{:put, key, val} | {:delete, key}, new_rng_state}. + # Weights: 50% create-or-overwrite, 30% update-existing, 20% delete-existing. + # Falls back to create when the shadow is empty. + @spec rand_op(:rand.state(), map(), [binary()]) :: + {{:put, binary(), CID.t()} | {:delete, binary()}, :rand.state()} + defp rand_op(rng, shadow, pool) do + {dice, rng} = :rand.uniform_s(100, rng) + existing = Map.keys(shadow) + n = length(existing) + + cond do + dice <= 50 or n == 0 -> + {idx, rng} = :rand.uniform_s(length(pool), rng) + {seed, rng} = :rand.uniform_s(1_000_000_000, rng) + key = Enum.at(pool, idx - 1) + val = CID.compute("#{key}:#{seed}", :raw) + {{:put, key, val}, rng} + + dice <= 80 -> + {idx, rng} = :rand.uniform_s(n, rng) + {seed, rng} = :rand.uniform_s(1_000_000_000, rng) + key = Enum.at(existing, idx - 1) + val = CID.compute("#{key}:#{seed}", :raw) + {{:put, key, val}, rng} + + true -> + {idx, rng} = :rand.uniform_s(n, rng) + key = Enum.at(existing, idx - 1) + {{:delete, key}, rng} + end + end + + # Apply a single op to both the tree and shadow map. + @spec apply_op(Tree.t(), map(), {:put, binary(), CID.t()} | {:delete, binary()}) :: + {Tree.t(), map()} + defp apply_op(tree, shadow, {:put, key, val}) do + {:ok, tree} = Tree.put(tree, key, val) + {tree, Map.put(shadow, key, val)} + end + + defp apply_op(tree, shadow, {:delete, key}) do + {:ok, tree} = Tree.delete(tree, key) + {tree, Map.delete(shadow, key)} + end + + # Assert every key in `shadow` is present in `tree` with the correct value, + # that the sizes match, and that `to_list/1` returns keys in sorted order. + @spec assert_matches_shadow(Tree.t(), map()) :: :ok + defp assert_matches_shadow(tree, shadow) do + {:ok, pairs} = Tree.to_list(tree) + + assert length(pairs) == map_size(shadow), + "size mismatch: tree has #{length(pairs)} keys, shadow has #{map_size(shadow)}" + + keys = Enum.map(pairs, &elem(&1, 0)) + assert keys == Enum.sort(keys), "to_list/1 returned keys out of order" + + for {key, expected_val} <- shadow do + assert {:ok, ^expected_val} = Tree.get(tree, key), + "wrong value for #{inspect(key)}" + end + + :ok + end + + # Build a fresh tree from a shadow map (sorted insertion order) and return + # the root CID. Used to verify determinism: the evolved tree and the + # freshly-built tree must share the same root. + @spec root_from_shadow(map()) :: CID.t() | nil + defp root_from_shadow(shadow) do + shadow + |> Enum.sort_by(&elem(&1, 0)) + |> Enum.reduce(new_tree(), fn {key, val}, acc -> + {:ok, t} = Tree.put(acc, key, val) + t + end) + |> Map.fetch!(:root) + end + + # --------------------------------------------------------------------------- + # Tests + # --------------------------------------------------------------------------- + + describe "stress" do + # ------------------------------------------------------------------------- + # Small: 100 ops, verified after every single mutation + # ------------------------------------------------------------------------- + + @tag :slow + test "100 random ops over 20-key seed, verified after each op" do + rng = :rand.seed_s(:exsss, {42, 1337, 99}) + + seed_keys = Enum.take(@full_pool, 20) + + {tree, shadow} = + Enum.reduce(seed_keys, {new_tree(), %{}}, fn key, {t, s} -> + val = CID.compute("seed:#{key}", :raw) + {:ok, t} = Tree.put(t, key, val) + {t, Map.put(s, key, val)} + end) + + assert_matches_shadow(tree, shadow) + + {tree, shadow, _rng} = + Enum.reduce(1..100, {tree, shadow, rng}, fn _i, {t, s, rng} -> + {op, rng} = rand_op(rng, s, @full_pool) + {t, s} = apply_op(t, s, op) + assert_matches_shadow(t, s) + {t, s, rng} + end) + + assert tree.root == root_from_shadow(shadow), + "evolved tree root differs from scratch-rebuilt root — history-dependence bug" + end + + # ------------------------------------------------------------------------- + # Large: 300 ops in batches of 10, verified after each batch + # ------------------------------------------------------------------------- + + @tag :slow + test "300 random ops over 50-key seed, verified per batch of 10" do + rng = :rand.seed_s(:exsss, {7, 13, 21}) + + seed_keys = Enum.take(@full_pool, 50) + + {tree, shadow} = + Enum.reduce(seed_keys, {new_tree(), %{}}, fn key, {t, s} -> + val = CID.compute("seed:#{key}", :raw) + {:ok, t} = Tree.put(t, key, val) + {t, Map.put(s, key, val)} + end) + + {tree, shadow, _rng} = + Enum.reduce(1..30, {tree, shadow, rng}, fn _batch, {t, s, rng} -> + {t, s, rng} = + Enum.reduce(1..10, {t, s, rng}, fn _i, {t, s, rng} -> + {op, rng} = rand_op(rng, s, @full_pool) + {t, s} = apply_op(t, s, op) + {t, s, rng} + end) + + assert_matches_shadow(t, s) + {t, s, rng} + end) + + assert tree.root == root_from_shadow(shadow), + "evolved tree root differs from scratch-rebuilt root — history-dependence bug" + end + + # ------------------------------------------------------------------------- + # High-height: 50 ops over a pool spanning heights 0–8, per-op verified + # ------------------------------------------------------------------------- + + @tag :slow + test "50 random ops over height-spanning pool (h0–h8), verified after each op" do + # This specifically targets the intermediate empty-node paths: keys at + # heights 6 and 8 force multiple levels of empty wrappers. Deleting them + # exercises trim_top and the recursive merge across those levels. + rng = :rand.seed_s(:exsss, {100, 200, 300}) + + {tree, shadow} = + Enum.reduce(@height_spanning_keys, {new_tree(), %{}}, fn key, {t, s} -> + val = CID.compute("seed:#{key}", :raw) + {:ok, t} = Tree.put(t, key, val) + {t, Map.put(s, key, val)} + end) + + assert_matches_shadow(tree, shadow) + + {tree, shadow, _rng} = + Enum.reduce(1..50, {tree, shadow, rng}, fn _i, {t, s, rng} -> + {op, rng} = rand_op(rng, s, @height_spanning_keys) + {t, s} = apply_op(t, s, op) + assert_matches_shadow(t, s) + {t, s, rng} + end) + + assert tree.root == root_from_shadow(shadow), + "evolved tree root differs from scratch-rebuilt root — history-dependence bug" + end + + # ------------------------------------------------------------------------- + # Delete-all then re-insert: evolved root must match original + # ------------------------------------------------------------------------- + + @tag :slow + test "delete all keys then re-insert in reverse order produces original root" do + # Covers the full tree lifecycle: grow → shrink to nil → regrow. + # Uses height-spanning keys so intermediate empty nodes are created and + # destroyed at every level. + keys = @height_spanning_keys ++ Enum.take(@key_pool, 22) + + {original_tree, shadow} = + Enum.reduce(keys, {new_tree(), %{}}, fn key, {t, s} -> + val = CID.compute("v:#{key}", :raw) + {:ok, t} = Tree.put(t, key, val) + {t, Map.put(s, key, val)} + end) + + assert_matches_shadow(original_tree, shadow) + + empty = + Enum.reduce(keys, original_tree, fn key, t -> + {:ok, t} = Tree.delete(t, key) + t + end) + + assert empty.root == nil + assert {:ok, []} = Tree.to_list(empty) + + rebuilt = + Enum.reduce(Enum.reverse(keys), new_tree(), fn key, t -> + {:ok, t} = Tree.put(t, key, shadow[key]) + t + end) + + assert rebuilt.root == original_tree.root, + "re-insert in reverse order produced a different root — determinism bug" + end + end +end