diff --git a/bundle/operations.go b/bundle/operations.go index db2c7fb..13bee1f 100644 --- a/bundle/operations.go +++ b/bundle/operations.go @@ -67,8 +67,7 @@ func (op *Operations) ParseJSONL(data []byte) ([]plc.PLCOperation, error) { } var operation plc.PLCOperation - // Use sonic instead of json.Unmarshal - if err := json.Unmarshal(line, &operation); err != nil { + if err := json.UnmarshalNoEscape(line, &operation); err != nil { return nil, fmt.Errorf("failed to parse line: %w", err) } diff --git a/cmd/plcbundle/detector.go b/cmd/plcbundle/detector.go index 0d0d454..f8b21f4 100644 --- a/cmd/plcbundle/detector.go +++ b/cmd/plcbundle/detector.go @@ -545,11 +545,18 @@ func parseAndLoadDetectors(detectorNames []string, confidence float64) (*detecto // detectOperation runs all detectors on an operation and returns labels + confidence func detectOperation(ctx context.Context, detectors []detector.Detector, op plc.PLCOperation, minConfidence float64) ([]string, float64) { + // Parse Operation ONCE before running detectors + opData, err := op.GetOperationData() + if err != nil { + return nil, 0 + } + op.ParsedOperation = opData // Set for detectors to use + var matchedLabels []string var maxConfidence float64 for _, det := range detectors { - match, err := det.Detect(ctx, op) + match, err := det.Detect(ctx, op) // ← op now has ParsedOperation set if err != nil || match == nil || match.Confidence < minConfidence { continue } diff --git a/plc/types.go b/plc/types.go index c0b3a35..eb38527 100644 --- a/plc/types.go +++ b/plc/types.go @@ -8,15 +8,14 @@ import ( // PLCOperation represents a single operation from the PLC directory type PLCOperation struct { - DID string `json:"did"` - //Operation map[string]interface{} `json:"operation"` - Operation json.RawMessage `json:"operation"` + DID string `json:"did"` + Operation json.RawMessage `json:"operation"` // Lazy CID string `json:"cid"` Nullified interface{} `json:"nullified,omitempty"` CreatedAt time.Time `json:"createdAt"` - // RawJSON stores the original JSON bytes for exact reproduction - RawJSON []byte `json:"-"` + RawJSON []byte `json:"-"` + ParsedOperation map[string]interface{} `json:"-"` // Pre-parsed cache } // IsNullified checks if this operation has been nullified @@ -92,14 +91,30 @@ type EndpointInfo struct { Endpoint string } -// GetOperationMap parses Operation RawMessage into a map -func (op *PLCOperation) GetOperationMap() (map[string]interface{}, error) { +// GetOperationData parses Operation into map (with caching) +func (op *PLCOperation) GetOperationData() (map[string]interface{}, error) { + // Return cached if already parsed + if op.ParsedOperation != nil { + return op.ParsedOperation, nil + } + + // Parse on first call if len(op.Operation) == 0 { return nil, nil } - var result map[string]interface{} - if err := json.Unmarshal(op.Operation, &result); err != nil { + + var data map[string]interface{} + if err := json.UnmarshalNoEscape(op.Operation, &data); err != nil { return nil, err } - return result, nil + + // Cache it + op.ParsedOperation = data + + return data, nil +} + +// GetOperationMap is an alias for compatibility +func (op *PLCOperation) GetOperationMap() (map[string]interface{}, error) { + return op.GetOperationData() } diff --git a/scripts/benchmark-detector.go b/scripts/benchmark-detector.go index 012a1ef..2606caa 100644 --- a/scripts/benchmark-detector.go +++ b/scripts/benchmark-detector.go @@ -16,11 +16,11 @@ import ( // Minimal operation struct type Operation struct { - DID string `json:"did"` - //Operation map[string]interface{} `json:"operation"` - CID string `json:"cid"` - Nullified interface{} `json:"nullified,omitempty"` - CreatedAt time.Time `json:"createdAt"` + DID string `json:"did"` + Operation map[string]interface{} `json:"operation"` + CID string `json:"cid"` + Nullified interface{} `json:"nullified,omitempty"` + CreatedAt time.Time `json:"createdAt"` // RawJSON stores the original JSON bytes for exact reproduction RawJSON []byte `json:"-"` @@ -34,6 +34,11 @@ func detect(op *Operation) []string { labels = append(labels, "test") } + // Log operation.sig (like console.log in JavaScript) + if sig, ok := op.Operation["sig"]; ok { + fmt.Fprintf(os.Stderr, "%v\n", sig) + } + return labels } diff --git a/scripts/benchmark-detector.js b/scripts/benchmark-detector.js index 467e12a..c50196e 100644 --- a/scripts/benchmark-detector.js +++ b/scripts/benchmark-detector.js @@ -4,20 +4,19 @@ function detect({ op }) { const labels = []; - if (op.did.match(/^did:plc:aa/)) { + if (op.did.startsWith('did:plc:aa')) { labels.push('test') } + + console.log(op.operation.sig) return labels; } // ========================================== -// Pure Bun bundle processor +// Pure Bun bundle processor with native zstd // ========================================== -import { spawn } from 'bun'; -import { readdir } from 'fs/promises'; - const BUNDLE_DIR = process.argv[2] || './'; const START_BUNDLE = parseInt(process.argv[3]) || 1; const END_BUNDLE = parseInt(process.argv[4]) || 100; @@ -39,9 +38,14 @@ for (let bundleNum = START_BUNDLE; bundleNum <= END_BUNDLE; bundleNum++) { const bundleFile = `${BUNDLE_DIR}/${bundleNum.toString().padStart(6, '0')}.jsonl.zst`; try { - // Decompress bundle using zstd command - const proc = spawn(['zstd', '-d', '-c', bundleFile]); - const text = await new Response(proc.stdout).text(); + // Read compressed bundle + const compressed = await Bun.file(bundleFile).arrayBuffer(); + + // Decompress using native Bun zstd (FAST!) + const decompressed = Bun.zstdDecompressSync(compressed); + + // Convert to text + const text = new TextDecoder().decode(decompressed); const lines = text.split('\n').filter(line => line.trim()); diff --git a/scripts/benchmark_detector_rs/.gitignore b/scripts/benchmark_detector_rs/.gitignore new file mode 100644 index 0000000..1de5659 --- /dev/null +++ b/scripts/benchmark_detector_rs/.gitignore @@ -0,0 +1 @@ +target \ No newline at end of file diff --git a/scripts/benchmark_detector_rs/Cargo.lock b/scripts/benchmark_detector_rs/Cargo.lock new file mode 100644 index 0000000..f102e7a --- /dev/null +++ b/scripts/benchmark_detector_rs/Cargo.lock @@ -0,0 +1,554 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "benchmark_detector" +version = "0.1.0" +dependencies = [ + "serde", + "sonic-rs", + "zstd", +] + +[[package]] +name = "bumpalo" +version = "3.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" + +[[package]] +name = "bytes" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" + +[[package]] +name = "cc" +version = "1.2.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "739eb0f94557554b3ca9a86d2d37bebd49c5e6d0c1d2bda35ba5bdac830befc2" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "faststr" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baec6a0289d7f1fe5665586ef7340af82e3037207bef60f5785e57569776f0c8" +dependencies = [ + "bytes", + "rkyv", + "serde", + "simdutf8", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127" + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" + +[[package]] +name = "hashbrown" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" + +[[package]] +name = "indexmap" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6717a8d2a5a929a1a2eb43a12812498ed141a0bcfb7e8f7844fbdbe4303bba9f" +dependencies = [ + "equivalent", + "hashbrown 0.16.0", +] + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b011eec8cc36da2aab2d5cff675ec18454fad408585853910a202391cf9f8e65" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.177" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" + +[[package]] +name = "munge" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e17401f259eba956ca16491461b6e8f72913a0a114e39736ce404410f915a0c" +dependencies = [ + "munge_macro", +] + +[[package]] +name = "munge_macro" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "proc-macro2" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "ptr_meta" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b9a0cf95a1196af61d4f1cbdab967179516d9a4a4312af1f31948f8f6224a79" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "quote" +version = "1.0.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rancor" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a063ea72381527c2a0561da9c80000ef822bdd7c3241b1cc1b12100e3df081ee" +dependencies = [ + "ptr_meta", +] + +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "rend" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cadadef317c2f20755a64d7fdc48f9e7178ee6b0e1f7fce33fa60f1d68a276e6" + +[[package]] +name = "rkyv" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a640b26f007713818e9a9b65d34da1cf58538207b052916a83d80e43f3ffa4" +dependencies = [ + "bytes", + "hashbrown 0.15.5", + "indexmap", + "munge", + "ptr_meta", + "rancor", + "rend", + "rkyv_derive", + "tinyvec", + "uuid", +] + +[[package]] +name = "rkyv_derive" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd83f5f173ff41e00337d97f6572e416d022ef8a19f371817259ae960324c482" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "sonic-number" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8a74044c092f4f43ca7a6cfd62854cf9fb5ac8502b131347c990bf22bef1dfe" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "sonic-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22540d56ba14521e4878ad436d498518c59698c39a89d5905c694932f0bf7134" +dependencies = [ + "ahash", + "bumpalo", + "bytes", + "cfg-if", + "faststr", + "itoa", + "ref-cast", + "ryu", + "serde", + "simdutf8", + "sonic-number", + "sonic-simd", + "thiserror", +] + +[[package]] +name = "sonic-simd" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b421f7b6aa4a5de8f685aaf398dfaa828346ee639d2b1c1061ab43d40baa6223" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "syn" +version = "2.0.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da58917d35242480a05c2897064da0a80589a2a0476c9a3f2fdc83b53502e917" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinyvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "unicode-ident" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" + +[[package]] +name = "uuid" +version = "1.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasip2" +version = "1.0.1+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da95793dfc411fbbd93f5be7715b0578ec61fe87cb1a42b12eb625caa5c5ea60" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04264334509e04a7bf8690f2384ef5265f05143a4bff3889ab7a3269adab59c2" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "420bc339d9f322e562942d52e115d57e950d12d88983a14c79b86859ee6c7ebc" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76f218a38c84bcb33c25ec7059b07847d465ce0e0a76b995e134a45adcb6af76" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wit-bindgen" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" + +[[package]] +name = "zerocopy" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/scripts/benchmark_detector_rs/Cargo.toml b/scripts/benchmark_detector_rs/Cargo.toml new file mode 100644 index 0000000..0de93b2 --- /dev/null +++ b/scripts/benchmark_detector_rs/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "benchmark_detector" +version = "0.1.0" +edition = "2021" + +[dependencies] +sonic-rs = "0.5" +serde = { version = "1.0", features = ["derive"] } +zstd = "0.13" + +[profile.release] +lto = true +codegen-units = 1 +opt-level = 3 + +[build] +rustflags = ["-C", "target-cpu=native"] \ No newline at end of file diff --git a/scripts/benchmark_detector_rs/src/main.rs b/scripts/benchmark_detector_rs/src/main.rs new file mode 100644 index 0000000..4b6600f --- /dev/null +++ b/scripts/benchmark_detector_rs/src/main.rs @@ -0,0 +1,123 @@ +use std::env; +use std::fs::File; +use std::io::{self, BufRead, BufReader, Write}; +use std::time::Instant; +use sonic_rs::{Deserialize, from_str, JsonValueTrait, Value}; + +#[derive(Deserialize)] +struct Operation { + did: String, + cid: String, + operation: Value, +} + +fn detect(op: &Operation) -> Vec { + let mut labels = Vec::new(); + + if op.did.starts_with("did:plc:aa") { + labels.push("test".to_string()); + } + + if let Some(sig) = op.operation.get("sig") { + eprintln!("{}", sig); + } + + labels +} + +fn main() -> io::Result<()> { + let args: Vec = env::args().collect(); + let bundle_dir = args.get(1).map(|s| s.as_str()).unwrap_or("./"); + let start_bundle: usize = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(1); + let end_bundle: usize = args.get(3).and_then(|s| s.parse().ok()).unwrap_or(100); + + eprintln!("Processing bundles {}-{} from {}", start_bundle, end_bundle, bundle_dir); + eprintln!(); + + let stdout = io::stdout(); + let mut writer = io::BufWriter::with_capacity(512 * 1024, stdout.lock()); + writeln!(writer, "bundle,position,cid,size,confidence,labels")?; + + let mut total_ops = 0; + let mut match_count = 0; + let mut total_bytes: u64 = 0; + let mut matched_bytes: u64 = 0; + let start_time = Instant::now(); + + for bundle_num in start_bundle..=end_bundle { + let bundle_file = format!("{}/{:06}.jsonl.zst", bundle_dir, bundle_num); + + let file = match File::open(&bundle_file) { + Ok(f) => f, + Err(_) => continue, + }; + + let decoder = match zstd::Decoder::new(file) { + Ok(d) => d, + Err(_) => continue, + }; + + let reader = BufReader::new(decoder); + + for (position, line) in reader.lines().enumerate() { + let line = match line { + Ok(l) => l, + Err(_) => continue, + }; + + if line.is_empty() { + continue; + } + + total_ops += 1; + let op_size = line.len(); + total_bytes += op_size as u64; + + let op: Operation = match from_str(&line) { + Ok(o) => o, + Err(_) => continue, + }; + + let labels = detect(&op); + + if !labels.is_empty() { + match_count += 1; + matched_bytes += op_size as u64; + + let cid_short = if op.cid.len() > 4 { + &op.cid[op.cid.len()-4..] + } else { + &op.cid + }; + + writeln!(writer, "{},{},{},{},0.95,{}", + bundle_num, position, cid_short, op_size, labels.join(";"))?; + } + } + + if bundle_num % 10 == 0 { + let elapsed = start_time.elapsed().as_secs_f64(); + let ops_per_sec = total_ops as f64 / elapsed; + eprint!("Processed {}/{} bundles | {} ops | {:.0} ops/sec\r", + bundle_num, end_bundle, total_ops, ops_per_sec); + } + } + + let elapsed = start_time.elapsed().as_secs_f64(); + writer.flush()?; + + eprintln!("\n\nāœ“ Detection complete"); + eprintln!(" Total operations: {}", total_ops); + eprintln!(" Matches found: {} ({:.2}%)", match_count, + match_count as f64 / total_ops as f64 * 100.0); + eprintln!(" Total size: {:.1} MB", total_bytes as f64 / 1e6); + eprintln!(" Matched size: {:.1} MB ({:.2}%)", + matched_bytes as f64 / 1e6, + matched_bytes as f64 / total_bytes as f64 * 100.0); + eprintln!(); + eprintln!(" Time elapsed: {:.2}s", elapsed); + eprintln!(" Throughput: {:.0} ops/sec", total_ops as f64 / elapsed); + eprintln!(" Speed: {:.1} MB/sec", total_bytes as f64 / elapsed / 1e6); + + Ok(()) +} diff --git a/scripts/detector-template.js b/scripts/detector-template.js index ae64667..cf73147 100644 --- a/scripts/detector-template.js +++ b/scripts/detector-template.js @@ -10,6 +10,8 @@ function detect({ op }) { // Return array of label strings // Return empty array [] for no match + //console.log(JSON.stringify(op)) + if (op.did.match(/^did:plc:aaa/)) { labels.push('test') }