diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 1408125..0000000 --- a/.dockerignore +++ /dev/null @@ -1,47 +0,0 @@ -# Git -.git -.gitignore - -# Build artifacts -*.exe -*.exe~ -*.dll -*.so -*.dylib -*.test -*.out -/bin/ -/dist/ - -# Data directories -/data/ -*.jsonl.zst -*.jsonl -*.json -plc_mempool_* -plc_bundles.json - -# IDE -.vscode/ -.idea/ -*.swp -*.swo -*~ - -# OS -.DS_Store -Thumbs.db - -# Docs -*.md -docs/ -examples/ - -# CI/CD -.github/ -.gitlab-ci.yml - -# Docker -Dockerfile* -docker-compose*.yml -.dockerignore diff --git a/.goreleaser.yml b/.goreleaser.yml deleted file mode 100644 index 27e2bee..0000000 --- a/.goreleaser.yml +++ /dev/null @@ -1,18 +0,0 @@ -version: 2 - -builds: - - main: ./cmd/plcbundle - binary: plcbundle - env: - - CGO_ENABLED=1 - goos: [linux] - goarch: [amd64, arm64] - ldflags: - - -s -w - - -X main.version={{.Version}} - -archives: - - formats: ["tar.gz"] - -release: - disable: true diff --git a/.tangled/workflows/build.yml b/.tangled/workflows/build.yml deleted file mode 100644 index 5f6ecd3..0000000 --- a/.tangled/workflows/build.yml +++ /dev/null @@ -1,35 +0,0 @@ -# Build workflow - test compilation on pull requests - -when: - - event: ["push", "pull_request"] - branch: ["main", "develop"] - -engine: "nixery" - -dependencies: - nixpkgs: - - go - - gcc - - git - - zstd - - zstd.dev - -environment: - CGO_ENABLED: "1" - GOPATH: "/workspace/go" - -steps: - - name: "Download dependencies" - command: | - go mod download - go mod verify - - - name: "Run tests" - command: | - go test -v ./... - - - name: "Build binary" - command: | - go build -o plcbundle ./cmd/plcbundle - ./plcbundle version - echo "✓ Build successful" diff --git a/.tangled/workflows/docker-release.yml b/.tangled/workflows/docker-release.yml deleted file mode 100644 index b2d00ed..0000000 --- a/.tangled/workflows/docker-release.yml +++ /dev/null @@ -1,72 +0,0 @@ -# .tangled/workflows/docker-release.yml - -when: - - event: ["push"] - branch: ["refs/tags/*"] # Trigger on any tag push - - event: ["manual"] - branch: ["main"] # Allow manual trigger from main - -engine: "nixery" - -dependencies: - nixpkgs: - - docker - - git - - bash - -environment: - DOCKER_REGISTRY: "docker.io" - DOCKER_IMAGE: "atscan/plcbundle" - -steps: - - name: "Extract version info" - command: | - # Extract version from tag or git - if [[ "${TANGLED_REF}" =~ ^refs/tags/(.+)$ ]]; then - VERSION="${BASH_REMATCH[1]}" - else - VERSION=$(git describe --tags --always --dirty) - fi - - GIT_COMMIT=$(git rev-parse --short HEAD) - BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") - - echo "VERSION=${VERSION}" >> $TANGLED_ENV - echo "GIT_COMMIT=${GIT_COMMIT}" >> $TANGLED_ENV - echo "BUILD_DATE=${BUILD_DATE}" >> $TANGLED_ENV - - echo "Building version: ${VERSION}" - echo "Commit: ${GIT_COMMIT}" - echo "Date: ${BUILD_DATE}" - - - name: "Set up Docker Buildx" - command: | - docker buildx create --use --name multiarch --driver docker-container - docker buildx inspect --bootstrap - - - name: "Login to Docker Hub" - command: | - echo "${DOCKER_PASSWORD}" | docker login -u "${DOCKER_USERNAME}" --password-stdin - environment: - DOCKER_USERNAME: "${{ secrets.DOCKER_USERNAME }}" - DOCKER_PASSWORD: "${{ secrets.DOCKER_PASSWORD }}" - - - name: "Build and push multi-platform images" - command: | - docker buildx build \ - --platform linux/amd64,linux/arm64 \ - --build-arg VERSION="${VERSION}" \ - --build-arg GIT_COMMIT="${GIT_COMMIT}" \ - --build-arg BUILD_DATE="${BUILD_DATE}" \ - --tag ${DOCKER_IMAGE}:${VERSION} \ - --tag ${DOCKER_IMAGE}:latest \ - --push \ - . - - echo "✓ Pushed ${DOCKER_IMAGE}:${VERSION}" - echo "✓ Pushed ${DOCKER_IMAGE}:latest" - - - name: "Verify images" - command: | - echo "Verifying pushed images..." - docker buildx imagetools inspect ${DOCKER_IMAGE}:${VERSION} diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 8c56ce2..0000000 --- a/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -# syntax=docker/dockerfile:1 - -FROM golang:1.25.3-alpine AS builder - -RUN apk add --no-cache git gcc musl-dev zstd-dev - -WORKDIR /build - -COPY go.mod go.sum ./ -RUN go mod download - -COPY . . - -ARG VERSION=dev -ARG GIT_COMMIT=unknown -ARG BUILD_DATE=unknown - -RUN CGO_ENABLED=1 go build \ - -ldflags="-w -s -X 'main.version=${VERSION}' -X 'main.gitCommit=${GIT_COMMIT}' -X 'main.buildDate=${BUILD_DATE}'" \ - -trimpath \ - -o plcbundle \ - ./cmd/plcbundle - -FROM alpine:3.19 - -RUN apk add --no-cache ca-certificates zstd-libs - -COPY --from=builder /build/plcbundle /usr/local/bin/plcbundle - -WORKDIR /data - -ENTRYPOINT ["plcbundle"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..8b4b832 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 ATScan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile deleted file mode 100644 index b9fb8aa..0000000 --- a/Makefile +++ /dev/null @@ -1,218 +0,0 @@ -.PHONY: all build install test clean fmt lint help version bump-patch bump-minor bump-major release release-build -.PHONY: docker-build docker-buildx docker-push docker-run docker-clean docker-shell compose-up compose-down compose-logs - -# Binary name -BINARY_NAME=plcbundle -INSTALL_PATH=$(GOPATH)/bin - -# Docker configuration -DOCKER_IMAGE=plcbundle -DOCKER_TAG=$(VERSION) -DOCKER_REGISTRY?=atscan -DOCKER_FULL_IMAGE=$(if $(DOCKER_REGISTRY),$(DOCKER_REGISTRY)/,)$(DOCKER_IMAGE):$(DOCKER_TAG) -DOCKER_LATEST=$(if $(DOCKER_REGISTRY),$(DOCKER_REGISTRY)/,)$(DOCKER_IMAGE):latest -DOCKER_PLATFORMS?=linux/amd64,linux/arm64 - -# Version information -VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev") -GIT_COMMIT := $(shell git rev-parse --short HEAD 2>/dev/null || echo "unknown") -BUILD_DATE := $(shell date -u +"%Y-%m-%dT%H:%M:%SZ") - -# Go commands -GOCMD=go -GOBUILD=$(GOCMD) build -GOINSTALL=$(GOCMD) install -GOCLEAN=$(GOCMD) clean -GOTEST=$(GOCMD) test -GOGET=$(GOCMD) get -GOFMT=$(GOCMD) fmt -GOMOD=$(GOCMD) mod - -# Test runner - auto-detect gotestsum -GOTESTSUM := $(shell command -v gotestsum 2> /dev/null) - -# Build flags -LDFLAGS=-ldflags "-X main.version=$(VERSION) -X main.gitCommit=$(GIT_COMMIT) -X main.buildDate=$(BUILD_DATE)" - -# Default target -all: build - -# Build the CLI tool with version info -build: - @echo "Building $(BINARY_NAME) $(VERSION)..." - $(GOBUILD) $(LDFLAGS) -o $(BINARY_NAME) ./cmd/plcbundle - -# Install the CLI tool globally -install: - @echo "Installing $(BINARY_NAME) ..." - $(GOINSTALL) $(LDFLAGS) ./cmd/plcbundle - -# Run tests -test: -ifdef GOTESTSUM - @gotestsum -- ./... -else - @echo "Running tests (install gotestsum for better output: go install gotest.tools/gotestsum@latest)" - @$(GOTEST) -v ./... -endif - -# Run tests with coverage -test-coverage: -ifdef GOTESTSUM - @gotestsum --format testname -- -cover ./... -else - @$(GOTEST) -v -cover ./... -endif - -# Clean build artifacts -clean: - @echo "Cleaning..." - $(GOCLEAN) - rm -f $(BINARY_NAME) - rm -rf dist/ - -# Format code -fmt: - @echo "Formatting code..." - $(GOFMT) ./... - -# Download dependencies -deps: - @echo "Downloading dependencies..." - $(GOMOD) download - $(GOMOD) tidy - -# Verify dependencies -verify: - @echo "Verifying dependencies..." - $(GOMOD) verify - -# Show current version -version: - @echo "Current version: $(VERSION)" - @echo "Git commit: $(GIT_COMMIT)" - @echo "Build date: $(BUILD_DATE)" - -# Bump patch version (0.1.0 -> 0.1.1) -bump-patch: - @echo "Bumping patch version..." - @./scripts/bump-version.sh patch - -# Bump minor version (0.1.0 -> 0.2.0) -bump-minor: - @echo "Bumping minor version..." - @./scripts/bump-version.sh minor - -# Bump major version (0.1.0 -> 1.0.0) -bump-major: - @echo "Bumping major version..." - @./scripts/bump-version.sh major - -# Bump alpha version (0.1.0 -> 0.2.0-alpha.1) -bump-alpha: - @echo "Bumping major version..." - @./scripts/bump-version.sh alpha - -# Create release -release: - @echo "Creating release for version $(VERSION)..." - @./scripts/release.sh - -# ============================================================================ -# Docker Commands -# ============================================================================ - -# Build Docker image (local) -docker-build: - @echo "Building Docker image $(DOCKER_FULL_IMAGE)..." - docker build \ - --build-arg VERSION=$(VERSION) \ - --build-arg GIT_COMMIT=$(GIT_COMMIT) \ - --build-arg BUILD_DATE="$(BUILD_DATE)" \ - -t $(DOCKER_FULL_IMAGE) \ - -t $(DOCKER_LATEST) \ - . - @echo "✓ Built: $(DOCKER_FULL_IMAGE)" - -# Build multi-platform and push -docker-buildx: - @echo "Building multi-platform image $(DOCKER_FULL_IMAGE)..." - docker buildx build \ - --platform $(DOCKER_PLATFORMS) \ - --build-arg VERSION=$(VERSION) \ - --build-arg GIT_COMMIT=$(GIT_COMMIT) \ - --build-arg BUILD_DATE="$(BUILD_DATE)" \ - --tag $(DOCKER_FULL_IMAGE) \ - --tag $(DOCKER_LATEST) \ - --push \ - . - @echo "✓ Built and pushed: $(DOCKER_FULL_IMAGE) ($(DOCKER_PLATFORMS))" - -# Push Docker image -docker-push: - @echo "Pushing Docker image..." - docker push $(DOCKER_FULL_IMAGE) - docker push $(DOCKER_LATEST) - @echo "✓ Pushed: $(DOCKER_FULL_IMAGE)" - -# Run Docker container as CLI -docker-run: - @docker run --rm -v $(PWD)/data:/data $(DOCKER_FULL_IMAGE) plcbundle $(CMD) - -# Shortcuts -docker-info: - @docker run --rm -v $(PWD)/data:/data $(DOCKER_FULL_IMAGE) plcbundle info - -docker-fetch: - @docker run --rm -v $(PWD)/data:/data $(DOCKER_FULL_IMAGE) plcbundle fetch - -docker-verify: - @docker run --rm -v $(PWD)/data:/data $(DOCKER_FULL_IMAGE) plcbundle verify - -# Run as server -docker-serve: - docker run --rm -it -p 8080:8080 -v $(PWD)/data:/data $(DOCKER_FULL_IMAGE) plcbundle serve --host 0.0.0.0 - -# Open shell -docker-shell: - docker run --rm -it -v $(PWD)/data:/data $(DOCKER_FULL_IMAGE) sh - -# Clean Docker artifacts -docker-clean: - @echo "Cleaning Docker images..." - docker rmi $(DOCKER_FULL_IMAGE) $(DOCKER_LATEST) 2>/dev/null || true - docker image prune -f - -# ============================================================================ -# Help -# ============================================================================ - -help: - @echo "Available targets:" - @echo "" - @echo "Build & Install:" - @echo " make build - Build the CLI tool" - @echo " make install - Install CLI tool globally" - @echo " make clean - Clean build artifacts" - @echo "" - @echo "Development:" - @echo " make test - Run tests" - @echo " make test-coverage - Run tests with coverage" - @echo " make fmt - Format code" - @echo " make deps - Download dependencies" - @echo "" - @echo "Release:" - @echo " make version - Show current version" - @echo " make bump-patch - Bump patch (0.1.0 -> 0.1.1)" - @echo " make bump-minor - Bump minor (0.1.0 -> 0.2.0)" - @echo " make bump-major - Bump major (0.1.0 -> 1.0.0)" - @echo " make release - Push tag to trigger release" - @echo "" - @echo "Docker:" - @echo " make docker-build - Build image (current platform)" - @echo " make docker-buildx - Build multi-platform and push" - @echo " make docker-run - Run CLI (CMD='info')" - @echo " make docker-info - Show bundle info" - @echo " make docker-serve - Run as server" - @echo " make docker-shell - Open shell" - @echo "" diff --git a/README.md b/README.md index c49a3f5..750c565 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,3 @@ -# PLC Bundle - ``` ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠄⠀⡀⠀⠀⠀⠀⠀⠀⢀⠀⠀⡀⠀⢀⠀⢀⡀⣤⡢⣤⡤⡀⡄⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⡄⡄⠐⡀⠈⣀⠀⡠⡠⠀⣢⣆⢌⡾⢙⠺⢽⠾⡋⣻⡷⡫⢵⣭⢦⣴⠦⠀⢠⠀⠀⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ @@ -26,153 +24,169 @@ ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠋⢀⠋⢈⡿⠿⠁⠉⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ ``` -> ⚠️ **Preview Version - Do Not Use in Production!** +# plcbundle -plcbundle archives AT Protocol's [DID PLC Directory](https://plc.directory/) operations into immutable, cryptographically-chained bundles of 10,000 operations. Each bundle is hashed (SHA-256), compressed (zstd), and linked to the previous bundle, creating a verifiable chain of DID operations. +> A Transparent and Verifiable Way to Sync the AT Protocol's PLC Directory -* 📄 [Technical Specification](./docs/specification.md) -* 📚 [Library Documentation](./docs/library.md) -* 💻 [CLI Guide](./docs/cli.md) -* 📰 [Announcement Article](https://leaflet.pub/feb982b4-64cb-4549-9d25-d7e68cecb11a) +plcbundle is an ecosystem of tools and specifications for archiving [AT Protocol](https://atproto.com/)'s [PLC Directory](https://plc.directory/) into verifiable, distributed bundles. Operations are grouped into immutable bundles of 10,000 each, cryptographically chained together, enabling anyone to host mirrors, verify data integrity, and analyze the DID ecosystem. -## What is plcbundle? +**Why plcbundle?** +- 🔒 **Verifiable** - Cryptographic chain ensures data integrity +- 🌐 **Distributed** - Host your own mirror, reduce central dependencies +- 📦 **Efficient** - ~5x compression, optimized for storage and transfer +- 🔍 **Transparent** - Open format, reproducible by anyone +- 🛠️ **Ecosystem** - Multiple implementations, tools, and libraries -plcbundle is a **format specification** for archiving PLC directory operations: +## Current Implementations -- **Bundling**: Groups 10,000 operations into compressed, immutable files -- **Chaining**: Each bundle is cryptographically linked to the previous one -- **Verifiable**: SHA-256 hashes ensure data integrity throughout the chain -- **Efficient**: Zstandard compression with ~5x compression ratios +| Repository | Language | Status | Features | Maintainer | +|-----------|----------|--------|----------|-------------| +| **[plcbundle-rs](https://tangled.org/atscan.net/plcbundle-rs)** | **Rust** | ✅ Active (unstable) | Library, CLI, Server, Sync | [@tree.fail](https://bsky.app/profile/tree.fail) | +| [plcbundle-go](https://tangled.org/atscan.net/plcbundle-go) | **Go** | *Deprecated* | Library, CLI, Server, Sync | [@tree.fail](https://bsky.app/profile/tree.fail) | +| [plcbundle-bun](https://tangled.org/atscan.net/plcbundle-bun) | **Bun (TypeScript)** | *Experiment* | Library, CLI | [@tree.fail](https://bsky.app/profile/tree.fail) | +| [plcbundle-ref](https://tangled.org/atscan.net/plcbundle-ref) | **TypeScript/Python/Ruby** | - | Examples | [@tree.fail](https://bsky.app/profile/tree.fail) | -### Implementations +## Quick Start -- **Go** (this repository) - Full-featured CLI tool, library, and HTTP server -- [TypeScript, Python, Ruby](https://tangled.org/@atscan.net/plcbundle-ref/) - Reference implementations +### Installation ---- +```bash +# Rust +cargo install plcbundle -## This Implementation (Go) +# Docker +docker pull atscan/plcbundle:latest +``` -This Go implementation provides: +### Basic Usage -- 📚 **Library** - Embed bundle operations in your Go applications -- 💻 **CLI Tool** - Command-line interface for all operations -- 🌐 **HTTP Server** - Host and serve bundles over HTTP -- 🔄 **Auto-sync** - Automatically fetch and bundle new operations -- 🔌 **WebSocket** - Stream operations in real-time -- 🔍 **Spam Detection** - Built-in and custom JavaScript detectors -- ⚡ **Performance** - Parallel processing, efficient compression -- 🐳 **Docker** - Ready-to-deploy containers +```bash +# Sync bundles from plc.directory +plcbundle sync -## Installation +# Verify chain integrity +plcbundle verify -### Go Install +# Get information +plcbundle info -```bash -# CLI tool -go install tangled.org/atscan.net/plcbundle/cmd/plcbundle@latest +# Get DID Document +plcbundle resolve tree.fail +plcbundle audit did:plc:524tuhdhh3m7li5gycdn6boe -# Library -go get tangled.org/atscan.net/plcbundle +# Run as HTTP mirror +plcbundle server --sync --resolver --port 8080 ``` -### Docker +[Full getting started guide →](./docs/getting-started.md) -[![Docker Hub](https://img.shields.io/docker/v/atscan/plcbundle?label=docker&logo=docker)](https://hub.docker.com/r/atscan/plcbundle) -[![Docker Image Size](https://img.shields.io/docker/image-size/atscan/plcbundle/latest)](https://hub.docker.com/r/atscan/plcbundle) -[![Docker Pulls](https://img.shields.io/docker/pulls/atscan/plcbundle)](https://hub.docker.com/r/atscan/plcbundle) +## How It Works -```bash -# Pull pre-built image -docker pull atscan/plcbundle:latest +``` +PLC Directory Bundle Chain Your Mirror + │ │ │ + │ Operations ┌────▼────┐ │ + │ 0-9,999 ────▶ │Bundle #0│ │ + │ │Hash: abc│──┐ │ + │ └─────────┘ │ │ + │ Operations │ Cryptographically │ + │ 10,000-19,999 ─────────────▶│ Linked │ + │ ┌────▼────┐ │ │ + │ │Bundle #1│◀─┘ │ + │ │Prev: abc│────────── Sync ────────▶ │ + │ │Hash: def│ │ + │ └─────────┘ │ + ▼ ▼ ▼ +``` -# Run CLI -docker run --rm -v $(pwd)/data:/data atscan/plcbundle info +Each bundle is: +- 📦 Exactly 10,000 operations +- 🔗 Linked to previous bundle via hash +- 🗜️ Compressed with zstd +- ✅ Independently verifiable +- 🔄 Reproducible from source -# Run as server -docker run -d -p 8080:8080 -v $(pwd)/data:/data atscan/plcbundle serve --host 0.0.0.0 +[Technical details →](./spec/format.md) -# Or use docker compose -curl -O https://tangled.org/@atscan.net/plcbundle/raw/main/docker-compose.yaml -docker compose up -d -``` +## Use Cases -### From Source +### 🪞 Host a Mirror +Run your own PLC directory mirror to reduce dependency on central infrastructure and improve availability. -```bash -git clone https://tangled.org/@atscan.net/plcbundle -cd plcbundle -make build -sudo make install -``` +### 🗄️ Archive & Backup +Create verifiable backups with cryptographic proof of integrity. Perfect for compliance and long-term archival. -## Quick Start +### 🔬 Research & Analytics +Analyze historical DID creation patterns, migrations, and network growth over time. -### As a Library +### 📊 Monitoring & Detection +Track operations, detect anomalies, identify spam patterns, monitor network health. -```go -import plcbundle "tangled.org/atscan.net/plcbundle" +## Specifications -mgr, _ := plcbundle.New("./plc_data", "https://plc.directory") -defer mgr.Close() +### Core Specifications +- **[Bundle Format](./spec/format.md)** - Bundle structure, encoding, and compression (v1.0) +- **[HTTP API](./spec/http-api.md)** - Server API for hosting mirrors (v1.0) -bundle, _ := mgr.FetchNext(context.Background()) -// Process bundle.Operations -``` +### Future Specifications +- WebSocket API - Real-time operation streaming +- P2P Protocol - Peer-to-peer bundle distribution -[See full library documentation →](./docs/library.md) +[All specifications →](./spec/) -### As a CLI Tool +## Related Tools -```bash -# Fetch bundles from plc.directory -plcbundle sync +*Building something? [Add it here](https://tangled.org/atscan.net/plcbundle/pulls)* -# Clone from remote -plcbundle clone https://plc.example.com +## Public Instances -# Verify integrity -plcbundle verify -``` +You can view all public instances on [plcbundle-watch.pages.dev](https://plcbundle-watch.pages.dev/) -[See full CLI reference →](./docs/cli.md) +*Host a mirror? [Submit a PR](https://tangled.org/atscan.net/plcbundle/plcbundle/pulls) to add it here* -### With Docker +## Security & Trust -```bash -# CLI usage -docker run --rm -v $(pwd)/data:/data plcbundle info +plcbundle provides cryptographic verification but requires external validation: -# Server mode -docker-compose up -d -curl http://localhost:8080/ -``` +**✅ Guarantees:** +- Bundle integrity via SHA-256 hashing +- Chain continuity via previous bundle links +- Reproducibility from source operations -## Use Cases +**⚠️ You must verify:** +- Operations against original plc.directory +- Bundles from multiple independent sources +- Published root and head hashes -- **Archiving**: Create verifiable backups of PLC operations -- **Mirroring**: Host your own PLC directory mirror -- **Research**: Analyze historical DID operations -- **Compliance**: Maintain tamper-evident audit trails +**Never trust a single source.** Always verify. -## Security Model +[Full security model →](./docs/security.md) -Bundles are cryptographically chained but require external verification: -- ✅ Verify against original PLC directory -- ✅ Compare with multiple independent mirrors -- ✅ Check published root and head hashes -- ✅ Anyone can reproduce bundles from PLC directory +## Contributing -## Documentation +We welcome contributions of all kinds: -- [Library Guide](./docs/library.md) - Comprehensive API documentation -- [CLI Guide](./docs/cli.md) - Command-line tool usage -- [Specification](./docs/specification.md) - Technical format specification +- 🔨 **Implementations** - Create clients in new languages ([guide](./docs/contributing.md)) +- 📖 **Documentation** - Improve guides, fix typos, add examples +- 🐛 **Issues** - [Report bugs](https://tangled.org/atscan/plcbundle/issues) or suggest features +- 🧪 **Test Vectors** - Help ensure implementation compatibility +- 🪞 **Mirrors** - Host bundles and share your endpoint +- 💬 **Discussion** - Share your ideas -## License +[Contributing guide →](./docs/contributing.md) -MIT +## Community & Support -## Contributing +- 🐛 [Issue Tracker](https://github.com/yourusername/plcbundle/issues) - Report bugs +- 📰 [Announcement Article](https://leaflet.pub/feb982b4-64cb-4549-9d25-d7e68cecb11a) - Background and motivation + +## Resources + +- 📐 [AT Protocol Documentation](https://atproto.com/) +- 🆔 [DID PLC Specification](https://web.plc.directory/) + +## License + +MIT License - See [LICENSE](./LICENSE) for details. -Contributions welcome! Please open an issue or PR. +Individual implementations may have different licenses - check their repositories. \ No newline at end of file diff --git a/bundle.go b/bundle.go deleted file mode 100644 index 37162ad..0000000 --- a/bundle.go +++ /dev/null @@ -1,57 +0,0 @@ -package plcbundle - -import ( - "context" - "io" - - "tangled.org/atscan.net/plcbundle/bundle" -) - -// Manager is the main entry point for plcbundle operations -type Manager struct { - internal *bundle.Manager -} - -// New creates a new plcbundle manager -func New(opts ...Option) (*Manager, error) { - config := defaultConfig() - for _, opt := range opts { - opt(config) - } - - mgr, err := bundle.NewManager(config.bundleConfig, config.plcClient) - if err != nil { - return nil, err - } - - return &Manager{internal: mgr}, nil -} - -// LoadBundle loads a bundle by number -func (m *Manager) LoadBundle(ctx context.Context, bundleNumber int) (*Bundle, error) { - b, err := m.internal.LoadBundle(ctx, bundleNumber) - if err != nil { - return nil, err - } - return toBundlePublic(b), nil -} - -// StreamBundleRaw streams raw compressed bundle data -func (m *Manager) StreamBundleRaw(ctx context.Context, bundleNumber int) (io.ReadCloser, error) { - return m.internal.StreamBundleRaw(ctx, bundleNumber) -} - -// Close closes the manager and releases resources -func (m *Manager) Close() error { - m.internal.Close() - return nil -} - -// FetchNextBundle fetches the next bundle from PLC directory -func (m *Manager) FetchNextBundle(ctx context.Context) (*Bundle, error) { - b, _, err := m.internal.FetchNextBundle(ctx, false, false) - if err != nil { - return nil, err - } - return toBundlePublic(b), nil -} diff --git a/bundle/bundle_test.go b/bundle/bundle_test.go deleted file mode 100644 index 9f943e4..0000000 --- a/bundle/bundle_test.go +++ /dev/null @@ -1,453 +0,0 @@ -package bundle_test - -import ( - "path/filepath" - "testing" - "time" - - "tangled.org/atscan.net/plcbundle/bundle" - "tangled.org/atscan.net/plcbundle/internal/bundleindex" - "tangled.org/atscan.net/plcbundle/internal/mempool" - "tangled.org/atscan.net/plcbundle/internal/plcclient" - "tangled.org/atscan.net/plcbundle/internal/storage" - "tangled.org/atscan.net/plcbundle/internal/types" -) - -var ( - bundleInfo = &storage.BundleInfo{ - BundleNumber: 1, - Origin: "test-origin", - ParentHash: "", - Cursor: "", - CreatedBy: "test", - Hostname: "test-host", - } -) - -// TestIndex tests index operations -func TestIndex(t *testing.T) { - - t.Run("CreateNewIndex", func(t *testing.T) { - idx := bundleindex.NewIndex("test-origin") - if idx == nil { - t.Fatal("NewIndex returned nil") - } - if idx.Version != types.INDEX_VERSION { - t.Errorf("expected version %s, got %s", types.INDEX_VERSION, idx.Version) - } - if idx.Count() != 0 { - t.Errorf("expected empty index, got count %d", idx.Count()) - } - }) - - t.Run("AddBundle", func(t *testing.T) { - idx := bundleindex.NewIndex("test-origin") - meta := &bundleindex.BundleMetadata{ - BundleNumber: 1, - StartTime: time.Now(), - EndTime: time.Now().Add(time.Hour), - OperationCount: types.BUNDLE_SIZE, - DIDCount: 1000, - Hash: "abc123", - CompressedHash: "def456", - } - - idx.AddBundle(meta) - - if idx.Count() != 1 { - t.Errorf("expected count 1, got %d", idx.Count()) - } - - retrieved, err := idx.GetBundle(1) - if err != nil { - t.Fatalf("GetBundle failed: %v", err) - } - if retrieved.Hash != meta.Hash { - t.Errorf("expected hash %s, got %s", meta.Hash, retrieved.Hash) - } - }) - - t.Run("SaveAndLoad", func(t *testing.T) { - tmpDir := t.TempDir() - indexPath := filepath.Join(tmpDir, "test_index.json") - - // Create and save - idx := bundleindex.NewIndex("test-origin") - idx.AddBundle(&bundleindex.BundleMetadata{ - BundleNumber: 1, - StartTime: time.Now(), - EndTime: time.Now().Add(time.Hour), - OperationCount: types.BUNDLE_SIZE, - Hash: "test123", - }) - - if err := idx.Save(indexPath); err != nil { - t.Fatalf("Save failed: %v", err) - } - - // Load - loaded, err := bundleindex.LoadIndex(indexPath) - if err != nil { - t.Fatalf("LoadIndex failed: %v", err) - } - - if loaded.Count() != 1 { - t.Errorf("expected count 1, got %d", loaded.Count()) - } - }) - - t.Run("GetBundleRange", func(t *testing.T) { - idx := bundleindex.NewIndex("test-origin") - for i := 1; i <= 5; i++ { - idx.AddBundle(&bundleindex.BundleMetadata{ - BundleNumber: i, - StartTime: time.Now(), - EndTime: time.Now().Add(time.Hour), - OperationCount: types.BUNDLE_SIZE, - }) - } - - bundles := idx.GetBundleRange(2, 4) - if len(bundles) != 3 { - t.Errorf("expected 3 bundles, got %d", len(bundles)) - } - if bundles[0].BundleNumber != 2 || bundles[2].BundleNumber != 4 { - t.Errorf("unexpected bundle range") - } - }) - - t.Run("FindGaps", func(t *testing.T) { - idx := bundleindex.NewIndex("test-origin") - // Add bundles 1, 2, 4, 5 (missing 3) - for _, num := range []int{1, 2, 4, 5} { - idx.AddBundle(&bundleindex.BundleMetadata{ - BundleNumber: num, - StartTime: time.Now(), - EndTime: time.Now().Add(time.Hour), - OperationCount: types.BUNDLE_SIZE, - }) - } - - gaps := idx.FindGaps() - if len(gaps) != 1 { - t.Errorf("expected 1 gap, got %d", len(gaps)) - } - if len(gaps) > 0 && gaps[0] != 3 { - t.Errorf("expected gap at 3, got %d", gaps[0]) - } - }) -} - -// TestBundle tests bundle operations -func TestBundle(t *testing.T) { - t.Run("ValidateForSave", func(t *testing.T) { - tests := []struct { - name string - bundle *bundle.Bundle - wantErr bool - }{ - { - name: "valid bundle", - bundle: &bundle.Bundle{ - BundleNumber: 1, - StartTime: time.Now(), - EndTime: time.Now().Add(time.Hour), - Operations: makeTestOperations(types.BUNDLE_SIZE), - }, - wantErr: false, - }, - { - name: "invalid bundle number", - bundle: &bundle.Bundle{ - BundleNumber: 0, - Operations: makeTestOperations(types.BUNDLE_SIZE), - }, - wantErr: true, - }, - { - name: "wrong operation count", - bundle: &bundle.Bundle{ - BundleNumber: 1, - Operations: makeTestOperations(100), - }, - wantErr: true, - }, - { - name: "start after end", - bundle: &bundle.Bundle{ - BundleNumber: 1, - StartTime: time.Now().Add(time.Hour), - EndTime: time.Now(), - Operations: makeTestOperations(types.BUNDLE_SIZE), - }, - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := tt.bundle.ValidateForSave() - if (err != nil) != tt.wantErr { - t.Errorf("ValidateForSave() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } - }) - - t.Run("CompressionRatio", func(t *testing.T) { - b := &bundle.Bundle{ - CompressedSize: 1000, - UncompressedSize: 5000, - } - ratio := b.CompressionRatio() - if ratio != 5.0 { - t.Errorf("expected ratio 5.0, got %f", ratio) - } - }) -} - -// TestMempool tests mempool operations -func TestMempool(t *testing.T) { - tmpDir := t.TempDir() - logger := &testLogger{t: t} - - t.Run("CreateAndAdd", func(t *testing.T) { - minTime := time.Now().Add(-time.Hour) - m, err := mempool.NewMempool(tmpDir, 1, minTime, logger, false) - if err != nil { - t.Fatalf("NewMempool failed: %v", err) - } - - ops := makeTestOperations(100) - added, err := m.Add(ops) - if err != nil { - t.Fatalf("Add failed: %v", err) - } - if added != 100 { - t.Errorf("expected 100 added, got %d", added) - } - if m.Count() != 100 { - t.Errorf("expected count 100, got %d", m.Count()) - } - }) - - t.Run("ChronologicalValidation", func(t *testing.T) { - minTime := time.Now().Add(-time.Hour) - m, err := mempool.NewMempool(tmpDir, 2, minTime, logger, false) - if err != nil { - t.Fatalf("NewMempool failed: %v", err) - } - - // Add operations in order - ops := makeTestOperations(10) - _, err = m.Add(ops) - if err != nil { - t.Fatalf("Add failed: %v", err) - } - - // Try to add operation before last one (should fail) - oldOp := []plcclient.PLCOperation{ - { - DID: "did:plc:old", - CID: "old123", - CreatedAt: time.Now().Add(-2 * time.Hour), - }, - } - _, err = m.Add(oldOp) - if err == nil { - t.Error("expected chronological validation error") - } - }) - - t.Run("TakeOperations", func(t *testing.T) { - minTime := time.Now().Add(-time.Hour) - m, err := mempool.NewMempool(tmpDir, 3, minTime, logger, false) - if err != nil { - t.Fatalf("NewMempool failed: %v", err) - } - - ops := makeTestOperations(100) - m.Add(ops) - - taken, err := m.Take(50) - if err != nil { - t.Fatalf("Take failed: %v", err) - } - if len(taken) != 50 { - t.Errorf("expected 50 operations, got %d", len(taken)) - } - if m.Count() != 50 { - t.Errorf("expected 50 remaining, got %d", m.Count()) - } - }) - - t.Run("SaveAndLoad", func(t *testing.T) { - minTime := time.Now().Add(-time.Hour) - m, err := mempool.NewMempool(tmpDir, 4, minTime, logger, false) - if err != nil { - t.Fatalf("NewMempool failed: %v", err) - } - - ops := makeTestOperations(50) - m.Add(ops) - - if err := m.Save(); err != nil { - t.Fatalf("Save failed: %v", err) - } - - // Create new mempool and load - m2, err := mempool.NewMempool(tmpDir, 4, minTime, logger, false) - if err != nil { - t.Fatalf("NewMempool failed: %v", err) - } - - if m2.Count() != 50 { - t.Errorf("expected 50 operations after load, got %d", m2.Count()) - } - }) - - t.Run("Validate", func(t *testing.T) { - minTime := time.Now().Add(-time.Hour) - m, err := mempool.NewMempool(tmpDir, 5, minTime, logger, false) - if err != nil { - t.Fatalf("NewMempool failed: %v", err) - } - - ops := makeTestOperations(10) - m.Add(ops) - - if err := m.Validate(); err != nil { - t.Errorf("Validate failed: %v", err) - } - }) -} - -// TestOperations tests low-level operations -func TestOperations(t *testing.T) { - tmpDir := t.TempDir() - logger := &testLogger{t: t} - - ops, err := storage.NewOperations(logger, false) - if err != nil { - t.Fatalf("NewOperations failed: %v", err) - } - defer ops.Close() - - t.Run("SerializeJSONL", func(t *testing.T) { - operations := makeTestOperations(10) - data := ops.SerializeJSONL(operations) - if len(data) == 0 { - t.Error("SerializeJSONL returned empty data") - } - }) - - t.Run("Hash", func(t *testing.T) { - data := []byte("test data") - hash := ops.Hash(data) - if len(hash) != 64 { // SHA256 hex = 64 chars - t.Errorf("expected hash length 64, got %d", len(hash)) - } - - // Same data should produce same hash - hash2 := ops.Hash(data) - if hash != hash2 { - t.Error("same data produced different hashes") - } - }) - - t.Run("SaveAndLoadBundle", func(t *testing.T) { - operations := makeTestOperations(types.BUNDLE_SIZE) - path := filepath.Join(tmpDir, "test_bundle.jsonl.zst") - - // Save - uncompHash, compHash, uncompSize, compSize, err := ops.SaveBundle(path, operations, bundleInfo) - if err != nil { - t.Fatalf("SaveBundle failed: %v", err) - } - - if uncompHash == "" || compHash == "" { - t.Error("empty hashes returned") - } - if uncompSize == 0 || compSize == 0 { - t.Error("zero sizes returned") - } - if compSize >= uncompSize { - t.Error("compressed size should be smaller than uncompressed") - } - - // Load - loaded, err := ops.LoadBundle(path) - if err != nil { - t.Fatalf("LoadBundle failed: %v", err) - } - - if len(loaded) != len(operations) { - t.Errorf("expected %d operations, got %d", len(operations), len(loaded)) - } - }) - - t.Run("ExtractUniqueDIDs", func(t *testing.T) { - operations := []plcclient.PLCOperation{ - {DID: "did:plc:1"}, - {DID: "did:plc:2"}, - {DID: "did:plc:1"}, // duplicate - {DID: "did:plc:3"}, - } - - dids := ops.ExtractUniqueDIDs(operations) - if len(dids) != 3 { - t.Errorf("expected 3 unique DIDs, got %d", len(dids)) - } - }) - - t.Run("GetBoundaryCIDs", func(t *testing.T) { - baseTime := time.Now() - operations := []plcclient.PLCOperation{ - {CID: "cid1", CreatedAt: baseTime}, - {CID: "cid2", CreatedAt: baseTime.Add(time.Second)}, - {CID: "cid3", CreatedAt: baseTime.Add(2 * time.Second)}, - {CID: "cid4", CreatedAt: baseTime.Add(2 * time.Second)}, // same as cid3 - {CID: "cid5", CreatedAt: baseTime.Add(2 * time.Second)}, // same as cid3 - } - - boundaryTime, cids := ops.GetBoundaryCIDs(operations) - if !boundaryTime.Equal(baseTime.Add(2 * time.Second)) { - t.Error("unexpected boundary time") - } - if len(cids) != 3 { // cid3, cid4, cid5 - t.Errorf("expected 3 boundary CIDs, got %d", len(cids)) - } - }) -} - -// Helper functions - -func makeTestOperations(count int) []plcclient.PLCOperation { - ops := make([]plcclient.PLCOperation, count) - baseTime := time.Now().Add(-time.Hour) - - for i := 0; i < count; i++ { - ops[i] = plcclient.PLCOperation{ - DID: "did:plc:test" + string(rune(i)), - CID: "bafytest" + string(rune(i)), - CreatedAt: baseTime.Add(time.Duration(i) * time.Second), - /*Operation: map[string]interface{}{ - "type": "create", - },*/ - } - } - - return ops -} - -type testLogger struct { - t *testing.T -} - -func (l *testLogger) Printf(format string, v ...interface{}) { - l.t.Logf(format, v...) -} - -func (l *testLogger) Println(v ...interface{}) { - l.t.Log(v...) -} diff --git a/bundle/helpers.go b/bundle/helpers.go deleted file mode 100644 index 4799090..0000000 --- a/bundle/helpers.go +++ /dev/null @@ -1,43 +0,0 @@ -package bundle - -import ( - "fmt" - "time" -) - -// formatTimeDistance formats a duration as "X ago" or "live" -func formatTimeDistance(d time.Duration) string { - if d < 10*time.Second { - return "live" - } - if d < time.Minute { - return fmt.Sprintf("%ds ago", int(d.Seconds())) - } - if d < time.Hour { - return fmt.Sprintf("%dm ago", int(d.Minutes())) - } - if d < 24*time.Hour { - hours := int(d.Hours()) - mins := int(d.Minutes()) % 60 - if mins > 0 { - return fmt.Sprintf("%dh%dm ago", hours, mins) - } - return fmt.Sprintf("%dh ago", hours) - } - days := int(d.Hours() / 24) - if days == 1 { - return "1 day ago" - } - if days < 7 { - return fmt.Sprintf("%d days ago", days) - } - weeks := days / 7 - if weeks < 4 { - return fmt.Sprintf("%d weeks ago", weeks) - } - months := days / 30 - if months < 12 { - return fmt.Sprintf("%d months ago", months) - } - return fmt.Sprintf("%.1f years ago", float64(days)/365) -} diff --git a/bundle/manager.go b/bundle/manager.go deleted file mode 100644 index 3a2ab3d..0000000 --- a/bundle/manager.go +++ /dev/null @@ -1,1862 +0,0 @@ -package bundle - -import ( - "context" - "fmt" - "io" - "log" - "os" - "path/filepath" - "runtime" - "runtime/debug" - "sort" - "strings" - "sync" - "sync/atomic" - "time" - - "tangled.org/atscan.net/plcbundle/internal/bundleindex" - "tangled.org/atscan.net/plcbundle/internal/didindex" - "tangled.org/atscan.net/plcbundle/internal/handleresolver" - "tangled.org/atscan.net/plcbundle/internal/mempool" - "tangled.org/atscan.net/plcbundle/internal/plcclient" - "tangled.org/atscan.net/plcbundle/internal/storage" - internalsync "tangled.org/atscan.net/plcbundle/internal/sync" - "tangled.org/atscan.net/plcbundle/internal/types" -) - -// defaultLogger is a simple logger implementation -type defaultLogger struct{} - -func (d defaultLogger) Printf(format string, v ...interface{}) { - log.Printf(format, v...) -} - -func (d defaultLogger) Println(v ...interface{}) { - log.Println(v...) -} - -// Manager handles bundle operations -type Manager struct { - config *Config - operations *storage.Operations - index *bundleindex.Index - indexPath string - logger types.Logger - mempool *mempool.Mempool - didIndex *didindex.Manager - - plcClient *plcclient.Client - handleResolver *handleresolver.Client - - syncer *internalsync.Fetcher - cloner *internalsync.Cloner - - bundleCache map[int]*Bundle - cacheMu sync.RWMutex - maxCacheSize int - - // Resolver performance tracking - resolverStats struct { - sync.Mutex - totalResolutions int64 - mempoolHits int64 - bundleHits int64 - errors int64 - - // Timing (in microseconds) - totalTime int64 - totalMempoolTime int64 - totalIndexTime int64 - totalLoadOpTime int64 - - // Recent timings (circular buffer) - recentTimes []resolverTiming - recentIdx int - recentSize int - } -} - -// NewManager creates a new bundle manager -func NewManager(config *Config, plcClient *plcclient.Client) (*Manager, error) { - if config == nil { - config = DefaultConfig("./plc_bundles") - } - - if config.Logger == nil { - config.Logger = defaultLogger{} - } - - // CHECK: Don't auto-create if repository doesn't exist - repoExists := repositoryExists(config.BundleDir) - - if !repoExists && !config.AutoInit { - return nil, fmt.Errorf( - "no plcbundle repository found in: %s\n\n"+ - "Initialize a new repository with:\n"+ - " plcbundle clone # Clone from remote\n"+ - " plcbundle sync # Fetch from PLC directory", - config.BundleDir, - ) - } - - // Ensure directory exists (only if repo exists OR AutoInit is enabled) - if err := os.MkdirAll(config.BundleDir, 0755); err != nil { - return nil, fmt.Errorf("failed to create bundle directory: %w", err) - } - - // Initialize operations handler - ops, err := storage.NewOperations(config.Logger, config.Verbose) - if err != nil { - return nil, fmt.Errorf("failed to initialize operations: %w", err) - } - - // Determine origin - var origin string - if plcClient != nil { - origin = plcClient.GetBaseURL() - } - - // Load or create index - indexPath := filepath.Join(config.BundleDir, bundleindex.INDEX_FILE) - index, err := bundleindex.LoadIndex(indexPath) - - // Check for bundle files in directory - bundleFiles, _ := filepath.Glob(filepath.Join(config.BundleDir, "*.jsonl.zst")) - bundleFiles = filterBundleFiles(bundleFiles) - hasBundleFiles := len(bundleFiles) > 0 - - // Check if clone/download is in progress (look for .tmp files) - tmpFiles, _ := filepath.Glob(filepath.Join(config.BundleDir, "*.tmp")) - cloneInProgress := len(tmpFiles) > 0 - - needsRebuild := false - - if err != nil { - // Index doesn't exist or is invalid - if hasBundleFiles { - if cloneInProgress { - config.Logger.Printf("Clone/download in progress, skipping auto-rebuild") - } else { - config.Logger.Printf("No valid index found, but detected %d bundle files", len(bundleFiles)) - needsRebuild = true - } - } else { - // No index and no bundles - create fresh index - config.Logger.Printf("Creating new index at %s", indexPath) - index = bundleindex.NewIndex(origin) - if err := index.Save(indexPath); err != nil { - return nil, fmt.Errorf("failed to save new index: %w", err) - } - } - } else { - // Index exists - auto-populate origin if missing - if index.Origin == "" { - if origin != "" { - config.Logger.Printf("⚠️ Upgrading old index: setting origin to %s", origin) - index.Origin = origin - if err := index.Save(indexPath); err != nil { - return nil, fmt.Errorf("failed to update index with origin: %w", err) - } - } else { - config.Logger.Printf("⚠️ Warning: index has no origin and no PLC client configured") - } - } - - // Validate origin matches if both are set - if index.Origin != "" && origin != "" && index.Origin != origin { - return nil, fmt.Errorf( - "origin mismatch: index has origin %q but PLC client points to %q\n"+ - "Cannot mix bundles from different sources. Use a different directory or reconfigure PLC client", - index.Origin, origin, - ) - } - - if config.Verbose { - config.Logger.Printf("Loaded index with %d bundles (origin: %s)", index.Count(), index.Origin) - } - - // Check if there are bundle files not in the index - if hasBundleFiles && len(bundleFiles) > index.Count() { - if cloneInProgress { - config.Logger.Printf("Clone/download in progress (%d .tmp files), skipping auto-rebuild", len(tmpFiles)) - } else { - config.Logger.Printf("Detected %d bundle files but index only has %d entries - rebuilding", - len(bundleFiles), index.Count()) - needsRebuild = true - } - } - } - - if index != nil && plcClient != nil { - currentOrigin := plcClient.GetBaseURL() - - // Check if origins match - if index.Origin != "" && index.Origin != currentOrigin { - return nil, fmt.Errorf( - "origin mismatch: index has origin %q but PLC client points to %q. "+ - "Cannot mix bundles from different sources", - index.Origin, currentOrigin, - ) - } - - // Set origin if not set (for backward compatibility with old indexes) - if index.Origin == "" && currentOrigin != "" { - index.Origin = currentOrigin - config.Logger.Printf("Setting origin for existing index: %s", currentOrigin) - if err := index.Save(indexPath); err != nil { - return nil, fmt.Errorf("failed to update index with origin: %w", err) - } - } - } - - // Perform rebuild if needed (using parallel scan) - if needsRebuild && config.AutoRebuild { - config.Logger.Printf("Rebuilding index from %d bundle files...", len(bundleFiles)) - - // Create temporary manager for scanning - tempMgr := &Manager{ - config: config, - operations: ops, - index: bundleindex.NewIndex("test-origin"), - indexPath: indexPath, - logger: config.Logger, - } - - // Use parallel scan with auto-detected CPU count - workers := config.RebuildWorkers - if workers <= 0 { - workers = runtime.NumCPU() - if workers < 1 { - workers = 1 - } - } - - config.Logger.Printf("Using %d workers for parallel scan", workers) - - // Create progress callback wrapper with new signature - var progressCallback func(current, total int, bytesProcessed int64) - if config.RebuildProgress != nil { - // Wrap the old-style callback to work with new signature - oldCallback := config.RebuildProgress - progressCallback = func(current, total int, bytesProcessed int64) { - oldCallback(current, total) - } - } else { - // Default: log every 100 bundles - progressCallback = func(current, total int, bytesProcessed int64) { - if current%100 == 0 || current == total { - mbProcessed := float64(bytesProcessed) / (1024 * 1024) - config.Logger.Printf("Rebuild progress: %d/%d bundles (%.1f%%), %.1f MB processed", - current, total, float64(current)/float64(total)*100, mbProcessed) - } - } - } - - start := time.Now() - - // Scan directory to rebuild index (parallel) - result, err := tempMgr.ScanDirectoryParallel(workers, progressCallback) - if err != nil { - return nil, fmt.Errorf("failed to rebuild index: %w", err) - } - - elapsed := time.Since(start) - - // Reload the rebuilt index - index, err = bundleindex.LoadIndex(indexPath) - if err != nil { - return nil, fmt.Errorf("failed to load rebuilt index: %w", err) - } - - // Calculate throughput - mbPerSec := float64(result.TotalUncompressed) / elapsed.Seconds() / (1024 * 1024) - - config.Logger.Printf("✓ Index rebuilt with %d bundles in %s", - index.Count(), elapsed.Round(time.Millisecond)) - config.Logger.Printf(" Speed: %.1f bundles/sec, %.1f MB/s (uncompressed)", - float64(result.BundleCount)/elapsed.Seconds(), mbPerSec) - - // Verify all chain hashes are present - bundles := index.GetBundles() - missingHashes := 0 - for i, meta := range bundles { - if meta.ContentHash == "" { - missingHashes++ - } - if i > 0 && meta.Hash == "" { - missingHashes++ - } - } - if missingHashes > 0 { - config.Logger.Printf("⚠️ Warning: %d bundles have missing hashes", missingHashes) - } - } - - if index == nil { - index = bundleindex.NewIndex("test-origin") - } - - // Initialize mempool for next bundle - lastBundle := index.GetLastBundle() - nextBundleNum := 1 - var minTimestamp time.Time - - if lastBundle != nil { - nextBundleNum = lastBundle.BundleNumber + 1 - minTimestamp = lastBundle.EndTime - } - - mempool, err := mempool.NewMempool(config.BundleDir, nextBundleNum, minTimestamp, config.Logger, config.Verbose) - if err != nil { - return nil, fmt.Errorf("failed to initialize mempool: %w", err) - } - - // Initialize DID index manager - didIndex := didindex.NewManager(config.BundleDir, config.Logger) - - // Initialize sync components - fetcher := internalsync.NewFetcher(plcClient, ops, config.Logger) - cloner := internalsync.NewCloner(ops, config.BundleDir, config.Logger) - - // Initialize handle resolver if configured - var handleResolver *handleresolver.Client - if config.HandleResolverURL != "" { - handleResolver = handleresolver.NewClient(config.HandleResolverURL) - } - - m := &Manager{ - config: config, - operations: ops, - index: index, - indexPath: indexPath, - logger: config.Logger, - mempool: mempool, - didIndex: didIndex, // Updated type - bundleCache: make(map[int]*Bundle), - maxCacheSize: 10, - syncer: fetcher, - cloner: cloner, - plcClient: plcClient, - handleResolver: handleResolver, - } - // Initialize resolver stats - m.resolverStats.recentSize = 1000 - m.resolverStats.recentTimes = make([]resolverTiming, 1000) - - return m, nil -} - -// Close cleans up resources -func (m *Manager) Close() { - if m.operations != nil { - m.operations.Close() - } - if m.plcClient != nil { - m.plcClient.Close() - } - if m.mempool != nil { - if err := m.mempool.Save(); err != nil { - m.logger.Printf("Warning: failed to save mempool: %v", err) - } - } - if m.didIndex != nil { - m.didIndex.Close() - } -} - -// GetIndex returns the current index -func (m *Manager) GetIndex() *bundleindex.Index { - return m.index -} - -// SaveIndex saves the index to disk -func (m *Manager) SaveIndex() error { - return m.index.Save(m.indexPath) -} - -// LoadBundle with caching -func (m *Manager) LoadBundle(ctx context.Context, bundleNumber int) (*Bundle, error) { - // Check cache first - m.cacheMu.RLock() - if cached, ok := m.bundleCache[bundleNumber]; ok { - m.cacheMu.RUnlock() - return cached, nil - } - m.cacheMu.RUnlock() - - // Load from disk (existing code) - bundle, err := m.loadBundleFromDisk(ctx, bundleNumber) - if err != nil { - return nil, err - } - - // Add to cache - m.cacheMu.Lock() - m.bundleCache[bundleNumber] = bundle - - // Simple LRU: if cache too big, remove oldest - if len(m.bundleCache) > m.maxCacheSize { - // Remove a random one (or implement proper LRU) - for k := range m.bundleCache { - delete(m.bundleCache, k) - break - } - } - m.cacheMu.Unlock() - - return bundle, nil -} - -// loadBundleFromDisk loads a bundle from disk -func (m *Manager) loadBundleFromDisk(_ context.Context, bundleNumber int) (*Bundle, error) { - // Get metadata from index - meta, err := m.index.GetBundle(bundleNumber) - if err != nil { - return nil, fmt.Errorf("bundle not in index: %w", err) - } - - // Load file - path := filepath.Join(m.config.BundleDir, fmt.Sprintf("%06d.jsonl.zst", bundleNumber)) - if !m.operations.FileExists(path) { - return nil, fmt.Errorf("bundle file not found: %s", path) - } - - // Verify hash if enabled - if m.config.VerifyOnLoad { - valid, actualHash, err := m.operations.VerifyHash(path, meta.CompressedHash) - if err != nil { - return nil, fmt.Errorf("failed to verify hash: %w", err) - } - if !valid { - return nil, fmt.Errorf("hash mismatch: expected %s, got %s", meta.CompressedHash, actualHash) - } - } - - // Load operations - operations, err := m.operations.LoadBundle(path) - if err != nil { - return nil, fmt.Errorf("failed to load bundle: %w", err) - } - - // Create bundle struct - bundle := &Bundle{ - BundleNumber: meta.BundleNumber, - StartTime: meta.StartTime, - EndTime: meta.EndTime, - Operations: operations, - DIDCount: meta.DIDCount, - Hash: meta.Hash, - ContentHash: meta.ContentHash, - Parent: meta.Parent, - CompressedHash: meta.CompressedHash, - CompressedSize: meta.CompressedSize, - UncompressedSize: meta.UncompressedSize, - Cursor: meta.Cursor, - Compressed: true, - CreatedAt: meta.CreatedAt, - } - - return bundle, nil -} - -// SaveBundle saves a bundle to disk and updates the index -func (m *Manager) SaveBundle(ctx context.Context, bundle *Bundle, verbose bool, quiet bool, stats types.BundleProductionStats, skipDIDIndex bool) (time.Duration, error) { - - totalStart := time.Now() - if err := bundle.ValidateForSave(); err != nil { - return 0, fmt.Errorf("bundle validation failed: %w", err) - } - - path := filepath.Join(m.config.BundleDir, fmt.Sprintf("%06d.jsonl.zst", bundle.BundleNumber)) - - // Get parent - var parent string - if bundle.BundleNumber > 1 { - prevBundle := m.index.GetLastBundle() - if prevBundle != nil { - parent = prevBundle.Hash - } else { - if prevMeta, err := m.index.GetBundle(bundle.BundleNumber - 1); err == nil { - parent = prevMeta.Hash - } - } - } - bundle.Parent = parent - - // Get origin - origin := m.index.Origin - if m.plcClient != nil { - origin = m.plcClient.GetBaseURL() - } - - // Get version - version := "dev" - if info, ok := debug.ReadBuildInfo(); ok && info.Main.Version != "" && info.Main.Version != "(devel)" { - version = info.Main.Version - } - - // Get hostname - hostname, _ := os.Hostname() - - // Create BundleInfo - bundleInfo := &storage.BundleInfo{ - BundleNumber: bundle.BundleNumber, - Origin: origin, - ParentHash: parent, - Cursor: bundle.Cursor, - CreatedBy: fmt.Sprintf("plcbundle/%s", version), - Hostname: hostname, - } - - if m.config.Verbose { - m.logger.Printf("DEBUG: Calling operations.SaveBundle with bundle=%d", bundleInfo.BundleNumber) - } - - // Save to disk with 3 parameters - uncompressedHash, compressedHash, uncompressedSize, compressedSize, err := m.operations.SaveBundle(path, bundle.Operations, bundleInfo) - if err != nil { - m.logger.Printf("DEBUG: SaveBundle FAILED: %v", err) - return 0, fmt.Errorf("failed to save bundle: %w", err) - } - - if m.config.Verbose { - m.logger.Printf("DEBUG: SaveBundle SUCCESS, setting bundle fields") - } - - bundle.Hash = m.operations.CalculateChainHash(parent, bundle.ContentHash) - bundle.ContentHash = uncompressedHash - bundle.CompressedHash = compressedHash - bundle.UncompressedSize = uncompressedSize - bundle.CompressedSize = compressedSize - bundle.CreatedAt = time.Now().UTC() - bundle.Hash = m.operations.CalculateChainHash(parent, bundle.ContentHash) - - if m.config.Verbose { - m.logger.Printf("DEBUG: Adding bundle %d to index", bundle.BundleNumber) - } - - // Add to index - m.index.AddBundle(bundle.ToMetadata()) - - if m.config.Verbose { - m.logger.Printf("DEBUG: Index now has %d bundles", m.index.Count()) - } - - // Save index - if err := m.SaveIndex(); err != nil { - m.logger.Printf("DEBUG: SaveIndex FAILED: %v", err) - return 0, fmt.Errorf("failed to save index: %w", err) - } - - if m.config.Verbose { - m.logger.Printf("DEBUG: Index saved, last bundle = %d", m.index.GetLastBundle().BundleNumber) - } - - saveDuration := time.Since(totalStart) - - // Clean up old mempool - oldMempoolFile := m.mempool.GetFilename() - if err := m.mempool.Delete(); err != nil && !quiet { - m.logger.Printf("Warning: failed to delete old mempool %s: %v", oldMempoolFile, err) - } - - // Create new mempool - nextBundle := bundle.BundleNumber + 1 - minTimestamp := bundle.EndTime - - newMempool, err := mempool.NewMempool(m.config.BundleDir, nextBundle, minTimestamp, m.logger, m.config.Verbose) - if err != nil { - return 0, fmt.Errorf("failed to create new mempool: %w", err) - } - - m.mempool = newMempool - - // DID index update (if enabled) - var indexUpdateDuration time.Duration - if !skipDIDIndex && m.didIndex != nil && m.didIndex.Exists() { - indexUpdateStart := time.Now() - if err := m.updateDIDIndexForBundle(ctx, bundle); err != nil { - m.logger.Printf("Warning: failed to update DID index: %v", err) - } else { - indexUpdateDuration = time.Since(indexUpdateStart) - if !quiet && m.config.Verbose { - m.logger.Printf(" [DID Index] Updated in %s", indexUpdateDuration) - } - } - } - - if !quiet { - msg := fmt.Sprintf("→ Bundle %06d | %s | fetch: %s (%d reqs)", - bundle.BundleNumber, - bundle.Hash[0:7], - stats.TotalDuration.Round(time.Millisecond), - stats.TotalFetches, - ) - if indexUpdateDuration > 0 { - msg += fmt.Sprintf(" | index: %s", indexUpdateDuration.Round(time.Millisecond)) - } - msg += fmt.Sprintf(" | %s", formatTimeDistance(time.Since(bundle.EndTime))) - m.logger.Println(msg) - } - - if m.config.Verbose { - m.logger.Printf("DEBUG: Bundle done = %d, finish duration = %s", - m.index.GetLastBundle().BundleNumber, - saveDuration.Round(time.Millisecond)) - } - - return indexUpdateDuration, nil -} - -// GetMempoolStats returns mempool statistics -func (m *Manager) GetMempoolStats() map[string]interface{} { - return m.mempool.Stats() -} - -// GetMempoolOperations returns all operations currently in mempool -func (m *Manager) GetMempoolOperations() ([]plcclient.PLCOperation, error) { - if m.mempool == nil { - return nil, fmt.Errorf("mempool not initialized") - } - - count := m.mempool.Count() - if count == 0 { - return []plcclient.PLCOperation{}, nil - } - - return m.mempool.Peek(count), nil -} - -// Add to Bundle type to implement BundleData interface -func (b *Bundle) GetBundleNumber() int { - return b.BundleNumber -} - -// VerifyBundle verifies a bundle's integrity -func (m *Manager) VerifyBundle(ctx context.Context, bundleNumber int) (*VerificationResult, error) { - result := &VerificationResult{ - BundleNumber: bundleNumber, - } - - // Get from index - meta, err := m.index.GetBundle(bundleNumber) - if err != nil { - result.Error = err - return result, nil - } - - result.ExpectedHash = meta.CompressedHash - - // Check file exists - path := filepath.Join(m.config.BundleDir, fmt.Sprintf("%06d.jsonl.zst", bundleNumber)) - result.FileExists = m.operations.FileExists(path) - if !result.FileExists { - result.Error = fmt.Errorf("file not found") - return result, nil - } - - // Verify BOTH compressed and content hashes - compHash, compSize, contentHash, contentSize, err := m.operations.CalculateFileHashes(path) - if err != nil { - result.Error = err - return result, nil - } - - result.LocalHash = compHash - - // Verify compressed hash - if compHash != meta.CompressedHash { - result.HashMatch = false - result.Valid = false - result.Error = fmt.Errorf("compressed hash mismatch: expected %s, got %s", meta.CompressedHash, compHash) - return result, nil - } - - // Verify content hash - if contentHash != meta.ContentHash { - result.HashMatch = false - result.Valid = false - result.Error = fmt.Errorf("content hash mismatch: expected %s, got %s", meta.ContentHash, contentHash) - return result, nil - } - - // Verify sizes match - if compSize != meta.CompressedSize { - result.Valid = false - result.Error = fmt.Errorf("compressed size mismatch: expected %d, got %d", meta.CompressedSize, compSize) - return result, nil - } - - if contentSize != meta.UncompressedSize { - result.Valid = false - result.Error = fmt.Errorf("uncompressed size mismatch: expected %d, got %d", meta.UncompressedSize, contentSize) - return result, nil - } - - result.HashMatch = true - result.Valid = true - - return result, nil -} - -// GetInfo returns information about the bundle manager -func (m *Manager) GetInfo() map[string]interface{} { - stats := m.index.GetStats() - stats["bundle_dir"] = m.config.BundleDir - stats["index_path"] = m.indexPath - stats["verify_on_load"] = m.config.VerifyOnLoad - return stats -} - -// ExportOperations exports operations from bundles -func (m *Manager) ExportOperations(ctx context.Context, afterTime time.Time, count int) ([]plcclient.PLCOperation, error) { - if count <= 0 { - count = 1000 - } - - var result []plcclient.PLCOperation - seenCIDs := make(map[string]bool) - - bundles := m.index.GetBundles() - - for _, meta := range bundles { - if result != nil && len(result) >= count { - break - } - - // Skip bundles before afterTime - if !afterTime.IsZero() && meta.EndTime.Before(afterTime) { - continue - } - - // Load bundle - bundle, err := m.LoadBundle(ctx, meta.BundleNumber) - if err != nil { - m.logger.Printf("Warning: failed to load bundle %d: %v", meta.BundleNumber, err) - continue - } - - // Add operations - for _, op := range bundle.Operations { - if !afterTime.IsZero() && op.CreatedAt.Before(afterTime) { - continue - } - - if seenCIDs[op.CID] { - continue - } - - seenCIDs[op.CID] = true - result = append(result, op) - - if len(result) >= count { - break - } - } - } - - return result, nil -} - -// IsBundleIndexed checks if a bundle is already in the index -func (m *Manager) IsBundleIndexed(bundleNumber int) bool { - _, err := m.index.GetBundle(bundleNumber) - return err == nil -} - -// RefreshMempool reloads mempool from disk -func (m *Manager) RefreshMempool() error { - if m.mempool == nil { - return fmt.Errorf("mempool not initialized") - } - return m.mempool.Load() -} - -// ClearMempool clears all operations from the mempool and saves -func (m *Manager) ClearMempool() error { - if m.mempool == nil { - return fmt.Errorf("mempool not initialized") - } - - m.logger.Printf("Clearing mempool...") - - count := m.mempool.Count() - - m.mempool.Clear() - - if err := m.mempool.Save(); err != nil { - return fmt.Errorf("failed to save mempool: %w", err) - } - - m.logger.Printf("Cleared %d operations from mempool", count) - - return nil -} - -// ValidateMempool validates mempool -func (m *Manager) ValidateMempool() error { - if m.mempool == nil { - return fmt.Errorf("mempool not initialized") - } - return m.mempool.Validate() -} - -// StreamBundleRaw streams the raw compressed bundle file -func (m *Manager) StreamBundleRaw(ctx context.Context, bundleNumber int) (io.ReadCloser, error) { - // Get metadata from index - meta, err := m.index.GetBundle(bundleNumber) - if err != nil { - return nil, fmt.Errorf("bundle not in index: %w", err) - } - - // Build file path - path := filepath.Join(m.config.BundleDir, fmt.Sprintf("%06d.jsonl.zst", bundleNumber)) - if !m.operations.FileExists(path) { - return nil, fmt.Errorf("bundle file not found: %s", path) - } - - // Optionally verify hash before streaming - if m.config.VerifyOnLoad { - valid, actualHash, err := m.operations.VerifyHash(path, meta.CompressedHash) - if err != nil { - return nil, fmt.Errorf("failed to verify hash: %w", err) - } - if !valid { - return nil, fmt.Errorf("hash mismatch: expected %s, got %s", meta.CompressedHash, actualHash) - } - } - - return m.operations.StreamRaw(path) -} - -// StreamBundleDecompressed streams the decompressed bundle data as JSONL -func (m *Manager) StreamBundleDecompressed(ctx context.Context, bundleNumber int) (io.ReadCloser, error) { - // Get metadata from index - _, err := m.index.GetBundle(bundleNumber) - if err != nil { - return nil, fmt.Errorf("bundle not in index: %w", err) - } - - // Build file path - path := filepath.Join(m.config.BundleDir, fmt.Sprintf("%06d.jsonl.zst", bundleNumber)) - if !m.operations.FileExists(path) { - return nil, fmt.Errorf("bundle file not found: %s", path) - } - - return m.operations.StreamDecompressed(path) -} - -// RefreshIndex reloads the index from disk if it has been modified -func (m *Manager) RefreshIndex() error { - // Check if index file has been modified - info, err := os.Stat(m.indexPath) - if err != nil { - return err - } - - // If index was modified after we loaded it, reload - if info.ModTime().After(m.index.UpdatedAt) { - m.logger.Printf("Index file modified, reloading...") - - newIndex, err := bundleindex.LoadIndex(m.indexPath) - if err != nil { - return fmt.Errorf("failed to reload index: %w", err) - } - - m.index = newIndex - m.logger.Printf("Index reloaded: %d bundles", m.index.Count()) - } - - return nil -} - -// GetMempool returns the current mempool -func (m *Manager) GetMempool() *mempool.Mempool { - return m.mempool -} - -// SaveMempool saves the current mempool state to disk -func (m *Manager) SaveMempool() error { - if m.mempool == nil { - return fmt.Errorf("mempool not initialized") - } - return m.mempool.Save() -} - -// GetPLCOrigin returns the PLC directory origin URL -func (m *Manager) GetPLCOrigin() string { - if m.plcClient == nil { - return "" - } - return m.plcClient.GetBaseURL() -} - -// GetCurrentCursor returns the current latest cursor position (including mempool) -func (m *Manager) GetCurrentCursor() int { - index := m.GetIndex() - bundles := index.GetBundles() - cursor := len(bundles) * types.BUNDLE_SIZE - - // Add mempool operations - mempoolStats := m.GetMempoolStats() - if count, ok := mempoolStats["count"].(int); ok { - cursor += count - } - - return cursor -} - -// LoadOperation loads a single operation from a bundle efficiently -func (m *Manager) LoadOperation(ctx context.Context, bundleNumber int, position int) (*plcclient.PLCOperation, error) { - // Validate position - if position < 0 || position >= types.BUNDLE_SIZE { - return nil, fmt.Errorf("invalid position: %d (must be 0-%d)", position, types.BUNDLE_SIZE-1) - } - - // Validate bundle exists in index - _, err := m.index.GetBundle(bundleNumber) - if err != nil { - return nil, fmt.Errorf("bundle not in index: %w", err) - } - - // Build file path - path := filepath.Join(m.config.BundleDir, fmt.Sprintf("%06d.jsonl.zst", bundleNumber)) - if !m.operations.FileExists(path) { - return nil, fmt.Errorf("bundle file not found: %s", path) - } - - // Load just the one operation (optimized - decompresses only until position) - return m.operations.LoadOperationAtPosition(path, position) -} - -// LoadOperations loads multiple operations from a bundle efficiently -func (m *Manager) LoadOperations(ctx context.Context, bundleNumber int, positions []int) (map[int]*plcclient.PLCOperation, error) { - // Build file path - path := filepath.Join(m.config.BundleDir, fmt.Sprintf("%06d.jsonl.zst", bundleNumber)) - if !m.operations.FileExists(path) { - return nil, fmt.Errorf("bundle file not found: %s", path) - } - - return m.operations.LoadOperationsAtPositions(path, positions) -} - -// filterBundleFiles filters out files starting with . or _ -func filterBundleFiles(files []string) []string { - filtered := make([]string, 0, len(files)) - for _, file := range files { - basename := filepath.Base(file) - if len(basename) > 0 && (basename[0] == '.' || basename[0] == '_') { - continue - } - filtered = append(filtered, file) - } - return filtered -} - -// ========================================== -// DID INDEX INTEGRATION (adapter methods) -// ========================================== - -// Implement BundleProvider interface for didindex -func (m *Manager) LoadBundleForDIDIndex(ctx context.Context, bundleNumber int) (*didindex.BundleData, error) { - bundle, err := m.LoadBundle(ctx, bundleNumber) - if err != nil { - return nil, err - } - - return &didindex.BundleData{ - BundleNumber: bundle.BundleNumber, - Operations: bundle.Operations, - }, nil -} - -func (m *Manager) GetBundleIndex() didindex.BundleIndexProvider { - return &bundleIndexAdapter{index: m.index} -} - -// bundleIndexAdapter adapts Index to BundleIndexProvider interface -type bundleIndexAdapter struct { - index *bundleindex.Index -} - -func (a *bundleIndexAdapter) GetBundles() []*didindex.BundleMetadata { - bundles := a.index.GetBundles() - result := make([]*didindex.BundleMetadata, len(bundles)) - for i, b := range bundles { - result[i] = &didindex.BundleMetadata{ - BundleNumber: b.BundleNumber, - StartTime: b.StartTime, - EndTime: b.EndTime, - } - } - return result -} - -func (a *bundleIndexAdapter) GetBundle(bundleNumber int) (*didindex.BundleMetadata, error) { - meta, err := a.index.GetBundle(bundleNumber) - if err != nil { - return nil, err - } - return &didindex.BundleMetadata{ - BundleNumber: meta.BundleNumber, - StartTime: meta.StartTime, - EndTime: meta.EndTime, - }, nil -} - -func (a *bundleIndexAdapter) GetLastBundle() *didindex.BundleMetadata { - meta := a.index.GetLastBundle() - if meta == nil { - return nil - } - return &didindex.BundleMetadata{ - BundleNumber: meta.BundleNumber, - StartTime: meta.StartTime, - EndTime: meta.EndTime, - } -} - -// GetDIDIndex returns the DID index manager -func (m *Manager) GetDIDIndex() *didindex.Manager { - return m.didIndex -} - -// BuildDIDIndex builds the complete DID index -func (m *Manager) BuildDIDIndex(ctx context.Context, progressCallback func(current, total int)) error { - if m.didIndex == nil { - m.didIndex = didindex.NewManager(m.config.BundleDir, m.logger) - } - - return m.didIndex.BuildIndexFromScratch(ctx, m, progressCallback) -} - -// updateDIDIndexForBundle updates index when a new bundle is added -func (m *Manager) updateDIDIndexForBundle(ctx context.Context, bundle *Bundle) error { - if m.didIndex == nil { - return nil - } - - // Convert to didindex.BundleData - bundleData := &didindex.BundleData{ - BundleNumber: bundle.BundleNumber, - Operations: bundle.Operations, - } - - return m.didIndex.UpdateIndexForBundle(ctx, bundleData) -} - -// GetDIDIndexStats returns DID index statistics -func (m *Manager) GetDIDIndexStats() map[string]interface{} { - if m.didIndex == nil { - return map[string]interface{}{ - "enabled": false, - } - } - - stats := m.didIndex.GetStats() - stats["enabled"] = true - stats["exists"] = m.didIndex.Exists() - - indexedDIDs := stats["total_dids"].(int64) - - // Get unique DIDs from mempool - mempoolDIDCount := int64(0) - if m.mempool != nil { - mempoolStats := m.GetMempoolStats() - if didCount, ok := mempoolStats["did_count"].(int); ok { - mempoolDIDCount = int64(didCount) - } - } - - stats["indexed_dids"] = indexedDIDs - stats["mempool_dids"] = mempoolDIDCount - stats["total_dids"] = indexedDIDs + mempoolDIDCount - - return stats -} - -// GetDIDOperations retrieves all operations for a DID (bundles + mempool combined) -// Returns: operations only, operations with locations, error -func (m *Manager) GetDIDOperations(ctx context.Context, did string, verbose bool) ([]plcclient.PLCOperation, []PLCOperationWithLocation, error) { - if err := plcclient.ValidateDIDFormat(did); err != nil { - return nil, nil, err - } - - // Set verbose mode - if m.didIndex != nil { - m.didIndex.SetVerbose(verbose) - } - - // Get bundled operations from DID index (includes nullified) - bundledOpsWithLoc, err := m.didIndex.GetDIDOperations(ctx, did, m) - if err != nil { - return nil, nil, err - } - - // Convert to bundle types - opsWithLoc := make([]PLCOperationWithLocation, len(bundledOpsWithLoc)) - bundledOps := make([]plcclient.PLCOperation, len(bundledOpsWithLoc)) - for i, r := range bundledOpsWithLoc { - opsWithLoc[i] = PLCOperationWithLocation{ - Operation: r.Operation, - Bundle: r.Bundle, - Position: r.Position, - } - bundledOps[i] = r.Operation - } - - // Get mempool operations - mempoolOps, err := m.GetDIDOperationsFromMempool(did) - if err != nil { - return nil, nil, err - } - - if len(mempoolOps) > 0 && verbose { - m.logger.Printf("DEBUG: Found %d operations in mempool", len(mempoolOps)) - } - - // Combine operations (for the slice return) - allOps := append(bundledOps, mempoolOps...) - - sort.Slice(allOps, func(i, j int) bool { - return allOps[i].CreatedAt.Before(allOps[j].CreatedAt) - }) - - return allOps, opsWithLoc, nil -} - -// GetDIDOperationsFromMempool retrieves operations for a DID from mempool only -func (m *Manager) GetDIDOperationsFromMempool(did string) ([]plcclient.PLCOperation, error) { - if m.mempool == nil { - return []plcclient.PLCOperation{}, nil - } - - // Use direct search - only copies matching operations - return m.mempool.FindDIDOperations(did), nil -} - -// GetLatestDIDOperation returns only the most recent non-nullified operation -func (m *Manager) GetLatestDIDOperation(ctx context.Context, did string) (*plcclient.PLCOperation, error) { - if err := plcclient.ValidateDIDFormat(did); err != nil { - return nil, err - } - - // Check mempool first (most recent data) - mempoolOps, _ := m.GetDIDOperationsFromMempool(did) - if len(mempoolOps) > 0 { - for i := len(mempoolOps) - 1; i >= 0; i-- { - if !mempoolOps[i].IsNullified() { - return &mempoolOps[i], nil - } - } - } - - // Delegate to DID index for bundled operations - return m.didIndex.GetLatestDIDOperation(ctx, did, m) -} - -// VerifyChain verifies the entire bundle chain -func (m *Manager) VerifyChain(ctx context.Context) (*ChainVerificationResult, error) { - result := &ChainVerificationResult{ - VerifiedBundles: make([]int, 0), - } - - bundles := m.index.GetBundles() - if len(bundles) == 0 { - result.Valid = true - return result, nil - } - - result.ChainLength = len(bundles) - - for i, meta := range bundles { - // Verify file hash - vr, err := m.VerifyBundle(ctx, meta.BundleNumber) - if err != nil || !vr.Valid { - result.Error = fmt.Sprintf("Bundle %d hash verification failed", meta.BundleNumber) - result.BrokenAt = meta.BundleNumber - return result, nil - } - - // Verify chain link - if i > 0 { - prevMeta := bundles[i-1] - - // Check parent reference - if meta.Parent != prevMeta.Hash { - result.Error = fmt.Sprintf("Chain broken at bundle %d: parent mismatch", meta.BundleNumber) - result.BrokenAt = meta.BundleNumber - return result, nil - } - - // Verify chain hash calculation - expectedHash := m.operations.CalculateChainHash(prevMeta.Hash, meta.ContentHash) - if meta.Hash != expectedHash { - result.Error = fmt.Sprintf("Chain broken at bundle %d: hash mismatch", meta.BundleNumber) - result.BrokenAt = meta.BundleNumber - return result, nil - } - } - - result.VerifiedBundles = append(result.VerifiedBundles, meta.BundleNumber) - } - - result.Valid = true - return result, nil -} - -// FetchNextBundle fetches operations and creates a bundle, looping until caught up -func (m *Manager) FetchNextBundle(ctx context.Context, verbose bool, quiet bool) (*Bundle, types.BundleProductionStats, error) { - if m.plcClient == nil { - return nil, types.BundleProductionStats{}, fmt.Errorf("PLC client not configured") - } - - lastBundle := m.index.GetLastBundle() - nextBundleNum := 1 - var afterTime string - var prevBoundaryCIDs map[string]bool - var prevBundleHash string - - if lastBundle != nil { - nextBundleNum = lastBundle.BundleNumber + 1 - afterTime = lastBundle.EndTime.Format(time.RFC3339Nano) - prevBundleHash = lastBundle.Hash - - // ALWAYS get boundaries from last bundle initially - prevBundle, err := m.LoadBundle(ctx, lastBundle.BundleNumber) - if err == nil { - _, prevBoundaryCIDs = m.operations.GetBoundaryCIDs(prevBundle.Operations) - if verbose { - m.logger.Printf("Loaded %d boundary CIDs from bundle %06d (at %s)", - len(prevBoundaryCIDs), lastBundle.BundleNumber, - lastBundle.EndTime.Format(time.RFC3339)[:19]) - } - } - } - - // If mempool has operations, update cursor but KEEP boundaries from bundle - // (mempool operations already had boundary dedup applied when they were added) - if m.mempool.Count() > 0 { - mempoolLastTime := m.mempool.GetLastTime() - if mempoolLastTime != "" { - if verbose { - m.logger.Printf("[DEBUG] Mempool has %d ops, resuming from %s", - m.mempool.Count(), mempoolLastTime[:19]) - } - afterTime = mempoolLastTime - - // Calculate boundaries from MEMPOOL for next fetch - mempoolOps, _ := m.GetMempoolOperations() - if len(mempoolOps) > 0 { - _, mempoolBoundaries := m.operations.GetBoundaryCIDs(mempoolOps) - prevBoundaryCIDs = mempoolBoundaries - if verbose { - m.logger.Printf("Using %d boundary CIDs from mempool", len(prevBoundaryCIDs)) - } - } - } - } - - if verbose { - m.logger.Printf("[DEBUG] Preparing bundle %06d (mempool: %d ops)...", nextBundleNum, m.mempool.Count()) - m.logger.Printf("[DEBUG] Starting cursor: %s", afterTime) - } - - totalFetches := 0 - maxAttempts := 50 - attempt := 0 - caughtUp := false - attemptStart := time.Now() - - for m.mempool.Count() < types.BUNDLE_SIZE && attempt < maxAttempts { - attempt++ - needed := types.BUNDLE_SIZE - m.mempool.Count() - - if !quiet && attempt > 1 { - m.logger.Printf(" Attempt %d: Need %d more ops, cursor: %s", - attempt, needed, afterTime[:19]) - } - - newOps, fetchCount, err := m.syncer.FetchToMempool( - ctx, - afterTime, - prevBoundaryCIDs, - needed, - !verbose, - m.mempool, - totalFetches, - ) - - totalFetches += fetchCount - - // Check if we got an incomplete batch - gotIncompleteBatch := len(newOps) > 0 && len(newOps) < needed && err == nil - - // Update cursor from mempool if we got new ops - if len(newOps) > 0 && m.mempool.Count() > 0 { - afterTime = m.mempool.GetLastTime() - } - - // Stop if caught up or error - if err != nil || len(newOps) == 0 || gotIncompleteBatch { - caughtUp = true - if verbose && totalFetches > 0 { - m.logger.Printf("DEBUG: Caught up to latest PLC data") - } - break - } - - if m.mempool.Count() >= types.BUNDLE_SIZE { - break - } - } - - totalDuration := time.Since(attemptStart) - - if m.mempool.Count() < types.BUNDLE_SIZE { - if caughtUp { - return nil, types.BundleProductionStats{}, fmt.Errorf("insufficient operations: have %d, need %d (caught up to latest PLC data)", - m.mempool.Count(), types.BUNDLE_SIZE) - } else { - return nil, types.BundleProductionStats{}, fmt.Errorf("insufficient operations: have %d, need %d (max attempts reached)", - m.mempool.Count(), types.BUNDLE_SIZE) - } - } - - // Create bundle - operations, err := m.mempool.Take(types.BUNDLE_SIZE) - if err != nil { - return nil, types.BundleProductionStats{}, err - } - - syncBundle := internalsync.CreateBundle(nextBundleNum, operations, afterTime, prevBundleHash, m.operations) - - bundle := &Bundle{ - BundleNumber: syncBundle.BundleNumber, - StartTime: syncBundle.StartTime, - EndTime: syncBundle.EndTime, - Operations: syncBundle.Operations, - DIDCount: syncBundle.DIDCount, - Cursor: syncBundle.Cursor, - Parent: syncBundle.Parent, - BoundaryCIDs: syncBundle.BoundaryCIDs, - Compressed: syncBundle.Compressed, - CreatedAt: syncBundle.CreatedAt, - } - - stats := types.BundleProductionStats{ - TotalFetches: totalFetches, - TotalDuration: totalDuration, - AvgPerFetch: float64(types.BUNDLE_SIZE) / float64(totalFetches), - Throughput: float64(types.BUNDLE_SIZE) / totalDuration.Seconds(), - } - - return bundle, stats, nil -} - -// CloneFromRemote clones bundles from a remote endpoint -func (m *Manager) CloneFromRemote(ctx context.Context, opts internalsync.CloneOptions) (*internalsync.CloneResult, error) { - // Define index update callback inline - updateIndexCallback := func(bundleNumbers []int, remoteMeta map[int]*bundleindex.BundleMetadata, verbose bool) error { - if len(bundleNumbers) == 0 { - return nil - } - - // Create file existence checker - fileExists := func(bundleNum int) bool { - path := filepath.Join(m.config.BundleDir, fmt.Sprintf("%06d.jsonl.zst", bundleNum)) - return m.operations.FileExists(path) - } - - // Update index with remote metadata - if err := m.index.UpdateFromRemote(bundleNumbers, remoteMeta, fileExists, verbose, m.logger); err != nil { - return err - } - - // Save index - return m.SaveIndex() - } - - // Delegate to cloner with inline callback - return m.cloner.Clone(ctx, opts, m.index, updateIndexCallback) -} - -// ResolveDID resolves a DID to its current document with detailed timing metrics -func (m *Manager) ResolveDID(ctx context.Context, did string) (*ResolveDIDResult, error) { - if err := plcclient.ValidateDIDFormat(did); err != nil { - atomic.AddInt64(&m.resolverStats.errors, 1) - return nil, err - } - - result := &ResolveDIDResult{} - totalStart := time.Now() - - // STEP 1: Check mempool first - mempoolStart := time.Now() - var latestMempoolOp *plcclient.PLCOperation - if m.mempool != nil { - latestMempoolOp = m.mempool.FindLatestDIDOperation(did) - } - result.MempoolTime = time.Since(mempoolStart) - - // Early return if found in mempool - if latestMempoolOp != nil { - doc, err := plcclient.ResolveDIDDocument(did, []plcclient.PLCOperation{*latestMempoolOp}) - if err != nil { - atomic.AddInt64(&m.resolverStats.errors, 1) - return nil, fmt.Errorf("resolution failed: %w", err) - } - - result.Document = doc - result.LatestOperation = latestMempoolOp - result.Source = "mempool" - result.TotalTime = time.Since(totalStart) - - m.recordResolverTiming(result, nil) - return result, nil - } - - // STEP 2: Index lookup - if m.didIndex == nil || !m.didIndex.Exists() { - atomic.AddInt64(&m.resolverStats.errors, 1) - return nil, fmt.Errorf("DID index not available - run 'plcbundle index build' to enable DID resolution") - } - - indexStart := time.Now() - locations, err := m.didIndex.GetDIDLocations(did) - result.IndexTime = time.Since(indexStart) - - if err != nil { - atomic.AddInt64(&m.resolverStats.errors, 1) - return nil, err - } - - if len(locations) == 0 { - atomic.AddInt64(&m.resolverStats.errors, 1) - return nil, fmt.Errorf("DID not found") - } - - // Find latest non-nullified location - var latestLoc *didindex.OpLocation - for i := range locations { - if locations[i].Nullified() { - continue - } - if latestLoc == nil || locations[i].IsAfter(*latestLoc) { - latestLoc = &locations[i] - } - } - - if latestLoc == nil { - atomic.AddInt64(&m.resolverStats.errors, 1) - return nil, fmt.Errorf("no valid operations (all nullified)") - } - - // STEP 3: Load operation - opStart := time.Now() - op, err := m.LoadOperation(ctx, latestLoc.BundleInt(), latestLoc.PositionInt()) - result.LoadOpTime = time.Since(opStart) - - if err != nil { - atomic.AddInt64(&m.resolverStats.errors, 1) - return nil, fmt.Errorf("failed to load operation: %w", err) - } - - result.BundleNumber = latestLoc.BundleInt() - result.Position = latestLoc.PositionInt() - - // STEP 4: Resolve document - doc, err := plcclient.ResolveDIDDocument(did, []plcclient.PLCOperation{*op}) - if err != nil { - atomic.AddInt64(&m.resolverStats.errors, 1) - return nil, fmt.Errorf("resolution failed: %w", err) - } - - result.Document = doc - result.LatestOperation = op - result.Source = "bundle" - result.TotalTime = time.Since(totalStart) - - m.recordResolverTiming(result, nil) - return result, nil -} - -// GetLastBundleNumber returns the last bundle number (0 if no bundles) -func (m *Manager) GetLastBundleNumber() int { - lastBundle := m.index.GetLastBundle() - if lastBundle == nil { - return 0 - } - return lastBundle.BundleNumber -} - -// GetMempoolCount returns the number of operations in mempool -func (m *Manager) GetMempoolCount() int { - return m.mempool.Count() -} - -func (m *Manager) FetchAndSaveNextBundle(ctx context.Context, verbose bool, quiet bool, skipDIDIndex bool) (int, *types.BundleProductionStats, error) { - bundle, stats, err := m.FetchNextBundle(ctx, verbose, quiet) - if err != nil { - return 0, nil, err - } - - indexTime, err := m.SaveBundle(ctx, bundle, verbose, quiet, stats, skipDIDIndex) - if err != nil { - return 0, nil, err - } - stats.IndexTime = indexTime - - return bundle.BundleNumber, &types.BundleProductionStats{}, nil -} - -// RunSyncLoop runs continuous sync loop (delegates to internal/sync) -func (m *Manager) RunSyncLoop(ctx context.Context, config *internalsync.SyncLoopConfig) error { - // Manager itself implements the SyncManager interface - return internalsync.RunSyncLoop(ctx, m, config) -} - -// RunSyncOnce performs a single sync cycle -func (m *Manager) RunSyncOnce(ctx context.Context, config *internalsync.SyncLoopConfig) (int, error) { - // Manager itself implements the SyncManager interface - return internalsync.SyncOnce(ctx, m, config) -} - -// EnsureDIDIndex ensures DID index is built and up-to-date -// Returns true if index was built/rebuilt, false if already up-to-date -func (m *Manager) EnsureDIDIndex(ctx context.Context, progressCallback func(current, total int)) (bool, error) { - // Build index - m.UpdateDIDIndexSmart(ctx, progressCallback) - return true, nil -} - -// Add this helper function at the top of manager.go -func repositoryExists(bundleDir string) bool { - indexPath := filepath.Join(bundleDir, bundleindex.INDEX_FILE) - - // Check for index file - if _, err := os.Stat(indexPath); err == nil { - return true - } - - // Check for bundle files - bundleFiles, _ := filepath.Glob(filepath.Join(bundleDir, "*.jsonl.zst")) - bundleFiles = filterBundleFiles(bundleFiles) - - return len(bundleFiles) > 0 -} - -// ResolveHandleOrDID resolves input that can be either a handle or DID -// Returns: (did, handleResolveTime, error) -func (m *Manager) ResolveHandleOrDID(ctx context.Context, input string) (string, time.Duration, error) { - input = strings.TrimSpace(input) - - // Normalize handle format (remove at://, @ prefixes) - if !strings.HasPrefix(input, "did:") { - input = strings.TrimPrefix(input, "at://") - input = strings.TrimPrefix(input, "@") - } - - // If already a DID, validate and return - if strings.HasPrefix(input, "did:plc:") { - if err := plcclient.ValidateDIDFormat(input); err != nil { - return "", 0, err - } - return input, 0, nil // No resolution needed - } - - // Support did:web too - if strings.HasPrefix(input, "did:web:") { - return input, 0, nil - } - - // It's a handle - need resolver - if m.handleResolver == nil { - return "", 0, fmt.Errorf( - "input '%s' appears to be a handle, but handle resolver is not configured\n\n"+ - "Configure resolver with:\n"+ - " plcbundle --handle-resolver https://quickdid.smokesignal.tools did resolve %s\n\n"+ - "Or set default in config", - input, input) - } - - resolveStart := time.Now() - if !m.config.Quiet { - m.logger.Printf("Resolving handle: %s", input) - } - did, err := m.handleResolver.ResolveHandle(ctx, input) - resolveTime := time.Since(resolveStart) - - if err != nil { - return "", resolveTime, fmt.Errorf("failed to resolve handle '%s': %w", input, err) - } - - if !m.config.Quiet { - m.logger.Printf("Resolved: %s → %s (in %s)", input, did, resolveTime) - } - return did, resolveTime, nil -} - -// GetHandleResolver returns the handle resolver (can be nil) -func (m *Manager) GetHandleResolver() *handleresolver.Client { - return m.handleResolver -} - -// recordResolverTiming records resolver performance metrics -func (m *Manager) recordResolverTiming(result *ResolveDIDResult, _ error) { - m.resolverStats.Lock() - defer m.resolverStats.Unlock() - - // Increment counters - atomic.AddInt64(&m.resolverStats.totalResolutions, 1) - - switch result.Source { - case "mempool": - atomic.AddInt64(&m.resolverStats.mempoolHits, 1) - case "bundle": - atomic.AddInt64(&m.resolverStats.bundleHits, 1) - } - - // Record timings - timing := resolverTiming{ - totalTime: result.TotalTime.Microseconds(), - mempoolTime: result.MempoolTime.Microseconds(), - indexTime: result.IndexTime.Microseconds(), - loadOpTime: result.LoadOpTime.Microseconds(), - source: result.Source, - } - - atomic.AddInt64(&m.resolverStats.totalTime, timing.totalTime) - atomic.AddInt64(&m.resolverStats.totalMempoolTime, timing.mempoolTime) - atomic.AddInt64(&m.resolverStats.totalIndexTime, timing.indexTime) - atomic.AddInt64(&m.resolverStats.totalLoadOpTime, timing.loadOpTime) - - // Add to circular buffer - m.resolverStats.recentTimes[m.resolverStats.recentIdx] = timing - m.resolverStats.recentIdx = (m.resolverStats.recentIdx + 1) % m.resolverStats.recentSize -} - -// GetResolverStats returns resolver performance statistics -func (m *Manager) GetResolverStats() map[string]interface{} { - totalResolutions := atomic.LoadInt64(&m.resolverStats.totalResolutions) - - if totalResolutions == 0 { - return map[string]interface{}{ - "total_resolutions": 0, - } - } - - mempoolHits := atomic.LoadInt64(&m.resolverStats.mempoolHits) - bundleHits := atomic.LoadInt64(&m.resolverStats.bundleHits) - errors := atomic.LoadInt64(&m.resolverStats.errors) - - totalTime := atomic.LoadInt64(&m.resolverStats.totalTime) - totalMempoolTime := atomic.LoadInt64(&m.resolverStats.totalMempoolTime) - totalIndexTime := atomic.LoadInt64(&m.resolverStats.totalIndexTime) - totalLoadOpTime := atomic.LoadInt64(&m.resolverStats.totalLoadOpTime) - - // Calculate overall averages - avgTotalMs := float64(totalTime) / float64(totalResolutions) / 1000.0 - avgMempoolMs := float64(totalMempoolTime) / float64(totalResolutions) / 1000.0 - - stats := map[string]interface{}{ - "total_resolutions": totalResolutions, - "mempool_hits": mempoolHits, - "bundle_hits": bundleHits, - "errors": errors, - "success_rate": float64(totalResolutions-errors) / float64(totalResolutions), - "mempool_hit_rate": float64(mempoolHits) / float64(totalResolutions), - - // Overall averages - "avg_total_time_ms": avgTotalMs, - "avg_mempool_time_ms": avgMempoolMs, - } - - // Only include bundle-specific stats if we have bundle hits - if bundleHits > 0 { - avgIndexMs := float64(totalIndexTime) / float64(bundleHits) / 1000.0 - avgLoadMs := float64(totalLoadOpTime) / float64(bundleHits) / 1000.0 - - stats["avg_index_time_ms"] = avgIndexMs - stats["avg_load_op_time_ms"] = avgLoadMs - } - - // Recent statistics - m.resolverStats.Lock() - recentCopy := make([]resolverTiming, m.resolverStats.recentSize) - copy(recentCopy, m.resolverStats.recentTimes) - m.resolverStats.Unlock() - - // Filter valid entries - validRecent := make([]resolverTiming, 0) - for _, t := range recentCopy { - if t.totalTime > 0 { - validRecent = append(validRecent, t) - } - } - - if len(validRecent) > 0 { - // Extract total times for percentiles - totalTimes := make([]int64, len(validRecent)) - for i, t := range validRecent { - totalTimes[i] = t.totalTime - } - sort.Slice(totalTimes, func(i, j int) bool { - return totalTimes[i] < totalTimes[j] - }) - - // Calculate recent average - var recentSum int64 - var recentMempoolSum int64 - var recentIndexSum int64 - var recentLoadSum int64 - recentBundleCount := 0 - - for _, t := range validRecent { - recentSum += t.totalTime - recentMempoolSum += t.mempoolTime - if t.source == "bundle" { - recentIndexSum += t.indexTime - recentLoadSum += t.loadOpTime - recentBundleCount++ - } - } - - stats["recent_avg_total_time_ms"] = float64(recentSum) / float64(len(validRecent)) / 1000.0 - stats["recent_avg_mempool_time_ms"] = float64(recentMempoolSum) / float64(len(validRecent)) / 1000.0 - - if recentBundleCount > 0 { - stats["recent_avg_index_time_ms"] = float64(recentIndexSum) / float64(recentBundleCount) / 1000.0 - stats["recent_avg_load_time_ms"] = float64(recentLoadSum) / float64(recentBundleCount) / 1000.0 - } - - stats["recent_sample_size"] = len(validRecent) - - // Percentiles - p50idx := len(totalTimes) * 50 / 100 - p95idx := len(totalTimes) * 95 / 100 - p99idx := len(totalTimes) * 99 / 100 - - stats["min_total_time_ms"] = float64(totalTimes[0]) / 1000.0 - stats["max_total_time_ms"] = float64(totalTimes[len(totalTimes)-1]) / 1000.0 - - if p50idx < len(totalTimes) { - stats["p50_total_time_ms"] = float64(totalTimes[p50idx]) / 1000.0 - } - if p95idx < len(totalTimes) { - stats["p95_total_time_ms"] = float64(totalTimes[p95idx]) / 1000.0 - } - if p99idx < len(totalTimes) { - stats["p99_total_time_ms"] = float64(totalTimes[p99idx]) / 1000.0 - } - } - - return stats -} - -// ResetResolverStats resets resolver performance statistics -func (m *Manager) ResetResolverStats() { - m.resolverStats.Lock() - defer m.resolverStats.Unlock() - - atomic.StoreInt64(&m.resolverStats.totalResolutions, 0) - atomic.StoreInt64(&m.resolverStats.mempoolHits, 0) - atomic.StoreInt64(&m.resolverStats.bundleHits, 0) - atomic.StoreInt64(&m.resolverStats.errors, 0) - atomic.StoreInt64(&m.resolverStats.totalTime, 0) - atomic.StoreInt64(&m.resolverStats.totalMempoolTime, 0) - atomic.StoreInt64(&m.resolverStats.totalIndexTime, 0) - atomic.StoreInt64(&m.resolverStats.totalLoadOpTime, 0) - - m.resolverStats.recentTimes = make([]resolverTiming, m.resolverStats.recentSize) - m.resolverStats.recentIdx = 0 -} - -func (m *Manager) SetQuiet(quiet bool) { - m.config.Quiet = quiet -} - -// ShouldRebuildDIDIndex checks if DID index needs rebuilding -// Returns: (needsRebuild bool, reason string, canUpdateIncrementally bool) -func (m *Manager) ShouldRebuildDIDIndex() (bool, string, bool) { - if m.didIndex == nil { - return false, "DID index disabled", false - } - - needsRebuild, reason := m.didIndex.NeedsRebuild(m.GetBundleIndex()) - - if needsRebuild { - return true, reason, false - } - - // Check if incremental update is better - canIncremental, behindBy := m.didIndex.ShouldUpdateIncrementally(m.GetBundleIndex()) - if canIncremental { - return false, fmt.Sprintf("can update incrementally (%d bundles)", behindBy), true - } - - return false, "index is up to date", false -} - -// UpdateDIDIndexSmart updates DID index intelligently (rebuild vs incremental) -func (m *Manager) UpdateDIDIndexSmart(ctx context.Context, progressCallback func(current, total int)) error { - needsRebuild, reason, canIncremental := m.ShouldRebuildDIDIndex() - - if !needsRebuild && !canIncremental { - if m.config.Verbose { - m.logger.Printf("DID index is up to date") - } - return nil - } - - if needsRebuild { - m.logger.Printf("Rebuilding DID index: %s", reason) - return m.BuildDIDIndex(ctx, progressCallback) - } - - if canIncremental { - m.logger.Printf("Updating DID index incrementally: %s", reason) - return m.updateDIDIndexIncremental(ctx, progressCallback) - } - - return nil -} - -// updateDIDIndexIncremental updates index for missing bundles only -func (m *Manager) updateDIDIndexIncremental(ctx context.Context, progressCallback func(current, total int)) error { - config := m.didIndex.GetConfig() - lastBundle := m.index.GetLastBundle() - - if lastBundle == nil || config.LastBundle >= lastBundle.BundleNumber { - return nil - } - - start := config.LastBundle + 1 - end := lastBundle.BundleNumber - total := end - start + 1 - - m.logger.Printf("Updating DID index for bundles %d-%d (%d bundles)", start, end, total) - - for bundleNum := start; bundleNum <= end; bundleNum++ { - bundle, err := m.LoadBundle(ctx, bundleNum) - if err != nil { - return fmt.Errorf("failed to load bundle %d: %w", bundleNum, err) - } - - bundleData := &didindex.BundleData{ - BundleNumber: bundle.BundleNumber, - Operations: bundle.Operations, - } - - if err := m.didIndex.UpdateIndexForBundle(ctx, bundleData); err != nil { - return fmt.Errorf("failed to update bundle %d: %w", bundleNum, err) - } - - if progressCallback != nil { - progressCallback(bundleNum-start+1, total) - } - } - - return nil -} diff --git a/bundle/metadata.go b/bundle/metadata.go deleted file mode 100644 index 42134e4..0000000 --- a/bundle/metadata.go +++ /dev/null @@ -1,178 +0,0 @@ -package bundle - -import ( - "bufio" - "encoding/json" - "fmt" - "os" - "time" - - "tangled.org/atscan.net/plcbundle/internal/bundleindex" - "tangled.org/atscan.net/plcbundle/internal/plcclient" - "tangled.org/atscan.net/plcbundle/internal/storage" -) - -// CalculateBundleMetadata calculates complete metadata for a bundle -func (m *Manager) CalculateBundleMetadata(bundleNumber int, path string, operations []plcclient.PLCOperation, parent string, cursor string) (*bundleindex.BundleMetadata, error) { - if len(operations) == 0 { - return nil, fmt.Errorf("bundle is empty") - } - - // Get file info - info, err := os.Stat(path) - if err != nil { - return nil, fmt.Errorf("failed to stat file: %w", err) - } - - // Extract unique DIDs - dids := m.operations.ExtractUniqueDIDs(operations) - - // Serialize to JSONL and calculate content hash - jsonlData := m.operations.SerializeJSONL(operations) - contentSize := int64(len(jsonlData)) - contentHash := m.operations.Hash(jsonlData) - - // Read compressed file and calculate compressed hash - compressedData, err := os.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("failed to read compressed file: %w", err) - } - compressedHash := m.operations.Hash(compressedData) - compressedSize := info.Size() - - // Calculate chain hash - chainHash := m.operations.CalculateChainHash(parent, contentHash) - - return &bundleindex.BundleMetadata{ - BundleNumber: bundleNumber, - StartTime: operations[0].CreatedAt, - EndTime: operations[len(operations)-1].CreatedAt, - OperationCount: len(operations), - DIDCount: len(dids), - Hash: chainHash, - ContentHash: contentHash, - Parent: parent, - CompressedHash: compressedHash, - CompressedSize: compressedSize, - UncompressedSize: contentSize, - Cursor: cursor, - CreatedAt: time.Now().UTC(), - }, nil -} - -// CalculateBundleMetadataFast calculates metadata quickly without chain hash -func (m *Manager) CalculateBundleMetadataFast(bundleNumber int, path string, operations []plcclient.PLCOperation, cursor string) (*bundleindex.BundleMetadata, error) { - if len(operations) == 0 { - return nil, fmt.Errorf("bundle is empty") - } - - // Calculate hashes efficiently - compressedHash, compressedSize, contentHash, contentSize, err := m.operations.CalculateFileHashes(path) - if err != nil { - return nil, err - } - - // Extract unique DIDs - dids := m.operations.ExtractUniqueDIDs(operations) - - return &bundleindex.BundleMetadata{ - BundleNumber: bundleNumber, - StartTime: operations[0].CreatedAt, - EndTime: operations[len(operations)-1].CreatedAt, - OperationCount: len(operations), - DIDCount: len(dids), - Hash: "", - ContentHash: contentHash, - Parent: "", - CompressedHash: compressedHash, - CompressedSize: compressedSize, - UncompressedSize: contentSize, - Cursor: cursor, - CreatedAt: time.Now().UTC(), - }, nil -} - -// CalculateMetadataStreaming calculates metadata by streaming (NO full load) -func (m *Manager) CalculateMetadataStreaming(bundleNumber int, path string) (*bundleindex.BundleMetadata, error) { - // STEP 1: Stream to get times + counts (minimal memory) - opCount, didCount, startTime, endTime, err := m.streamBundleInfo(path) - if err != nil { - return nil, err - } - - // STEP 2: Calculate hashes from file - compressedHash, compressedSize, contentHash, contentSize, err := m.operations.CalculateFileHashes(path) - if err != nil { - return nil, err - } - - return &bundleindex.BundleMetadata{ - BundleNumber: bundleNumber, - StartTime: startTime, - EndTime: endTime, - OperationCount: opCount, - DIDCount: didCount, - Hash: "", // Calculated later in sequential phase - ContentHash: contentHash, - Parent: "", // Calculated later - CompressedHash: compressedHash, - CompressedSize: compressedSize, - UncompressedSize: contentSize, - Cursor: "", - CreatedAt: time.Now().UTC(), - }, nil -} - -// streamBundleInfo extracts metadata by streaming (minimal memory) -func (m *Manager) streamBundleInfo(path string) (opCount, didCount int, startTime, endTime time.Time, err error) { - file, err := os.Open(path) - if err != nil { - return 0, 0, time.Time{}, time.Time{}, err - } - defer file.Close() - - // Use abstracted reader from storage package - reader, err := storage.NewStreamingReader(file) - if err != nil { - return 0, 0, time.Time{}, time.Time{}, fmt.Errorf("failed to create reader: %w", err) - } - defer reader.Release() - - scanner := bufio.NewScanner(reader) - buf := make([]byte, 64*1024) - scanner.Buffer(buf, 1024*1024) - - didSet := make(map[string]bool) - lineNum := 0 - - for scanner.Scan() { - line := scanner.Bytes() - if len(line) == 0 { - continue - } - - // Only parse minimal fields (DID + time) - var op struct { - DID string `json:"did"` - CreatedAt time.Time `json:"createdAt"` - } - - if err := json.Unmarshal(line, &op); err != nil { - continue - } - - if lineNum == 0 { - startTime = op.CreatedAt - } - endTime = op.CreatedAt - - didSet[op.DID] = true - lineNum++ - } - - if err := scanner.Err(); err != nil { - return 0, 0, time.Time{}, time.Time{}, err - } - - return lineNum, len(didSet), startTime, endTime, nil -} diff --git a/bundle/scanner.go b/bundle/scanner.go deleted file mode 100644 index 8bfe7fd..0000000 --- a/bundle/scanner.go +++ /dev/null @@ -1,341 +0,0 @@ -package bundle - -import ( - "fmt" - "path/filepath" - "sort" - "strconv" - "strings" - "sync" - "time" - - "tangled.org/atscan.net/plcbundle/internal/bundleindex" -) - -// ScanDirectory scans the bundle directory and rebuilds the index -func (m *Manager) ScanDirectory() (*DirectoryScanResult, error) { - result := &DirectoryScanResult{ - BundleDir: m.config.BundleDir, - } - - m.logger.Printf("Scanning directory: %s", m.config.BundleDir) - - // Find all bundle files - files, err := filepath.Glob(filepath.Join(m.config.BundleDir, "*.jsonl.zst")) - if err != nil { - return nil, fmt.Errorf("failed to scan directory: %w", err) - } - files = filterBundleFiles(files) - - if len(files) == 0 { - m.logger.Printf("No bundle files found") - return result, nil - } - - // Parse bundle numbers - var bundleNumbers []int - for _, file := range files { - base := filepath.Base(file) - numStr := strings.TrimSuffix(base, ".jsonl.zst") - num, err := strconv.Atoi(numStr) - if err != nil { - m.logger.Printf("Warning: skipping invalid filename: %s", base) - continue - } - bundleNumbers = append(bundleNumbers, num) - } - - sort.Ints(bundleNumbers) - - result.BundleCount = len(bundleNumbers) - if len(bundleNumbers) > 0 { - result.FirstBundle = bundleNumbers[0] - result.LastBundle = bundleNumbers[len(bundleNumbers)-1] - } - - // Find gaps - if len(bundleNumbers) > 1 { - for i := result.FirstBundle; i <= result.LastBundle; i++ { - found := false - for _, num := range bundleNumbers { - if num == i { - found = true - break - } - } - if !found { - result.MissingGaps = append(result.MissingGaps, i) - } - } - } - - m.logger.Printf("Found %d bundles (gaps: %d)", result.BundleCount, len(result.MissingGaps)) - - // Load each bundle and rebuild index - var newMetadata []*bundleindex.BundleMetadata - var totalSize int64 - - for _, num := range bundleNumbers { - path := filepath.Join(m.config.BundleDir, fmt.Sprintf("%06d.jsonl.zst", num)) - - // Load bundle - ops, err := m.operations.LoadBundle(path) - if err != nil { - m.logger.Printf("Warning: failed to load bundle %d: %v", num, err) - continue - } - - // Get file size - size, _ := m.operations.GetFileSize(path) - totalSize += size - - // Calculate parent and cursor from previous bundle - var parent string - var cursor string - if num > 1 && len(newMetadata) > 0 { - prevMeta := newMetadata[len(newMetadata)-1] - parent = prevMeta.Hash - cursor = prevMeta.EndTime.Format(time.RFC3339Nano) - } - - // Use the ONE method for metadata calculation - meta, err := m.CalculateBundleMetadata(num, path, ops, parent, cursor) - if err != nil { - m.logger.Printf("Warning: failed to calculate metadata for bundle %d: %v", num, err) - continue - } - - newMetadata = append(newMetadata, meta) - - m.logger.Printf(" Scanned bundle %06d: %d ops, %d DIDs", num, len(ops), meta.DIDCount) - } - - result.TotalSize = totalSize - - // Rebuild index - m.index.Rebuild(newMetadata) - - // Save index - if err := m.SaveIndex(); err != nil { - return nil, fmt.Errorf("failed to save index: %w", err) - } - - result.IndexUpdated = true - - m.logger.Printf("Index rebuilt with %d bundles", len(newMetadata)) - - return result, nil -} - -// ScanDirectoryParallel scans the bundle directory in parallel and rebuilds the index -func (m *Manager) ScanDirectoryParallel(workers int, progressCallback func(current, total int, bytesProcessed int64)) (*DirectoryScanResult, error) { - result := &DirectoryScanResult{ - BundleDir: m.config.BundleDir, - } - - m.logger.Printf("Scanning directory (parallel, %d workers): %s", workers, m.config.BundleDir) - - // Find all bundle files - files, err := filepath.Glob(filepath.Join(m.config.BundleDir, "*.jsonl.zst")) - if err != nil { - return nil, fmt.Errorf("failed to scan directory: %w", err) - } - files = filterBundleFiles(files) - - if len(files) == 0 { - m.logger.Printf("No bundle files found") - return result, nil - } - - // Parse bundle numbers - var bundleNumbers []int - for _, file := range files { - base := filepath.Base(file) - numStr := strings.TrimSuffix(base, ".jsonl.zst") - num, err := strconv.Atoi(numStr) - if err != nil { - m.logger.Printf("Warning: skipping invalid filename: %s", base) - continue - } - bundleNumbers = append(bundleNumbers, num) - } - - sort.Ints(bundleNumbers) - - result.BundleCount = len(bundleNumbers) - if len(bundleNumbers) > 0 { - result.FirstBundle = bundleNumbers[0] - result.LastBundle = bundleNumbers[len(bundleNumbers)-1] - } - - // Find gaps - if len(bundleNumbers) > 1 { - for i := result.FirstBundle; i <= result.LastBundle; i++ { - found := false - for _, num := range bundleNumbers { - if num == i { - found = true - break - } - } - if !found { - result.MissingGaps = append(result.MissingGaps, i) - } - } - } - - m.logger.Printf("Found %d bundles (gaps: %d)", result.BundleCount, len(result.MissingGaps)) - - // Process bundles in parallel - type bundleResult struct { - index int - meta *bundleindex.BundleMetadata - err error - } - - jobs := make(chan int, len(bundleNumbers)) - results := make(chan bundleResult, len(bundleNumbers)) - - // Start workers - var wg sync.WaitGroup - for w := 0; w < workers; w++ { - wg.Add(1) - go func() { - defer wg.Done() - for num := range jobs { - path := filepath.Join(m.config.BundleDir, fmt.Sprintf("%06d.jsonl.zst", num)) - - // Stream metadata WITHOUT loading all operations - meta, err := m.CalculateMetadataStreaming(num, path) - if err != nil { - results <- bundleResult{index: num, err: err} - continue - } - - results <- bundleResult{index: num, meta: meta} - } - }() - } - - // Send jobs - for _, num := range bundleNumbers { - jobs <- num - } - close(jobs) - - // Wait for all workers to finish - go func() { - wg.Wait() - close(results) - }() - - // Collect results - metadataMap := make(map[int]*bundleindex.BundleMetadata) - var totalSize int64 - var totalUncompressed int64 - processed := 0 - - for result := range results { - processed++ - - // Update progress WITH bytes - if progressCallback != nil { - if result.meta != nil { - totalUncompressed += result.meta.UncompressedSize - } - progressCallback(processed, len(bundleNumbers), totalUncompressed) - } - - if result.err != nil { - m.logger.Printf("Warning: failed to process bundle %d: %v", result.index, result.err) - continue - } - metadataMap[result.index] = result.meta - totalSize += result.meta.CompressedSize - } - - // Build ordered metadata slice and calculate chain hashes - var newMetadata []*bundleindex.BundleMetadata - var parent string - - for i, num := range bundleNumbers { - meta, ok := metadataMap[num] - if !ok { - continue - } - - // Set cursor from previous bundle's EndTime - if i > 0 && len(newMetadata) > 0 { - prevMeta := newMetadata[len(newMetadata)-1] - meta.Cursor = prevMeta.EndTime.Format(time.RFC3339Nano) - } - - // Calculate chain hash (must be done sequentially) - meta.Hash = m.operations.CalculateChainHash(parent, meta.ContentHash) - meta.Parent = parent - - newMetadata = append(newMetadata, meta) - parent = meta.Hash - } - - result.TotalSize = totalSize - result.TotalUncompressed = totalUncompressed - - // Rebuild index - m.index.Rebuild(newMetadata) - - // Save index - if err := m.SaveIndex(); err != nil { - return nil, fmt.Errorf("failed to save index: %w", err) - } - - result.IndexUpdated = true - - m.logger.Printf("Index rebuilt with %d bundles", len(newMetadata)) - - return result, nil -} - -// ScanBundle scans a single bundle file and returns its metadata -func (m *Manager) ScanBundle(path string, bundleNumber int) (*bundleindex.BundleMetadata, error) { - // Load bundle file - operations, err := m.operations.LoadBundle(path) - if err != nil { - return nil, fmt.Errorf("failed to load bundle: %w", err) - } - - if len(operations) == 0 { - return nil, fmt.Errorf("bundle is empty") - } - - // Get parent chain hash and cursor from previous bundle - var parent string - var cursor string - if bundleNumber > 1 { - if prevMeta, err := m.index.GetBundle(bundleNumber - 1); err == nil { - parent = prevMeta.Hash - cursor = prevMeta.EndTime.Format(time.RFC3339Nano) - } - } - - // Use the ONE method - return m.CalculateBundleMetadata(bundleNumber, path, operations, parent, cursor) -} - -// ScanAndIndexBundle scans a bundle file and adds it to the index -func (m *Manager) ScanAndIndexBundle(path string, bundleNumber int) (*bundleindex.BundleMetadata, error) { - meta, err := m.ScanBundle(path, bundleNumber) - if err != nil { - return nil, err - } - - // Add to index - m.index.AddBundle(meta) - - // Save index - if err := m.SaveIndex(); err != nil { - return nil, fmt.Errorf("failed to save index: %w", err) - } - - return meta, nil -} diff --git a/bundle/types.go b/bundle/types.go deleted file mode 100644 index 1cdfef2..0000000 --- a/bundle/types.go +++ /dev/null @@ -1,218 +0,0 @@ -package bundle - -import ( - "fmt" - "path/filepath" - "time" - - "tangled.org/atscan.net/plcbundle/internal/bundleindex" - "tangled.org/atscan.net/plcbundle/internal/plcclient" - "tangled.org/atscan.net/plcbundle/internal/types" -) - -// Bundle represents a PLC bundle -type Bundle struct { - BundleNumber int `json:"bundle_number"` - StartTime time.Time `json:"start_time"` - EndTime time.Time `json:"end_time"` - Operations []plcclient.PLCOperation `json:"-"` - DIDCount int `json:"did_count"` - - Hash string `json:"hash"` // Chain hash (primary) - ContentHash string `json:"content_hash"` // Content hash - Parent string `json:"parent"` - - CompressedHash string `json:"compressed_hash"` - CompressedSize int64 `json:"compressed_size"` - UncompressedSize int64 `json:"uncompressed_size"` - Cursor string `json:"cursor"` - BoundaryCIDs []string `json:"boundary_cids,omitempty"` - Compressed bool `json:"compressed"` - CreatedAt time.Time `json:"created_at"` -} - -// GetFilePath returns the file path for this bundle -func (b *Bundle) GetFilePath(bundleDir string) string { - return filepath.Join(bundleDir, fmt.Sprintf("%06d.jsonl.zst", b.BundleNumber)) -} - -// GetUncompressedFilePath returns the uncompressed file path -func (b *Bundle) GetUncompressedFilePath(bundleDir string) string { - return filepath.Join(bundleDir, fmt.Sprintf("%06d.jsonl", b.BundleNumber)) -} - -// OperationCount returns the number of operations (always BUNDLE_SIZE for complete bundles) -func (b *Bundle) OperationCount() int { - if len(b.Operations) > 0 { - return len(b.Operations) - } - return types.BUNDLE_SIZE -} - -// CompressionRatio returns the compression ratio -func (b *Bundle) CompressionRatio() float64 { - if b.CompressedSize == 0 { - return 0 - } - return float64(b.UncompressedSize) / float64(b.CompressedSize) -} - -// ValidateForSave performs validation before saving (no hash required) -func (b *Bundle) ValidateForSave() error { - if b.BundleNumber < 1 { - return fmt.Errorf("invalid bundle number: %d", b.BundleNumber) - } - if len(b.Operations) != types.BUNDLE_SIZE { - return fmt.Errorf("invalid operation count: expected %d, got %d", types.BUNDLE_SIZE, len(b.Operations)) - } - if b.StartTime.After(b.EndTime) { - return fmt.Errorf("start_time is after end_time") - } - return nil -} - -// Validate performs full validation (including hashes) -func (b *Bundle) Validate() error { - if err := b.ValidateForSave(); err != nil { - return err - } - if b.Hash == "" { - return fmt.Errorf("missing hash") - } - if b.CompressedHash == "" { - return fmt.Errorf("missing compressed hash") - } - return nil -} - -// VerificationResult contains the result of bundle verification -type VerificationResult struct { - BundleNumber int - Valid bool - HashMatch bool - FileExists bool - Error error - LocalHash string - ExpectedHash string -} - -// ChainVerificationResult contains the result of chain verification -type ChainVerificationResult struct { - Valid bool - ChainLength int - BrokenAt int - Error string - VerifiedBundles []int -} - -// DirectoryScanResult contains results from scanning a directory -type DirectoryScanResult struct { - BundleDir string - BundleCount int - FirstBundle int - LastBundle int - MissingGaps []int - TotalSize int64 // Compressed size - TotalUncompressed int64 - IndexUpdated bool -} - -// Config holds configuration for bundle operations -type Config struct { - BundleDir string - HandleResolverURL string - VerifyOnLoad bool - AutoRebuild bool - AutoInit bool // Allow auto-creating empty repository - RebuildWorkers int // Number of workers for parallel rebuild (0 = auto-detect) - RebuildProgress func(current, total int) // Progress callback for rebuild - Logger types.Logger - Verbose bool - Quiet bool -} - -// DefaultConfig returns default configuration -func DefaultConfig(bundleDir string) *Config { - return &Config{ - BundleDir: bundleDir, - HandleResolverURL: "https://quickdid.atscan.net", - VerifyOnLoad: true, - AutoRebuild: true, - AutoInit: false, - RebuildWorkers: 0, // 0 means auto-detect CPU count - RebuildProgress: nil, // No progress callback by default - Logger: nil, - Verbose: false, - Quiet: false, - } -} - -// CloneOptions configures cloning behavior -type CloneOptions struct { - RemoteURL string - Workers int - SkipExisting bool - ProgressFunc func(downloaded, total int, bytesDownloaded, bytesTotal int64) - SaveInterval time.Duration - Verbose bool - Logger types.Logger -} - -// CloneResult contains cloning results -type CloneResult struct { - RemoteBundles int - Downloaded int - Failed int - Skipped int - TotalBytes int64 - Duration time.Duration - Interrupted bool - FailedBundles []int -} - -// PLCOperationWithLocation contains an operation with its bundle/position metadata -type PLCOperationWithLocation struct { - Operation plcclient.PLCOperation - Bundle int - Position int -} - -func (b *Bundle) ToMetadata() *bundleindex.BundleMetadata { - return &bundleindex.BundleMetadata{ - BundleNumber: b.BundleNumber, - StartTime: b.StartTime, - EndTime: b.EndTime, - OperationCount: b.OperationCount(), - DIDCount: b.DIDCount, - Hash: b.Hash, // Chain hash - ContentHash: b.ContentHash, // Content hash - Parent: b.Parent, - CompressedHash: b.CompressedHash, - CompressedSize: b.CompressedSize, - UncompressedSize: b.UncompressedSize, - Cursor: b.Cursor, - CreatedAt: b.CreatedAt, - } -} - -// ResolveDIDResult contains DID resolution with timing metrics -type ResolveDIDResult struct { - Document *plcclient.DIDDocument - LatestOperation *plcclient.PLCOperation - MempoolTime time.Duration - IndexTime time.Duration - LoadOpTime time.Duration - TotalTime time.Duration - ResolvedHandle string - Source string // "mempool" or "bundle" - BundleNumber int // if from bundle - Position int // if from bundle -} - -type resolverTiming struct { - totalTime int64 - mempoolTime int64 - indexTime int64 - loadOpTime int64 - source string // "mempool" or "bundle" -} diff --git a/cmd/plcbundle/commands/clean.go b/cmd/plcbundle/commands/clean.go deleted file mode 100644 index baeefd3..0000000 --- a/cmd/plcbundle/commands/clean.go +++ /dev/null @@ -1,474 +0,0 @@ -package commands - -import ( - "fmt" - "os" - "path/filepath" - "sort" - "strings" - "time" - - "github.com/spf13/cobra" -) - -func NewCleanCommand() *cobra.Command { - var ( - dryRun bool - force bool - aggressive bool - keepTemp bool - keepOldMempool bool - olderThanDays int - ) - - cmd := &cobra.Command{ - Use: "clean [flags]", - Aliases: []string{"cleanup", "gc"}, - Short: "Clean up temporary and orphaned files", - Long: `Clean up temporary and orphaned files from repository - -Removes: - • Temporary files (.tmp, .tmp.*) - • Old mempool files (keeps current) - • Orphaned bundle files (not in index) - • DID index cache (if --aggressive) - -By default, performs a safe clean. Use --aggressive for deeper cleaning. -Always use --dry-run first to preview what will be deleted.`, - - Example: ` # Preview what would be cleaned (RECOMMENDED FIRST) - plcbundle clean --dry-run - - # Safe clean (remove temp files and old mempools) - plcbundle clean - - # Aggressive clean (also remove orphaned bundles and caches) - plcbundle clean --aggressive - - # Clean but keep temporary download files - plcbundle clean --keep-temp - - # Force clean without confirmation - plcbundle clean --force - - # Clean files older than 7 days only - plcbundle clean --older-than 7 - - # Full cleanup with preview - plcbundle clean --aggressive --dry-run`, - - RunE: func(cmd *cobra.Command, args []string) error { - verbose, _ := cmd.Root().PersistentFlags().GetBool("verbose") - - mgr, dir, err := getManager(&ManagerOptions{Cmd: cmd}) - if err != nil { - return err - } - defer mgr.Close() - - return runClean(mgr, dir, cleanOptions{ - dryRun: dryRun, - force: force, - aggressive: aggressive, - keepTemp: keepTemp, - keepOldMempool: keepOldMempool, - olderThanDays: olderThanDays, - verbose: verbose, - }) - }, - } - - // Flags - cmd.Flags().BoolVarP(&dryRun, "dry-run", "n", false, "Show what would be deleted without deleting") - cmd.Flags().BoolVarP(&force, "force", "f", false, "Skip confirmation prompt") - cmd.Flags().BoolVar(&aggressive, "aggressive", false, "Also remove orphaned bundles and caches") - cmd.Flags().BoolVar(&keepTemp, "keep-temp", false, "Don't delete temporary files") - cmd.Flags().BoolVar(&keepOldMempool, "keep-old-mempool", false, "Don't delete old mempool files") - cmd.Flags().IntVar(&olderThanDays, "older-than", 0, "Only clean files older than N days (0 = all)") - - return cmd -} - -type cleanOptions struct { - dryRun bool - force bool - aggressive bool - keepTemp bool - keepOldMempool bool - olderThanDays int - verbose bool -} - -type cleanPlan struct { - tempFiles []fileInfo - oldMempoolFiles []fileInfo - orphanedBundles []fileInfo - didIndexCache bool - totalSize int64 - totalFiles int -} - -type fileInfo struct { - path string - size int64 - modTime time.Time - reason string -} - -func runClean(mgr BundleManager, dir string, opts cleanOptions) error { - fmt.Printf("Analyzing repository: %s\n\n", dir) - - // Calculate age cutoff - var cutoffTime time.Time - if opts.olderThanDays > 0 { - cutoffTime = time.Now().AddDate(0, 0, -opts.olderThanDays) - fmt.Printf("Only cleaning files older than %d days (%s)\n\n", - opts.olderThanDays, cutoffTime.Format("2006-01-02")) - } - - // Build clean plan - plan, err := buildCleanPlan(mgr, dir, opts, cutoffTime) - if err != nil { - return fmt.Errorf("failed to build clean plan: %w", err) - } - - // Display plan - displayCleanPlan(plan, opts) - - // If dry-run, stop here - if opts.dryRun { - fmt.Printf("\n💡 This was a dry-run. No files were deleted.\n") - fmt.Printf(" Run without --dry-run to perform cleanup.\n") - return nil - } - - // If nothing to clean - if plan.totalFiles == 0 { - fmt.Printf("✓ Repository is clean (nothing to remove)\n") - return nil - } - - // Confirm unless forced - if !opts.force { - if !confirmClean(plan) { - fmt.Println("Cancelled") - return nil - } - fmt.Println() - } - - // Execute cleanup - return executeClean(plan, opts) -} - -func buildCleanPlan(mgr BundleManager, dir string, opts cleanOptions, cutoffTime time.Time) (*cleanPlan, error) { - plan := &cleanPlan{ - tempFiles: make([]fileInfo, 0), - oldMempoolFiles: make([]fileInfo, 0), - orphanedBundles: make([]fileInfo, 0), - } - - // Get current mempool info - mempoolStats := mgr.GetMempoolStats() - currentMempoolBundle := mempoolStats["target_bundle"].(int) - currentMempoolFile := fmt.Sprintf("plc_mempool_%06d.jsonl", currentMempoolBundle) - - // Scan directory - entries, err := os.ReadDir(dir) - if err != nil { - return nil, err - } - - // Get index for orphan detection - var indexedBundles map[int]bool - if opts.aggressive { - indexedBundles = make(map[int]bool) - index := mgr.GetIndex() - for _, meta := range index.GetBundles() { - indexedBundles[meta.BundleNumber] = true - } - } - - for _, entry := range entries { - if entry.IsDir() { - continue - } - - name := entry.Name() - fullPath := filepath.Join(dir, name) - info, err := entry.Info() - if err != nil { - continue - } - - // Skip if newer than cutoff - if !cutoffTime.IsZero() && info.ModTime().After(cutoffTime) { - continue - } - - // 1. Temporary files - if !opts.keepTemp && isTempFile(name) { - plan.tempFiles = append(plan.tempFiles, fileInfo{ - path: fullPath, - size: info.Size(), - modTime: info.ModTime(), - reason: "temporary file", - }) - plan.totalSize += info.Size() - plan.totalFiles++ - continue - } - - // 2. Old mempool files - if !opts.keepOldMempool && isMempoolFile(name) && name != currentMempoolFile { - plan.oldMempoolFiles = append(plan.oldMempoolFiles, fileInfo{ - path: fullPath, - size: info.Size(), - modTime: info.ModTime(), - reason: "old mempool file", - }) - plan.totalSize += info.Size() - plan.totalFiles++ - continue - } - - // 3. Orphaned bundles (aggressive mode only) - if opts.aggressive && isBundleFile(name) { - bundleNum := extractBundleNumber(name) - if bundleNum > 0 && !indexedBundles[bundleNum] { - plan.orphanedBundles = append(plan.orphanedBundles, fileInfo{ - path: fullPath, - size: info.Size(), - modTime: info.ModTime(), - reason: "not in index", - }) - plan.totalSize += info.Size() - plan.totalFiles++ - } - } - } - - // 4. DID index cache (aggressive mode only) - if opts.aggressive { - didIndexDir := filepath.Join(dir, ".plcbundle") - if stat, err := os.Stat(didIndexDir); err == nil && stat.IsDir() { - plan.didIndexCache = true - } - } - - return plan, nil -} - -func displayCleanPlan(plan *cleanPlan, opts cleanOptions) { - if opts.dryRun { - fmt.Printf("╔════════════════════════════════════════════════════════════════╗\n") - fmt.Printf("║ DRY-RUN MODE ║\n") - fmt.Printf("║ (showing what would be deleted) ║\n") - fmt.Printf("╚════════════════════════════════════════════════════════════════╝\n\n") - } else { - fmt.Printf("Clean Plan\n") - fmt.Printf("══════════\n\n") - } - - // Display by category - if len(plan.tempFiles) > 0 { - displayFileCategory("Temporary Files", plan.tempFiles, opts.verbose) - } - - if len(plan.oldMempoolFiles) > 0 { - displayFileCategory("Old Mempool Files", plan.oldMempoolFiles, opts.verbose) - } - - if len(plan.orphanedBundles) > 0 { - displayFileCategory("Orphaned Bundles", plan.orphanedBundles, opts.verbose) - } - - if plan.didIndexCache { - fmt.Printf("📁 DID Index Cache\n") - fmt.Printf(" .plcbundle/ directory (will be preserved but shards unloaded)\n\n") - } - - // Summary - if plan.totalFiles > 0 { - fmt.Printf("Summary\n") - fmt.Printf("───────\n") - fmt.Printf(" Files to delete: %s\n", formatNumber(plan.totalFiles)) - fmt.Printf(" Space to free: %s\n", formatBytes(plan.totalSize)) - fmt.Printf("\n") - } else { - fmt.Printf("✓ No files to clean\n\n") - } -} - -func displayFileCategory(title string, files []fileInfo, verbose bool) { - if len(files) == 0 { - return - } - - totalSize := int64(0) - for _, f := range files { - totalSize += f.size - } - - fmt.Printf("🗑️ %s (%d files, %s)\n", title, len(files), formatBytes(totalSize)) - - if verbose { - // Sort by size (largest first) - sorted := make([]fileInfo, len(files)) - copy(sorted, files) - sort.Slice(sorted, func(i, j int) bool { - return sorted[i].size > sorted[j].size - }) - - displayCount := len(sorted) - if displayCount > 20 { - displayCount = 20 - } - - for i := 0; i < displayCount; i++ { - f := sorted[i] - age := time.Since(f.modTime) - fmt.Printf(" • %-40s %10s %s ago\n", - filepath.Base(f.path), - formatBytes(f.size), - formatDurationShort(age)) - } - - if len(sorted) > displayCount { - fmt.Printf(" ... and %d more\n", len(sorted)-displayCount) - } - } else { - // Compact summary - fmt.Printf(" %d file(s) • %s\n", len(files), formatBytes(totalSize)) - } - fmt.Printf("\n") -} - -func confirmClean(plan *cleanPlan) bool { - fmt.Printf("⚠️ This will delete %d file(s) (%s)\n", - plan.totalFiles, formatBytes(plan.totalSize)) - fmt.Printf("Type 'clean' to confirm: ") - - var response string - fmt.Scanln(&response) - - return strings.TrimSpace(response) == "clean" -} - -func executeClean(plan *cleanPlan, opts cleanOptions) error { - deleted := 0 - var totalFreed int64 - failed := 0 - var firstError error - - // Delete temp files - for _, f := range plan.tempFiles { - if opts.verbose { - fmt.Printf(" Deleting: %s\n", filepath.Base(f.path)) - } - - if err := os.Remove(f.path); err != nil { - failed++ - if firstError == nil { - firstError = err - } - if opts.verbose { - fmt.Printf(" ✗ Failed: %v\n", err) - } - } else { - deleted++ - totalFreed += f.size - } - } - - // Delete old mempool files - for _, f := range plan.oldMempoolFiles { - if opts.verbose { - fmt.Printf(" Deleting: %s\n", filepath.Base(f.path)) - } - - if err := os.Remove(f.path); err != nil { - failed++ - if firstError == nil { - firstError = err - } - if opts.verbose { - fmt.Printf(" ✗ Failed: %v\n", err) - } - } else { - deleted++ - totalFreed += f.size - } - } - - // Delete orphaned bundles (aggressive) - if len(plan.orphanedBundles) > 0 { - fmt.Printf("\nRemoving orphaned bundles...\n") - for _, f := range plan.orphanedBundles { - if opts.verbose { - fmt.Printf(" Deleting: %s\n", filepath.Base(f.path)) - } - - if err := os.Remove(f.path); err != nil { - failed++ - if firstError == nil { - firstError = err - } - if opts.verbose { - fmt.Printf(" ✗ Failed: %v\n", err) - } - } else { - deleted++ - totalFreed += f.size - } - } - } - - // Summary - fmt.Printf("\n") - if failed > 0 { - fmt.Printf("⚠️ Cleanup completed with errors\n") - fmt.Printf(" Deleted: %d files\n", deleted) - fmt.Printf(" Failed: %d files\n", failed) - fmt.Printf(" Space freed: %s\n", formatBytes(totalFreed)) - if firstError != nil { - fmt.Printf(" First error: %v\n", firstError) - } - return fmt.Errorf("cleanup completed with %d errors", failed) - } - - fmt.Printf("✓ Cleanup complete\n") - fmt.Printf(" Deleted: %d files\n", deleted) - fmt.Printf(" Space freed: %s\n", formatBytes(totalFreed)) - - return nil -} - -// Helper functions - -func isTempFile(name string) bool { - return strings.HasSuffix(name, ".tmp") || - strings.Contains(name, ".tmp.") || - strings.HasPrefix(name, "._") || - strings.HasPrefix(name, ".~") -} - -func isMempoolFile(name string) bool { - return strings.HasPrefix(name, "plc_mempool_") && - strings.HasSuffix(name, ".jsonl") -} - -func isBundleFile(name string) bool { - return strings.HasSuffix(name, ".jsonl.zst") && - len(name) == len("000000.jsonl.zst") -} - -func extractBundleNumber(name string) int { - if !strings.HasSuffix(name, ".jsonl.zst") { - return 0 - } - numStr := strings.TrimSuffix(name, ".jsonl.zst") - var num int - fmt.Sscanf(numStr, "%d", &num) - return num -} diff --git a/cmd/plcbundle/commands/clone.go b/cmd/plcbundle/commands/clone.go deleted file mode 100644 index 73c5a49..0000000 --- a/cmd/plcbundle/commands/clone.go +++ /dev/null @@ -1,228 +0,0 @@ -package commands - -import ( - "context" - "fmt" - "os" - "os/signal" - "path/filepath" - "strings" - "sync" - "syscall" - "time" - - "github.com/spf13/cobra" - "tangled.org/atscan.net/plcbundle/cmd/plcbundle/ui" - internalsync "tangled.org/atscan.net/plcbundle/internal/sync" -) - -func NewCloneCommand() *cobra.Command { - var ( - workers int - resume bool - ) - - cmd := &cobra.Command{ - Use: "clone [directory]", - Short: "Clone bundles from remote HTTP endpoint", - Long: `Clone bundles from remote plcbundle HTTP endpoint - -Download all bundles from a remote plcbundle server to your local -repository. Similar to 'git clone' - creates a complete copy of -the remote repository. - -If directory is not specified, bundles will be cloned into './bundles' - -The clone process: - 1. Fetches remote index - 2. Downloads missing bundles in parallel - 3. Verifies hashes - 4. Updates local index - 5. Can be interrupted and resumed safely`, - - Args: cobra.RangeArgs(1, 2), - - Example: ` # Clone into default 'bundles' directory - plcbundle clone https://plc.example.com - - # Clone into specific directory - plcbundle clone https://plc.example.com my-plc-data - - # Clone with more parallel workers (faster) - plcbundle clone https://plc.example.com --workers 8 - - # Resume interrupted clone - plcbundle clone https://plc.example.com --resume - - # Verbose output (shows each bundle) - plcbundle clone https://plc.example.com my-bundles -v`, - - RunE: func(cmd *cobra.Command, args []string) error { - remoteURL := strings.TrimSuffix(args[0], "/") - - targetDir := "bundles" - if len(args) > 1 { - targetDir = args[1] - } - - // Get global verbose flag - verbose, _ := cmd.Root().PersistentFlags().GetBool("verbose") - - return runClone(remoteURL, targetDir, cloneOptions{ - workers: workers, - verbose: verbose, - resume: resume, - }) - }, - } - - // Local flags - cmd.Flags().IntVarP(&workers, "workers", "w", 4, - "Number of parallel download workers (higher = faster, more bandwidth)") - cmd.Flags().BoolVarP(&resume, "resume", "r", false, - "Resume interrupted clone (automatically skips existing bundles)") - - return cmd -} - -type cloneOptions struct { - workers int - verbose bool - resume bool -} - -func runClone(remoteURL string, targetDir string, opts cloneOptions) error { - // Create target directory if it doesn't exist - absDir, err := filepath.Abs(targetDir) - if err != nil { - return fmt.Errorf("invalid directory path: %w", err) - } - - // Clone creates new repository in specific directory - mgr, dir, err := getManager(&ManagerOptions{ - Dir: absDir, - PLCURL: "https://plc.directory", - AutoInit: true, - }) - if err != nil { - return err - } - defer mgr.Close() - - fmt.Printf("Cloning from: %s\n", remoteURL) - fmt.Printf("Target directory: %s\n", dir) - fmt.Printf("Workers: %d\n", opts.workers) - - if opts.resume { - fmt.Printf("Mode: resume (skipping existing bundles)\n") - } - - fmt.Printf("(Press Ctrl+C to safely interrupt - progress will be saved)\n\n") - - // Set up signal handling - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) - - // Set up progress bar - var progress *ui.ProgressBar - var progressMu sync.Mutex - progressActive := true - - go func() { - <-sigChan - progressMu.Lock() - progressActive = false - if progress != nil { - fmt.Println() - } - progressMu.Unlock() - - fmt.Printf("\n⚠️ Interrupt received! Finishing current downloads and saving progress...\n") - cancel() - }() - - // Clone with library - result, err := mgr.CloneFromRemote(ctx, internalsync.CloneOptions{ - RemoteURL: remoteURL, - Workers: opts.workers, - SkipExisting: opts.resume, - SaveInterval: 5 * time.Second, - Verbose: opts.verbose, - ProgressFunc: func(downloaded, total int, bytesDownloaded, bytesTotal int64) { - progressMu.Lock() - defer progressMu.Unlock() - - if !progressActive { - return - } - - if progress == nil { - progress = ui.NewProgressBarWithBytes(total, bytesTotal) - } - progress.SetWithBytes(downloaded, bytesDownloaded) - }, - }) - - // Ensure progress is stopped - progressMu.Lock() - progressActive = false - if progress != nil { - progress.Finish() - } - progressMu.Unlock() - - if err != nil { - return fmt.Errorf("clone failed: %w", err) - } - - // Display results - if result.Interrupted { - fmt.Printf("⚠️ Download interrupted by user\n") - } else { - fmt.Printf("\n✓ Clone complete in %s\n", result.Duration.Round(time.Millisecond)) - } - - fmt.Printf("\nResults:\n") - fmt.Printf(" Remote bundles: %d\n", result.RemoteBundles) - if result.Skipped > 0 { - fmt.Printf(" Skipped (existing): %d\n", result.Skipped) - } - fmt.Printf(" Downloaded: %d\n", result.Downloaded) - if result.Failed > 0 { - fmt.Printf(" Failed: %d\n", result.Failed) - } - fmt.Printf(" Total size: %s\n", formatBytes(result.TotalBytes)) - - if result.Duration.Seconds() > 0 && result.Downloaded > 0 { - mbPerSec := float64(result.TotalBytes) / result.Duration.Seconds() / (1024 * 1024) - bundlesPerSec := float64(result.Downloaded) / result.Duration.Seconds() - fmt.Printf(" Average speed: %.1f MB/s (%.1f bundles/s)\n", mbPerSec, bundlesPerSec) - } - - if result.Failed > 0 { - fmt.Printf("\n⚠️ Failed bundles: ") - for i, num := range result.FailedBundles { - if i > 0 { - fmt.Printf(", ") - } - if i > 10 { - fmt.Printf("... and %d more", len(result.FailedBundles)-10) - break - } - fmt.Printf("%06d", num) - } - fmt.Printf("\nRe-run the clone command to retry failed bundles.\n") - return fmt.Errorf("clone completed with errors") - } - - if result.Interrupted { - fmt.Printf("\n✓ Progress saved. Re-run the clone command to resume.\n") - return fmt.Errorf("clone interrupted") - } - - fmt.Printf("\n✓ Clone complete!\n") - return nil -} diff --git a/cmd/plcbundle/commands/common.go b/cmd/plcbundle/commands/common.go deleted file mode 100644 index 6cfb8d1..0000000 --- a/cmd/plcbundle/commands/common.go +++ /dev/null @@ -1,293 +0,0 @@ -package commands - -import ( - "context" - "fmt" - "os" - "path/filepath" - "strings" - "time" - - "github.com/spf13/cobra" - "tangled.org/atscan.net/plcbundle/bundle" - "tangled.org/atscan.net/plcbundle/internal/bundleindex" - "tangled.org/atscan.net/plcbundle/internal/didindex" - "tangled.org/atscan.net/plcbundle/internal/plcclient" - internalsync "tangled.org/atscan.net/plcbundle/internal/sync" - "tangled.org/atscan.net/plcbundle/internal/types" -) - -// BundleManager interface (for testing/mocking) -type BundleManager interface { - Close() - GetIndex() *bundleindex.Index - LoadBundle(ctx context.Context, bundleNumber int) (*bundle.Bundle, error) - VerifyBundle(ctx context.Context, bundleNumber int) (*bundle.VerificationResult, error) - VerifyChain(ctx context.Context) (*bundle.ChainVerificationResult, error) - GetInfo() map[string]interface{} - GetMempoolStats() map[string]interface{} - GetMempoolOperations() ([]plcclient.PLCOperation, error) - ValidateMempool() error - RefreshMempool() error - ClearMempool() error - FetchNextBundle(ctx context.Context, verbose bool, quiet bool) (*bundle.Bundle, types.BundleProductionStats, error) - SaveBundle(ctx context.Context, bundle *bundle.Bundle, verbose bool, quiet bool, stats types.BundleProductionStats, skipDIDIndex bool) (time.Duration, error) - SaveIndex() error - GetDIDIndexStats() map[string]interface{} - GetDIDIndex() *didindex.Manager - BuildDIDIndex(ctx context.Context, progress func(int, int)) error - GetDIDOperations(ctx context.Context, did string, verbose bool) ([]plcclient.PLCOperation, []PLCOperationWithLocation, error) - GetDIDOperationsFromMempool(did string) ([]plcclient.PLCOperation, error) - GetLatestDIDOperation(ctx context.Context, did string) (*plcclient.PLCOperation, error) - LoadOperation(ctx context.Context, bundleNum, position int) (*plcclient.PLCOperation, error) - LoadOperations(ctx context.Context, bundleNumber int, positions []int) (map[int]*plcclient.PLCOperation, error) - CloneFromRemote(ctx context.Context, opts internalsync.CloneOptions) (*internalsync.CloneResult, error) - ResolveDID(ctx context.Context, did string) (*bundle.ResolveDIDResult, error) - RunSyncOnce(ctx context.Context, config *internalsync.SyncLoopConfig) (int, error) - RunSyncLoop(ctx context.Context, config *internalsync.SyncLoopConfig) error - GetBundleIndex() didindex.BundleIndexProvider - ScanDirectoryParallel(workers int, progressCallback func(current, total int, bytesProcessed int64)) (*bundle.DirectoryScanResult, error) - LoadBundleForDIDIndex(ctx context.Context, bundleNumber int) (*didindex.BundleData, error) - ResolveHandleOrDID(ctx context.Context, input string) (string, time.Duration, error) - SetQuiet(quiet bool) -} - -// PLCOperationWithLocation wraps operation with location info -type PLCOperationWithLocation = bundle.PLCOperationWithLocation - -// ============================================================================ -// MANAGER OPTIONS STRUCT -// ============================================================================ - -// ManagerOptions configures manager creation -type ManagerOptions struct { - Cmd *cobra.Command // Optional: for reading --dir flag - Dir string // Optional: explicit directory (overrides Cmd flag and cwd) - PLCURL string // Optional: PLC directory URL - HandleResolverURL string // Optional: Handle resolver URL (XRPC) - AutoInit bool // Optional: allow creating new empty repository (default: false) -} - -// ============================================================================ -// SINGLE UNIFIED getManager METHOD -// ============================================================================ - -// getManager creates or opens a bundle manager -// Pass nil for default options (read-only, current directory, no PLC client) -// -// Examples: -// -// getManager(nil) // All defaults -// getManager(&ManagerOptions{Cmd: cmd}) // Use --dir flag -// getManager(&ManagerOptions{PLCURL: url}) // Add PLC client -// getManager(&ManagerOptions{AutoInit: true}) // Allow creating repo -// getManager(&ManagerOptions{Dir: "/path", AutoInit: true}) // Explicit dir + create -func getManager(opts *ManagerOptions) (*bundle.Manager, string, error) { - - // Silence usage for operational errors - if opts.Cmd != nil { - opts.Cmd.SilenceUsage = true - } - - // Use defaults if nil - if opts == nil { - opts = &ManagerOptions{} - } - - // Determine directory (priority: explicit Dir > Cmd flag > current working dir) - var dir string - - if opts.Dir != "" { - // Explicit directory provided - dir = opts.Dir - } else if opts.Cmd != nil { - // Try to get from command --dir flag - dir, _ = opts.Cmd.Root().PersistentFlags().GetString("dir") - } - - // Fallback to current working directory - if dir == "" { - var err error - dir, err = os.Getwd() - if err != nil { - return nil, "", err - } - } - - // Convert to absolute path - absDir, err := filepath.Abs(dir) - if err != nil { - return nil, "", fmt.Errorf("invalid directory path: %w", err) - } - - // Create config - config := bundle.DefaultConfig(absDir) - config.AutoInit = opts.AutoInit - - // Check BOTH global AND local verbose flags - if opts.Cmd != nil { - globalVerbose, _ := opts.Cmd.Root().PersistentFlags().GetBool("verbose") - localVerbose, _ := opts.Cmd.Flags().GetBool("verbose") - globalQuiet, _ := opts.Cmd.Root().PersistentFlags().GetBool("quiet") - localQuiet, _ := opts.Cmd.Flags().GetBool("quiet") - - // Use OR logic: verbose if EITHER flag is set - config.Verbose = globalVerbose || localVerbose - config.Quiet = globalQuiet || localQuiet - } - - // Create PLC client if URL provided - var client *plcclient.Client - if opts.PLCURL != "" { - // Build user agent with version - userAgent := fmt.Sprintf("plcbundle/%s", - GetVersion()) - - client = plcclient.NewClient( - opts.PLCURL, - plcclient.WithUserAgent(userAgent), - ) - } - - // Set handle resolver URL from flag or option - handleResolverURL := opts.HandleResolverURL - if handleResolverURL == "" && opts.Cmd != nil { - handleResolverURL, _ = opts.Cmd.Root().PersistentFlags().GetString("handle-resolver") - } - // Only override default if explicitly provided - if handleResolverURL != "" { - config.HandleResolverURL = handleResolverURL - } - - // Create manager - mgr, err := bundle.NewManager(config, client) - if err != nil { - return nil, "", err - } - - return mgr, absDir, nil -} - -// parseBundleRange parses bundle range string -func parseBundleRange(rangeStr string) (start, end int, err error) { - if !strings.Contains(rangeStr, "-") { - var num int - _, err = fmt.Sscanf(rangeStr, "%d", &num) - if err != nil { - return 0, 0, fmt.Errorf("invalid bundle number: %w", err) - } - return num, num, nil - } - - parts := strings.Split(rangeStr, "-") - if len(parts) != 2 { - return 0, 0, fmt.Errorf("invalid range format (expected: N or start-end)") - } - - _, err = fmt.Sscanf(parts[0], "%d", &start) - if err != nil { - return 0, 0, fmt.Errorf("invalid start: %w", err) - } - - _, err = fmt.Sscanf(parts[1], "%d", &end) - if err != nil { - return 0, 0, fmt.Errorf("invalid end: %w", err) - } - - if start > end { - return 0, 0, fmt.Errorf("start must be <= end") - } - - return start, end, nil -} - -// Formatting helpers - -func formatBytes(bytes int64) string { - if bytes < 0 { - return fmt.Sprintf("-%s", formatBytes(-bytes)) - } - - const unit = 1000 - if bytes < unit { - return fmt.Sprintf("%d B", bytes) - } - div, exp := int64(unit), 0 - for n := bytes / unit; n >= unit; n /= unit { - div *= unit - exp++ - } - return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "KMGTPE"[exp]) -} - -func formatDuration(d time.Duration) string { - if d < time.Minute { - return fmt.Sprintf("%.0f seconds", d.Seconds()) - } - if d < time.Hour { - return fmt.Sprintf("%.1f minutes", d.Minutes()) - } - if d < 24*time.Hour { - return fmt.Sprintf("%.1f hours", d.Hours()) - } - days := d.Hours() / 24 - if days < 30 { - return fmt.Sprintf("%.1f days", days) - } - if days < 365 { - return fmt.Sprintf("%.1f months", days/30) - } - return fmt.Sprintf("%.1f years", days/365) -} - -func formatNumber(n int) string { - s := fmt.Sprintf("%d", n) - var result []byte - for i, c := range s { - if i > 0 && (len(s)-i)%3 == 0 { - result = append(result, ',') - } - result = append(result, byte(c)) - } - return string(result) -} - -// commandLogger adapts to types.Logger -type commandLogger struct { - quiet bool -} - -func (l *commandLogger) Printf(format string, v ...interface{}) { - if !l.quiet { - fmt.Fprintf(os.Stderr, format+"\n", v...) - } -} - -func (l *commandLogger) Println(v ...interface{}) { - if !l.quiet { - fmt.Fprintln(os.Stderr, v...) - } -} - -// formatCount formats count with color coding -func formatCount(count int) string { - if count == 0 { - return "\033[32m0 ✓\033[0m" - } - return fmt.Sprintf("\033[33m%d ⚠️\033[0m", count) -} - -// formatCountCritical formats count with critical color coding -func formatCountCritical(count int) string { - if count == 0 { - return "\033[32m0 ✓\033[0m" - } - return fmt.Sprintf("\033[31m%d ✗\033[0m", count) -} - -func min(a, b int) int { - if a < b { - return a - } - return b -} diff --git a/cmd/plcbundle/commands/detector.go b/cmd/plcbundle/commands/detector.go deleted file mode 100644 index 789cfa6..0000000 --- a/cmd/plcbundle/commands/detector.go +++ /dev/null @@ -1,789 +0,0 @@ -package commands - -import ( - "bufio" - "context" - "fmt" - "net/http" - _ "net/http/pprof" - "os" - "runtime" - "sort" - "strings" - "sync" - "sync/atomic" - - "github.com/goccy/go-json" - "github.com/spf13/cobra" - "tangled.org/atscan.net/plcbundle/cmd/plcbundle/ui" - "tangled.org/atscan.net/plcbundle/detector" - "tangled.org/atscan.net/plcbundle/internal/plcclient" -) - -func NewDetectorCommand() *cobra.Command { - cmd := &cobra.Command{ - Use: "detector", - Aliases: []string{"detect"}, - Short: "Detect and filter spam/invalid operations", - Long: `Detect and filter spam/invalid operations - -Run detection algorithms on PLC operations to identify spam, -invalid handles, service abuse, and other problematic patterns. - -Built-in detectors: - • invalid_handle - Invalid handle patterns (underscores, etc) - • aka_spam - Excessive/garbage alsoKnownAs entries - • spam_pds - Known spam PDS endpoints - • service_abuse - Abused service structures - • noop - Benchmark detector (returns no matches) - -Custom detectors: - Load JavaScript detectors from .js files with a detect() function.`, - - Example: ` # List available detectors - plcbundle detector list - - # Run detector on bundles - plcbundle detector run invalid_handle --bundles 1-100 - - # Run with parallel processing - plcbundle detector run invalid_handle --bundles 1-100 --workers 8 - - # Run custom detector script - plcbundle detector run ./my_detector.js --bundles 1-100 - - # Run multiple detectors - plcbundle detector run invalid_handle aka_spam --bundles 1-100 - - # Run all detectors - plcbundle detector run all --bundles 1-100 - - # Filter JSONL from stdin - cat ops.jsonl | plcbundle detector filter invalid_handle > clean.jsonl - - # Get detector info - plcbundle detector info invalid_handle`, - } - - // Add subcommands - cmd.AddCommand(newDetectorListCommand()) - cmd.AddCommand(newDetectorTestCommand()) - cmd.AddCommand(newDetectorRunCommand()) - cmd.AddCommand(newDetectorFilterCommand()) - cmd.AddCommand(newDetectorInfoCommand()) - - return cmd -} - -// ============================================================================ -// DETECTOR LIST -// ============================================================================ - -func newDetectorListCommand() *cobra.Command { - return &cobra.Command{ - Use: "list", - Short: "List available detectors", - Long: `List all available built-in and loaded detectors`, - - Example: ` # List all detectors - plcbundle detector list`, - - RunE: func(cmd *cobra.Command, args []string) error { - registry := detector.DefaultRegistry() - detectors := registry.List() - - sort.Slice(detectors, func(i, j int) bool { - return detectors[i].Name() < detectors[j].Name() - }) - - fmt.Printf("Available detectors:\n\n") - for _, d := range detectors { - fmt.Printf(" %-20s %s (v%s)\n", d.Name(), d.Description(), d.Version()) - } - fmt.Printf("\nUse 'plcbundle detector info ' for details\n") - - return nil - }, - } -} - -// ============================================================================ -// DETECTOR TEST -// ============================================================================ - -func newDetectorTestCommand() *cobra.Command { - var ( - bundleNum int - confidence float64 - verbose bool - ) - - cmd := &cobra.Command{ - Use: "test ", - Short: "Test detector on specific bundle", - Long: `Test a detector on a specific bundle and show results`, - - Example: ` # Test on bundle 42 - plcbundle detector test invalid_handle --bundle 42 - - # Verbose output with samples - plcbundle detector test aka_spam --bundle 100 -v - - # Custom confidence threshold - plcbundle detector test spam_pds --bundle 50 --confidence 0.85`, - - Args: cobra.ExactArgs(1), - - RunE: func(cmd *cobra.Command, args []string) error { - detectorName := args[0] - - mgr, _, err := getManager(&ManagerOptions{Cmd: cmd}) - if err != nil { - return err - } - defer mgr.Close() - - ctx := context.Background() - bundle, err := mgr.LoadBundle(ctx, bundleNum) - if err != nil { - return fmt.Errorf("failed to load bundle: %w", err) - } - - fmt.Printf("Testing detector '%s' on bundle %06d...\n", detectorName, bundleNum) - fmt.Printf("Min confidence: %.2f\n\n", confidence) - - registry := detector.DefaultRegistry() - config := detector.DefaultConfig() - config.MinConfidence = confidence - - runner := detector.NewRunner(registry, config, &commandLogger{}) - results, err := runner.RunOnBundle(ctx, detectorName, bundle) - if err != nil { - return fmt.Errorf("detection failed: %w", err) - } - - stats := detector.CalculateStats(results, len(bundle.Operations)) - - fmt.Printf("Results:\n") - fmt.Printf(" Total operations: %d\n", stats.TotalOperations) - fmt.Printf(" Matches found: %d (%.2f%%)\n\n", stats.MatchedCount, stats.MatchRate*100) - - if len(stats.ByReason) > 0 { - fmt.Printf("Breakdown by reason:\n") - for reason, count := range stats.ByReason { - pct := float64(count) / float64(stats.MatchedCount) * 100 - fmt.Printf(" %-25s %d (%.1f%%)\n", reason, count, pct) - } - fmt.Printf("\n") - } - - if verbose && len(results) > 0 { - fmt.Printf("Sample matches (first 10):\n") - displayCount := min(10, len(results)) - - for i := 0; i < displayCount; i++ { - res := results[i] - globalPos := (res.BundleNumber * 10000) + res.Position - fmt.Printf(" %d. Position %d: %s\n", i+1, globalPos, res.DID) - fmt.Printf(" Reason: %s (confidence: %.2f)\n", res.Match.Reason, res.Match.Confidence) - if res.Match.Note != "" { - fmt.Printf(" Note: %s\n", res.Match.Note) - } - } - } - - return nil - }, - } - - cmd.Flags().IntVar(&bundleNum, "bundle", 0, "Bundle number to test (required)") - cmd.Flags().Float64Var(&confidence, "confidence", 0.90, "Minimum confidence threshold") - cmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "Show sample matches") - - cmd.MarkFlagRequired("bundle") - - return cmd -} - -// ============================================================================ -// DETECTOR RUN (with parallel processing) -// ============================================================================ - -func newDetectorRunCommand() *cobra.Command { - var ( - bundleRange string - confidence float64 - pprofPort string - workers int - noProgress bool - ) - - cmd := &cobra.Command{ - Use: "run [detector2...] [flags]", - Short: "Run detector(s) and output CSV results", - Long: `Run one or more detectors on bundles and output results as CSV - -Output format uses global position: (bundleNumber × 10,000) + position -Example: 88410345 = bundle 8841, position 345 - -Supports parallel processing across multiple workers for better performance. - -Detectors can be: - • Built-in detector names (invalid_handle, aka_spam, etc) - • Path to JavaScript detector file (./my_detector.js) - • Special keyword 'all' to run all built-in detectors`, - - Example: ` # Run single detector - plcbundle detector run invalid_handle --bundles 1-100 - - # Run with 8 parallel workers (faster) - plcbundle detector run invalid_handle --bundles 1-1000 --workers 8 - - # Run multiple detectors in parallel - plcbundle detector run invalid_handle aka_spam --bundles 1-100 -w 4 - - # Run custom script - plcbundle detector run ./my_detector.js --bundles 1-100 - - # Run all built-in detectors - plcbundle detector run all --bundles 1-100 --workers 8 - - # Save results to file - plcbundle detector run all --bundles 1-100 -w 8 > results.csv - - # Disable progress bar (for scripting) - plcbundle detector run spam --bundles 1-100 --no-progress - - # Enable profiling - plcbundle detector run all --bundles 1-100 --pprof :6060`, - - Args: cobra.MinimumNArgs(1), - - RunE: func(cmd *cobra.Command, args []string) error { - detectorNames := args - - // Start pprof if requested - if pprofPort != "" { - go func() { - fmt.Fprintf(os.Stderr, "pprof server starting on http://localhost%s/debug/pprof/\n", pprofPort) - http.ListenAndServe(pprofPort, nil) - }() - } - - // Auto-detect workers if not specified - if workers <= 0 { - workers = runtime.NumCPU() - if workers < 1 { - workers = 1 - } - } - - mgr, _, err := getManager(&ManagerOptions{Cmd: cmd}) - if err != nil { - return err - } - defer mgr.Close() - - // Determine range - var start, end int - if bundleRange == "" { - index := mgr.GetIndex() - bundles := index.GetBundles() - if len(bundles) == 0 { - return fmt.Errorf("no bundles available") - } - start = bundles[0].BundleNumber - end = bundles[len(bundles)-1].BundleNumber - fmt.Fprintf(os.Stderr, "Using all bundles: %d-%d\n", start, end) - } else { - start, end, err = parseBundleRange(bundleRange) - if err != nil { - return err - } - } - - // Load detectors - setup, err := parseAndLoadDetectors(detectorNames, confidence) - if err != nil { - return err - } - defer setup.cleanup() - - fmt.Fprintf(os.Stderr, "Running %d detector(s) on bundles %d-%d (%d workers)\n", - len(setup.detectors), start, end, workers) - fmt.Fprintf(os.Stderr, "Min confidence: %.2f\n\n", confidence) - - return runDetectionParallel(cmd.Context(), mgr, setup, start, end, workers, !noProgress) - }, - } - - cmd.Flags().StringVar(&bundleRange, "bundles", "", "Bundle range (e.g., '1-100', default: all)") - cmd.Flags().Float64Var(&confidence, "confidence", 0.90, "Minimum confidence threshold") - cmd.Flags().IntVarP(&workers, "workers", "w", 0, "Number of parallel workers (0 = auto-detect CPU count)") - cmd.Flags().BoolVar(&noProgress, "no-progress", false, "Disable progress bar") - cmd.Flags().StringVar(&pprofPort, "pprof", "", "Enable pprof on port (e.g., ':6060')") - - return cmd -} - -// ============================================================================ -// DETECTOR FILTER -// ============================================================================ - -func newDetectorFilterCommand() *cobra.Command { - var confidence float64 - - cmd := &cobra.Command{ - Use: "filter [detector2...]", - Short: "Filter JSONL operations from stdin", - Long: `Filter JSONL operations from stdin using detectors - -Reads operations from stdin, runs detectors, and outputs only -operations that DO NOT match (clean operations). - -Perfect for cleaning datasets or pre-processing.`, - - Example: ` # Filter with built-in detector - cat ops.jsonl | plcbundle detector filter invalid_handle > clean.jsonl - - # Filter with custom script - plcbundle export --all | plcbundle detector filter ./spam.js > clean.jsonl - - # Chain multiple detectors - cat ops.jsonl | plcbundle detector filter invalid_handle aka_spam > clean.jsonl - - # Custom confidence - cat ops.jsonl | plcbundle detector filter spam_pds --confidence 0.95 > clean.jsonl`, - - Args: cobra.MinimumNArgs(1), - - RunE: func(cmd *cobra.Command, args []string) error { - detectorNames := args - - setup, err := parseAndLoadDetectors(detectorNames, confidence) - if err != nil { - return err - } - defer setup.cleanup() - - fmt.Fprintf(os.Stderr, "Filtering with %d detector(s), min confidence: %.2f\n\n", len(setup.detectors), confidence) - - return filterFromStdin(cmd.Context(), setup) - }, - } - - cmd.Flags().Float64Var(&confidence, "confidence", 0.90, "Minimum confidence threshold") - - return cmd -} - -// ============================================================================ -// DETECTOR INFO -// ============================================================================ - -func newDetectorInfoCommand() *cobra.Command { - return &cobra.Command{ - Use: "info ", - Short: "Show detailed detector information", - Long: `Show detailed information about a specific detector`, - - Example: ` # Show detector info - plcbundle detector info invalid_handle - plcbundle detector info aka_spam`, - - Args: cobra.ExactArgs(1), - - RunE: func(cmd *cobra.Command, args []string) error { - detectorName := args[0] - - registry := detector.DefaultRegistry() - d, err := registry.Get(detectorName) - if err != nil { - return err - } - - fmt.Printf("Detector: %s\n", d.Name()) - fmt.Printf("Version: %s\n", d.Version()) - fmt.Printf("Description: %s\n\n", d.Description()) - - fmt.Printf("Usage examples:\n") - fmt.Printf(" # Test on single bundle\n") - fmt.Printf(" plcbundle detector test %s --bundle 42\n\n", d.Name()) - fmt.Printf(" # Run on range and save\n") - fmt.Printf(" plcbundle detector run %s --bundles 1-100 > results.csv\n\n", d.Name()) - fmt.Printf(" # Filter JSONL stream\n") - fmt.Printf(" cat ops.jsonl | plcbundle detector filter %s > clean.jsonl\n\n", d.Name()) - - return nil - }, - } -} - -// ============================================================================ -// PARALLEL DETECTION IMPLEMENTATION -// ============================================================================ - -type detectionMatch struct { - globalPos int - cid string - size int - confidence float64 - labels []string -} - -// detectionResult holds results from processing a bundle -type detectionResult struct { - bundleNum int - matches []detectionMatch - totalOps int - totalBytes int64 - matchedBytes int64 - err error -} - -func runDetectionParallel(ctx context.Context, mgr BundleManager, setup *detectorSetup, start, end int, workers int, showProgress bool) error { - totalBundles := end - start + 1 - - // Don't create more workers than bundles - if workers > totalBundles { - workers = totalBundles - } - - // Use unbuffered channels to avoid blocking issues - jobs := make(chan int, workers*2) // Small buffer for job numbers only - results := make(chan detectionResult, workers*2) - - // Shared counters - var ( - totalOps int64 - matchCount int64 - totalBytes int64 - matchedBytes int64 - ) - - // Progress tracking - var progress *ui.ProgressBar - if showProgress { - progress = ui.NewProgressBar(totalBundles) - } - - // CSV header - fmt.Println("position,cid,size,confidence,labels") - - // Start workers BEFORE sending jobs - var wg sync.WaitGroup - for w := 0; w < workers; w++ { - wg.Add(1) - go func(workerID int) { - defer wg.Done() - - for bundleNum := range jobs { - select { - case <-ctx.Done(): - return - default: - } - - // Process bundle - res := processBundleForDetection(ctx, mgr, setup, bundleNum) - - // Send result (may block if collector is slow, but that's OK) - select { - case results <- res: - case <-ctx.Done(): - return - } - } - }(w) - } - - // Send jobs in separate goroutine - go func() { - defer close(jobs) - for bundleNum := start; bundleNum <= end; bundleNum++ { - select { - case jobs <- bundleNum: - case <-ctx.Done(): - return - } - } - }() - - // Collect results in separate goroutine - done := make(chan struct{}) - go func() { - defer close(done) - wg.Wait() - close(results) - }() - - // Process results in main goroutine - processed := 0 - allMatches := make([]detectionMatch, 0, 1000) - - for res := range results { - processed++ - - if res.err != nil { - if res.err != context.Canceled { - fmt.Fprintf(os.Stderr, "\nWarning: bundle %06d failed: %v\n", res.bundleNum, res.err) - } - } else { - // Update counters - atomic.AddInt64(&totalOps, int64(res.totalOps)) - atomic.AddInt64(&matchCount, int64(len(res.matches))) - atomic.AddInt64(&totalBytes, res.totalBytes) - atomic.AddInt64(&matchedBytes, res.matchedBytes) - - // Collect matches - allMatches = append(allMatches, res.matches...) - - // Output immediately (don't buffer too much) - if len(allMatches) >= 500 { - for _, match := range allMatches { - fmt.Printf("%d,%s,%d,%.2f,%s\n", - match.globalPos, match.cid, match.size, - match.confidence, strings.Join(match.labels, ";")) - } - allMatches = allMatches[:0] - } - } - - if progress != nil { - progress.Set(processed) - } - } - - // Flush remaining matches - for _, match := range allMatches { - fmt.Printf("%d,%s,%d,%.2f,%s\n", - match.globalPos, match.cid, match.size, - match.confidence, strings.Join(match.labels, ";")) - } - - if progress != nil { - progress.Finish() - } - - // Wait for cleanup - <-done - - // Summary - finalTotalOps := atomic.LoadInt64(&totalOps) - finalMatchCount := atomic.LoadInt64(&matchCount) - finalTotalBytes := atomic.LoadInt64(&totalBytes) - finalMatchedBytes := atomic.LoadInt64(&matchedBytes) - - if finalTotalOps == 0 { - fmt.Fprintf(os.Stderr, "\n⚠️ No operations processed\n") - return nil - } - - fmt.Fprintf(os.Stderr, "\n✓ Detection complete\n") - fmt.Fprintf(os.Stderr, " Total operations: %d\n", finalTotalOps) - fmt.Fprintf(os.Stderr, " Matches found: %d (%.2f%%)\n", - finalMatchCount, float64(finalMatchCount)/float64(finalTotalOps)*100) - fmt.Fprintf(os.Stderr, " Total size: %s\n", formatBytes(finalTotalBytes)) - fmt.Fprintf(os.Stderr, " Matched size: %s (%.2f%%)\n", - formatBytes(finalMatchedBytes), - float64(finalMatchedBytes)/float64(finalTotalBytes)*100) - - return nil -} - -func processBundleForDetection(ctx context.Context, mgr BundleManager, setup *detectorSetup, bundleNum int) detectionResult { - res := detectionResult{bundleNum: bundleNum} - - bundle, err := mgr.LoadBundle(ctx, bundleNum) - if err != nil { - res.err = err - return res - } - - res.totalOps = len(bundle.Operations) - matches := make([]detectionMatch, 0) - - for position, op := range bundle.Operations { - // Calculate global position - globalPos := (bundleNum * 10000) + position - - opSize := len(op.RawJSON) - if opSize == 0 { - data, _ := json.Marshal(op) - opSize = len(data) - } - res.totalBytes += int64(opSize) - - labels, conf := detectOperation(ctx, setup.detectors, op, setup.confidence) - - if len(labels) > 0 { - res.matchedBytes += int64(opSize) - - cidShort := op.CID - if len(cidShort) > 4 { - cidShort = cidShort[len(cidShort)-4:] - } - - matches = append(matches, detectionMatch{ - globalPos: globalPos, - cid: cidShort, - size: opSize, - confidence: conf, - labels: labels, - }) - } - } - - res.matches = matches - return res -} - -// ============================================================================ -// FILTER FROM STDIN -// ============================================================================ - -func filterFromStdin(ctx context.Context, setup *detectorSetup) error { - scanner := bufio.NewScanner(os.Stdin) - buf := make([]byte, 0, 64*1024) - scanner.Buffer(buf, 1024*1024) - - cleanCount, filteredCount, totalCount := 0, 0, 0 - totalBytes, filteredBytes := int64(0), int64(0) - - for scanner.Scan() { - line := scanner.Bytes() - if len(line) == 0 { - continue - } - - totalCount++ - totalBytes += int64(len(line)) - - var op plcclient.PLCOperation - if err := json.Unmarshal(line, &op); err != nil { - continue - } - - labels, _ := detectOperation(ctx, setup.detectors, op, setup.confidence) - - if len(labels) == 0 { - // Clean - output to stdout - cleanCount++ - fmt.Println(string(line)) - } else { - // Matched - filter out - filteredCount++ - filteredBytes += int64(len(line)) - } - - if totalCount%1000 == 0 { - fmt.Fprintf(os.Stderr, "Processed: %d | Clean: %d | Filtered: %d\r", totalCount, cleanCount, filteredCount) - } - } - - fmt.Fprintf(os.Stderr, "\n\n✓ Filter complete\n") - fmt.Fprintf(os.Stderr, " Total: %d | Clean: %d (%.2f%%) | Filtered: %d (%.2f%%)\n", - totalCount, cleanCount, float64(cleanCount)/float64(totalCount)*100, - filteredCount, float64(filteredCount)/float64(totalCount)*100) - fmt.Fprintf(os.Stderr, " Size saved: %s (%.2f%%)\n", formatBytes(filteredBytes), float64(filteredBytes)/float64(totalBytes)*100) - - return nil -} - -// ============================================================================ -// SHARED HELPERS -// ============================================================================ - -type detectorSetup struct { - detectors []detector.Detector - scriptDetectors []interface{ Close() error } - confidence float64 -} - -func (ds *detectorSetup) cleanup() { - for _, sd := range ds.scriptDetectors { - sd.Close() - } -} - -func parseAndLoadDetectors(detectorNames []string, confidence float64) (*detectorSetup, error) { - registry := detector.DefaultRegistry() - - // Handle "all" keyword - if len(detectorNames) == 1 && detectorNames[0] == "all" { - detectorNames = registry.Names() - fmt.Fprintf(os.Stderr, "Using all detectors: %s\n", strings.Join(detectorNames, ", ")) - } - - setup := &detectorSetup{ - detectors: make([]detector.Detector, 0, len(detectorNames)), - scriptDetectors: make([]interface{ Close() error }, 0), - confidence: confidence, - } - - for _, name := range detectorNames { - // JavaScript detector - if strings.HasSuffix(name, ".js") { - sd, err := detector.NewScriptDetector(name) - if err != nil { - setup.cleanup() - return nil, fmt.Errorf("failed to load script %s: %w", name, err) - } - setup.scriptDetectors = append(setup.scriptDetectors, sd) - registry.Register(sd) - setup.detectors = append(setup.detectors, sd) - fmt.Fprintf(os.Stderr, "✓ Loaded detector: %s\n", sd.Name()) - } else { - // Built-in detector - d, err := registry.Get(name) - if err != nil { - setup.cleanup() - return nil, err - } - setup.detectors = append(setup.detectors, d) - } - } - - return setup, nil -} - -func detectOperation(ctx context.Context, detectors []detector.Detector, op plcclient.PLCOperation, minConfidence float64) ([]string, float64) { - opData, err := op.GetOperationData() - if err != nil { - return nil, 0 - } - op.ParsedOperation = opData - - var matchedLabels []string - var maxConfidence float64 - - for _, det := range detectors { - match, err := det.Detect(ctx, op) - if err != nil || match == nil || match.Confidence < minConfidence { - continue - } - - // Extract labels - var labels []string - if labelList, ok := match.Metadata["labels"].([]string); ok { - labels = labelList - } else if labelList, ok := match.Metadata["labels"].([]interface{}); ok { - for _, l := range labelList { - if str, ok := l.(string); ok { - labels = append(labels, str) - } - } - } - - if len(labels) == 0 { - labels = []string{det.Name()} - } - - matchedLabels = append(matchedLabels, labels...) - if match.Confidence > maxConfidence { - maxConfidence = match.Confidence - } - } - - return matchedLabels, maxConfidence -} diff --git a/cmd/plcbundle/commands/did.go b/cmd/plcbundle/commands/did.go deleted file mode 100644 index f49ad2a..0000000 --- a/cmd/plcbundle/commands/did.go +++ /dev/null @@ -1,1298 +0,0 @@ -package commands - -import ( - "bufio" - "context" - "fmt" - "os" - "strings" - "sync" - "time" - - "github.com/goccy/go-json" - "github.com/spf13/cobra" - "tangled.org/atscan.net/plcbundle/cmd/plcbundle/ui" - "tangled.org/atscan.net/plcbundle/internal/didindex" - "tangled.org/atscan.net/plcbundle/internal/plcclient" -) - -func NewDIDCommand() *cobra.Command { - cmd := &cobra.Command{ - Use: "did", - Aliases: []string{"d"}, - Short: "DID operations and queries", - Long: `DID operations and queries - -Query and analyze DIDs in the bundle repository. All commands -require a DID index to be built for optimal performance.`, - - Example: ` # Lookup all operations for a DID - plcbundle did lookup did:plc:524tuhdhh3m7li5gycdn6boe - - # Resolve to current DID document - plcbundle did resolve did:plc:524tuhdhh3m7li5gycdn6boe - - # Show complete audit log - plcbundle did history did:plc:524tuhdhh3m7li5gycdn6boe - - # Show DID statistics - plcbundle did stats did:plc:524tuhdhh3m7li5gycdn6boe - - # Batch process from file - plcbundle did batch dids.txt`, - } - - // Add subcommands - cmd.AddCommand(newDIDLookupCommand()) - cmd.AddCommand(newDIDResolveCommand()) - cmd.AddCommand(newDIDHistoryCommand()) - cmd.AddCommand(newDIDBatchCommand()) - cmd.AddCommand(newDIDStatsCommand()) - - return cmd -} - -// ============================================================================ -// DID LOOKUP - Find all operations for a DID -// ============================================================================ - -func newDIDLookupCommand() *cobra.Command { - var ( - verbose bool - showJSON bool - ) - - cmd := &cobra.Command{ - Use: "lookup ", - Aliases: []string{"find", "get"}, - Short: "Find all operations for a DID or handle", - Long: `Find all operations for a DID or handle - -Retrieves all operations (both bundled and mempool) for a specific DID. -Accepts either: - • DID: did:plc:524tuhdhh3m7li5gycdn6boe - • Handle: tree.fail (resolves via configured resolver) - -Requires DID index to be built.`, - - Example: ` # Lookup by DID - plcbundle did lookup did:plc:524tuhdhh3m7li5gycdn6boe - - # Lookup by handle - plcbundle did lookup tree.fail - plcbundle did lookup ngerakines.me - - # With non-default handle resolver configured - plcbundle --handle-resolver https://quickdid.smokesignal.tools did lookup tree.fail`, - - Args: cobra.ExactArgs(1), - - RunE: func(cmd *cobra.Command, args []string) error { - input := args[0] - - mgr, _, err := getManager(&ManagerOptions{Cmd: cmd}) - if err != nil { - return err - } - defer mgr.Close() - - // Resolve handle to DID with timing - ctx := context.Background() - did, _, err := mgr.ResolveHandleOrDID(ctx, input) - if err != nil { - return err - } - - stats := mgr.GetDIDIndexStats() - if !stats["exists"].(bool) { - fmt.Fprintf(os.Stderr, "⚠️ DID index not found. Run: plcbundle index build\n") - fmt.Fprintf(os.Stderr, " Falling back to full scan (slow)...\n\n") - } - - totalStart := time.Now() - - // Lookup operations - lookupStart := time.Now() - _, opsWithLoc, err := mgr.GetDIDOperations(ctx, did, verbose) - if err != nil { - return err - } - lookupElapsed := time.Since(lookupStart) - - // Check mempool - mempoolStart := time.Now() - mempoolOps, err := mgr.GetDIDOperationsFromMempool(did) - if err != nil { - return fmt.Errorf("error checking mempool: %w", err) - } - mempoolElapsed := time.Since(mempoolStart) - - totalElapsed := time.Since(totalStart) - - if len(opsWithLoc) == 0 && len(mempoolOps) == 0 { - if showJSON { - fmt.Println("{\"found\": false, \"operations\": []}") - } else { - fmt.Printf("DID not found (searched in %s)\n", totalElapsed) - } - return nil - } - - if showJSON { - return outputLookupJSON(did, opsWithLoc, mempoolOps, totalElapsed, lookupElapsed, mempoolElapsed) - } - - return displayLookupResults(did, opsWithLoc, mempoolOps, totalElapsed, lookupElapsed, mempoolElapsed, verbose, stats) - }, - } - - cmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "Verbose debug output") - cmd.Flags().BoolVar(&showJSON, "json", false, "Output as JSON") - - return cmd -} - -// ============================================================================ -// DID RESOLVE - Resolve to current document -// ============================================================================ - -func newDIDResolveCommand() *cobra.Command { - var ( - verbose bool - raw bool - ) - - cmd := &cobra.Command{ - Use: "resolve ", - Aliases: []string{"doc", "document"}, - Short: "Resolve DID to current document", - Long: `Resolve DID to current W3C DID document - -Resolves a DID to its current state by applying all non-nullified -operations in chronological order. Returns standard W3C DID document. - -Optimized for speed: checks mempool first, then uses DID index for -O(1) lookup of latest operation.`, - - Example: ` # Resolve DID - plcbundle did resolve did:plc:524tuhdhh3m7li5gycdn6boe - - # Show timings and other details - plcbundle did resolve did:plc:524tuhdhh3m7li5gycdn6boe --verbose - - # Get raw PLC state (not W3C format) - plcbundle did resolve did:plc:524tuhdhh3m7li5gycdn6boe --raw - - # Pipe to jq - plcbundle did resolve did:plc:524tuhdhh3m7li5gycdn6boe | jq .service - - # Resolve by handle - plcbundle did resolve tree.fail`, - - Args: cobra.ExactArgs(1), - - RunE: func(cmd *cobra.Command, args []string) error { - input := args[0] - - mgr, _, err := getManager(&ManagerOptions{Cmd: cmd}) - if err != nil { - return err - } - defer mgr.Close() - - ctx := context.Background() - - // Resolve handle to DID with timing - did, handleResolveTime, err := mgr.ResolveHandleOrDID(ctx, input) - if err != nil { - return err - } - - if verbose { - fmt.Fprintf(os.Stderr, "Resolving DID: %s\n", did) - mgr.GetDIDIndex().SetVerbose(true) - } - - result, err := mgr.ResolveDID(ctx, did) - if err != nil { - return err - } - - // Display timing if requested - if verbose { - if handleResolveTime > 0 { - fmt.Fprintf(os.Stderr, "Handle: %s | ", handleResolveTime) - } - if result.Source == "mempool" { - fmt.Fprintf(os.Stderr, "Mempool check: %s (✓ found)\n", result.MempoolTime) - fmt.Fprintf(os.Stderr, "Total: %s\n\n", result.TotalTime) - } else { - fmt.Fprintf(os.Stderr, "Mempool: %s | Index: %s | Load: %s | Total: %s\n", - result.MempoolTime, result.IndexTime, result.LoadOpTime, result.TotalTime) - fmt.Fprintf(os.Stderr, "Source: bundle %06d, position %d\n\n", - result.BundleNumber, result.Position) - } - } - - // Output document - data, _ := json.MarshalIndent(result.Document, "", " ") - fmt.Println(string(data)) - - return nil - }, - } - - cmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "Verbose debug output") - cmd.Flags().BoolVar(&raw, "raw", false, "Output raw PLC state (not W3C document)") - - return cmd -} - -// ============================================================================ -// DID HISTORY - Show complete audit log -// ============================================================================ - -func newDIDHistoryCommand() *cobra.Command { - var ( - verbose bool - showJSON bool - compact bool - includeNullified bool - ) - - cmd := &cobra.Command{ - Use: "history ", - Aliases: []string{"log", "audit"}, - Short: "Show complete DID audit log", - Long: `Show complete DID audit log - -Displays all operations for a DID in chronological order, showing -the complete history including nullified operations. - -This provides a full audit trail of all changes to the DID.`, - - Example: ` # Show full history - plcbundle did history did:plc:524tuhdhh3m7li5gycdn6boe - - # Include nullified operations - plcbundle did history did:plc:524tuhdhh3m7li5gycdn6boe --include-nullified - - # Compact one-line format - plcbundle did history did:plc:524tuhdhh3m7li5gycdn6boe --compact - - # JSON output - plcbundle did history did:plc:524tuhdhh3m7li5gycdn6boe --json`, - - Args: cobra.ExactArgs(1), - - RunE: func(cmd *cobra.Command, args []string) error { - did := args[0] - - mgr, _, err := getManager(&ManagerOptions{Cmd: cmd}) - if err != nil { - return err - } - defer mgr.Close() - - ctx := context.Background() - - // Get all operations with locations - _, opsWithLoc, err := mgr.GetDIDOperations(ctx, did, verbose) - if err != nil { - return err - } - - // Get mempool operations - mempoolOps, err := mgr.GetDIDOperationsFromMempool(did) - if err != nil { - return err - } - - if len(opsWithLoc) == 0 && len(mempoolOps) == 0 { - fmt.Fprintf(os.Stderr, "DID not found: %s\n", did) - return nil - } - - if showJSON { - return outputHistoryJSON(did, opsWithLoc, mempoolOps) - } - - return displayHistory(did, opsWithLoc, mempoolOps, compact, includeNullified) - }, - } - - cmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "Verbose output") - cmd.Flags().BoolVar(&showJSON, "json", false, "Output as JSON") - cmd.Flags().BoolVar(&compact, "compact", false, "Compact one-line format") - cmd.Flags().BoolVar(&includeNullified, "include-nullified", false, "Show nullified operations") - - return cmd -} - -// ============================================================================ -// DID BATCH - Process multiple DIDs from file or stdin -// ============================================================================ - -func newDIDBatchCommand() *cobra.Command { - var ( - action string - workers int - outputFile string - fromStdin bool - ) - - cmd := &cobra.Command{ - Use: "batch [file]", - Short: "Process multiple DIDs from file or stdin", - Long: `Process multiple DIDs from file or stdin - -Read DIDs from a file (one per line) or stdin and perform batch operations. -Supports parallel processing for better performance. - -Actions: - lookup - Lookup all DIDs and show summary - resolve - Resolve all DIDs to documents - export - Export all operations to JSONL - -Input formats: - - File path: reads DIDs from file - - "-" or --stdin: reads DIDs from stdin - - Omit file + use --stdin: reads from stdin`, - - Example: ` # Batch lookup from file - plcbundle did batch dids.txt --action lookup - - # Read from stdin - cat dids.txt | plcbundle did batch --stdin --action lookup - cat dids.txt | plcbundle did batch - --action resolve - - # Export operations for DIDs from stdin - echo "did:plc:524tuhdhh3m7li5gycdn6boe" | plcbundle did batch - --action export - - # Pipe results - plcbundle did batch dids.txt --action resolve -o resolved.jsonl - - # Parallel processing - cat dids.txt | plcbundle did batch --stdin --action lookup --workers 8 - - # Chain commands - grep "did:plc:" some_file.txt | plcbundle did batch - --action export > ops.jsonl`, - - Args: cobra.MaximumNArgs(1), - - RunE: func(cmd *cobra.Command, args []string) error { - var filename string - - // Determine input source - if len(args) > 0 { - filename = args[0] - if filename == "-" { - fromStdin = true - } - } else if !fromStdin { - return fmt.Errorf("either provide filename or use --stdin flag\n" + - "Examples:\n" + - " plcbundle did batch dids.txt\n" + - " plcbundle did batch --stdin\n" + - " cat dids.txt | plcbundle did batch -") - } - - mgr, _, err := getManager(&ManagerOptions{Cmd: cmd}) - if err != nil { - return err - } - defer mgr.Close() - - return processBatchDIDs(mgr, filename, batchOptions{ - action: action, - workers: workers, - outputFile: outputFile, - fromStdin: fromStdin, - }) - }, - } - - cmd.Flags().StringVar(&action, "action", "lookup", "Action: lookup, resolve, export") - cmd.Flags().IntVar(&workers, "workers", 4, "Number of parallel workers") - cmd.Flags().StringVarP(&outputFile, "output", "o", "", "Output file (default: stdout)") - cmd.Flags().BoolVar(&fromStdin, "stdin", false, "Read DIDs from stdin") - - return cmd -} - -// ============================================================================ -// DID STATS - Show DID activity statistics -// ============================================================================ - -func newDIDStatsCommand() *cobra.Command { - var ( - showGlobal bool - showJSON bool - ) - - cmd := &cobra.Command{ - Use: "stats [did]", - Short: "Show DID activity statistics", - Long: `Show DID activity statistics - -Display statistics for a specific DID or global DID index stats. - -With DID: shows operation count, first/last activity, bundle distribution -Without DID: shows global index statistics`, - - Example: ` # Stats for specific DID - plcbundle did stats did:plc:524tuhdhh3m7li5gycdn6boe - - # Global index stats - plcbundle did stats --global - plcbundle did stats - - # JSON output - plcbundle did stats did:plc:524tuhdhh3m7li5gycdn6boe --json`, - - Args: cobra.MaximumNArgs(1), - - RunE: func(cmd *cobra.Command, args []string) error { - mgr, dir, err := getManager(&ManagerOptions{Cmd: cmd}) - if err != nil { - return err - } - defer mgr.Close() - - // Global stats - if len(args) == 0 || showGlobal { - return showGlobalDIDStats(mgr, dir, showJSON) - } - - // Specific DID stats - did := args[0] - return showDIDStats(mgr, did, showJSON) - }, - } - - cmd.Flags().BoolVar(&showGlobal, "global", false, "Show global index stats") - cmd.Flags().BoolVar(&showJSON, "json", false, "Output as JSON") - - return cmd -} - -// ============================================================================ -// Helper Functions -// ============================================================================ - -type batchOptions struct { - action string - workers int - outputFile string - fromStdin bool -} - -func processBatchDIDs(mgr BundleManager, filename string, opts batchOptions) error { - // Determine input source - var input *os.File - var err error - - if opts.fromStdin { - input = os.Stdin - fmt.Fprintf(os.Stderr, "Reading DIDs from stdin...\n") - } else { - input, err = os.Open(filename) - if err != nil { - return fmt.Errorf("failed to open file: %w", err) - } - defer input.Close() - fmt.Fprintf(os.Stderr, "Reading DIDs from: %s\n", filename) - } - - // Read DIDs - var dids []string - scanner := bufio.NewScanner(input) - - // Increase buffer size for large input - buf := make([]byte, 64*1024) - scanner.Buffer(buf, 1024*1024) - - lineNum := 0 - for scanner.Scan() { - lineNum++ - line := strings.TrimSpace(scanner.Text()) - - // Skip empty lines and comments - if line == "" || strings.HasPrefix(line, "#") { - continue - } - - // Basic validation - if !strings.HasPrefix(line, "did:plc:") { - fmt.Fprintf(os.Stderr, "⚠️ Line %d: skipping invalid DID: %s\n", lineNum, line) - continue - } - - dids = append(dids, line) - } - - if err := scanner.Err(); err != nil { - return fmt.Errorf("error reading input: %w", err) - } - - if len(dids) == 0 { - return fmt.Errorf("no valid DIDs found in input") - } - - fmt.Fprintf(os.Stderr, "Processing %d DIDs with action '%s' (%d workers)\n\n", - len(dids), opts.action, opts.workers) - - // Setup output - var output *os.File - if opts.outputFile != "" { - output, err = os.Create(opts.outputFile) - if err != nil { - return fmt.Errorf("failed to create output file: %w", err) - } - defer output.Close() - fmt.Fprintf(os.Stderr, "Output: %s\n\n", opts.outputFile) - } else { - output = os.Stdout - } - - // Process based on action - switch opts.action { - case "lookup": - return batchLookup(mgr, dids, output, opts.workers) - case "resolve": - return batchResolve(mgr, dids, output, opts.workers) - case "export": - return batchExport(mgr, dids, output, opts.workers) - default: - return fmt.Errorf("unknown action: %s (valid: lookup, resolve, export)", opts.action) - } -} - -func showGlobalDIDStats(mgr BundleManager, dir string, showJSON bool) error { - stats := mgr.GetDIDIndexStats() - - if !stats["exists"].(bool) { - fmt.Printf("DID index does not exist\n") - fmt.Printf("Run: plcbundle index build\n") - return nil - } - - if showJSON { - data, _ := json.MarshalIndent(stats, "", " ") - fmt.Println(string(data)) - return nil - } - - indexedDIDs := stats["indexed_dids"].(int64) - mempoolDIDs := stats["mempool_dids"].(int64) - totalDIDs := stats["total_dids"].(int64) - - fmt.Printf("\nDID Index Statistics\n") - fmt.Printf("════════════════════\n\n") - fmt.Printf(" Location: %s/.plcbundle/\n", dir) - - if mempoolDIDs > 0 { - fmt.Printf(" Indexed DIDs: %s (in bundles)\n", formatNumber(int(indexedDIDs))) - fmt.Printf(" Mempool DIDs: %s (not yet bundled)\n", formatNumber(int(mempoolDIDs))) - fmt.Printf(" Total DIDs: %s\n", formatNumber(int(totalDIDs))) - } else { - fmt.Printf(" Total DIDs: %s\n", formatNumber(int(totalDIDs))) - } - - fmt.Printf(" Shard count: %d\n", stats["shard_count"]) - fmt.Printf(" Last bundle: %06d\n", stats["last_bundle"]) - fmt.Printf(" Updated: %s\n\n", stats["updated_at"].(time.Time).Format("2006-01-02 15:04:05")) - - fmt.Printf(" Cached shards: %d / %d\n", stats["cached_shards"], stats["cache_limit"]) - - if cachedList, ok := stats["cache_order"].([]int); ok && len(cachedList) > 0 { - fmt.Printf(" Hot shards: ") - for i, shard := range cachedList { - if i > 0 { - fmt.Printf(", ") - } - if i >= 10 { - fmt.Printf("... (+%d more)", len(cachedList)-10) - break - } - fmt.Printf("%02x", shard) - } - fmt.Printf("\n") - } - - fmt.Printf("\n") - return nil -} - -func showDIDStats(mgr BundleManager, did string, showJSON bool) error { - ctx := context.Background() - - // Get operations - _, opsWithLoc, err := mgr.GetDIDOperations(ctx, did, false) - if err != nil { - return err - } - - mempoolOps, err := mgr.GetDIDOperationsFromMempool(did) - if err != nil { - return err - } - - if len(opsWithLoc) == 0 && len(mempoolOps) == 0 { - fmt.Fprintf(os.Stderr, "DID not found: %s\n", did) - return nil - } - - // Calculate stats - totalOps := len(opsWithLoc) + len(mempoolOps) - nullifiedCount := 0 - for _, owl := range opsWithLoc { - if owl.Operation.IsNullified() { - nullifiedCount++ - } - } - - bundleSpan := 0 - if len(opsWithLoc) > 0 { - bundles := make(map[int]bool) - for _, owl := range opsWithLoc { - bundles[owl.Bundle] = true - } - bundleSpan = len(bundles) - } - - if showJSON { - output := map[string]interface{}{ - "did": did, - "total_operations": totalOps, - "bundled": len(opsWithLoc), - "mempool": len(mempoolOps), - "nullified": nullifiedCount, - "active": totalOps - nullifiedCount, - "bundle_span": bundleSpan, - } - data, _ := json.MarshalIndent(output, "", " ") - fmt.Println(string(data)) - return nil - } - - fmt.Printf("\nDID Statistics\n") - fmt.Printf("══════════════\n\n") - fmt.Printf(" DID: %s\n\n", did) - fmt.Printf(" Total operations: %d\n", totalOps) - fmt.Printf(" Active: %d\n", totalOps-nullifiedCount) - if nullifiedCount > 0 { - fmt.Printf(" Nullified: %d\n", nullifiedCount) - } - if len(opsWithLoc) > 0 { - fmt.Printf(" Bundled: %d\n", len(opsWithLoc)) - fmt.Printf(" Bundle span: %d bundles\n", bundleSpan) - } - if len(mempoolOps) > 0 { - fmt.Printf(" Mempool: %d\n", len(mempoolOps)) - } - fmt.Printf("\n") - - return nil -} - -func displayHistory(did string, opsWithLoc []PLCOperationWithLocation, mempoolOps []plcclient.PLCOperation, compact bool, includeNullified bool) error { - if compact { - return displayHistoryCompact(did, opsWithLoc, mempoolOps, includeNullified) - } - return displayHistoryDetailed(did, opsWithLoc, mempoolOps, includeNullified) -} - -func displayHistoryCompact(did string, opsWithLoc []PLCOperationWithLocation, mempoolOps []plcclient.PLCOperation, includeNullified bool) error { - fmt.Printf("DID History: %s\n\n", did) - - for _, owl := range opsWithLoc { - if !includeNullified && owl.Operation.IsNullified() { - continue - } - - status := "✓" - if owl.Operation.IsNullified() { - status = "✗" - } - - fmt.Printf("%s [%06d:%04d] %s %s\n", - status, - owl.Bundle, - owl.Position, - owl.Operation.CreatedAt.Format("2006-01-02 15:04:05"), - owl.Operation.CID) - } - - for _, op := range mempoolOps { - fmt.Printf("✓ [mempool ] %s %s\n", - op.CreatedAt.Format("2006-01-02 15:04:05"), - op.CID) - } - - return nil -} - -func displayHistoryDetailed(did string, opsWithLoc []PLCOperationWithLocation, mempoolOps []plcclient.PLCOperation, includeNullified bool) error { - fmt.Printf("═══════════════════════════════════════════════════════════════\n") - fmt.Printf(" DID Audit Log\n") - fmt.Printf("═══════════════════════════════════════════════════════════════\n\n") - fmt.Printf("DID: %s\n\n", did) - - for i, owl := range opsWithLoc { - if !includeNullified && owl.Operation.IsNullified() { - continue - } - - op := owl.Operation - status := "✓ Active" - if op.IsNullified() { - status = "✗ Nullified" - } - - fmt.Printf("Operation %d [Bundle %06d, Position %04d]\n", i+1, owl.Bundle, owl.Position) - fmt.Printf(" CID: %s\n", op.CID) - fmt.Printf(" Created: %s\n", op.CreatedAt.Format("2006-01-02 15:04:05.000 MST")) - fmt.Printf(" Status: %s\n", status) - - if opData, err := op.GetOperationData(); err == nil && opData != nil { - showOperationDetails(&op) - } - - fmt.Printf("\n") - } - - if len(mempoolOps) > 0 { - fmt.Printf("Mempool Operations (%d)\n", len(mempoolOps)) - fmt.Printf("══════════════════════════════════════════════════════════════\n\n") - - for i, op := range mempoolOps { - fmt.Printf("Operation %d [Mempool]\n", i+1) - fmt.Printf(" CID: %s\n", op.CID) - fmt.Printf(" Created: %s\n", op.CreatedAt.Format("2006-01-02 15:04:05.000 MST")) - fmt.Printf(" Status: ✓ Active\n") - fmt.Printf("\n") - } - } - - return nil -} - -func outputHistoryJSON(did string, opsWithLoc []PLCOperationWithLocation, mempoolOps []plcclient.PLCOperation) error { - output := map[string]interface{}{ - "did": did, - "bundled": make([]map[string]interface{}, 0), - "mempool": make([]map[string]interface{}, 0), - } - - for _, owl := range opsWithLoc { - output["bundled"] = append(output["bundled"].([]map[string]interface{}), map[string]interface{}{ - "bundle": owl.Bundle, - "position": owl.Position, - "cid": owl.Operation.CID, - "nullified": owl.Operation.IsNullified(), - "created_at": owl.Operation.CreatedAt.Format(time.RFC3339Nano), - }) - } - - for _, op := range mempoolOps { - output["mempool"] = append(output["mempool"].([]map[string]interface{}), map[string]interface{}{ - "cid": op.CID, - "nullified": op.IsNullified(), - "created_at": op.CreatedAt.Format(time.RFC3339Nano), - }) - } - - data, _ := json.MarshalIndent(output, "", " ") - fmt.Println(string(data)) - - return nil -} - -func batchLookup(mgr BundleManager, dids []string, output *os.File, _ int) error { - progress := ui.NewProgressBar(len(dids)) - ctx := context.Background() - - // CSV header - fmt.Fprintf(output, "did,status,operation_count,bundled,mempool,nullified\n") - - found := 0 - notFound := 0 - errorCount := 0 - - for i, did := range dids { - _, opsWithLoc, err := mgr.GetDIDOperations(ctx, did, false) - if err != nil { - errorCount++ - fmt.Fprintf(output, "%s,error,0,0,0,0\n", did) - progress.Set(i + 1) - continue - } - - mempoolOps, _ := mgr.GetDIDOperationsFromMempool(did) - - if len(opsWithLoc) == 0 && len(mempoolOps) == 0 { - notFound++ - fmt.Fprintf(output, "%s,not_found,0,0,0,0\n", did) - } else { - found++ - - // Count nullified - nullified := 0 - for _, owl := range opsWithLoc { - if owl.Operation.IsNullified() { - nullified++ - } - } - - fmt.Fprintf(output, "%s,found,%d,%d,%d,%d\n", - did, - len(opsWithLoc)+len(mempoolOps), - len(opsWithLoc), - len(mempoolOps), - nullified) - } - - progress.Set(i + 1) - } - - progress.Finish() - - fmt.Fprintf(os.Stderr, "\n✓ Batch lookup complete\n") - fmt.Fprintf(os.Stderr, " DIDs input: %d\n", len(dids)) - fmt.Fprintf(os.Stderr, " Found: %d\n", found) - fmt.Fprintf(os.Stderr, " Not found: %d\n", notFound) - if errorCount > 0 { - fmt.Fprintf(os.Stderr, " Errors: %d\n", errorCount) - } - - return nil -} - -func batchResolve(mgr BundleManager, dids []string, output *os.File, workers int) error { - ctx := context.Background() - overallStart := time.Now() - - // ================================================================= - // PHASE 1: Batch lookup locations - // ================================================================= - fmt.Fprintf(os.Stderr, "Phase 1/2: Looking up %d DID locations...\n", len(dids)) - phase1Start := time.Now() - - type didLocation struct { - did string - bundle int - position int - fromMempool bool - mempoolOp *plcclient.PLCOperation - err error - } - - locations := make([]didLocation, len(dids)) - - // Mempool check - for i, did := range dids { - if mempoolOp := findLatestInMempool(mgr, did); mempoolOp != nil { - locations[i] = didLocation{did: did, fromMempool: true, mempoolOp: mempoolOp} - } - } - - // Batch index lookup - needsLookup := make([]string, 0) - lookupMap := make(map[string]int) - for i, did := range dids { - if !locations[i].fromMempool { - needsLookup = append(needsLookup, did) - lookupMap[did] = i - } - } - - if len(needsLookup) > 0 { - batchResults, _ := mgr.GetDIDIndex().BatchGetDIDLocations(needsLookup) - for did, locs := range batchResults { - var latest *didindex.OpLocation - for i := range locs { - if !locs[i].Nullified() && (latest == nil || locs[i].IsAfter(*latest)) { - latest = &locs[i] - } - } - - idx := lookupMap[did] - if latest != nil { - locations[idx] = didLocation{did: did, bundle: latest.BundleInt(), position: latest.PositionInt()} - } else { - locations[idx] = didLocation{did: did, err: fmt.Errorf("not found")} - } - } - - for _, did := range needsLookup { - if idx := lookupMap[did]; locations[idx].bundle == 0 && locations[idx].err == nil { - locations[idx] = didLocation{did: did, err: fmt.Errorf("not found")} - } - } - } - - phase1Duration := time.Since(phase1Start) - fmt.Fprintf(os.Stderr, " ✓ %s\n\n", phase1Duration.Round(time.Millisecond)) - - // ================================================================= - // PHASE 2: Group by bundle, load ops, resolve (MERGED, parallel) - // ================================================================= - fmt.Fprintf(os.Stderr, "Phase 2/2: Loading and resolving (%d workers)...\n", workers) - - // Group DIDs by bundle - type bundleGroup struct { - bundleNum int - dids []int // indices into locations array - } - - bundleMap := make(map[int][]int) - mempoolDIDs := make([]int, 0) - errorDIDs := make([]int, 0) - - for i, loc := range locations { - if loc.err != nil { - errorDIDs = append(errorDIDs, i) - } else if loc.fromMempool { - mempoolDIDs = append(mempoolDIDs, i) - } else { - bundleMap[loc.bundle] = append(bundleMap[loc.bundle], i) - } - } - - bundles := make([]bundleGroup, 0, len(bundleMap)) - for bn, didIndices := range bundleMap { - bundles = append(bundles, bundleGroup{bundleNum: bn, dids: didIndices}) - } - - fmt.Fprintf(os.Stderr, " %d bundles, %d mempool, %d errors\n", - len(bundles), len(mempoolDIDs), len(errorDIDs)) - - // Setup output - writer := bufio.NewWriterSize(output, 512*1024) - defer writer.Flush() - - var ( - resolved int - failed int - processed int - mu sync.Mutex // Single lock for all counters - ) - - progress := ui.NewProgressBar(len(dids)) - - writeResult := func(doc *plcclient.DIDDocument, err error) { - mu.Lock() - defer mu.Unlock() - - processed++ - progress.Set(processed) - - if err != nil { - failed++ - } else { - resolved++ - data, _ := json.Marshal(doc) - writer.Write(data) - writer.WriteByte('\n') - if resolved%100 == 0 { - writer.Flush() - } - } - } - - // Process mempool DIDs (already have ops) - for _, idx := range mempoolDIDs { - loc := locations[idx] - doc, err := plcclient.ResolveDIDDocument(loc.did, []plcclient.PLCOperation{*loc.mempoolOp}) - writeResult(doc, err) - } - - // Process errors - for range errorDIDs { - writeResult(nil, fmt.Errorf("not found")) - } - - // Process bundles in parallel - LoadOperations once per bundle - bundleJobs := make(chan bundleGroup, len(bundles)) - var wg sync.WaitGroup - - for w := 0; w < workers; w++ { - wg.Add(1) - go func() { - defer wg.Done() - - for job := range bundleJobs { - // Collect all positions needed from this bundle - positions := make([]int, len(job.dids)) - for i, didIdx := range job.dids { - positions[i] = locations[didIdx].position - } - - // Load operations once for this bundle - ops, err := mgr.LoadOperations(ctx, job.bundleNum, positions) - - if err != nil { - // All DIDs from this bundle fail - for range job.dids { - writeResult(nil, err) - } - continue - } - - // Resolve each DID using loaded operations - for i, didIdx := range job.dids { - loc := locations[didIdx] - - if op, ok := ops[positions[i]]; ok { - doc, resolveErr := plcclient.ResolveDIDDocument(loc.did, []plcclient.PLCOperation{*op}) - writeResult(doc, resolveErr) - } else { - writeResult(nil, fmt.Errorf("operation not loaded")) - } - } - } - }() - } - - // Send bundle jobs - for _, bg := range bundles { - bundleJobs <- bg - } - close(bundleJobs) - - wg.Wait() - writer.Flush() - progress.Finish() - - totalDuration := time.Since(overallStart) - - fmt.Fprintf(os.Stderr, "\n✓ Batch resolve complete\n") - fmt.Fprintf(os.Stderr, " Resolved: %d/%d\n", resolved, len(dids)) - if failed > 0 { - fmt.Fprintf(os.Stderr, " Failed: %d\n", failed) - } - fmt.Fprintf(os.Stderr, " Total: %s (%.1f DIDs/sec)\n", - totalDuration.Round(time.Millisecond), - float64(resolved)/totalDuration.Seconds()) - - return nil -} - -// Helper function to find latest non-nullified op in mempool -func findLatestInMempool(mgr BundleManager, did string) *plcclient.PLCOperation { - ops, err := mgr.GetDIDOperationsFromMempool(did) - if err != nil || len(ops) == 0 { - return nil - } - - // Search backwards from most recent - for i := len(ops) - 1; i >= 0; i-- { - if !ops[i].IsNullified() { - return &ops[i] - } - } - - return nil -} - -func batchExport(mgr BundleManager, dids []string, output *os.File, _ int) error { - progress := ui.NewProgressBar(len(dids)) - ctx := context.Background() - - totalOps := 0 - processedDIDs := 0 - errorCount := 0 - - // Use buffered writer for better performance - writer := bufio.NewWriterSize(output, 512*1024) - defer writer.Flush() - - for i, did := range dids { - _, opsWithLoc, err := mgr.GetDIDOperations(ctx, did, false) - if err != nil { - errorCount++ - if i < 10 { // Only log first few errors - fmt.Fprintf(os.Stderr, "Error processing %s: %v\n", did, err) - } - progress.Set(i + 1) - continue - } - - // Get mempool operations too - mempoolOps, _ := mgr.GetDIDOperationsFromMempool(did) - - if len(opsWithLoc) == 0 && len(mempoolOps) == 0 { - progress.Set(i + 1) - continue - } - - processedDIDs++ - - // Export bundled operations - for _, owl := range opsWithLoc { - if len(owl.Operation.RawJSON) > 0 { - writer.Write(owl.Operation.RawJSON) - } else { - data, _ := json.Marshal(owl.Operation) - writer.Write(data) - } - writer.WriteByte('\n') - totalOps++ - } - - // Export mempool operations - for _, op := range mempoolOps { - if len(op.RawJSON) > 0 { - writer.Write(op.RawJSON) - } else { - data, _ := json.Marshal(op) - writer.Write(data) - } - writer.WriteByte('\n') - totalOps++ - } - - // Flush periodically - if i%100 == 0 { - writer.Flush() - } - - progress.Set(i + 1) - } - - writer.Flush() - progress.Finish() - - fmt.Fprintf(os.Stderr, "\n✓ Batch export complete\n") - fmt.Fprintf(os.Stderr, " DIDs input: %d\n", len(dids)) - fmt.Fprintf(os.Stderr, " DIDs processed: %d\n", processedDIDs) - fmt.Fprintf(os.Stderr, " Operations: %s\n", formatNumber(totalOps)) - if errorCount > 0 { - fmt.Fprintf(os.Stderr, " Errors: %d\n", errorCount) - } - - return nil -} - -// ============================================================================ -// Shared Helper Functions (used by both DID and legacy index commands) -// ============================================================================ - -func outputLookupJSON(did string, opsWithLoc []PLCOperationWithLocation, mempoolOps []plcclient.PLCOperation, totalElapsed, lookupElapsed, mempoolElapsed time.Duration) error { - output := map[string]interface{}{ - "found": true, - "did": did, - "timing": map[string]interface{}{ - "total_ms": totalElapsed.Milliseconds(), - "lookup_ms": lookupElapsed.Milliseconds(), - "mempool_ms": mempoolElapsed.Milliseconds(), - }, - "bundled": make([]map[string]interface{}, 0), - "mempool": make([]map[string]interface{}, 0), - } - - for _, owl := range opsWithLoc { - output["bundled"] = append(output["bundled"].([]map[string]interface{}), map[string]interface{}{ - "bundle": owl.Bundle, - "position": owl.Position, - "cid": owl.Operation.CID, - "nullified": owl.Operation.IsNullified(), - "created_at": owl.Operation.CreatedAt.Format(time.RFC3339Nano), - }) - } - - for _, op := range mempoolOps { - output["mempool"] = append(output["mempool"].([]map[string]interface{}), map[string]interface{}{ - "cid": op.CID, - "nullified": op.IsNullified(), - "created_at": op.CreatedAt.Format(time.RFC3339Nano), - }) - } - - data, _ := json.MarshalIndent(output, "", " ") - fmt.Println(string(data)) - - return nil -} - -func displayLookupResults(did string, opsWithLoc []PLCOperationWithLocation, mempoolOps []plcclient.PLCOperation, totalElapsed, lookupElapsed, mempoolElapsed time.Duration, verbose bool, _ map[string]interface{}) error { - nullifiedCount := 0 - for _, owl := range opsWithLoc { - if owl.Operation.IsNullified() { - nullifiedCount++ - } - } - - totalOps := len(opsWithLoc) + len(mempoolOps) - activeOps := len(opsWithLoc) - nullifiedCount + len(mempoolOps) - - fmt.Printf("═══════════════════════════════════════════════════════════════\n") - fmt.Printf(" DID Lookup Results\n") - fmt.Printf("═══════════════════════════════════════════════════════════════\n\n") - fmt.Printf("DID: %s\n\n", did) - - fmt.Printf("Summary\n───────\n") - fmt.Printf(" Total operations: %d\n", totalOps) - fmt.Printf(" Active operations: %d\n", activeOps) - if nullifiedCount > 0 { - fmt.Printf(" Nullified: %d\n", nullifiedCount) - } - if len(opsWithLoc) > 0 { - fmt.Printf(" Bundled: %d\n", len(opsWithLoc)) - } - if len(mempoolOps) > 0 { - fmt.Printf(" Mempool: %d\n", len(mempoolOps)) - } - fmt.Printf("\n") - - fmt.Printf("Performance\n───────────\n") - fmt.Printf(" Index lookup: %s\n", lookupElapsed) - fmt.Printf(" Mempool check: %s\n", mempoolElapsed) - fmt.Printf(" Total time: %s\n\n", totalElapsed) - - // Show operations - if len(opsWithLoc) > 0 { - fmt.Printf("Bundled Operations (%d total)\n", len(opsWithLoc)) - fmt.Printf("══════════════════════════════════════════════════════════════\n\n") - - for i, owl := range opsWithLoc { - op := owl.Operation - status := "✓ Active" - if op.IsNullified() { - status = "✗ Nullified" - } - - fmt.Printf("Operation %d [Bundle %06d, Position %04d]\n", i+1, owl.Bundle, owl.Position) - fmt.Printf(" CID: %s\n", op.CID) - fmt.Printf(" Created: %s\n", op.CreatedAt.Format("2006-01-02 15:04:05.000 MST")) - fmt.Printf(" Status: %s\n", status) - - if verbose && !op.IsNullified() { - showOperationDetails(&op) - } - - fmt.Printf("\n") - } - } - - fmt.Printf("✓ Lookup complete in %s\n", totalElapsed) - return nil -} - -func showOperationDetails(op *plcclient.PLCOperation) { - if opData, err := op.GetOperationData(); err == nil && opData != nil { - if opType, ok := opData["type"].(string); ok { - fmt.Printf(" Type: %s\n", opType) - } - - if handle, ok := opData["handle"].(string); ok { - fmt.Printf(" Handle: %s\n", handle) - } else if aka, ok := opData["alsoKnownAs"].([]interface{}); ok && len(aka) > 0 { - if akaStr, ok := aka[0].(string); ok { - handle := strings.TrimPrefix(akaStr, "at://") - fmt.Printf(" Handle: %s\n", handle) - } - } - } -} diff --git a/cmd/plcbundle/commands/diff.go b/cmd/plcbundle/commands/diff.go deleted file mode 100644 index a52eb49..0000000 --- a/cmd/plcbundle/commands/diff.go +++ /dev/null @@ -1,798 +0,0 @@ -package commands - -import ( - "context" - "fmt" - "io" - "net/http" - "sort" - "strings" - "time" - - "github.com/goccy/go-json" - "github.com/spf13/cobra" - "tangled.org/atscan.net/plcbundle/bundle" - "tangled.org/atscan.net/plcbundle/internal/bundleindex" - "tangled.org/atscan.net/plcbundle/internal/plcclient" -) - -func NewDiffCommand() *cobra.Command { - var ( - verbose bool - bundleNum int - showOperations bool - showSample int - ) - - cmd := &cobra.Command{ - Use: "diff ", - Aliases: []string{"compare"}, - Short: "Compare repositories", - Long: `Compare local repository against remote or local target - -Compares bundle indexes to find differences such as: - • Missing bundles (in target but not local) - • Extra bundles (in local but not target) - • Hash mismatches (different content) - • Content mismatches (different data) - -For deeper analysis of specific bundles, use --bundle flag to see -detailed differences in metadata and operations. - -The target can be: - • Remote HTTP URL (e.g., https://plc.example.com) - • Remote index URL (e.g., https://plc.example.com/index.json) - • Local file path (e.g., /path/to/plc_bundles.json)`, - - Example: ` # High-level comparison - plcbundle diff https://plc.example.com - - # Show all differences (verbose) - plcbundle diff https://plc.example.com -v - - # Deep dive into specific bundle - plcbundle diff https://plc.example.com --bundle 23 - - # Compare bundle with operation samples - plcbundle diff https://plc.example.com --bundle 23 --show-operations - - # Show first 50 operations - plcbundle diff https://plc.example.com --bundle 23 --sample 50 - - # Using alias - plcbundle compare https://plc.example.com`, - - Args: cobra.ExactArgs(1), - - RunE: func(cmd *cobra.Command, args []string) error { - target := args[0] - - mgr, dir, err := getManager(&ManagerOptions{Cmd: cmd}) - if err != nil { - return err - } - defer mgr.Close() - - // If specific bundle requested, do detailed diff - if bundleNum > 0 { - return diffSpecificBundle(mgr, target, bundleNum, showOperations, showSample) - } - - // Otherwise, do high-level index comparison - return diffIndexes(mgr, dir, target, verbose) - }, - } - - // Flags - cmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "Show all differences (verbose output)") - cmd.Flags().IntVarP(&bundleNum, "bundle", "b", 0, "Deep diff of specific bundle") - cmd.Flags().BoolVar(&showOperations, "show-operations", false, "Show operation differences (use with --bundle)") - cmd.Flags().IntVar(&showSample, "sample", 10, "Number of sample operations to show (use with --bundle)") - - return cmd -} - -// diffIndexes performs high-level index comparison -func diffIndexes(mgr BundleManager, dir string, target string, verbose bool) error { - fmt.Printf("Comparing: %s\n", dir) - fmt.Printf(" Against: %s\n\n", target) - - // Load local index - localIndex := mgr.GetIndex() - - // Load target index - fmt.Printf("Loading target index...\n") - targetIndex, err := loadTargetIndex(target) - if err != nil { - return fmt.Errorf("error loading target index: %w", err) - } - - // Perform comparison - comparison := compareIndexes(localIndex, targetIndex) - - // Display results - displayComparison(comparison, verbose) - - // If there are hash mismatches, suggest deep dive - if len(comparison.HashMismatches) > 0 { - fmt.Printf("\n💡 Tip: Use --bundle flag to investigate specific mismatches:\n") - fmt.Printf(" plcbundle diff %s --bundle %d --show-operations\n", - target, comparison.HashMismatches[0].BundleNumber) - } - - if comparison.HasDifferences() { - return fmt.Errorf("indexes have differences") - } - - return nil -} - -// diffSpecificBundle performs detailed comparison of a specific bundle -func diffSpecificBundle(mgr BundleManager, target string, bundleNum int, showOps bool, sampleSize int) error { - ctx := context.Background() - - fmt.Printf("Deep Diff: Bundle %06d\n", bundleNum) - fmt.Printf("═══════════════════════════════════════════════════════════════\n\n") - - // Load local bundle - fmt.Printf("Loading local bundle %06d...\n", bundleNum) - localBundle, err := mgr.LoadBundle(ctx, bundleNum) - if err != nil { - return fmt.Errorf("failed to load local bundle: %w", err) - } - - // Load remote index to get metadata - fmt.Printf("Loading remote index...\n") - remoteIndex, err := loadTargetIndex(target) - if err != nil { - return fmt.Errorf("failed to load remote index: %w", err) - } - - remoteMeta, err := remoteIndex.GetBundle(bundleNum) - if err != nil { - return fmt.Errorf("bundle not found in remote index: %w", err) - } - - // Load remote bundle - fmt.Printf("Loading remote bundle %06d...\n\n", bundleNum) - remoteOps, err := loadRemoteBundle(target, bundleNum) - if err != nil { - return fmt.Errorf("failed to load remote bundle: %w", err) - } - - // Compare metadata - displayBundleMetadataComparison(localBundle, remoteMeta) - - // Compare operations - if showOps { - fmt.Printf("\n") - displayOperationComparison(localBundle.Operations, remoteOps, sampleSize) - } - - // Compare hashes in detail - fmt.Printf("\n") - displayHashAnalysis(localBundle, remoteMeta) - - return nil -} - -// displayBundleMetadataComparison shows metadata comparison -func displayBundleMetadataComparison(local *bundle.Bundle, remote *bundleindex.BundleMetadata) { - fmt.Printf("Metadata Comparison\n") - fmt.Printf("───────────────────\n\n") - - // Basic info - fmt.Printf(" Bundle Number: %06d\n", local.BundleNumber) - - // Times - timeMatch := local.StartTime.Equal(remote.StartTime) && local.EndTime.Equal(remote.EndTime) - fmt.Printf(" Start Time: %s %s\n", - formatTimeDiff(local.StartTime, remote.StartTime), - statusIcon(timeMatch)) - fmt.Printf(" Local: %s\n", local.StartTime.Format(time.RFC3339)) - fmt.Printf(" Remote: %s\n", remote.StartTime.Format(time.RFC3339)) - - fmt.Printf(" End Time: %s %s\n", - formatTimeDiff(local.EndTime, remote.EndTime), - statusIcon(timeMatch)) - fmt.Printf(" Local: %s\n", local.EndTime.Format(time.RFC3339)) - fmt.Printf(" Remote: %s\n", remote.EndTime.Format(time.RFC3339)) - - // Counts - opCountMatch := len(local.Operations) == remote.OperationCount - didCountMatch := local.DIDCount == remote.DIDCount - - fmt.Printf(" Operation Count: %s %s\n", - formatCountDiff(len(local.Operations), remote.OperationCount), - statusIcon(opCountMatch)) - fmt.Printf(" DID Count: %s %s\n", - formatCountDiff(local.DIDCount, remote.DIDCount), - statusIcon(didCountMatch)) - - // Sizes - sizeMatch := local.CompressedSize == remote.CompressedSize - fmt.Printf(" Compressed Size: %s %s\n", - formatSizeDiff(local.CompressedSize, remote.CompressedSize), - statusIcon(sizeMatch)) - - uncompMatch := local.UncompressedSize == remote.UncompressedSize - fmt.Printf(" Uncompressed Size: %s %s\n", - formatSizeDiff(local.UncompressedSize, remote.UncompressedSize), - statusIcon(uncompMatch)) -} - -// displayHashAnalysis shows detailed hash comparison -func displayHashAnalysis(local *bundle.Bundle, remote *bundleindex.BundleMetadata) { - fmt.Printf("Hash Analysis\n") - fmt.Printf("═════════════\n\n") - - // Content hash (most important) - contentMatch := local.ContentHash == remote.ContentHash - fmt.Printf(" Content Hash: %s\n", statusIcon(contentMatch)) - fmt.Printf(" Local: %s\n", local.ContentHash) - fmt.Printf(" Remote: %s\n", remote.ContentHash) - if !contentMatch { - fmt.Printf(" ⚠️ Different bundle content!\n") - } - fmt.Printf("\n") - - // Compressed hash - compMatch := local.CompressedHash == remote.CompressedHash - fmt.Printf(" Compressed Hash: %s\n", statusIcon(compMatch)) - fmt.Printf(" Local: %s\n", local.CompressedHash) - fmt.Printf(" Remote: %s\n", remote.CompressedHash) - if !compMatch && contentMatch { - fmt.Printf(" ℹ️ Different compression (same content)\n") - } - fmt.Printf("\n") - - // Chain hash - chainMatch := local.Hash == remote.Hash - fmt.Printf(" Chain Hash: %s\n", statusIcon(chainMatch)) - fmt.Printf(" Local: %s\n", local.Hash) - fmt.Printf(" Remote: %s\n", remote.Hash) - if !chainMatch { - // Analyze why chain hash differs - parentMatch := local.Parent == remote.Parent - fmt.Printf("\n Chain Components:\n") - fmt.Printf(" Parent: %s\n", statusIcon(parentMatch)) - fmt.Printf(" Local: %s\n", local.Parent) - fmt.Printf(" Remote: %s\n", remote.Parent) - - if !parentMatch { - fmt.Printf(" ⚠️ Different parent → chain diverged at earlier bundle\n") - } else if !contentMatch { - fmt.Printf(" ⚠️ Same parent but different content → different operations\n") - } - } -} - -// displayOperationComparison shows operation differences -func displayOperationComparison(localOps []plcclient.PLCOperation, remoteOps []plcclient.PLCOperation, sampleSize int) { - fmt.Printf("Operation Comparison\n") - fmt.Printf("════════════════════\n\n") - - if len(localOps) != len(remoteOps) { - fmt.Printf(" ⚠️ Different operation counts: local=%d, remote=%d\n\n", - len(localOps), len(remoteOps)) - } - - // Build CID sets for comparison - localCIDs := make(map[string]int) - remoteCIDs := make(map[string]int) - - for i, op := range localOps { - localCIDs[op.CID] = i - } - for i, op := range remoteOps { - remoteCIDs[op.CID] = i - } - - // Find differences - store as position+CID pairs - type cidWithPos struct { - cid string - pos int - } - - var missingInLocal []cidWithPos - var missingInRemote []cidWithPos - var positionMismatches []cidWithPos - - for cid, remotePos := range remoteCIDs { - if localPos, exists := localCIDs[cid]; !exists { - missingInLocal = append(missingInLocal, cidWithPos{cid, remotePos}) - } else if localPos != remotePos { - positionMismatches = append(positionMismatches, cidWithPos{cid, localPos}) - } - } - - for cid, localPos := range localCIDs { - if _, exists := remoteCIDs[cid]; !exists { - missingInRemote = append(missingInRemote, cidWithPos{cid, localPos}) - } - } - - // Sort by position - sort.Slice(missingInLocal, func(i, j int) bool { - return missingInLocal[i].pos < missingInLocal[j].pos - }) - sort.Slice(missingInRemote, func(i, j int) bool { - return missingInRemote[i].pos < missingInRemote[j].pos - }) - sort.Slice(positionMismatches, func(i, j int) bool { - return positionMismatches[i].pos < positionMismatches[j].pos - }) - - // Display differences - if len(missingInLocal) > 0 { - fmt.Printf(" Missing in Local (%d operations):\n", len(missingInLocal)) - displaySample := min(sampleSize, len(missingInLocal)) - for i := 0; i < displaySample; i++ { - item := missingInLocal[i] - fmt.Printf(" - [%04d] %s\n", item.pos, item.cid) - } - if len(missingInLocal) > displaySample { - fmt.Printf(" ... and %d more\n", len(missingInLocal)-displaySample) - } - fmt.Printf("\n") - } - - if len(missingInRemote) > 0 { - fmt.Printf(" Missing in Remote (%d operations):\n", len(missingInRemote)) - displaySample := min(sampleSize, len(missingInRemote)) - for i := 0; i < displaySample; i++ { - item := missingInRemote[i] - fmt.Printf(" + [%04d] %s\n", item.pos, item.cid) - } - if len(missingInRemote) > displaySample { - fmt.Printf(" ... and %d more\n", len(missingInRemote)-displaySample) - } - fmt.Printf("\n") - } - - if len(positionMismatches) > 0 { - fmt.Printf(" Position Mismatches (%d operations):\n", len(positionMismatches)) - displaySample := min(sampleSize, len(positionMismatches)) - for i := 0; i < displaySample; i++ { - item := positionMismatches[i] - remotePos := remoteCIDs[item.cid] - fmt.Printf(" ~ %s\n", item.cid) - fmt.Printf(" Local: position %04d\n", item.pos) - fmt.Printf(" Remote: position %04d\n", remotePos) - } - if len(positionMismatches) > displaySample { - fmt.Printf(" ... and %d more\n", len(positionMismatches)-displaySample) - } - fmt.Printf("\n") - } - - if len(missingInLocal) == 0 && len(missingInRemote) == 0 && len(positionMismatches) == 0 { - fmt.Printf(" ✓ All operations match (same CIDs, same order)\n\n") - } - - // Show sample operations for context - if len(localOps) > 0 { - fmt.Printf("Sample Operations (first %d):\n", min(sampleSize, len(localOps))) - fmt.Printf("────────────────────────────────\n") - for i := 0; i < min(sampleSize, len(localOps)); i++ { - op := localOps[i] - remoteMatch := "" - if remotePos, exists := remoteCIDs[op.CID]; exists { - if remotePos == i { - remoteMatch = " ✓" - } else { - remoteMatch = fmt.Sprintf(" ~ (remote pos: %04d)", remotePos) - } - } else { - remoteMatch = " ✗ (missing in remote)" - } - - fmt.Printf(" [%04d] %s%s\n", i, op.CID, remoteMatch) - fmt.Printf(" DID: %s\n", op.DID) - fmt.Printf(" Time: %s\n", op.CreatedAt.Format(time.RFC3339)) - } - fmt.Printf("\n") - } -} - -// loadRemoteBundle loads a bundle from remote server -func loadRemoteBundle(baseURL string, bundleNum int) ([]plcclient.PLCOperation, error) { - // Determine the data URL - url := baseURL - if !strings.HasSuffix(url, ".json") { - url = strings.TrimSuffix(url, "/") - } - url = strings.TrimSuffix(url, "/index.json") - url = strings.TrimSuffix(url, "/plc_bundles.json") - url = fmt.Sprintf("%s/jsonl/%d", url, bundleNum) - - client := &http.Client{Timeout: 60 * time.Second} - - resp, err := client.Get(url) - if err != nil { - return nil, fmt.Errorf("failed to download: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode) - } - - // Parse JSONL - var operations []plcclient.PLCOperation - decoder := json.NewDecoder(resp.Body) - - for { - var op plcclient.PLCOperation - if err := decoder.Decode(&op); err != nil { - if err == io.EOF { - break - } - return nil, fmt.Errorf("failed to parse operation: %w", err) - } - operations = append(operations, op) - } - - return operations, nil -} - -// loadTargetIndex loads index from URL or local path -func loadTargetIndex(target string) (*bundleindex.Index, error) { - if strings.HasPrefix(target, "http://") || strings.HasPrefix(target, "https://") { - return loadIndexFromURL(target) - } - return bundleindex.LoadIndex(target) -} - -// loadIndexFromURL loads index from remote URL -func loadIndexFromURL(url string) (*bundleindex.Index, error) { - if !strings.HasSuffix(url, ".json") { - url = strings.TrimSuffix(url, "/") + "/index.json" - } - - client := &http.Client{Timeout: 30 * time.Second} - - resp, err := client.Get(url) - if err != nil { - return nil, fmt.Errorf("failed to download: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode) - } - - data, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read response: %w", err) - } - - var idx bundleindex.Index - if err := json.Unmarshal(data, &idx); err != nil { - return nil, fmt.Errorf("failed to parse index: %w", err) - } - - return &idx, nil -} - -// compareIndexes performs the actual comparison -func compareIndexes(local, target *bundleindex.Index) *IndexComparison { - localBundles := local.GetBundles() - targetBundles := target.GetBundles() - - localMap := make(map[int]*bundleindex.BundleMetadata) - targetMap := make(map[int]*bundleindex.BundleMetadata) - - for _, b := range localBundles { - localMap[b.BundleNumber] = b - } - for _, b := range targetBundles { - targetMap[b.BundleNumber] = b - } - - comparison := &IndexComparison{ - LocalCount: len(localBundles), - TargetCount: len(targetBundles), - MissingBundles: make([]int, 0), - ExtraBundles: make([]int, 0), - HashMismatches: make([]HashMismatch, 0), - ContentMismatches: make([]HashMismatch, 0), - } - - // Get ranges - if len(localBundles) > 0 { - comparison.LocalRange = [2]int{localBundles[0].BundleNumber, localBundles[len(localBundles)-1].BundleNumber} - comparison.LocalUpdated = local.UpdatedAt - comparison.LocalTotalSize = local.TotalSize - } - - if len(targetBundles) > 0 { - comparison.TargetRange = [2]int{targetBundles[0].BundleNumber, targetBundles[len(targetBundles)-1].BundleNumber} - comparison.TargetUpdated = target.UpdatedAt - comparison.TargetTotalSize = target.TotalSize - } - - // Find missing bundles - for bundleNum := range targetMap { - if _, exists := localMap[bundleNum]; !exists { - comparison.MissingBundles = append(comparison.MissingBundles, bundleNum) - } - } - sort.Ints(comparison.MissingBundles) - - // Find extra bundles - for bundleNum := range localMap { - if _, exists := targetMap[bundleNum]; !exists { - comparison.ExtraBundles = append(comparison.ExtraBundles, bundleNum) - } - } - sort.Ints(comparison.ExtraBundles) - - // Compare hashes - for bundleNum, localMeta := range localMap { - if targetMeta, exists := targetMap[bundleNum]; exists { - comparison.CommonCount++ - - chainMismatch := localMeta.Hash != targetMeta.Hash - contentMismatch := localMeta.ContentHash != targetMeta.ContentHash - - if chainMismatch || contentMismatch { - mismatch := HashMismatch{ - BundleNumber: bundleNum, - LocalHash: localMeta.Hash, - TargetHash: targetMeta.Hash, - LocalContentHash: localMeta.ContentHash, - TargetContentHash: targetMeta.ContentHash, - } - - if chainMismatch { - comparison.HashMismatches = append(comparison.HashMismatches, mismatch) - } - if contentMismatch && !chainMismatch { - comparison.ContentMismatches = append(comparison.ContentMismatches, mismatch) - } - } - } - } - - // ADD THIS: Sort mismatches by bundle number - sort.Slice(comparison.HashMismatches, func(i, j int) bool { - return comparison.HashMismatches[i].BundleNumber < comparison.HashMismatches[j].BundleNumber - }) - sort.Slice(comparison.ContentMismatches, func(i, j int) bool { - return comparison.ContentMismatches[i].BundleNumber < comparison.ContentMismatches[j].BundleNumber - }) - - return comparison -} - -// displayComparison shows comparison results -func displayComparison(c *IndexComparison, verbose bool) { - fmt.Printf("Comparison Results\n") - fmt.Printf("══════════════════\n\n") - - fmt.Printf("Summary\n───────\n") - fmt.Printf(" Local bundles: %d\n", c.LocalCount) - fmt.Printf(" Target bundles: %d\n", c.TargetCount) - fmt.Printf(" Common bundles: %d\n", c.CommonCount) - fmt.Printf(" Missing bundles: %s\n", formatCount(len(c.MissingBundles))) - fmt.Printf(" Extra bundles: %s\n", formatCount(len(c.ExtraBundles))) - fmt.Printf(" Hash mismatches: %s\n", formatCountCritical(len(c.HashMismatches))) - fmt.Printf(" Content mismatches: %s\n", formatCount(len(c.ContentMismatches))) - - if c.LocalCount > 0 { - fmt.Printf("\n Local range: %06d - %06d\n", c.LocalRange[0], c.LocalRange[1]) - fmt.Printf(" Local size: %.2f MB\n", float64(c.LocalTotalSize)/(1024*1024)) - fmt.Printf(" Local updated: %s\n", c.LocalUpdated.Format("2006-01-02 15:04:05")) - } - - if c.TargetCount > 0 { - fmt.Printf("\n Target range: %06d - %06d\n", c.TargetRange[0], c.TargetRange[1]) - fmt.Printf(" Target size: %.2f MB\n", float64(c.TargetTotalSize)/(1024*1024)) - fmt.Printf(" Target updated: %s\n", c.TargetUpdated.Format("2006-01-02 15:04:05")) - } - - // Show differences - if len(c.HashMismatches) > 0 { - showHashMismatches(c.HashMismatches, verbose) - } - - if len(c.MissingBundles) > 0 { - showMissingBundles(c.MissingBundles, verbose) - } - - if len(c.ExtraBundles) > 0 { - showExtraBundles(c.ExtraBundles, verbose) - } - - // Final status - fmt.Printf("\n") - if !c.HasDifferences() { - fmt.Printf("✓ Indexes are identical\n") - } else { - fmt.Printf("✗ Indexes have differences\n") - if len(c.HashMismatches) > 0 { - fmt.Printf("\n⚠️ WARNING: Chain hash mismatches detected!\n") - fmt.Printf("This indicates different bundle content or chain integrity issues.\n") - } - } -} - -// showHashMismatches displays hash mismatches -func showHashMismatches(mismatches []HashMismatch, verbose bool) { - fmt.Printf("\n⚠️ CHAIN HASH MISMATCHES (CRITICAL)\n") - fmt.Printf("════════════════════════════════════\n\n") - - displayCount := len(mismatches) - if displayCount > 10 && !verbose { - displayCount = 10 - } - - for i := 0; i < displayCount; i++ { - m := mismatches[i] - fmt.Printf(" Bundle %06d:\n", m.BundleNumber) - fmt.Printf(" Chain Hash:\n") - fmt.Printf(" Local: %s\n", m.LocalHash) - fmt.Printf(" Target: %s\n", m.TargetHash) - - if m.LocalContentHash != m.TargetContentHash { - fmt.Printf(" Content Hash (also differs):\n") - fmt.Printf(" Local: %s\n", m.LocalContentHash) - fmt.Printf(" Target: %s\n", m.TargetContentHash) - } - fmt.Printf("\n") - } - - if len(mismatches) > displayCount { - fmt.Printf(" ... and %d more (use -v to show all)\n\n", len(mismatches)-displayCount) - } -} - -// showMissingBundles displays missing bundles -func showMissingBundles(bundles []int, verbose bool) { - fmt.Printf("\nMissing Bundles (in target but not local)\n") - fmt.Printf("──────────────────────────────────────────\n") - - if verbose || len(bundles) <= 20 { - displayCount := len(bundles) - if displayCount > 20 && !verbose { - displayCount = 20 - } - - for i := 0; i < displayCount; i++ { - fmt.Printf(" %06d\n", bundles[i]) - } - - if len(bundles) > displayCount { - fmt.Printf(" ... and %d more (use -v to show all)\n", len(bundles)-displayCount) - } - } else { - displayBundleRanges(bundles) - } -} - -// showExtraBundles displays extra bundles -func showExtraBundles(bundles []int, verbose bool) { - fmt.Printf("\nExtra Bundles (in local but not target)\n") - fmt.Printf("────────────────────────────────────────\n") - - if verbose || len(bundles) <= 20 { - displayCount := len(bundles) - if displayCount > 20 && !verbose { - displayCount = 20 - } - - for i := 0; i < displayCount; i++ { - fmt.Printf(" %06d\n", bundles[i]) - } - - if len(bundles) > displayCount { - fmt.Printf(" ... and %d more (use -v to show all)\n", len(bundles)-displayCount) - } - } else { - displayBundleRanges(bundles) - } -} - -// displayBundleRanges displays bundles as ranges -func displayBundleRanges(bundles []int) { - if len(bundles) == 0 { - return - } - - rangeStart := bundles[0] - rangeEnd := bundles[0] - - for i := 1; i < len(bundles); i++ { - if bundles[i] == rangeEnd+1 { - rangeEnd = bundles[i] - } else { - if rangeStart == rangeEnd { - fmt.Printf(" %06d\n", rangeStart) - } else { - fmt.Printf(" %06d - %06d\n", rangeStart, rangeEnd) - } - rangeStart = bundles[i] - rangeEnd = bundles[i] - } - } - - if rangeStart == rangeEnd { - fmt.Printf(" %06d\n", rangeStart) - } else { - fmt.Printf(" %06d - %06d\n", rangeStart, rangeEnd) - } -} - -// Helper formatting functions - -func statusIcon(match bool) string { - if match { - return "✓" - } - return "✗" -} - -func formatTimeDiff(local, remote time.Time) string { - if local.Equal(remote) { - return "identical" - } - diff := local.Sub(remote) - if diff < 0 { - diff = -diff - } - return fmt.Sprintf("differs by %s", diff) -} - -func formatCountDiff(local, remote int) string { - if local == remote { - return fmt.Sprintf("%d", local) - } - return fmt.Sprintf("local=%d, remote=%d", local, remote) -} - -func formatSizeDiff(local, remote int64) string { - if local == remote { - return formatBytes(local) - } - diff := local - remote - sign := "+" - if diff < 0 { - sign = "-" - diff = -diff - } - return fmt.Sprintf("local=%s, remote=%s (%s%s)", - formatBytes(local), formatBytes(remote), sign, formatBytes(diff)) -} - -// IndexComparison holds comparison results -type IndexComparison struct { - LocalCount int - TargetCount int - CommonCount int - MissingBundles []int - ExtraBundles []int - HashMismatches []HashMismatch - ContentMismatches []HashMismatch - LocalRange [2]int - TargetRange [2]int - LocalTotalSize int64 - TargetTotalSize int64 - LocalUpdated time.Time - TargetUpdated time.Time -} - -// HashMismatch represents a hash mismatch between bundles -type HashMismatch struct { - BundleNumber int - LocalHash string - TargetHash string - LocalContentHash string - TargetContentHash string -} - -// HasDifferences checks if there are any differences -func (ic *IndexComparison) HasDifferences() bool { - return len(ic.MissingBundles) > 0 || len(ic.ExtraBundles) > 0 || - len(ic.HashMismatches) > 0 || len(ic.ContentMismatches) > 0 -} diff --git a/cmd/plcbundle/commands/export.go b/cmd/plcbundle/commands/export.go deleted file mode 100644 index a1db4dd..0000000 --- a/cmd/plcbundle/commands/export.go +++ /dev/null @@ -1,356 +0,0 @@ -package commands - -import ( - "context" - "fmt" - "os" - "time" - - "github.com/goccy/go-json" - "github.com/spf13/cobra" - internalsync "tangled.org/atscan.net/plcbundle/internal/sync" -) - -func NewExportCommand() *cobra.Command { - var ( - all bool - rangeStr string - sync bool - plcURL string - count int - after string - ) - - cmd := &cobra.Command{ - Use: "export [flags]", - Aliases: []string{"stream", "backfill"}, - Short: "Export operations to stdout (JSONL)", - Long: `Export operations to stdout in JSONL format - -Outputs PLC operations as newline-delimited JSON to stdout. -Progress and status messages go to stderr. - -By default, exports only existing bundles. Use --sync to also fetch -new bundles from PLC directory until caught up. - -Supports filtering by count and timestamp for selective exports.`, - - Example: ` # Export all existing bundles - plcbundle export --all - - # Export and sync new bundles - plcbundle export --all --sync - - # Export specific range (existing only) - plcbundle export --range 1-100 - - # Export with limit - plcbundle export --all --count 50000 - - # Export after timestamp - plcbundle export --all --after 2024-01-01T00:00:00Z - - # Combine filters - plcbundle export --range 1-100 --count 10000 --after 2024-01-01T00:00:00Z - - # Export from specific PLC directory - plcbundle export --all --sync --plc https://plc.directory - - # Pipe to file - plcbundle export --all > operations.jsonl - - # Process with jq - plcbundle export --all | jq -r .did | sort | uniq -c - - # Sync and filter with detector - plcbundle export --all --sync | plcbundle detector filter spam - - # Using aliases (backwards compatible) - plcbundle stream --all --sync - plcbundle backfill --all`, - - RunE: func(cmd *cobra.Command, args []string) error { - verbose, _ := cmd.Root().PersistentFlags().GetBool("verbose") - quiet, _ := cmd.Root().PersistentFlags().GetBool("quiet") - - mgr, dir, err := getManager(&ManagerOptions{Cmd: cmd, PLCURL: plcURL}) - if err != nil { - return err - } - defer mgr.Close() - - if !quiet { - fmt.Fprintf(os.Stderr, "Exporting from: %s\n", dir) - } - - // Parse after timestamp if provided - var afterTime time.Time - if after != "" { - afterTime, err = time.Parse(time.RFC3339, after) - if err != nil { - return fmt.Errorf("invalid --after timestamp (use RFC3339 format): %w", err) - } - } - - // Determine bundle range - var start, end int - - if all { - index := mgr.GetIndex() - bundles := index.GetBundles() - - if len(bundles) == 0 { - if sync { - // No bundles but sync enabled - start from 1 - start = 1 - end = 0 // Will be updated after sync - } else { - if !quiet { - fmt.Fprintf(os.Stderr, "No bundles available (use --sync to fetch)\n") - } - return nil - } - } else { - start = bundles[0].BundleNumber - end = bundles[len(bundles)-1].BundleNumber - } - - } else if rangeStr != "" { - var err error - start, end, err = parseBundleRange(rangeStr) - if err != nil { - return err - } - - } else { - return fmt.Errorf("either --all or --range required") - } - - if !quiet { - if sync { - fmt.Fprintf(os.Stderr, "Mode: export existing + sync new bundles\n") - } else { - fmt.Fprintf(os.Stderr, "Mode: export existing only\n") - } - - if count > 0 { - fmt.Fprintf(os.Stderr, "Limit: %d operations\n", count) - } - if after != "" { - fmt.Fprintf(os.Stderr, "After: %s\n", after) - } - fmt.Fprintf(os.Stderr, "\n") - } - - return exportBundles(cmd.Context(), mgr, exportOptions{ - start: start, - end: end, - sync: sync, - count: count, - afterTime: afterTime, - verbose: verbose, - quiet: quiet, - }) - }, - } - - cmd.Flags().BoolVar(&all, "all", false, "Export all bundles") - cmd.Flags().StringVarP(&rangeStr, "range", "r", "", "Export bundle range (e.g., '1-100')") - cmd.Flags().BoolVar(&sync, "sync", false, "Also fetch new bundles from PLC (until caught up)") - cmd.Flags().StringVar(&plcURL, "plc", "https://plc.directory", "PLC directory URL (for --sync)") - cmd.Flags().IntVarP(&count, "count", "n", 0, "Limit number of operations (0 = unlimited)") - cmd.Flags().StringVar(&after, "after", "", "Only export operations after this timestamp (RFC3339)") - - return cmd -} - -type exportOptions struct { - start int - end int - sync bool - count int - afterTime time.Time - verbose bool - quiet bool -} - -func exportBundles(ctx context.Context, mgr BundleManager, opts exportOptions) error { - operationCount := 0 - exported := 0 - - // Phase 1: Export existing bundles - existingCount := 0 - if opts.end > 0 { - existingCount, exported = exportExistingBundles( - ctx, mgr, opts.start, opts.end, - &operationCount, opts.count, opts.afterTime, - opts.verbose, opts.quiet, - ) - } - - // Check if we hit the count limit - if opts.count > 0 && exported >= opts.count { - if !opts.quiet { - fmt.Fprintf(os.Stderr, "\n✓ Export complete (limit reached)\n") - fmt.Fprintf(os.Stderr, " Bundles: %d\n", existingCount) - fmt.Fprintf(os.Stderr, " Operations: %d\n", exported) - } - return nil - } - - // Phase 2: Sync and export new bundles (if enabled and not at limit) - fetchedCount := 0 - if opts.sync && (opts.count == 0 || exported < opts.count) { - if !opts.quiet { - fmt.Fprintf(os.Stderr, "\nSyncing new bundles from PLC...\n") - } - - logger := &exportLogger{quiet: opts.quiet} - config := &internalsync.SyncLoopConfig{ - MaxBundles: 0, - Verbose: opts.verbose, - Logger: logger, - OnBundleSynced: func(bundleNum, synced, mempoolCount int, duration, indexTime time.Duration) { - // Stream the bundle we just created - if bundle, err := mgr.LoadBundle(ctx, bundleNum); err == nil { - for _, op := range bundle.Operations { - // Apply filters - if !opts.afterTime.IsZero() && op.CreatedAt.Before(opts.afterTime) { - continue - } - - if opts.count > 0 && exported >= opts.count { - return // Stop when limit reached - } - - // Output operation - if len(op.RawJSON) > 0 { - fmt.Println(string(op.RawJSON)) - } else { - data, _ := json.Marshal(op) - fmt.Println(string(data)) - } - exported++ - operationCount++ - } - } - }, - } - - var err error - fetchedCount, err = mgr.RunSyncOnce(ctx, config) - if err != nil { - return err - } - } - - // Summary - if !opts.quiet { - fmt.Fprintf(os.Stderr, "\n✓ Export complete\n") - - if opts.sync && fetchedCount > 0 { - fmt.Fprintf(os.Stderr, " Bundles: %d (%d existing + %d synced)\n", - existingCount+fetchedCount, existingCount, fetchedCount) - } else if opts.sync { - fmt.Fprintf(os.Stderr, " Bundles: %d (already up to date)\n", existingCount) - } else { - fmt.Fprintf(os.Stderr, " Bundles: %d\n", existingCount) - } - - fmt.Fprintf(os.Stderr, " Operations: %d", exported) - if opts.count > 0 { - fmt.Fprintf(os.Stderr, " (limit: %d)", opts.count) - } - fmt.Fprintf(os.Stderr, "\n") - } - - return nil -} - -func exportExistingBundles( - ctx context.Context, - mgr BundleManager, - start, end int, - operationCount *int, - limit int, - afterTime time.Time, - verbose bool, - quiet bool, -) (bundleCount int, exported int) { - - processedCount := 0 - exportedOps := 0 - - for bundleNum := start; bundleNum <= end; bundleNum++ { - select { - case <-ctx.Done(): - return processedCount, exportedOps - default: - } - - bundle, err := mgr.LoadBundle(ctx, bundleNum) - if err != nil { - if verbose { - fmt.Fprintf(os.Stderr, "Bundle %06d: not found (skipped)\n", bundleNum) - } - continue - } - - // Export operations with filters - for _, op := range bundle.Operations { - // Filter by timestamp - if !afterTime.IsZero() && op.CreatedAt.Before(afterTime) { - continue - } - - // Check count limit - if limit > 0 && exportedOps >= limit { - if verbose { - fmt.Fprintf(os.Stderr, "Bundle %06d: limit reached, stopping\n", bundleNum) - } - return processedCount, exportedOps - } - - // Output operation to stdout (JSONL) - if len(op.RawJSON) > 0 { - fmt.Println(string(op.RawJSON)) - } else { - data, _ := json.Marshal(op) - fmt.Println(string(data)) - } - exportedOps++ - } - - *operationCount += len(bundle.Operations) - processedCount++ - - if verbose { - fmt.Fprintf(os.Stderr, "Bundle %06d: ✓ (%d ops, %d exported)\n", - bundleNum, len(bundle.Operations), exportedOps) - } else if !quiet && processedCount%100 == 0 { - fmt.Fprintf(os.Stderr, "Exported: %d bundles, %d ops\r", processedCount, exportedOps) - } - } - - if !quiet && !verbose && processedCount > 0 { - fmt.Fprintf(os.Stderr, "Existing: %d bundles, %d ops\n", processedCount, exportedOps) - } - - return processedCount, exportedOps -} - -type exportLogger struct { - quiet bool -} - -func (l *exportLogger) Printf(format string, v ...interface{}) { - if !l.quiet { - fmt.Fprintf(os.Stderr, format+"\n", v...) - } -} - -func (l *exportLogger) Println(v ...interface{}) { - if !l.quiet { - fmt.Fprintln(os.Stderr, v...) - } -} diff --git a/cmd/plcbundle/commands/index.go b/cmd/plcbundle/commands/index.go deleted file mode 100644 index 7ed19f4..0000000 --- a/cmd/plcbundle/commands/index.go +++ /dev/null @@ -1,388 +0,0 @@ -package commands - -import ( - "context" - "fmt" - "time" - - "github.com/goccy/go-json" - "github.com/spf13/cobra" - "tangled.org/atscan.net/plcbundle/cmd/plcbundle/ui" -) - -func NewIndexCommand() *cobra.Command { - cmd := &cobra.Command{ - Use: "index", - Short: "DID index management", - Long: `DID index management operations - -Manage the DID position index which maps DIDs to their bundle locations. -This index enables fast O(1) DID lookups and is required for DID -resolution and query operations.`, - - Example: ` # Build DID position index - plcbundle index build - - # Repair DID index (rebuild from bundles) - plcbundle index repair - - # Show DID index statistics - plcbundle index stats - - # Verify DID index integrity - plcbundle index verify`, - } - - cmd.AddCommand(newIndexBuildCommand()) - cmd.AddCommand(newIndexRepairCommand()) - cmd.AddCommand(newIndexStatsCommand()) - cmd.AddCommand(newIndexVerifyCommand()) - - return cmd -} - -// ============================================================================ -// INDEX BUILD - Build DID position index -// ============================================================================ - -func newIndexBuildCommand() *cobra.Command { - var force bool - - cmd := &cobra.Command{ - Use: "build", - Short: "Build DID position index", - Long: `Build DID position index from bundles - -Creates a sharded index mapping each DID to its bundle locations, -enabling fast O(1) DID lookups. Required for DID resolution. - -The index is built incrementally and auto-updates as new bundles -are added. Use --force to rebuild from scratch.`, - - Example: ` # Build index - plcbundle index build - - # Force rebuild from scratch - plcbundle index build --force`, - - RunE: func(cmd *cobra.Command, args []string) error { - mgr, dir, err := getManager(&ManagerOptions{Cmd: cmd}) - if err != nil { - return err - } - defer mgr.Close() - - stats := mgr.GetDIDIndexStats() - if stats["exists"].(bool) && !force { - fmt.Printf("DID index already exists (use --force to rebuild)\n") - fmt.Printf("Directory: %s\n", dir) - fmt.Printf("Total DIDs: %s\n", formatNumber(int(stats["total_dids"].(int64)))) - return nil - } - - index := mgr.GetIndex() - bundleCount := index.Count() - - if bundleCount == 0 { - fmt.Printf("No bundles to index\n") - return nil - } - - fmt.Printf("Building DID index in: %s\n", dir) - fmt.Printf("Indexing %d bundles...\n\n", bundleCount) - - progress := ui.NewProgressBar(bundleCount) - start := time.Now() - ctx := context.Background() - - err = mgr.BuildDIDIndex(ctx, func(current, total int) { - progress.Set(current) - }) - - progress.Finish() - - if err != nil { - return fmt.Errorf("build failed: %w", err) - } - - elapsed := time.Since(start) - stats = mgr.GetDIDIndexStats() - - fmt.Printf("\n✓ DID index built in %s\n", elapsed.Round(time.Millisecond)) - fmt.Printf(" Total DIDs: %s\n", formatNumber(int(stats["total_dids"].(int64)))) - fmt.Printf(" Shards: %d\n", stats["shard_count"]) - fmt.Printf(" Location: %s/.plcbundle/\n", dir) - - return nil - }, - } - - cmd.Flags().BoolVar(&force, "force", false, "Rebuild even if index exists") - - return cmd -} - -// ============================================================================ -// INDEX REPAIR - Repair/rebuild DID index -// ============================================================================ - -func newIndexRepairCommand() *cobra.Command { - cmd := &cobra.Command{ - Use: "repair", - Aliases: []string{"rebuild"}, - Short: "Repair DID index", - Long: `Repair DID index by rebuilding from bundles - -Rebuilds the DID index from scratch and verifies consistency. -Use this when: - • DID index is corrupted - • Index is out of sync with bundles - • After manual bundle operations - • Upgrade to new index version`, - - Example: ` # Repair DID index - plcbundle index repair - - # Verbose output - plcbundle index repair -v`, - - RunE: func(cmd *cobra.Command, args []string) error { - verbose, _ := cmd.Root().PersistentFlags().GetBool("verbose") - - mgr, dir, err := getManager(&ManagerOptions{Cmd: cmd}) - if err != nil { - return err - } - defer mgr.Close() - - stats := mgr.GetDIDIndexStats() - if !stats["exists"].(bool) { - fmt.Printf("DID index does not exist\n") - fmt.Printf("Use: plcbundle index build\n") - return nil - } - - fmt.Printf("Repairing DID index in: %s\n\n", dir) - - index := mgr.GetIndex() - bundleCount := index.Count() - - if bundleCount == 0 { - fmt.Printf("No bundles to index\n") - return nil - } - - fmt.Printf("Rebuilding index from %d bundles...\n\n", bundleCount) - - var progress *ui.ProgressBar - if !verbose { - progress = ui.NewProgressBar(bundleCount) - } - - start := time.Now() - ctx := context.Background() - - err = mgr.BuildDIDIndex(ctx, func(current, total int) { - if progress != nil { - progress.Set(current) - } else if current%100 == 0 || current == total { - fmt.Printf("Progress: %d/%d (%.1f%%) \r", - current, total, float64(current)/float64(total)*100) - } - }) - - if progress != nil { - progress.Finish() - } - - if err != nil { - return fmt.Errorf("repair failed: %w", err) - } - - // Verify consistency - fmt.Printf("\nVerifying consistency...\n") - if err := mgr.GetDIDIndex().VerifyAndRepairIndex(ctx, mgr); err != nil { - return fmt.Errorf("verification failed: %w", err) - } - - elapsed := time.Since(start) - stats = mgr.GetDIDIndexStats() - - fmt.Printf("\n✓ DID index repaired in %s\n", elapsed.Round(time.Millisecond)) - fmt.Printf(" Total DIDs: %s\n", formatNumber(int(stats["total_dids"].(int64)))) - fmt.Printf(" Shards: %d\n", stats["shard_count"]) - fmt.Printf(" Last bundle: %06d\n", stats["last_bundle"]) - - return nil - }, - } - - return cmd -} - -// ============================================================================ -// INDEX STATS - Show DID index statistics -// ============================================================================ - -func newIndexStatsCommand() *cobra.Command { - var showJSON bool - - cmd := &cobra.Command{ - Use: "stats", - Aliases: []string{"info"}, - Short: "Show DID index statistics", - Long: `Show DID index statistics - -Displays DID index information including total DIDs indexed, -shard distribution, cache statistics, and coverage.`, - - Example: ` # Show statistics - plcbundle index stats - - # JSON output - plcbundle index stats --json`, - - RunE: func(cmd *cobra.Command, args []string) error { - mgr, dir, err := getManager(&ManagerOptions{Cmd: cmd}) - if err != nil { - return err - } - defer mgr.Close() - - stats := mgr.GetDIDIndexStats() - - if showJSON { - data, _ := json.MarshalIndent(stats, "", " ") - fmt.Println(string(data)) - return nil - } - - if !stats["exists"].(bool) { - fmt.Printf("DID index does not exist\n") - fmt.Printf("Run: plcbundle index build\n") - return nil - } - - indexedDIDs := stats["indexed_dids"].(int64) - mempoolDIDs := stats["mempool_dids"].(int64) - totalDIDs := stats["total_dids"].(int64) - - fmt.Printf("\nDID Index Statistics\n") - fmt.Printf("════════════════════\n\n") - fmt.Printf(" Location: %s/.plcbundle/\n", dir) - - if mempoolDIDs > 0 { - fmt.Printf(" Indexed DIDs: %s (in bundles)\n", formatNumber(int(indexedDIDs))) - fmt.Printf(" Mempool DIDs: %s (not yet bundled)\n", formatNumber(int(mempoolDIDs))) - fmt.Printf(" Total DIDs: %s\n", formatNumber(int(totalDIDs))) - } else { - fmt.Printf(" Total DIDs: %s\n", formatNumber(int(totalDIDs))) - } - - fmt.Printf(" Shard count: %d\n", stats["shard_count"]) - fmt.Printf(" Last bundle: %06d\n", stats["last_bundle"]) - fmt.Printf(" Updated: %s\n\n", stats["updated_at"].(time.Time).Format("2006-01-02 15:04:05")) - - fmt.Printf(" Cached shards: %d / %d\n", stats["cached_shards"], stats["cache_limit"]) - - if cachedList, ok := stats["cache_order"].([]int); ok && len(cachedList) > 0 { - fmt.Printf(" Hot shards: ") - for i, shard := range cachedList { - if i > 0 { - fmt.Printf(", ") - } - if i >= 10 { - fmt.Printf("... (+%d more)", len(cachedList)-10) - break - } - fmt.Printf("%02x", shard) - } - fmt.Printf("\n") - } - - fmt.Printf("\n") - return nil - }, - } - - cmd.Flags().BoolVar(&showJSON, "json", false, "Output as JSON") - - return cmd -} - -// ============================================================================ -// INDEX VERIFY - Verify DID index integrity -// ============================================================================ - -func newIndexVerifyCommand() *cobra.Command { - var verbose bool - - cmd := &cobra.Command{ - Use: "verify", - Aliases: []string{"check"}, - Short: "Verify DID index integrity", - Long: `Verify DID index integrity and consistency - -Checks the DID index for consistency with bundles: - • Index version is current - • All bundles are indexed - • Shard files are valid - • No corruption detected - -Automatically repairs minor issues.`, - - Example: ` # Verify DID index - plcbundle index verify - - # Verbose output - plcbundle index verify -v`, - - RunE: func(cmd *cobra.Command, args []string) error { - mgr, dir, err := getManager(&ManagerOptions{Cmd: cmd}) - if err != nil { - return err - } - defer mgr.Close() - - stats := mgr.GetDIDIndexStats() - - if !stats["exists"].(bool) { - fmt.Printf("DID index does not exist\n") - fmt.Printf("Run: plcbundle index build\n") - return nil - } - - fmt.Printf("Verifying DID index in: %s\n\n", dir) - - ctx := context.Background() - - if verbose { - fmt.Printf("Index version: %d\n", mgr.GetDIDIndex().GetConfig().Version) - fmt.Printf("Total DIDs: %s\n", formatNumber(int(stats["total_dids"].(int64)))) - fmt.Printf("Shards: %d\n", stats["shard_count"]) - fmt.Printf("Last bundle: %06d\n\n", stats["last_bundle"]) - } - - fmt.Printf("Checking consistency with bundles...\n") - - if err := mgr.GetDIDIndex().VerifyAndRepairIndex(ctx, mgr); err != nil { - fmt.Printf("\n✗ DID index verification failed\n") - fmt.Printf(" Error: %v\n", err) - return fmt.Errorf("verification failed: %w", err) - } - - stats = mgr.GetDIDIndexStats() - - fmt.Printf("\n✓ DID index is valid\n") - fmt.Printf(" Total DIDs: %s\n", formatNumber(int(stats["total_dids"].(int64)))) - fmt.Printf(" Shards: %d\n", stats["shard_count"]) - fmt.Printf(" Last bundle: %06d\n", stats["last_bundle"]) - - return nil - }, - } - - cmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "Verbose output") - - return cmd -} diff --git a/cmd/plcbundle/commands/inspect.go b/cmd/plcbundle/commands/inspect.go deleted file mode 100644 index 19a99a9..0000000 --- a/cmd/plcbundle/commands/inspect.go +++ /dev/null @@ -1,982 +0,0 @@ -package commands - -import ( - "context" - "fmt" - "os" - "path/filepath" - "sort" - "strings" - "time" - - "github.com/goccy/go-json" - "github.com/spf13/cobra" - "tangled.org/atscan.net/plcbundle/internal/storage" -) - -// ============================================================================ -// TYPES (defined at package level to avoid conflicts) -// ============================================================================ - -type DIDActivity struct { - DID string `json:"did"` - Count int `json:"count"` -} - -type DomainCount struct { - Domain string `json:"domain"` - Count int `json:"count"` -} - -type EndpointCount struct { - Endpoint string `json:"endpoint"` - Count int `json:"count"` -} - -type TimeSlot struct { - Time time.Time `json:"time"` - Count int `json:"count"` -} - -type inspectOptions struct { - showJSON bool - verify bool - showSamples bool - sampleCount int - skipMetadata bool - skipPatterns bool - skipCrypto bool - verbose bool -} - -type inspectResult struct { - // Metadata - Metadata *storage.BundleMetadata `json:"metadata,omitempty"` - - // Basic stats - FilePath string `json:"file_path"` - FileSize int64 `json:"file_size"` - HasMetadataFrame bool `json:"has_metadata_frame"` - HasFrameIndex bool `json:"has_frame_index"` - - // Operation analysis - TotalOps int `json:"total_ops"` - NullifiedOps int `json:"nullified_ops"` - ActiveOps int `json:"active_ops"` - UniqueDIDs int `json:"unique_dids"` - OperationTypes map[string]int `json:"operation_types"` - - // DID patterns - TopDIDs []DIDActivity `json:"top_dids"` - SingleOpDIDs int `json:"single_op_dids"` - MultiOpDIDs int `json:"multi_op_dids"` - - // Handle patterns - TotalHandles int `json:"total_handles"` - TopDomains []DomainCount `json:"top_domains"` - InvalidHandles int `json:"invalid_handles"` - - // Service patterns - TotalServices int `json:"total_services"` - UniqueEndpoints int `json:"unique_endpoints"` - TopPDSEndpoints []EndpointCount `json:"top_pds_endpoints"` - - // Temporal - TimeDistribution *TimeDistributionSummary `json:"time_distribution,omitempty"` - AvgOpsPerMinute float64 `json:"avg_ops_per_minute"` - - // Size analysis - AvgOpSize int `json:"avg_op_size"` - MinOpSize int `json:"min_op_size"` - MaxOpSize int `json:"max_op_size"` - TotalOpSize int64 `json:"total_op_size"` - - // Crypto verification - ContentHashValid bool `json:"content_hash_valid"` - CompressedHashValid bool `json:"compressed_hash_valid"` - MetadataValid bool `json:"metadata_valid"` - - // Timing - LoadTime time.Duration `json:"load_time"` - AnalyzeTime time.Duration `json:"analyze_time"` - VerifyTime time.Duration `json:"verify_time"` - TotalTime time.Duration `json:"total_time"` -} - -type bundleAnalysis struct { - TotalOps int `json:"total_ops"` - NullifiedOps int `json:"nullified_ops"` - ActiveOps int `json:"active_ops"` - UniqueDIDs int `json:"unique_dids"` - OperationTypes map[string]int `json:"operation_types"` - SingleOpDIDs int `json:"single_op_dids"` - MultiOpDIDs int `json:"multi_op_dids"` - TotalHandles int `json:"total_handles"` - InvalidHandles int `json:"invalid_handles"` - TotalServices int `json:"total_services"` - UniqueEndpoints int `json:"unique_endpoints"` - AvgOpsPerMinute float64 `json:"avg_ops_per_minute"` - AvgOpSize int `json:"avg_op_size"` - MinOpSize int `json:"min_op_size"` - MaxOpSize int `json:"max_op_size"` - TotalOpSize int64 `json:"total_op_size"` - - // For top-N calculations (unexported, won't appear in JSON) - didActivity map[string]int - domainCounts map[string]int - endpointCounts map[string]int - - // For time calculations - timeSlots map[int64]int - - // Results - TopDIDs []DIDActivity `json:"top_dids"` - TopDomains []DomainCount `json:"top_domains"` - TopPDSEndpoints []EndpointCount `json:"top_pds_endpoints"` - TimeDistribution *TimeDistributionSummary `json:"time_distribution,omitempty"` -} - -type TimeDistributionSummary struct { - EarliestOp time.Time `json:"earliest_op"` - LatestOp time.Time `json:"latest_op"` - TimeSpan string `json:"time_span"` - PeakHour time.Time `json:"peak_hour"` - PeakHourOps int `json:"peak_hour_ops"` - TotalHours int `json:"total_hours"` -} - -// ============================================================================ -// COMMAND DEFINITION -// ============================================================================ - -func NewInspectCommand() *cobra.Command { - var ( - showJSON bool - verify bool - showSamples bool - sampleCount int - skipMetadata bool - skipPatterns bool - skipCrypto bool - ) - - cmd := &cobra.Command{ - Use: "inspect ", - Short: "Deep analysis of bundle contents", - Long: `Deep analysis of bundle contents - -Performs comprehensive analysis of a bundle including: - • Embedded metadata (from skippable frame) - • Operation type breakdown - • DID activity patterns - • Handle and domain statistics - • Service endpoint analysis - • Temporal distribution - • Cryptographic verification - • Size analysis - -Can inspect either by bundle number (from repository) or direct file path.`, - - Example: ` # Inspect from repository - plcbundle inspect 42 - - # Inspect specific file - plcbundle inspect /path/to/000042.jsonl.zst - plcbundle inspect 000042.jsonl.zst - - # Skip certain analysis sections - plcbundle inspect 42 --skip-patterns --skip-crypto - - # Show sample operations - plcbundle inspect 42 --samples --sample-count 20 - - # Verify all hashes - plcbundle inspect 42 --verify - - # JSON output (for scripting) - plcbundle inspect 42 --json`, - - Args: cobra.ExactArgs(1), - - RunE: func(cmd *cobra.Command, args []string) error { - input := args[0] - verbose, _ := cmd.Root().PersistentFlags().GetBool("verbose") - - return runInspect(cmd, input, inspectOptions{ - showJSON: showJSON, - verify: verify, - showSamples: showSamples, - sampleCount: sampleCount, - skipMetadata: skipMetadata, - skipPatterns: skipPatterns, - skipCrypto: skipCrypto, - verbose: verbose, - }) - }, - } - - cmd.Flags().BoolVar(&showJSON, "json", false, "Output as JSON") - cmd.Flags().BoolVar(&verify, "verify", false, "Verify cryptographic hashes") - cmd.Flags().BoolVar(&showSamples, "samples", false, "Show sample operations") - cmd.Flags().IntVar(&sampleCount, "sample-count", 10, "Number of samples to show") - cmd.Flags().BoolVar(&skipMetadata, "skip-metadata", false, "Skip embedded metadata section") - cmd.Flags().BoolVar(&skipPatterns, "skip-patterns", false, "Skip pattern analysis") - cmd.Flags().BoolVar(&skipCrypto, "skip-crypto", false, "Skip cryptographic verification") - - return cmd -} - -// ============================================================================ -// MAIN LOGIC -// ============================================================================ - -func runInspect(cmd *cobra.Command, input string, opts inspectOptions) error { - totalStart := time.Now() - - // Determine if input is bundle number or file path - bundlePath, bundleNum, err := resolveBundlePath(cmd, input) - if err != nil { - return err - } - - result := &inspectResult{ - FilePath: bundlePath, - OperationTypes: make(map[string]int), - TopDIDs: make([]DIDActivity, 0), - TopDomains: make([]DomainCount, 0), - TopPDSEndpoints: make([]EndpointCount, 0), - } - - // Check file exists - info, err := os.Stat(bundlePath) - if err != nil { - return fmt.Errorf("bundle file not found: %w", err) - } - result.FileSize = info.Size() - - // Check for frame index - ops, _ := storage.NewOperations(nil, opts.verbose) - - if _, err := ops.ExtractBundleMetadata(bundlePath); err == nil { - result.HasFrameIndex = true // Has embedded index - } else { - // Check for external .idx file (legacy) - indexPath := bundlePath + ".idx" - if _, err := os.Stat(indexPath); err == nil { - result.HasFrameIndex = true - } - } - - fmt.Fprintf(os.Stderr, "Inspecting: %s\n", filepath.Base(bundlePath)) - fmt.Fprintf(os.Stderr, "File size: %s\n\n", formatBytes(result.FileSize)) - - // SECTION 1: Extract embedded metadata (fast!) - if !opts.skipMetadata { - fmt.Fprintf(os.Stderr, "Reading embedded metadata...\n") - metaStart := time.Now() - - ops, _ := storage.NewOperations(nil, opts.verbose) - - meta, err := ops.ExtractBundleMetadata(bundlePath) - if err != nil { - if opts.verbose { - fmt.Fprintf(os.Stderr, " No embedded metadata: %v\n", err) - } - result.HasMetadataFrame = false - } else { - result.HasMetadataFrame = true - result.Metadata = meta - if opts.verbose { - fmt.Fprintf(os.Stderr, " ✓ Extracted in %s\n", time.Since(metaStart)) - } - } - fmt.Fprintf(os.Stderr, "\n") - } - - // SECTION 2: Load and analyze operations - fmt.Fprintf(os.Stderr, "Loading and analyzing operations...\n") - loadStart := time.Now() - - analysis, err := analyzeBundle(bundlePath, opts) - if err != nil { - return fmt.Errorf("analysis failed: %w", err) - } - - result.LoadTime = time.Since(loadStart) - result.TotalOps = analysis.TotalOps - result.NullifiedOps = analysis.NullifiedOps - result.ActiveOps = analysis.ActiveOps - result.UniqueDIDs = analysis.UniqueDIDs - result.OperationTypes = analysis.OperationTypes - result.TopDIDs = analysis.TopDIDs - result.SingleOpDIDs = analysis.SingleOpDIDs - result.MultiOpDIDs = analysis.MultiOpDIDs - result.TotalHandles = analysis.TotalHandles - result.TopDomains = analysis.TopDomains - result.InvalidHandles = analysis.InvalidHandles - result.TotalServices = analysis.TotalServices - result.UniqueEndpoints = analysis.UniqueEndpoints - result.TopPDSEndpoints = analysis.TopPDSEndpoints - result.TimeDistribution = analysis.TimeDistribution - result.AvgOpsPerMinute = analysis.AvgOpsPerMinute - result.AvgOpSize = analysis.AvgOpSize - result.MinOpSize = analysis.MinOpSize - result.MaxOpSize = analysis.MaxOpSize - result.TotalOpSize = analysis.TotalOpSize - - fmt.Fprintf(os.Stderr, " ✓ Analyzed in %s\n\n", result.LoadTime) - - // SECTION 3: Cryptographic verification - if opts.verify && !opts.skipCrypto { - fmt.Fprintf(os.Stderr, "Verifying cryptographic hashes...\n") - verifyStart := time.Now() - - // Pass cmd parameter - result.ContentHashValid, result.CompressedHashValid, result.MetadataValid = - verifyCrypto(cmd, bundlePath, result.Metadata, bundleNum, opts.verbose) - - result.VerifyTime = time.Since(verifyStart) - fmt.Fprintf(os.Stderr, " ✓ Verified in %s\n\n", result.VerifyTime) - } - - result.TotalTime = time.Since(totalStart) - - // Display results - if opts.showJSON { - return displayInspectJSON(result) - } - - return displayInspectHuman(result, analysis, opts) -} - -// ============================================================================ -// ANALYSIS FUNCTIONS -// ============================================================================ - -func analyzeBundle(path string, opts inspectOptions) (*bundleAnalysis, error) { - ops, _ := storage.NewOperations(nil, opts.verbose) - operations, err := ops.LoadBundle(path) - if err != nil { - return nil, err - } - - analysis := &bundleAnalysis{ - TotalOps: len(operations), - OperationTypes: make(map[string]int), - didActivity: make(map[string]int), - domainCounts: make(map[string]int), - endpointCounts: make(map[string]int), - timeSlots: make(map[int64]int), - } - - // Analyze each operation - for _, op := range operations { - // Nullification - if op.IsNullified() { - analysis.NullifiedOps++ - } else { - analysis.ActiveOps++ - } - - // DID activity - analysis.didActivity[op.DID]++ - - // Size stats - opSize := len(op.RawJSON) - if opSize == 0 { - data, _ := json.Marshal(op) - opSize = len(data) - } - - analysis.TotalOpSize += int64(opSize) - if analysis.MinOpSize == 0 || opSize < analysis.MinOpSize { - analysis.MinOpSize = opSize - } - if opSize > analysis.MaxOpSize { - analysis.MaxOpSize = opSize - } - - // Parse operation for detailed analysis - opData, err := op.GetOperationData() - if err != nil || opData == nil { - continue - } - - // Operation type - if opType, ok := opData["type"].(string); ok { - analysis.OperationTypes[opType]++ - } - - // Handle analysis - if !opts.skipPatterns { - analyzeHandles(opData, analysis) - analyzeServices(opData, analysis) - } - - // Time distribution (group by minute) - timeSlot := op.CreatedAt.Unix() / 60 - analysis.timeSlots[timeSlot]++ - } - - // Calculate derived stats - analysis.UniqueDIDs = len(analysis.didActivity) - if analysis.TotalOps > 0 { - analysis.AvgOpSize = int(analysis.TotalOpSize / int64(analysis.TotalOps)) - } - - // Count single vs multi-op DIDs - for _, count := range analysis.didActivity { - if count == 1 { - analysis.SingleOpDIDs++ - } else { - analysis.MultiOpDIDs++ - } - } - - // Top DIDs - analysis.TopDIDs = getTopDIDs(analysis.didActivity, 10) - - // Top domains - analysis.TopDomains = getTopDomains(analysis.domainCounts, 10) - - // Top endpoints - analysis.TopPDSEndpoints = getTopEndpoints(analysis.endpointCounts, 10) - - // Unique endpoints - analysis.UniqueEndpoints = len(analysis.endpointCounts) - - // Time distribution - analysis.TimeDistribution = calculateTimeDistributionSummary(analysis.timeSlots) - - // Calculate ops per minute - if len(operations) > 1 { - duration := operations[len(operations)-1].CreatedAt.Sub(operations[0].CreatedAt) - if duration.Minutes() > 0 { - analysis.AvgOpsPerMinute = float64(len(operations)) / duration.Minutes() - } - } - - return analysis, nil -} - -func analyzeHandles(opData map[string]interface{}, analysis *bundleAnalysis) { - if aka, ok := opData["alsoKnownAs"].([]interface{}); ok { - for _, a := range aka { - if akaStr, ok := a.(string); ok { - if strings.HasPrefix(akaStr, "at://") { - analysis.TotalHandles++ - - // Extract domain - handle := strings.TrimPrefix(akaStr, "at://") - if idx := strings.Index(handle, "/"); idx > 0 { - handle = handle[:idx] - } - - // Count domain (TLD) - parts := strings.Split(handle, ".") - if len(parts) >= 2 { - domain := parts[len(parts)-1] - if len(parts) >= 2 { - domain = parts[len(parts)-2] + "." + domain - } - analysis.domainCounts[domain]++ - } - - // Check for invalid patterns - if strings.Contains(handle, "_") { - analysis.InvalidHandles++ - } - } - } - } - } -} - -func analyzeServices(opData map[string]interface{}, analysis *bundleAnalysis) { - if services, ok := opData["services"].(map[string]interface{}); ok { - analysis.TotalServices += len(services) - - // Extract PDS endpoints - if pds, ok := services["atproto_pds"].(map[string]interface{}); ok { - if endpoint, ok := pds["endpoint"].(string); ok { - // Normalize endpoint - endpoint = strings.TrimPrefix(endpoint, "https://") - endpoint = strings.TrimPrefix(endpoint, "http://") - if idx := strings.Index(endpoint, "/"); idx > 0 { - endpoint = endpoint[:idx] - } - analysis.endpointCounts[endpoint]++ - } - } - } -} - -func getTopDIDs(didActivity map[string]int, limit int) []DIDActivity { - var results []DIDActivity - for did, count := range didActivity { - results = append(results, DIDActivity{DID: did, Count: count}) - } - - sort.Slice(results, func(i, j int) bool { - return results[i].Count > results[j].Count - }) - - if len(results) > limit { - results = results[:limit] - } - - return results -} - -func getTopDomains(domainCounts map[string]int, limit int) []DomainCount { - var results []DomainCount - for domain, count := range domainCounts { - results = append(results, DomainCount{Domain: domain, Count: count}) - } - - sort.Slice(results, func(i, j int) bool { - return results[i].Count > results[j].Count - }) - - if len(results) > limit { - results = results[:limit] - } - - return results -} - -func getTopEndpoints(endpointCounts map[string]int, limit int) []EndpointCount { - var results []EndpointCount - for endpoint, count := range endpointCounts { - results = append(results, EndpointCount{Endpoint: endpoint, Count: count}) - } - - sort.Slice(results, func(i, j int) bool { - return results[i].Count > results[j].Count - }) - - if len(results) > limit { - results = results[:limit] - } - - return results -} - -func calculateTimeDistributionSummary(timeSlots map[int64]int) *TimeDistributionSummary { - if len(timeSlots) == 0 { - return nil - } - - var earliest, latest int64 - var peakHour int64 - var peakCount int - - // Group by hour and find stats - hourlySlots := make(map[int64]int) - - for ts, count := range timeSlots { - // Track earliest/latest - if earliest == 0 || ts < earliest { - earliest = ts - } - if ts > latest { - latest = ts - } - - // Group by hour - hour := (ts / 3600) * 3600 // Truncate to hour - hourlySlots[hour] += count - } - - // Find peak hour - for hour, count := range hourlySlots { - if count > peakCount { - peakCount = count - peakHour = hour - } - } - - // Calculate time span - duration := time.Unix(latest, 0).Sub(time.Unix(earliest, 0)) - timeSpan := formatDuration(duration) - - // Calculate total hours covered - totalHours := len(hourlySlots) - - return &TimeDistributionSummary{ - EarliestOp: time.Unix(earliest, 0).UTC(), - LatestOp: time.Unix(latest, 0).UTC(), - TimeSpan: timeSpan, - PeakHour: time.Unix(peakHour, 0).UTC(), - PeakHourOps: peakCount, - TotalHours: totalHours, - } -} - -// ============================================================================ -// DISPLAY FUNCTIONS -// ============================================================================ - -func displayInspectHuman(result *inspectResult, _ *bundleAnalysis, opts inspectOptions) error { - fmt.Printf("\n") - fmt.Printf("═══════════════════════════════════════════════════════════════\n") - fmt.Printf(" Bundle Deep Inspection\n") - fmt.Printf("═══════════════════════════════════════════════════════════════\n\n") - - // File info - fmt.Printf("📁 File Information\n") - fmt.Printf("───────────────────\n") - fmt.Printf(" Path: %s\n", filepath.Base(result.FilePath)) - fmt.Printf(" Size: %s\n", formatBytes(result.FileSize)) - fmt.Printf(" Has metadata frame: %v\n", result.HasMetadataFrame) - fmt.Printf(" Has frame index: %v\n\n", result.HasFrameIndex) - - // Embedded metadata - if result.HasMetadataFrame && result.Metadata != nil && !opts.skipMetadata { - meta := result.Metadata - fmt.Printf("📋 Embedded Metadata (Skippable Frame)\n") - fmt.Printf("──────────────────────────────────────\n") - fmt.Printf(" Format: %s\n", meta.Format) - fmt.Printf(" Origin: %s\n", meta.Origin) - fmt.Printf(" Bundle Number: %06d\n", meta.BundleNumber) - if meta.CreatedBy != "" { - fmt.Printf(" Created by: %s\n", meta.CreatedBy) - } - if meta.CreatedByHost != "" { - fmt.Printf(" Created on: %s\n", meta.CreatedByHost) - } - fmt.Printf(" Created at: %s\n", meta.CreatedAt.Format("2006-01-02 15:04:05 MST")) - - fmt.Printf("\n Content:\n") - fmt.Printf(" Operations: %s\n", formatNumber(meta.OperationCount)) - fmt.Printf(" Unique DIDs: %s\n", formatNumber(meta.DIDCount)) - fmt.Printf(" Frames: %d × %d ops\n", meta.FrameCount, meta.FrameSize) - fmt.Printf(" Timespan: %s → %s\n", - meta.StartTime.Format("2006-01-02 15:04:05"), - meta.EndTime.Format("2006-01-02 15:04:05")) - fmt.Printf(" Duration: %s\n", - formatDuration(meta.EndTime.Sub(meta.StartTime))) - - fmt.Printf("\n Integrity:\n") - fmt.Printf(" Content hash: %s\n", meta.ContentHash) - if meta.ParentHash != "" { - fmt.Printf(" Parent hash: %s\n", meta.ParentHash) - } - - if len(meta.FrameOffsets) > 0 { - // Calculate metadata size (size of the metadata frame itself) - metadataSize := int64(0) - if result.HasMetadataFrame { - // Metadata is at the end of file, after all data frames - // Size = file size - last frame offset - if len(meta.FrameOffsets) > 0 { - lastFrameOffset := meta.FrameOffsets[len(meta.FrameOffsets)-1] - metadataSize = result.FileSize - lastFrameOffset - } - } - - // Print with fixes - fmt.Printf(" Ops Frame Index: %d offsets (embedded)\n", len(meta.FrameOffsets)) - fmt.Printf(" Metadata size: %s\n", formatBytes(metadataSize)) - fmt.Printf(" Frame offsets: %v\n", formatOffsetArray(meta.FrameOffsets, 5)) // Show first 5 - } - fmt.Printf("\n") - } - - // Operations breakdown - fmt.Printf("📊 Operations Analysis\n") - fmt.Printf("──────────────────────\n") - fmt.Printf(" Total operations: %s\n", formatNumber(result.TotalOps)) - fmt.Printf(" Active: %s (%.1f%%)\n", - formatNumber(result.ActiveOps), - float64(result.ActiveOps)/float64(result.TotalOps)*100) - if result.NullifiedOps > 0 { - fmt.Printf(" Nullified: %s (%.1f%%)\n", - formatNumber(result.NullifiedOps), - float64(result.NullifiedOps)/float64(result.TotalOps)*100) - } - - if len(result.OperationTypes) > 0 { - fmt.Printf("\n Operation Types:\n") - - // Sort by count - var types []struct { - name string - count int - } - for name, count := range result.OperationTypes { - types = append(types, struct { - name string - count int - }{name, count}) - } - sort.Slice(types, func(i, j int) bool { - return types[i].count > types[j].count - }) - - for _, t := range types { - pct := float64(t.count) / float64(result.TotalOps) * 100 - fmt.Printf(" %-25s %s (%.1f%%)\n", t.name, formatNumber(t.count), pct) - } - } - fmt.Printf("\n") - - // DID patterns - fmt.Printf("👤 DID Activity Patterns\n") - fmt.Printf("────────────────────────\n") - fmt.Printf(" Unique DIDs: %s\n", formatNumber(result.UniqueDIDs)) - fmt.Printf(" Single-op DIDs: %s (%.1f%%)\n", - formatNumber(result.SingleOpDIDs), - float64(result.SingleOpDIDs)/float64(result.UniqueDIDs)*100) - fmt.Printf(" Multi-op DIDs: %s (%.1f%%)\n", - formatNumber(result.MultiOpDIDs), - float64(result.MultiOpDIDs)/float64(result.UniqueDIDs)*100) - - if len(result.TopDIDs) > 0 { - fmt.Printf("\n Most Active DIDs:\n") - for i, da := range result.TopDIDs { - if i >= 5 { - break - } - fmt.Printf(" %d. %s (%d ops)\n", i+1, da.DID, da.Count) - } - } - fmt.Printf("\n") - - // Handle patterns - if !opts.skipPatterns && result.TotalHandles > 0 { - fmt.Printf("🏷️ Handle Statistics\n") - fmt.Printf("────────────────────\n") - fmt.Printf(" Total handles: %s\n", formatNumber(result.TotalHandles)) - if result.InvalidHandles > 0 { - fmt.Printf(" Invalid patterns: %s (%.1f%%)\n", - formatNumber(result.InvalidHandles), - float64(result.InvalidHandles)/float64(result.TotalHandles)*100) - } - - if len(result.TopDomains) > 0 { - fmt.Printf("\n Top Domains:\n") - for i, dc := range result.TopDomains { - if i >= 10 { - break - } - pct := float64(dc.Count) / float64(result.TotalHandles) * 100 - fmt.Printf(" %-25s %s (%.1f%%)\n", dc.Domain, formatNumber(dc.Count), pct) - } - } - fmt.Printf("\n") - } - - // Service patterns - if !opts.skipPatterns && result.TotalServices > 0 { - fmt.Printf("🌐 Service Endpoints\n") - fmt.Printf("────────────────────\n") - fmt.Printf(" Total services: %s\n", formatNumber(result.TotalServices)) - fmt.Printf(" Unique endpoints: %s\n", formatNumber(result.UniqueEndpoints)) - - if len(result.TopPDSEndpoints) > 0 { - fmt.Printf("\n Top PDS Endpoints:\n") - for i, ec := range result.TopPDSEndpoints { - if i >= 10 { - break - } - fmt.Printf(" %-40s %s ops\n", ec.Endpoint, formatNumber(ec.Count)) - } - } - fmt.Printf("\n") - } - - // Temporal analysis - fmt.Printf("⏱️ Time Distribution\n") - fmt.Printf("───────────────────────\n") - if result.TimeDistribution != nil { - td := result.TimeDistribution - fmt.Printf(" Earliest operation: %s\n", td.EarliestOp.Format(time.RFC3339)) - fmt.Printf(" Latest operation: %s\n", td.LatestOp.Format(time.RFC3339)) - fmt.Printf(" Time span: %s\n", td.TimeSpan) - fmt.Printf(" Peak hour: %s (%d ops)\n", - td.PeakHour.Format("2006-01-02 15:04"), td.PeakHourOps) - fmt.Printf(" Total active hours: %d\n", td.TotalHours) - fmt.Printf(" Avg ops/minute: %.1f\n", result.AvgOpsPerMinute) - } - fmt.Printf("\n") - - // Size analysis - fmt.Printf("📏 Size Analysis\n") - fmt.Printf("────────────────\n") - fmt.Printf(" Total data: %s\n", formatBytes(result.TotalOpSize)) - fmt.Printf(" Average per op: %s\n", formatBytes(int64(result.AvgOpSize))) - fmt.Printf(" Min operation: %s\n", formatBytes(int64(result.MinOpSize))) - fmt.Printf(" Max operation: %s\n\n", formatBytes(int64(result.MaxOpSize))) - - // Cryptographic verification - if opts.verify && !opts.skipCrypto { - fmt.Printf("🔐 Cryptographic Verification\n") - fmt.Printf("─────────────────────────────\n") - - status := func(valid bool) string { - if valid { - return "✓ Valid" - } - return "✗ Invalid" - } - - fmt.Printf(" Content hash: %s\n", status(result.ContentHashValid)) - fmt.Printf(" Compressed hash: %s\n", status(result.CompressedHashValid)) - if result.HasMetadataFrame { - fmt.Printf(" Metadata integrity: %s\n", status(result.MetadataValid)) - } - fmt.Printf("\n") - } - - // Performance summary - fmt.Printf("⚡ Performance\n") - fmt.Printf("──────────────\n") - fmt.Printf(" Load time: %s\n", result.LoadTime) - if opts.verify { - fmt.Printf(" Verify time: %s\n", result.VerifyTime) - } - fmt.Printf(" Total time: %s\n", result.TotalTime) - if result.LoadTime.Seconds() > 0 { - opsPerSec := float64(result.TotalOps) / result.LoadTime.Seconds() - mbPerSec := float64(result.TotalOpSize) / result.LoadTime.Seconds() / (1024 * 1024) - fmt.Printf(" Throughput: %.0f ops/sec, %.2f MB/s\n", opsPerSec, mbPerSec) - } - fmt.Printf("\n") - - return nil -} - -func displayInspectJSON(result *inspectResult) error { - data, _ := json.MarshalIndent(result, "", " ") - fmt.Println(string(data)) - return nil -} - -func verifyCrypto(cmd *cobra.Command, path string, meta *storage.BundleMetadata, bundleNum int, verbose bool) (contentValid, compressedValid, metadataValid bool) { - ops, _ := storage.NewOperations(nil, verbose) - - // Calculate actual hashes from file - compHash, compSize, contentHash, contentSize, err := ops.CalculateFileHashes(path) - if err != nil { - if verbose { - fmt.Fprintf(os.Stderr, " Hash calculation failed: %v\n", err) - } - return false, false, false - } - - contentValid = true - compressedValid = true - metadataValid = true - - // Verify against embedded metadata if available - if meta != nil { - // Check content hash (this is in the metadata) - if meta.ContentHash != "" && meta.ContentHash != contentHash { - contentValid = false - if verbose { - fmt.Fprintf(os.Stderr, " ✗ Content hash mismatch!\n") - fmt.Fprintf(os.Stderr, " Expected: %s\n", meta.ContentHash) - fmt.Fprintf(os.Stderr, " Actual: %s\n", contentHash) - } - } - - if meta.OperationCount > 0 { - // We can't verify this without loading, so skip - metadataValid = true - } - - // Note: We don't check compressed hash/size because they're not in metadata - // (The file IS the compressed data, so it's redundant) - - if verbose { - fmt.Fprintf(os.Stderr, " Embedded metadata:\n") - fmt.Fprintf(os.Stderr, " Content hash: %s\n", meta.ContentHash[:16]+"...") - fmt.Fprintf(os.Stderr, " Operations: %d\n", meta.OperationCount) - fmt.Fprintf(os.Stderr, " DIDs: %d\n", meta.DIDCount) - } - } - - // Also verify against repository index if bundle number is known - if bundleNum > 0 { - mgr, _, err := getManager(&ManagerOptions{Cmd: cmd}) - if err == nil { - defer mgr.Close() - - ctx := context.Background() - vr, err := mgr.VerifyBundle(ctx, bundleNum) - if err == nil && vr != nil { - // Index verification - indexContentValid := vr.Valid - indexHashMatch := vr.HashMatch - - if verbose { - fmt.Fprintf(os.Stderr, " Repository index:\n") - fmt.Fprintf(os.Stderr, " Content valid: %v\n", indexContentValid) - fmt.Fprintf(os.Stderr, " Hash match: %v\n", indexHashMatch) - } - - contentValid = contentValid && indexContentValid - compressedValid = compressedValid && indexHashMatch - } - } - } - - if verbose { - fmt.Fprintf(os.Stderr, " Calculated hashes:\n") - fmt.Fprintf(os.Stderr, " Content: %s (%s)\n", contentHash[:16]+"...", formatBytes(contentSize)) - fmt.Fprintf(os.Stderr, " Compressed: %s (%s)\n", compHash[:16]+"...", formatBytes(compSize)) - } - - return contentValid, compressedValid, metadataValid -} - -func resolveBundlePath(cmd *cobra.Command, input string) (path string, bundleNum int, err error) { - // Check if it's a file path - if strings.HasSuffix(input, ".zst") || strings.Contains(input, "/") || strings.Contains(input, "\\") { - absPath, err := filepath.Abs(input) - if err != nil { - return "", 0, err - } - - // Try to extract bundle number from filename - base := filepath.Base(absPath) - fmt.Sscanf(base, "%d", &bundleNum) - - return absPath, bundleNum, nil - } - - // Try to parse as bundle number - if _, err := fmt.Sscanf(input, "%d", &bundleNum); err == nil { - // Load from repository - mgr, dir, err := getManager(&ManagerOptions{Cmd: cmd}) - if err != nil { - return "", 0, err - } - defer mgr.Close() - - path := filepath.Join(dir, fmt.Sprintf("%06d.jsonl.zst", bundleNum)) - if _, err := os.Stat(path); err != nil { - return "", 0, fmt.Errorf("bundle %d not found in repository", bundleNum) - } - - return path, bundleNum, nil - } - - return "", 0, fmt.Errorf("invalid input: must be bundle number or file path") -} - -func formatOffsetArray(offsets []int64, maxShow int) string { - if len(offsets) == 0 { - return "[]" - } - - if len(offsets) <= maxShow { - return fmt.Sprintf("%v", offsets) - } - - // Show first maxShow elements - shown := make([]int64, maxShow) - copy(shown, offsets[:maxShow]) - return fmt.Sprintf("%v ... (%d more)", shown, len(offsets)-maxShow) -} diff --git a/cmd/plcbundle/commands/log.go b/cmd/plcbundle/commands/log.go deleted file mode 100644 index 9ff868a..0000000 --- a/cmd/plcbundle/commands/log.go +++ /dev/null @@ -1,427 +0,0 @@ -package commands - -import ( - "fmt" - "io" - "os" - "os/exec" - "strings" - "time" - - "github.com/spf13/cobra" - "golang.org/x/term" - "tangled.org/atscan.net/plcbundle/internal/bundleindex" -) - -func NewLogCommand() *cobra.Command { - var ( - last int - oneline bool - noHashes bool - reverse bool - noPager bool - ) - - cmd := &cobra.Command{ - Use: "log [flags]", - Aliases: []string{"history"}, - Short: "Show bundle history", - Long: `Show bundle history in chronological order - -Displays a log of all bundles in the repository, similar to 'git log'. -By default shows all bundles in reverse chronological order (newest first) -with chain hashes and automatically pipes through a pager. - -The log shows: - • Bundle number and timestamp - • Operation and DID counts - • Compressed and uncompressed sizes - • Chain and content hashes (default, use --no-hashes to hide) - • Coverage timespan - -Output is automatically piped through 'less' (or $PAGER) when stdout -is a terminal, just like 'git log'. Use --no-pager to disable.`, - - Example: ` # Show all bundles (newest first, auto-paged) - plcbundle log - - # Show last 10 bundles - plcbundle log --last 10 - plcbundle log -n 10 - - # One-line format - plcbundle log --oneline - - # Hide hashes - plcbundle log --no-hashes - - # Oldest first (ascending order) - plcbundle log --reverse - - # Disable pager (direct output) - plcbundle log --no-pager - - # Combination - plcbundle log -n 20 --oneline - - # Using alias - plcbundle history -n 5`, - - RunE: func(cmd *cobra.Command, args []string) error { - mgr, dir, err := getManager(&ManagerOptions{Cmd: cmd}) - if err != nil { - return err - } - defer mgr.Close() - - return showBundleLog(mgr, dir, logOptions{ - last: last, - oneline: oneline, - showHashes: !noHashes, // Invert: hashes ON by default - reverse: reverse, - noPager: noPager, - }) - }, - } - - // Flags - cmd.Flags().IntVarP(&last, "last", "n", 0, "Show only last N bundles (0 = all)") - cmd.Flags().BoolVar(&oneline, "oneline", false, "Show one line per bundle (compact)") - cmd.Flags().BoolVar(&noHashes, "no-hashes", false, "Hide chain and content hashes") - cmd.Flags().BoolVar(&reverse, "reverse", false, "Show oldest first (ascending order)") - cmd.Flags().BoolVar(&noPager, "no-pager", false, "Disable pager (output directly)") - - return cmd -} - -type logOptions struct { - last int - oneline bool - showHashes bool - reverse bool - noPager bool -} - -func showBundleLog(mgr BundleManager, dir string, opts logOptions) error { - index := mgr.GetIndex() - bundles := index.GetBundles() - - if len(bundles) == 0 { - fmt.Printf("No bundles in repository\n") - fmt.Printf("Directory: %s\n\n", dir) - fmt.Printf("Get started:\n") - fmt.Printf(" plcbundle clone Clone from remote\n") - fmt.Printf(" plcbundle sync Fetch from PLC directory\n") - return nil - } - - // Apply limit - displayBundles := bundles - if opts.last > 0 && opts.last < len(bundles) { - displayBundles = bundles[len(bundles)-opts.last:] - } - - // Reverse if not --reverse (default is newest first) - if !opts.reverse { - reversed := make([]*bundleindex.BundleMetadata, len(displayBundles)) - for i, b := range displayBundles { - reversed[len(displayBundles)-1-i] = b - } - displayBundles = reversed - } - - // Determine output destination (pager or stdout) - var output io.WriteCloser - var useColor bool - - if !opts.noPager && isTTY(os.Stdout) { - // Try to use pager - pager, err := startPager() - if err == nil { - output = pager - useColor = true - defer pager.Close() - } else { - output = os.Stdout - useColor = isTTY(os.Stdout) - } - } else { - output = os.Stdout - useColor = !opts.noPager && isTTY(os.Stdout) - } - - // Render to output - renderBundleLog(output, dir, index, bundles, displayBundles, opts, useColor) - - return nil -} - -func renderBundleLog(w io.Writer, dir string, index *bundleindex.Index, allBundles, displayBundles []*bundleindex.BundleMetadata, opts logOptions, useColor bool) { - // Color codes - var ( - colorReset = "" - colorBundleNum = "" - colorHash = "" - colorDate = "" - colorAge = "" - colorSize = "" - colorHeader = "" - colorDim = "" - ) - - if useColor { - colorReset = "\033[0m" - colorBundleNum = "\033[1;36m" // Bright cyan + bold - colorHash = "\033[33m" // Yellow - colorDate = "\033[32m" // Green - colorAge = "\033[90m" // Dim - colorSize = "\033[35m" // Magenta - colorHeader = "\033[1;37m" // Bright white + bold - colorDim = "\033[2m" // Dim - } - - // Header - if !opts.oneline { - fmt.Fprintf(w, "%sBundle History%s\n", colorHeader, colorReset) - fmt.Fprintf(w, "Directory: %s\n", dir) - if origin := index.Origin; origin != "" { - fmt.Fprintf(w, "Origin: %s\n", origin) - } - fmt.Fprintf(w, "Total: %d bundles", len(allBundles)) - if opts.last > 0 && opts.last < len(allBundles) { - fmt.Fprintf(w, " (showing last %d)", opts.last) - } - fmt.Fprintf(w, "\n\n") - } - - // Display bundles - for i, meta := range displayBundles { - if opts.oneline { - displayBundleOneLine(w, meta, opts.showHashes, colorBundleNum, colorHash, colorDate, colorAge, colorSize, colorReset) - } else { - displayBundleDetailed(w, meta, opts.showHashes, colorBundleNum, colorHash, colorDate, colorAge, colorSize, colorDim, colorReset) - - // Add separator between bundles (except last) - if i < len(displayBundles)-1 { - fmt.Fprintf(w, "%s────────────────────────────────────────────────────────%s\n\n", - colorDim, colorReset) - } - } - } - - // Summary footer - if !opts.oneline && len(displayBundles) > 0 { - fmt.Fprintf(w, "\n") - displayLogSummary(w, allBundles, displayBundles, opts.last, colorHeader, colorReset) - } -} - -func displayBundleOneLine(w io.Writer, meta *bundleindex.BundleMetadata, showHashes bool, colorBundle, colorHash, colorDate, colorAge, colorSize, colorReset string) { - age := time.Since(meta.EndTime) - ageStr := formatDurationShort(age) - - // Build hash preview - hashPreview := "" - if showHashes && len(meta.Hash) >= 12 { - hashPreview = fmt.Sprintf(" %s%s%s", colorHash, meta.Hash[:12], colorReset) - } - - fmt.Fprintf(w, "%s%06d%s%s %s%s%s %s%s ago%s %d ops, %d DIDs, %s%s%s\n", - colorBundle, meta.BundleNumber, colorReset, - hashPreview, - colorDate, meta.EndTime.Format("2006-01-02 15:04"), colorReset, - colorAge, ageStr, colorReset, - meta.OperationCount, - meta.DIDCount, - colorSize, formatBytes(meta.CompressedSize), colorReset) -} - -func displayBundleDetailed(w io.Writer, meta *bundleindex.BundleMetadata, showHashes bool, colorBundle, colorHash, colorDate, colorAge, colorSize, colorDim, colorReset string) { - fmt.Fprintf(w, "%sBundle %06d%s\n", colorBundle, meta.BundleNumber, colorReset) - - // Timestamp and age - age := time.Since(meta.EndTime) - fmt.Fprintf(w, " Date: %s%s%s %s(%s ago)%s\n", - colorDate, meta.EndTime.Format("2006-01-02 15:04:05"), colorReset, - colorAge, formatDuration(age), colorReset) - - // Timespan of bundle - duration := meta.EndTime.Sub(meta.StartTime) - fmt.Fprintf(w, " Timespan: %s → %s (%s)\n", - meta.StartTime.Format("15:04:05"), - meta.EndTime.Format("15:04:05"), - formatDuration(duration)) - - // Operations and DIDs - fmt.Fprintf(w, " Operations: %s\n", formatNumber(meta.OperationCount)) - fmt.Fprintf(w, " DIDs: %s unique\n", formatNumber(meta.DIDCount)) - - // Sizes - ratio := float64(meta.UncompressedSize) / float64(meta.CompressedSize) - fmt.Fprintf(w, " Size: %s%s%s → %s (%.2fx compression)\n", - colorSize, - formatBytes(meta.UncompressedSize), - colorReset, - formatBytes(meta.CompressedSize), - ratio) - - // Hashes (shown by default now) - if showHashes { - fmt.Fprintf(w, "\n Hashes:\n") - fmt.Fprintf(w, " %sChain:%s %s%s%s\n", - colorDim, colorReset, - colorHash, meta.Hash, colorReset) - fmt.Fprintf(w, " %sContent:%s %s%s%s\n", - colorDim, colorReset, - colorHash, meta.ContentHash, colorReset) - if meta.Parent != "" { - fmt.Fprintf(w, " %sParent:%s %s%s%s\n", - colorDim, colorReset, - colorHash, meta.Parent, colorReset) - } - } -} - -func displayLogSummary(w io.Writer, allBundles, displayedBundles []*bundleindex.BundleMetadata, limit int, colorHeader, colorReset string) { - first := displayedBundles[0] - last := displayedBundles[len(displayedBundles)-1] - - // Determine order for summary - var earliest, latest *bundleindex.BundleMetadata - if first.BundleNumber < last.BundleNumber { - earliest = first - latest = last - } else { - earliest = last - latest = first - } - - fmt.Fprintf(w, "%sSummary%s\n", colorHeader, colorReset) - fmt.Fprintf(w, "───────\n") - - if limit > 0 && limit < len(allBundles) { - fmt.Fprintf(w, " Showing: %d of %d bundles\n", len(displayedBundles), len(allBundles)) - } else { - fmt.Fprintf(w, " Bundles: %d total\n", len(allBundles)) - } - - fmt.Fprintf(w, " Range: %06d → %06d\n", earliest.BundleNumber, latest.BundleNumber) - - // Calculate total stats for displayed bundles - totalOps := 0 - totalSize := int64(0) - totalUncompressed := int64(0) - for _, meta := range displayedBundles { - totalOps += meta.OperationCount - totalSize += meta.CompressedSize - totalUncompressed += meta.UncompressedSize - } - - fmt.Fprintf(w, " Operations: %s\n", formatNumber(totalOps)) - fmt.Fprintf(w, " Size: %s (compressed)\n", formatBytes(totalSize)) - - timespan := latest.EndTime.Sub(earliest.StartTime) - fmt.Fprintf(w, " Timespan: %s\n", formatDuration(timespan)) -} - -// isTTY checks if the given file is a terminal -func isTTY(f *os.File) bool { - return term.IsTerminal(int(f.Fd())) -} - -// startPager starts a pager process and returns a WriteCloser -func startPager() (io.WriteCloser, error) { - // Check for PAGER environment variable - pagerCmd := os.Getenv("PAGER") - if pagerCmd == "" { - // Default to less with options - pagerCmd = "less" - } - - // Parse pager command (handle args) - parts := strings.Fields(pagerCmd) - if len(parts) == 0 { - return nil, fmt.Errorf("empty pager command") - } - - cmdName := parts[0] - cmdArgs := parts[1:] - - // Special handling for less - add color support flags - if cmdName == "less" || strings.HasSuffix(cmdName, "/less") { - // Add flags if not already present - hasR := false - for _, arg := range cmdArgs { - if strings.Contains(arg, "R") { - hasR = true - break - } - } - if !hasR { - // -R: enable raw control characters (for colors) - // -F: quit if output fits on one screen - // -X: don't clear screen on exit - cmdArgs = append([]string{"-R", "-F", "-X"}, cmdArgs...) - } - } - - cmd := exec.Command(cmdName, cmdArgs...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - stdin, err := cmd.StdinPipe() - if err != nil { - return nil, err - } - - if err := cmd.Start(); err != nil { - return nil, err - } - - // Return a WriteCloser that closes stdin and waits for process - return &pagerWriter{ - writer: stdin, - cmd: cmd, - started: true, - }, nil -} - -// pagerWriter wraps a pager process -type pagerWriter struct { - writer io.WriteCloser - cmd *exec.Cmd - started bool -} - -func (pw *pagerWriter) Write(p []byte) (n int, err error) { - return pw.writer.Write(p) -} - -func (pw *pagerWriter) Close() error { - pw.writer.Close() - if pw.started { - pw.cmd.Wait() - } - return nil -} - -// formatDurationShort formats duration in short form -func formatDurationShort(d time.Duration) string { - if d < time.Minute { - return fmt.Sprintf("%.0fs", d.Seconds()) - } - if d < time.Hour { - return fmt.Sprintf("%.0fm", d.Minutes()) - } - if d < 24*time.Hour { - return fmt.Sprintf("%.0fh", d.Hours()) - } - days := d.Hours() / 24 - if days < 30 { - return fmt.Sprintf("%.0fd", days) - } - if days < 365 { - return fmt.Sprintf("%.0fmo", days/30) - } - return fmt.Sprintf("%.1fy", days/365) -} diff --git a/cmd/plcbundle/commands/ls.go b/cmd/plcbundle/commands/ls.go deleted file mode 100644 index fc2ca79..0000000 --- a/cmd/plcbundle/commands/ls.go +++ /dev/null @@ -1,292 +0,0 @@ -package commands - -import ( - "fmt" - "strings" - "time" - - "github.com/spf13/cobra" - "tangled.org/atscan.net/plcbundle/internal/bundleindex" -) - -func NewLsCommand() *cobra.Command { - var ( - last int - reverse bool - format string - noHeader bool - separator string - ) - - cmd := &cobra.Command{ - Use: "ls [flags]", - Short: "List bundles (machine-readable)", - Long: `List bundles in machine-readable format - -Outputs bundle information in a clean, parseable format suitable for -piping to sed, awk, grep, or other text processing tools. - -No colors, no pager - just consistent, tab-separated data perfect for -shell scripts and automation.`, - - Example: ` # List all bundles - plcbundle ls - - # Last 10 bundles - plcbundle ls -n 10 - - # Oldest first - plcbundle ls --reverse - - # Custom format - plcbundle ls --format "bundle,hash,date,size" - - # CSV format - plcbundle ls --separator "," - - # Scripting examples - plcbundle ls | awk '{print $1}' # Just bundle numbers - plcbundle ls | grep 000150 # Find specific bundle - plcbundle ls -n 5 | cut -f1,4 # First and 4th columns - plcbundle ls --format bundle,hash # Custom columns - plcbundle ls --separator "," > bundles.csv # Export to CSV`, - - RunE: func(cmd *cobra.Command, args []string) error { - mgr, _, err := getManager(&ManagerOptions{Cmd: cmd}) - if err != nil { - return err - } - defer mgr.Close() - - return listBundles(mgr, lsOptions{ - last: last, - reverse: reverse, - format: format, - noHeader: noHeader, - separator: separator, - }) - }, - } - - // Flags - cmd.Flags().IntVarP(&last, "last", "n", 0, "Show only last N bundles (0 = all)") - cmd.Flags().BoolVar(&reverse, "reverse", false, "Show oldest first") - cmd.Flags().StringVar(&format, "format", "bundle,hash,date,ops,dids,size", - "Output format: bundle,hash,date,ops,dids,size,uncompressed,ratio,timespan") - cmd.Flags().BoolVar(&noHeader, "no-header", false, "Omit header row") - cmd.Flags().StringVar(&separator, "separator", "\t", "Field separator (default: tab)") - - return cmd -} - -type lsOptions struct { - last int - reverse bool - format string - noHeader bool - separator string -} - -func listBundles(mgr BundleManager, opts lsOptions) error { - index := mgr.GetIndex() - bundles := index.GetBundles() - - if len(bundles) == 0 { - return nil - } - - // Apply limit - displayBundles := bundles - if opts.last > 0 && opts.last < len(bundles) { - displayBundles = bundles[len(bundles)-opts.last:] - } - - // Reverse if not --reverse (default is newest first, like log) - if !opts.reverse { - reversed := make([]*bundleindex.BundleMetadata, len(displayBundles)) - for i, b := range displayBundles { - reversed[len(displayBundles)-1-i] = b - } - displayBundles = reversed - } - - // Parse format string - fields := parseFormatString(opts.format) - - // Print header (unless disabled) - if !opts.noHeader { - printHeader(fields, opts.separator) - } - - // Print each bundle - for _, meta := range displayBundles { - printBundleFields(meta, fields, opts.separator) - } - - return nil -} - -// parseFormatString parses comma-separated field names -func parseFormatString(format string) []string { - parts := strings.Split(format, ",") - fields := make([]string, 0, len(parts)) - for _, p := range parts { - field := strings.TrimSpace(p) - if field != "" { - fields = append(fields, field) - } - } - return fields -} - -// printHeader prints the header row -func printHeader(fields []string, sep string) { - headers := make([]string, len(fields)) - for i, field := range fields { - headers[i] = getFieldHeader(field) - } - fmt.Println(strings.Join(headers, sep)) -} - -// getFieldHeader returns the header name for a field -func getFieldHeader(field string) string { - switch field { - case "bundle": - return "bundle" - case "hash": - return "hash" - case "content": - return "content_hash" - case "parent": - return "parent_hash" - case "date", "time": - return "date" - case "age": - return "age" - case "ops", "operations": - return "ops" - case "dids": - return "dids" - case "size", "compressed": - return "size" - case "uncompressed": - return "uncompressed" - case "ratio": - return "ratio" - case "timespan", "duration": - return "timespan" - case "start": - return "start_time" - case "end": - return "end_time" - default: - return field - } -} - -// printBundleFields prints a bundle's fields -func printBundleFields(meta *bundleindex.BundleMetadata, fields []string, sep string) { - values := make([]string, len(fields)) - - for i, field := range fields { - values[i] = getFieldValue(meta, field) - } - - fmt.Println(strings.Join(values, sep)) -} - -// getFieldValue returns the value for a specific field -func getFieldValue(meta *bundleindex.BundleMetadata, field string) string { - switch field { - case "bundle": - return fmt.Sprintf("%06d", meta.BundleNumber) - - case "hash": - return meta.Hash - - case "hash_short": - if len(meta.Hash) >= 12 { - return meta.Hash[:12] - } - return meta.Hash - - case "content": - return meta.ContentHash - - case "content_short": - if len(meta.ContentHash) >= 12 { - return meta.ContentHash[:12] - } - return meta.ContentHash - - case "parent": - return meta.Parent - - case "parent_short": - if len(meta.Parent) >= 12 { - return meta.Parent[:12] - } - return meta.Parent - - case "date", "time": - return meta.EndTime.Format("2006-01-02T15:04:05Z") - - case "date_short": - return meta.EndTime.Format("2006-01-02") - - case "timestamp", "unix": - return fmt.Sprintf("%d", meta.EndTime.Unix()) - - case "age": - age := time.Since(meta.EndTime) - return formatDurationShort(age) - - case "age_seconds": - return fmt.Sprintf("%.0f", time.Since(meta.EndTime).Seconds()) - - case "ops", "operations": - return fmt.Sprintf("%d", meta.OperationCount) - - case "dids": - return fmt.Sprintf("%d", meta.DIDCount) - - case "size", "compressed": - return fmt.Sprintf("%d", meta.CompressedSize) - - case "size_mb": - return fmt.Sprintf("%.2f", float64(meta.CompressedSize)/(1024*1024)) - - case "uncompressed": - return fmt.Sprintf("%d", meta.UncompressedSize) - - case "uncompressed_mb": - return fmt.Sprintf("%.2f", float64(meta.UncompressedSize)/(1024*1024)) - - case "ratio": - if meta.CompressedSize > 0 { - ratio := float64(meta.UncompressedSize) / float64(meta.CompressedSize) - return fmt.Sprintf("%.2f", ratio) - } - return "0" - - case "timespan", "duration": - duration := meta.EndTime.Sub(meta.StartTime) - return formatDurationShort(duration) - - case "timespan_seconds": - duration := meta.EndTime.Sub(meta.StartTime) - return fmt.Sprintf("%.0f", duration.Seconds()) - - case "start": - return meta.StartTime.Format("2006-01-02T15:04:05Z") - - case "end": - return meta.EndTime.Format("2006-01-02T15:04:05Z") - - case "created": - return meta.CreatedAt.Format("2006-01-02T15:04:05Z") - - default: - return "" - } -} diff --git a/cmd/plcbundle/commands/mempool.go b/cmd/plcbundle/commands/mempool.go deleted file mode 100644 index 4f34b37..0000000 --- a/cmd/plcbundle/commands/mempool.go +++ /dev/null @@ -1,357 +0,0 @@ -package commands - -import ( - "fmt" - "os" - "path/filepath" - "strings" - "time" - - "github.com/goccy/go-json" - "github.com/spf13/cobra" - "tangled.org/atscan.net/plcbundle/internal/types" -) - -func NewMempoolCommand() *cobra.Command { - cmd := &cobra.Command{ - Use: "mempool [status]", - Aliases: []string{"mp"}, - Short: "Manage mempool operations", - Long: `Manage mempool operations - -The mempool stores operations waiting to be bundled. It maintains -strict chronological order and automatically validates consistency.`, - - Example: ` # Show mempool status - plcbundle mempool - plcbundle mempool status - - # Clear all operations - plcbundle mempool clear - - # Export operations as JSONL - plcbundle mempool dump - plcbundle mempool dump > operations.jsonl - - # Using alias - plcbundle mp status`, - - RunE: func(cmd *cobra.Command, args []string) error { - // Default to status subcommand - return mempoolStatus(cmd, args) - }, - } - - // Add subcommands - cmd.AddCommand(newMempoolStatusCommand()) - cmd.AddCommand(newMempoolClearCommand()) - cmd.AddCommand(newMempoolDumpCommand()) - - return cmd -} - -// ============================================================================ -// MEMPOOL STATUS - Show current state -// ============================================================================ - -func newMempoolStatusCommand() *cobra.Command { - var verbose bool - - cmd := &cobra.Command{ - Use: "status", - Aliases: []string{"s", "info"}, - Short: "Show mempool status", - Long: `Show mempool status and statistics - -Displays current mempool state including operation count, progress toward -next bundle, validation status, and memory usage.`, - - Example: ` # Show status - plcbundle mempool status - plcbundle mempool - - # Verbose output with samples - plcbundle mempool status -v`, - - RunE: func(cmd *cobra.Command, args []string) error { - return mempoolStatus(cmd, args) - }, - } - - cmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "Show sample operations") - - return cmd -} - -func mempoolStatus(cmd *cobra.Command, _ []string) error { - verbose, _ := cmd.Flags().GetBool("verbose") - if cmd.Parent() != nil { - // Called as subcommand, check parent's verbose flag - if v, err := cmd.Parent().Flags().GetBool("verbose"); err == nil && v { - verbose = true - } - } - - mgr, dir, err := getManager(&ManagerOptions{Cmd: cmd}) - if err != nil { - return err - } - defer mgr.Close() - - return showMempoolStatus(mgr, dir, verbose) -} - -// ============================================================================ -// MEMPOOL CLEAR - Remove all operations -// ============================================================================ - -func newMempoolClearCommand() *cobra.Command { - var force bool - - cmd := &cobra.Command{ - Use: "clear", - Aliases: []string{"c", "reset"}, - Short: "Clear mempool operations", - Long: `Clear all operations from mempool - -Removes all operations from the mempool and saves the empty state. -This is a destructive operation that cannot be undone. - -Use cases: - • Reset after testing - • Clear corrupted data - • Force fresh start`, - - Example: ` # Clear with confirmation - plcbundle mempool clear - - # Force clear without confirmation - plcbundle mempool clear --force - plcbundle mempool clear -f`, - - RunE: func(cmd *cobra.Command, args []string) error { - mgr, dir, err := getManager(&ManagerOptions{Cmd: cmd}) - if err != nil { - return err - } - defer mgr.Close() - - stats := mgr.GetMempoolStats() - count := stats["count"].(int) - - if count == 0 { - fmt.Println("Mempool is already empty") - return nil - } - - fmt.Printf("Working in: %s\n\n", dir) - - if !force { - fmt.Printf("⚠️ This will clear %d operations from the mempool.\n", count) - fmt.Printf("Are you sure? [y/N]: ") - var response string - fmt.Scanln(&response) - if strings.ToLower(strings.TrimSpace(response)) != "y" { - fmt.Println("Cancelled") - return nil - } - } - - if err := mgr.ClearMempool(); err != nil { - return fmt.Errorf("clear failed: %w", err) - } - - fmt.Printf("\n✓ Mempool cleared (%d operations removed)\n", count) - return nil - }, - } - - cmd.Flags().BoolVarP(&force, "force", "f", false, "Skip confirmation prompt") - - return cmd -} - -// ============================================================================ -// MEMPOOL DUMP - Export operations as JSONL -// ============================================================================ - -func newMempoolDumpCommand() *cobra.Command { - var outputFile string - - cmd := &cobra.Command{ - Use: "dump", - Aliases: []string{"export", "d"}, - Short: "Export mempool as JSONL", - Long: `Export mempool operations as JSONL - -Outputs all operations in the mempool as newline-delimited JSON. -Perfect for backup, analysis, or piping to other tools.`, - - Example: ` # Dump to stdout - plcbundle mempool dump - - # Save to file - plcbundle mempool dump > mempool.jsonl - plcbundle mempool dump -o mempool.jsonl - - # Pipe to jq - plcbundle mempool dump | jq -r .did - - # Count operations - plcbundle mempool dump | wc -l - - # Using alias - plcbundle mempool export`, - - RunE: func(cmd *cobra.Command, args []string) error { - mgr, _, err := getManager(&ManagerOptions{Cmd: cmd}) - if err != nil { - return err - } - defer mgr.Close() - - ops, err := mgr.GetMempoolOperations() - if err != nil { - return fmt.Errorf("failed to get mempool operations: %w", err) - } - - if len(ops) == 0 { - fmt.Fprintf(os.Stderr, "Mempool is empty\n") - return nil - } - - // Determine output destination - var output *os.File - if outputFile != "" { - output, err = os.Create(outputFile) - if err != nil { - return fmt.Errorf("failed to create output file: %w", err) - } - defer output.Close() - fmt.Fprintf(os.Stderr, "Exporting to: %s\n", outputFile) - } else { - output = os.Stdout - } - - // Write JSONL - for _, op := range ops { - if len(op.RawJSON) > 0 { - output.Write(op.RawJSON) - } else { - data, _ := json.Marshal(op) - output.Write(data) - } - output.Write([]byte("\n")) - } - - fmt.Fprintf(os.Stderr, "Exported %d operations from mempool\n", len(ops)) - return nil - }, - } - - cmd.Flags().StringVarP(&outputFile, "output", "o", "", "Output file (default: stdout)") - - return cmd -} - -// ============================================================================ -// HELPER FUNCTIONS -// ============================================================================ - -func showMempoolStatus(mgr BundleManager, dir string, verbose bool) error { - stats := mgr.GetMempoolStats() - count := stats["count"].(int) - canCreate := stats["can_create_bundle"].(bool) - targetBundle := stats["target_bundle"].(int) - minTimestamp := stats["min_timestamp"].(time.Time) - validated := stats["validated"].(bool) - - fmt.Printf("Mempool Status\n") - fmt.Printf("══════════════\n\n") - fmt.Printf(" Directory: %s\n", dir) - fmt.Printf(" Target bundle: %06d\n", targetBundle) - fmt.Printf(" Operations: %d / %d\n", count, types.BUNDLE_SIZE) - fmt.Printf(" Min timestamp: %s\n\n", minTimestamp.Format("2006-01-02 15:04:05")) - - // Validation status - validationIcon := "✓" - if !validated { - validationIcon = "⚠️" - } - fmt.Printf(" Validated: %s %v\n", validationIcon, validated) - - if count > 0 { - // Size information - if sizeBytes, ok := stats["size_bytes"].(int); ok { - fmt.Printf(" Size: %.2f KB\n", float64(sizeBytes)/1024) - } - - // Time range - if firstTime, ok := stats["first_time"].(time.Time); ok { - fmt.Printf(" First op: %s\n", firstTime.Format("2006-01-02 15:04:05")) - } - if lastTime, ok := stats["last_time"].(time.Time); ok { - fmt.Printf(" Last op: %s\n", lastTime.Format("2006-01-02 15:04:05")) - } - - fmt.Printf("\n") - - // Progress bar - progress := float64(count) / float64(types.BUNDLE_SIZE) * 100 - fmt.Printf(" Progress: %.1f%% (%d/%d)\n", progress, count, types.BUNDLE_SIZE) - - barWidth := 40 - filled := int(float64(barWidth) * float64(count) / float64(types.BUNDLE_SIZE)) - if filled > barWidth { - filled = barWidth - } - bar := strings.Repeat("█", filled) + strings.Repeat("░", barWidth-filled) - fmt.Printf(" [%s]\n\n", bar) - - // Bundle creation status - if canCreate { - fmt.Printf(" ✓ Ready to create bundle\n") - } else { - remaining := types.BUNDLE_SIZE - count - fmt.Printf(" Need %s more operations\n", formatNumber(remaining)) - } - } else { - fmt.Printf("\n (empty)\n") - } - - fmt.Printf("\n") - - // Verbose: Show sample operations - if verbose && count > 0 { - fmt.Printf("Sample Operations (first 10)\n") - fmt.Printf("────────────────────────────\n\n") - - ops, err := mgr.GetMempoolOperations() - if err != nil { - return fmt.Errorf("error getting operations: %w", err) - } - - showCount := 10 - if len(ops) < showCount { - showCount = len(ops) - } - - for i := 0; i < showCount; i++ { - op := ops[i] - fmt.Printf(" %d. DID: %s\n", i+1, op.DID) - fmt.Printf(" CID: %s\n", op.CID) - fmt.Printf(" Created: %s\n", op.CreatedAt.Format("2006-01-02 15:04:05.000")) - fmt.Printf("\n") - } - - if len(ops) > showCount { - fmt.Printf(" ... and %d more\n\n", len(ops)-showCount) - } - } - - // Show mempool file location - mempoolFilename := fmt.Sprintf("plc_mempool_%06d.jsonl", targetBundle) - fmt.Printf("File: %s\n", filepath.Join(dir, mempoolFilename)) - - return nil -} diff --git a/cmd/plcbundle/commands/migrate.go b/cmd/plcbundle/commands/migrate.go deleted file mode 100644 index 667e7b0..0000000 --- a/cmd/plcbundle/commands/migrate.go +++ /dev/null @@ -1,448 +0,0 @@ -package commands - -import ( - "encoding/binary" - "fmt" - "os" - "path/filepath" - "time" - - "github.com/spf13/cobra" - "tangled.org/atscan.net/plcbundle/cmd/plcbundle/ui" - "tangled.org/atscan.net/plcbundle/internal/bundleindex" - "tangled.org/atscan.net/plcbundle/internal/storage" -) - -func NewMigrateCommand() *cobra.Command { - var ( - dryRun bool - force bool - workers int - ) - - cmd := &cobra.Command{ - Use: "migrate [flags]", - Short: "Migrate bundles to new zstd frame format", - Long: `Migrate old single-frame zstd bundles to new multi-frame format - -This command converts bundles from the legacy single-frame zstd format -to the new multi-frame format with .idx index files. This enables: - • Faster random access to individual operations - • Reduced memory usage when loading specific positions - • Better performance for DID lookups - -The migration: - 1. Scans for bundles missing .idx files (legacy format) - 2. Re-compresses them using multi-frame format (100 ops/frame) - 3. Generates .idx frame offset index files - 4. Preserves all hashes and metadata - 5. Verifies content integrity - -Original files are replaced atomically. Use --dry-run to preview.`, - - Example: ` # Preview migration (recommended first) - plcbundle migrate --dry-run - - # Migrate all legacy bundles - plcbundle migrate - - # Force migration even if .idx files exist - plcbundle migrate --force - - # Parallel migration (faster) - plcbundle migrate --workers 8 - - # Verbose output - plcbundle migrate -v`, - - RunE: func(cmd *cobra.Command, args []string) error { - verbose, _ := cmd.Root().PersistentFlags().GetBool("verbose") - - mgr, dir, err := getManager(&ManagerOptions{Cmd: cmd}) - if err != nil { - return err - } - defer mgr.Close() - - return runMigration(mgr, dir, migrationOptions{ - dryRun: dryRun, - force: force, - workers: workers, - verbose: verbose, - }) - }, - } - - cmd.Flags().BoolVarP(&dryRun, "dry-run", "n", false, "Show what would be migrated without migrating") - cmd.Flags().BoolVarP(&force, "force", "f", false, "Re-migrate bundles that already have .idx files") - cmd.Flags().IntVarP(&workers, "workers", "w", 4, "Number of parallel workers") - - return cmd -} - -type migrationOptions struct { - dryRun bool - force bool - workers int - verbose bool -} - -type bundleMigrationInfo struct { - bundleNumber int - oldSize int64 - uncompressedSize int64 - oldFormat string - oldCompressionRatio float64 -} - -func runMigration(mgr BundleManager, dir string, opts migrationOptions) error { - fmt.Printf("Scanning for legacy bundles in: %s\n\n", dir) - - index := mgr.GetIndex() - bundles := index.GetBundles() - - if len(bundles) == 0 { - fmt.Println("No bundles to migrate") - return nil - } - - version := GetVersion() - ops := &storage.Operations{} - - var needsMigration []bundleMigrationInfo - var totalSize int64 - - for _, meta := range bundles { - bundlePath := filepath.Join(dir, fmt.Sprintf("%06d.jsonl.zst", meta.BundleNumber)) - embeddedMeta, err := ops.ExtractBundleMetadata(bundlePath) - - info := bundleMigrationInfo{ - bundleNumber: meta.BundleNumber, - oldSize: meta.CompressedSize, - uncompressedSize: meta.UncompressedSize, - } - - if meta.CompressedSize > 0 { - info.oldCompressionRatio = float64(meta.UncompressedSize) / float64(meta.CompressedSize) - } - - if err != nil { - info.oldFormat = "v0 (single-frame)" - } else { - info.oldFormat = embeddedMeta.Format - } - - if err != nil || opts.force { - needsMigration = append(needsMigration, info) - totalSize += meta.CompressedSize - } - } - - if len(needsMigration) == 0 { - fmt.Println("✓ All bundles already migrated") - fmt.Println("\nUse --force to re-migrate") - return nil - } - - // COMPACT PLAN - fmt.Printf("Migration Plan\n") - fmt.Printf("══════════════\n\n") - - formatCounts := make(map[string]int) - var totalUncompressed int64 - for _, info := range needsMigration { - formatCounts[info.oldFormat]++ - totalUncompressed += info.uncompressedSize - } - - fmt.Printf(" Format: ") - first := true - for format, count := range formatCounts { - if !first { - fmt.Printf(" + ") - } - fmt.Printf("%s (%d)", format, count) - first = false - } - fmt.Printf(" → v%d\n", storage.MetadataFormatVersion) - - fmt.Printf(" Bundles: %d\n", len(needsMigration)) - fmt.Printf(" Size: %s (%.3fx compression)\n", - formatBytes(totalSize), - float64(totalUncompressed)/float64(totalSize)) - fmt.Printf(" Workers: %d, Compression Level: %d\n\n", opts.workers, storage.CompressionLevel) - - if opts.dryRun { - fmt.Printf("💡 Dry-run mode\n") - return nil - } - - // Execute migration - fmt.Printf("Migrating...\n\n") - - start := time.Now() - progress := ui.NewProgressBar(len(needsMigration)) - - success := 0 - failed := 0 - var firstError error - hashChanges := make([]int, 0) - - var totalOldSize int64 - var totalNewSize int64 - var totalOldUncompressed int64 - var totalNewUncompressed int64 - - for i, info := range needsMigration { - totalOldSize += info.oldSize - totalOldUncompressed += info.uncompressedSize - - sizeDiff, newUncompressedSize, err := migrateBundle(dir, info.bundleNumber, index, version, opts.verbose) - if err != nil { - failed++ - if firstError == nil { - firstError = err - } - if opts.verbose { - fmt.Fprintf(os.Stderr, "\n✗ Bundle %06d failed: %v\n", info.bundleNumber, err) - } - } else { - success++ - hashChanges = append(hashChanges, info.bundleNumber) - - newSize := info.oldSize + sizeDiff - totalNewSize += newSize - totalNewUncompressed += newUncompressedSize - - if opts.verbose { - oldRatio := float64(info.uncompressedSize) / float64(info.oldSize) - newRatio := float64(newUncompressedSize) / float64(newSize) - - fmt.Fprintf(os.Stderr, "✓ %06d: %.3fx→%.3fx %+s\n", - info.bundleNumber, oldRatio, newRatio, formatBytes(sizeDiff)) - } - } - - progress.Set(i + 1) - } - - progress.Finish() - elapsed := time.Since(start) - - // Update index - if len(hashChanges) > 0 { - fmt.Printf("\nUpdating index...\n") - updateStart := time.Now() - - updated := 0 - for _, bundleNum := range hashChanges { - bundlePath := filepath.Join(dir, fmt.Sprintf("%06d.jsonl.zst", bundleNum)) - compHash, compSize, _, contentSize, err := ops.CalculateFileHashes(bundlePath) - if err != nil { - fmt.Fprintf(os.Stderr, " ⚠️ Failed to hash %06d: %v\n", bundleNum, err) - continue - } - - bundleMeta, err := index.GetBundle(bundleNum) - if err != nil { - continue - } - - bundleMeta.CompressedHash = compHash - bundleMeta.CompressedSize = compSize - bundleMeta.UncompressedSize = contentSize - - index.AddBundle(bundleMeta) - updated++ - } - - if err := mgr.SaveIndex(); err != nil { - fmt.Fprintf(os.Stderr, " ⚠️ Failed to save index: %v\n", err) - } else { - fmt.Printf(" ✓ %d entries in %s\n", updated, time.Since(updateStart).Round(time.Millisecond)) - } - } - - // COMPACT SUMMARY - fmt.Printf("\n") - if failed == 0 { - fmt.Printf("✓ Complete: %d bundles in %s\n\n", success, elapsed.Round(time.Millisecond)) - - if totalOldSize > 0 && success > 0 { - sizeDiff := totalNewSize - totalOldSize - oldRatio := float64(totalOldUncompressed) / float64(totalOldSize) - newRatio := float64(totalNewUncompressed) / float64(totalNewSize) - ratioDiff := newRatio - oldRatio - - // MEASURE ACTUAL METADATA SIZE (not estimated) - var totalActualMetadata int64 - for _, bundleNum := range hashChanges { - bundlePath := filepath.Join(dir, fmt.Sprintf("%06d.jsonl.zst", bundleNum)) - metaSize, _ := measureMetadataSize(bundlePath) - totalActualMetadata += metaSize - } - - // FIXED ALIGNMENT - fmt.Printf(" Old New Change\n") - fmt.Printf(" ──────── ──────── ─────────\n") - fmt.Printf("Size %-13s %-13s %+s (%.1f%%)\n", - formatBytes(totalOldSize), - formatBytes(totalNewSize), - formatBytes(sizeDiff), - float64(sizeDiff)/float64(totalOldSize)*100) - fmt.Printf("Ratio %-13s %-13s %+s\n", - fmt.Sprintf("%.3fx", oldRatio), fmt.Sprintf("%.3fx", newRatio), fmt.Sprintf("%+.3fx", ratioDiff)) - fmt.Printf("Avg/bundle %-13s %-13s %+s\n\n", - formatBytes(totalOldSize/int64(success)), - formatBytes(totalNewSize/int64(success)), - formatBytes(sizeDiff/int64(success))) - - // FIXED BREAKDOWN - use actual metadata size - if totalActualMetadata > 0 { - compressionEfficiency := sizeDiff - totalActualMetadata - - fmt.Printf("Breakdown:\n") - fmt.Printf(" Metadata: %+s (~%s/bundle, structural)\n", - formatBytes(totalActualMetadata), - formatBytes(totalActualMetadata/int64(success))) - - // FIX: Use absolute threshold based on old size, not metadata size - threshold := totalOldSize / 1000 // 0.1% of old size - - if abs(compressionEfficiency) > threshold { - if compressionEfficiency > 0 { - // Compression got worse - pctWorse := float64(compressionEfficiency) / float64(totalOldSize) * 100 - fmt.Printf(" Compression: %+s (%.2f%% worse)\n", - formatBytes(compressionEfficiency), pctWorse) - } else if compressionEfficiency < 0 { - // Compression improved - pctBetter := float64(-compressionEfficiency) / float64(totalOldSize) * 100 - fmt.Printf(" Compression: %s (%.2f%% better)\n", - formatBytes(compressionEfficiency), pctBetter) - } - } else { - // Truly negligible - fmt.Printf(" Compression: unchanged\n") - } - } - - fmt.Printf("\n") - } - } else { - fmt.Printf("⚠️ Failed: %d bundles\n", failed) - if firstError != nil { - fmt.Printf(" Error: %v\n", firstError) - } - return fmt.Errorf("migration failed for %d bundles", failed) - } - - return nil -} - -func migrateBundle(dir string, bundleNum int, index *bundleindex.Index, version string, verbose bool) (sizeDiff int64, newUncompressedSize int64, err error) { - bundlePath := filepath.Join(dir, fmt.Sprintf("%06d.jsonl.zst", bundleNum)) - backupPath := bundlePath + ".bak" - - meta, err := index.GetBundle(bundleNum) - if err != nil { - return 0, 0, fmt.Errorf("bundle not in index: %w", err) - } - - oldSize := meta.CompressedSize - - ops := &storage.Operations{} - operations, err := ops.LoadBundle(bundlePath) - if err != nil { - return 0, 0, fmt.Errorf("failed to load: %w", err) - } - - if err := os.Rename(bundlePath, backupPath); err != nil { - return 0, 0, fmt.Errorf("failed to backup: %w", err) - } - - hostname, _ := os.Hostname() - - bundleInfo := &storage.BundleInfo{ - BundleNumber: meta.BundleNumber, - Origin: index.Origin, - ParentHash: meta.Parent, - Cursor: meta.Cursor, - CreatedBy: fmt.Sprintf("plcbundle/%s", version), - Hostname: hostname, - } - - contentHash, _, contentSize, compSize, err := ops.SaveBundle(bundlePath, operations, bundleInfo) - if err != nil { - os.Rename(backupPath, bundlePath) - return 0, 0, fmt.Errorf("failed to save: %w", err) - } - - embeddedMeta, err := ops.ExtractBundleMetadata(bundlePath) - if err != nil { - os.Remove(bundlePath) - os.Rename(backupPath, bundlePath) - return 0, 0, fmt.Errorf("embedded metadata not created: %w", err) - } - - if len(embeddedMeta.FrameOffsets) == 0 { - os.Remove(bundlePath) - os.Rename(backupPath, bundlePath) - return 0, 0, fmt.Errorf("frame offsets missing in metadata") - } - - if contentHash != meta.ContentHash { - fmt.Fprintf(os.Stderr, " ⚠️ Content hash changed: %s → %s\n", - meta.ContentHash[:12], contentHash[:12]) - } - - os.Remove(backupPath) - - // Calculate changes - newSize := compSize - sizeDiff = newSize - oldSize - newUncompressedSize = contentSize - - if verbose { - oldRatio := float64(meta.UncompressedSize) / float64(oldSize) - newRatio := float64(contentSize) / float64(newSize) - - fmt.Fprintf(os.Stderr, " Frames: %d, Ratio: %.3fx→%.3fx, Size: %+s\n", - len(embeddedMeta.FrameOffsets)-1, oldRatio, newRatio, formatBytes(sizeDiff)) - } - - return sizeDiff, newUncompressedSize, nil -} - -func measureMetadataSize(bundlePath string) (int64, error) { - file, err := os.Open(bundlePath) - if err != nil { - return 0, err - } - defer file.Close() - - // Read magic (4 bytes) + size (4 bytes) - header := make([]byte, 8) - if _, err := file.Read(header); err != nil { - return 0, err - } - - // Check if it's a skippable frame - magic := binary.LittleEndian.Uint32(header[0:4]) - if magic < 0x184D2A50 || magic > 0x184D2A5F { - return 0, nil // No metadata frame - } - - // Get frame data size - frameSize := binary.LittleEndian.Uint32(header[4:8]) - - // Total metadata size = 4 (magic) + 4 (size) + frameSize (data) - return int64(8 + frameSize), nil -} - -func abs(n int64) int64 { - if n < 0 { - return -n - } - return n -} diff --git a/cmd/plcbundle/commands/op.go b/cmd/plcbundle/commands/op.go deleted file mode 100644 index 5a43bb1..0000000 --- a/cmd/plcbundle/commands/op.go +++ /dev/null @@ -1,482 +0,0 @@ -package commands - -import ( - "context" - "fmt" - "os" - "strconv" - "strings" - "time" - - "github.com/goccy/go-json" - "github.com/spf13/cobra" - "tangled.org/atscan.net/plcbundle/internal/plcclient" - "tangled.org/atscan.net/plcbundle/internal/types" -) - -func NewOpCommand() *cobra.Command { - cmd := &cobra.Command{ - Use: "op", - Aliases: []string{"operation", "record"}, - Short: "Operation queries and inspection", - Long: `Operation queries and inspection - -Direct access to individual operations within bundles using either: - • Bundle number + position (e.g., 42 1337) - • Global position (e.g., 420000) - -Global position format: (bundleNumber × 10,000) + position -Example: 88410345 = bundle 8841, position 345`, - - Example: ` # Get operation as JSON - plcbundle op get 42 1337 - plcbundle op get 420000 - - # Show operation (formatted) - plcbundle op show 42 1337 - plcbundle op show 88410345 - - # Find by CID - plcbundle op find bafyreig3...`, - } - - // Add subcommands - cmd.AddCommand(newOpGetCommand()) - cmd.AddCommand(newOpShowCommand()) - cmd.AddCommand(newOpFindCommand()) - - return cmd -} - -// ============================================================================ -// OP GET - Get operation as JSON -// ============================================================================ - -func newOpGetCommand() *cobra.Command { - var verbose bool - - cmd := &cobra.Command{ - Use: "get | ", - Short: "Get operation as JSON", - Long: `Get operation as JSON (machine-readable) - -Supports two input formats: - 1. Bundle number + position: get 42 1337 - 2. Global position: get 420000 - -Global position = (bundleNumber × 10,000) + position - -Use -v/--verbose to see detailed timing breakdown.`, - - Example: ` # By bundle + position - plcbundle op get 42 1337 - - # By global position - plcbundle op get 88410345 - - # With timing metrics - plcbundle op get 42 1337 -v - plcbundle op get 88410345 --verbose - - # Pipe to jq - plcbundle op get 42 1337 | jq .did`, - - Args: cobra.RangeArgs(1, 2), - - RunE: func(cmd *cobra.Command, args []string) error { - bundleNum, position, err := parseOpArgs(args) - if err != nil { - return err - } - - mgr, _, err := getManager(&ManagerOptions{Cmd: cmd}) - if err != nil { - return err - } - defer mgr.Close() - - ctx := context.Background() - - // Time the operation load - totalStart := time.Now() - op, err := mgr.LoadOperation(ctx, bundleNum, position) - totalDuration := time.Since(totalStart) - - if err != nil { - return err - } - - if verbose { - globalPos := (bundleNum * 10000) + position - - // Log-style output (compact, single-line friendly) - fmt.Fprintf(os.Stderr, "[Load] Bundle %06d:%04d (pos=%d) in %s", - bundleNum, position, globalPos, totalDuration) - fmt.Fprintf(os.Stderr, " | %d bytes", len(op.RawJSON)) - fmt.Fprintf(os.Stderr, "\n") - } - - // Output raw JSON to stdout - if len(op.RawJSON) > 0 { - fmt.Println(string(op.RawJSON)) - } else { - data, _ := json.Marshal(op) - fmt.Println(string(data)) - } - - return nil - }, - } - - cmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "Show timing metrics") - - return cmd -} - -// // ============================================================================ -// OP SHOW - Show operation (formatted) -// ============================================================================ - -func newOpShowCommand() *cobra.Command { - var verbose bool - - cmd := &cobra.Command{ - Use: "show | ", - Short: "Show operation (human-readable)", - Long: `Show operation with formatted output - -Displays operation in human-readable format with: - • Bundle location and global position - • DID and CID - • Timestamp and age - • Nullification status - • Parsed operation details - • Performance metrics (with -v)`, - - Example: ` # By bundle + position - plcbundle op show 42 1337 - - # By global position - plcbundle op show 88410345 - - # Verbose with timing and full JSON - plcbundle op show 42 1337 -v`, - - Args: cobra.RangeArgs(1, 2), - - RunE: func(cmd *cobra.Command, args []string) error { - bundleNum, position, err := parseOpArgs(args) - if err != nil { - return err - } - - mgr, _, err := getManager(&ManagerOptions{Cmd: cmd}) - if err != nil { - return err - } - defer mgr.Close() - - ctx := context.Background() - - // Time the operation - loadStart := time.Now() - op, err := mgr.LoadOperation(ctx, bundleNum, position) - loadDuration := time.Since(loadStart) - - if err != nil { - return err - } - - // Time the parsing - parseStart := time.Now() - opData, parseErr := op.GetOperationData() - parseDuration := time.Since(parseStart) - - return displayOperationWithTiming(bundleNum, position, op, opData, parseErr, - loadDuration, parseDuration, verbose) - }, - } - - cmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "Show timing metrics and full JSON") - - return cmd -} - -// ============================================================================ -// OP FIND - Find operation by CID -// ============================================================================ - -func newOpFindCommand() *cobra.Command { - cmd := &cobra.Command{ - Use: "find ", - Short: "Find operation by CID", - Long: `Find operation by CID across all bundles - -Searches the entire repository for an operation with the given CID -and returns its location (bundle + position). - -Note: This performs a full scan and can be slow on large repositories.`, - - Example: ` # Find by CID - plcbundle op find bafyreig3tg4k... - - # Pipe to op get - plcbundle op find bafyreig3... | awk '{print $3, $5}' | xargs plcbundle op get`, - - Args: cobra.ExactArgs(1), - - RunE: func(cmd *cobra.Command, args []string) error { - cid := args[0] - - mgr, _, err := getManager(&ManagerOptions{Cmd: cmd}) - if err != nil { - return err - } - defer mgr.Close() - - return findOperationByCID(mgr, cid) - }, - } - - return cmd -} - -// ============================================================================ -// Helper Functions -// ============================================================================ - -// parseOpArgs parses operation arguments (supports both formats) -func parseOpArgs(args []string) (bundleNum, position int, err error) { - if len(args) == 1 { - global, err := strconv.Atoi(args[0]) - if err != nil { - return 0, 0, fmt.Errorf("invalid position: %w", err) - } - - if global < types.BUNDLE_SIZE { - // Small numbers: shorthand for "bundle 1, position N" - // op get 1 → bundle 1, position 1 - // op get 100 → bundle 1, position 100 - return 1, global, nil - } - - // Large numbers: global position - // op get 10000 → bundle 1, position 0 - // op get 88410345 → bundle 8841, position 345 - bundleNum = global / types.BUNDLE_SIZE - position = global % types.BUNDLE_SIZE - - return bundleNum, position, nil - } - - if len(args) == 2 { - // Explicit: bundle + position - bundleNum, err = strconv.Atoi(args[0]) - if err != nil { - return 0, 0, fmt.Errorf("invalid bundle number: %w", err) - } - - position, err = strconv.Atoi(args[1]) - if err != nil { - return 0, 0, fmt.Errorf("invalid position: %w", err) - } - - return bundleNum, position, nil - } - - return 0, 0, fmt.Errorf("usage: op OR op ") -} - -// findOperationByCID searches for an operation by CID -func findOperationByCID(mgr BundleManager, cid string) error { - ctx := context.Background() - - // CHECK MEMPOOL FIRST (most recent data) - fmt.Fprintf(os.Stderr, "Checking mempool...\n") - mempoolOps, err := mgr.GetMempoolOperations() - if err == nil && len(mempoolOps) > 0 { - for pos, op := range mempoolOps { - if op.CID == cid { - fmt.Printf("Found in mempool: position %d\n\n", pos) - fmt.Printf(" DID: %s\n", op.DID) - fmt.Printf(" Created: %s\n", op.CreatedAt.Format("2006-01-02 15:04:05")) - - if op.IsNullified() { - fmt.Printf(" Status: ✗ Nullified") - if nullCID := op.GetNullifyingCID(); nullCID != "" { - fmt.Printf(" by %s", nullCID) - } - fmt.Printf("\n") - } else { - fmt.Printf(" Status: ✓ Active\n") - } - - return nil - } - } - } - - // Search bundles - index := mgr.GetIndex() - bundles := index.GetBundles() - - if len(bundles) == 0 { - fmt.Fprintf(os.Stderr, "No bundles to search\n") - return nil - } - - fmt.Fprintf(os.Stderr, "Searching %d bundles for CID: %s\n\n", len(bundles), cid) - - for _, meta := range bundles { - bundle, err := mgr.LoadBundle(ctx, meta.BundleNumber) - if err != nil { - continue - } - - for pos, op := range bundle.Operations { - if op.CID == cid { - globalPos := (meta.BundleNumber * types.BUNDLE_SIZE) + pos - - fmt.Printf("Found: bundle %06d, position %d\n", meta.BundleNumber, pos) - fmt.Printf("Global position: %d\n\n", globalPos) - - fmt.Printf(" DID: %s\n", op.DID) - fmt.Printf(" Created: %s\n", op.CreatedAt.Format("2006-01-02 15:04:05")) - - if op.IsNullified() { - fmt.Printf(" Status: ✗ Nullified") - if nullCID := op.GetNullifyingCID(); nullCID != "" { - fmt.Printf(" by %s", nullCID) - } - fmt.Printf("\n") - } else { - fmt.Printf(" Status: ✓ Active\n") - } - - return nil - } - } - - // Progress indicator - if meta.BundleNumber%100 == 0 { - fmt.Fprintf(os.Stderr, "Searched through bundle %06d...\r", meta.BundleNumber) - } - } - - fmt.Fprintf(os.Stderr, "\nCID not found: %s\n", cid) - fmt.Fprintf(os.Stderr, "(Searched %d bundles + mempool)\n", len(bundles)) - return fmt.Errorf("CID not found") -} - -// displayOperationWithTiming shows formatted operation details with timing -func displayOperationWithTiming(bundleNum, position int, op *plcclient.PLCOperation, - opData map[string]interface{}, _ error, - loadDuration, parseDuration time.Duration, verbose bool) error { - - globalPos := (bundleNum * types.BUNDLE_SIZE) + position - - fmt.Printf("═══════════════════════════════════════════════════════════════\n") - fmt.Printf(" Operation %d\n", globalPos) - fmt.Printf("═══════════════════════════════════════════════════════════════\n\n") - - fmt.Printf("Location\n") - fmt.Printf("────────\n") - fmt.Printf(" Bundle: %06d\n", bundleNum) - fmt.Printf(" Position: %d\n", position) - fmt.Printf(" Global position: %d\n\n", globalPos) - - fmt.Printf("Identity\n") - fmt.Printf("────────\n") - fmt.Printf(" DID: %s\n", op.DID) - fmt.Printf(" CID: %s\n\n", op.CID) - - fmt.Printf("Timestamp\n") - fmt.Printf("─────────\n") - fmt.Printf(" Created: %s\n", op.CreatedAt.Format("2006-01-02 15:04:05.000 MST")) - fmt.Printf(" Age: %s\n\n", formatDuration(time.Since(op.CreatedAt))) - - // Status - status := "✓ Active" - if op.IsNullified() { - status = "✗ Nullified" - if cid := op.GetNullifyingCID(); cid != "" { - status += fmt.Sprintf(" by %s", cid) - } - } - fmt.Printf("Status\n") - fmt.Printf("──────\n") - fmt.Printf(" %s\n\n", status) - - // Performance metrics (always shown if verbose) - if verbose { - totalTime := loadDuration + parseDuration - - fmt.Printf("Performance\n") - fmt.Printf("───────────\n") - fmt.Printf(" Load time: %s\n", loadDuration) - fmt.Printf(" Parse time: %s\n", parseDuration) - fmt.Printf(" Total time: %s\n", totalTime) - - if len(op.RawJSON) > 0 { - fmt.Printf(" Data size: %d bytes\n", len(op.RawJSON)) - mbPerSec := float64(len(op.RawJSON)) / loadDuration.Seconds() / (1024 * 1024) - fmt.Printf(" Load speed: %.2f MB/s\n", mbPerSec) - } - - fmt.Printf("\n") - } - - // Parse operation details - if opData != nil && !op.IsNullified() { - fmt.Printf("Details\n") - fmt.Printf("───────\n") - - if opType, ok := opData["type"].(string); ok { - fmt.Printf(" Type: %s\n", opType) - } - - if handle, ok := opData["handle"].(string); ok { - fmt.Printf(" Handle: %s\n", handle) - } else if aka, ok := opData["alsoKnownAs"].([]interface{}); ok && len(aka) > 0 { - if akaStr, ok := aka[0].(string); ok { - handle := strings.TrimPrefix(akaStr, "at://") - fmt.Printf(" Handle: %s\n", handle) - } - } - - if services, ok := opData["services"].(map[string]interface{}); ok { - if pds, ok := services["atproto_pds"].(map[string]interface{}); ok { - if endpoint, ok := pds["endpoint"].(string); ok { - fmt.Printf(" PDS: %s\n", endpoint) - } - } - } - - fmt.Printf("\n") - } - - // Verbose: show full JSON (pretty-printed) - if verbose { - fmt.Printf("Raw JSON\n") - fmt.Printf("────────\n") - - var data []byte - if len(op.RawJSON) > 0 { - // Parse and re-format the raw JSON - var temp interface{} - if err := json.Unmarshal(op.RawJSON, &temp); err == nil { - data, _ = json.MarshalIndent(temp, "", " ") - } else { - // Fallback to raw if parse fails - data = op.RawJSON - } - } else { - data, _ = json.MarshalIndent(op, "", " ") - } - - fmt.Println(string(data)) - fmt.Printf("\n") - } - - return nil -} diff --git a/cmd/plcbundle/commands/progress_helper.go b/cmd/plcbundle/commands/progress_helper.go deleted file mode 100644 index 48feda0..0000000 --- a/cmd/plcbundle/commands/progress_helper.go +++ /dev/null @@ -1,56 +0,0 @@ -package commands - -import ( - "tangled.org/atscan.net/plcbundle/cmd/plcbundle/ui" -) - -// NewBundleProgressBar creates a progress bar with auto-calculated bytes from bundles -func NewBundleProgressBar(mgr BundleManager, start, end int) *ui.ProgressBar { - total := end - start + 1 - - // Calculate total bytes from bundle metadata - index := mgr.GetIndex() - totalBytes := int64(0) - - for bundleNum := start; bundleNum <= end; bundleNum++ { - if meta, err := index.GetBundle(bundleNum); err == nil { - totalBytes += meta.UncompressedSize - } - } - - if totalBytes > 0 { - return ui.NewProgressBarWithBytes(total, totalBytes) - } - - // Fallback: estimate based on average - stats := index.GetStats() - if avgBytes := estimateAvgBundleSize(stats); avgBytes > 0 { - return ui.NewProgressBarWithBytesAuto(total, avgBytes) - } - - return ui.NewProgressBar(total) -} - -// estimateAvgBundleSize estimates average uncompressed bundle size -func estimateAvgBundleSize(stats map[string]interface{}) int64 { - if totalUncompressed, ok := stats["total_uncompressed_size"].(int64); ok { - if bundleCount, ok := stats["bundle_count"].(int); ok && bundleCount > 0 { - return totalUncompressed / int64(bundleCount) - } - } - return 0 -} - -// UpdateBundleProgress updates progress with bundle's actual size -func UpdateBundleProgress(pb *ui.ProgressBar, current int, bundle interface{}) { - // Try to extract size from bundle if available - type sizer interface { - GetUncompressedSize() int64 - } - - if b, ok := bundle.(sizer); ok { - pb.SetWithBytes(current, b.GetUncompressedSize()) - } else { - pb.Set(current) - } -} diff --git a/cmd/plcbundle/commands/query.go b/cmd/plcbundle/commands/query.go deleted file mode 100644 index 5a7d54c..0000000 --- a/cmd/plcbundle/commands/query.go +++ /dev/null @@ -1,610 +0,0 @@ -// cmd/plcbundle/commands/query.go -package commands - -import ( - "bytes" - "context" - "fmt" - "os" - "runtime" - "strconv" - "strings" - "sync" - "sync/atomic" - - "github.com/goccy/go-json" - "github.com/jmespath-community/go-jmespath" // Correct import - "github.com/spf13/cobra" - "tangled.org/atscan.net/plcbundle/cmd/plcbundle/ui" -) - -func NewQueryCommand() *cobra.Command { - var ( - bundleRange string - threads int - format string - limit int - noProgress bool - simple bool - ) - - cmd := &cobra.Command{ - Use: "query [flags]", - Aliases: []string{"q"}, - Short: "Query ops using JMESPath or simple dot notation", - Long: `Query operations using JMESPath expressions or simple dot notation - -Stream through operations in bundles and evaluate expressions. -Supports parallel processing for better performance. - -Simple Mode (--simple): - Fast field extraction using dot notation (no JMESPath parsing): - did Extract top-level field - operation.handle Nested object access - operation.services.atproto_pds.endpoint - alsoKnownAs[0] Array indexing - - Performance: should be faster than JMESPath mode - Limitations: No filters, functions, or projections - -JMESPath Mode (default): - Full JMESPath query language with filters, projections, functions.`, - - Example: ` # Simple mode (faster) - plcbundle query did --bundles 1-100 --simple - plcbundle query operation.handle --bundles 1-100 --simple - plcbundle query operation.services.atproto_pds.endpoint --simple --bundles 1-100 - - # JMESPath mode (powerful) - plcbundle query 'operation.services.*.endpoint' --bundles 1-100 - plcbundle query 'operation | {did: did, handle: handle}' --bundles 1-10`, - - Args: cobra.ExactArgs(1), - - RunE: func(cmd *cobra.Command, args []string) error { - expression := args[0] - - if threads <= 0 { - threads = runtime.NumCPU() - if threads < 1 { - threads = 1 - } - } - - mgr, _, err := getManager(&ManagerOptions{Cmd: cmd}) - if err != nil { - return err - } - defer mgr.Close() - - var start, end int - if bundleRange == "" { - index := mgr.GetIndex() - bundles := index.GetBundles() - if len(bundles) == 0 { - return fmt.Errorf("no bundles available") - } - start = bundles[0].BundleNumber - end = bundles[len(bundles)-1].BundleNumber - } else { - start, end, err = parseBundleRange(bundleRange) - if err != nil { - return err - } - } - - return runQuery(cmd.Context(), mgr, queryOptions{ - expression: expression, - start: start, - end: end, - threads: threads, - format: format, - limit: limit, - noProgress: noProgress, - simple: simple, - }) - }, - } - - cmd.Flags().StringVar(&bundleRange, "bundles", "", "Bundle selection: number (42) or range (1-50)") - cmd.Flags().IntVar(&threads, "threads", 0, "Number of worker threads (0 = auto-detect CPU cores)") - cmd.Flags().StringVar(&format, "format", "jsonl", "Output format: jsonl|count") - cmd.Flags().IntVar(&limit, "limit", 0, "Limit number of results (0 = unlimited)") - cmd.Flags().BoolVar(&noProgress, "no-progress", false, "Disable progress output") - cmd.Flags().BoolVar(&simple, "simple", false, "Use fast dot notation instead of JMESPath") - - return cmd -} - -type queryOptions struct { - expression string - start int - end int - threads int - format string - limit int - noProgress bool - simple bool -} - -func runQuery(ctx context.Context, mgr BundleManager, opts queryOptions) error { - totalBundles := opts.end - opts.start + 1 - - if opts.threads > totalBundles { - opts.threads = totalBundles - } - - fmt.Fprintf(os.Stderr, "Query: %s\n", opts.expression) - if opts.simple { - fmt.Fprintf(os.Stderr, "Mode: simple (fast dot notation)\n") - } else { - fmt.Fprintf(os.Stderr, "Mode: JMESPath\n") - } - fmt.Fprintf(os.Stderr, "Bundles: %d-%d (%d total)\n", opts.start, opts.end, totalBundles) - fmt.Fprintf(os.Stderr, "Threads: %d\n", opts.threads) - fmt.Fprintf(os.Stderr, "Format: %s\n", opts.format) - if opts.limit > 0 { - fmt.Fprintf(os.Stderr, "Limit: %d\n", opts.limit) - } - fmt.Fprintf(os.Stderr, "\n") - - // FIXED: Use interface type, not pointer - var compiled jmespath.JMESPath // NOT *jmespath.JMESPath - var simpleQuery *simpleFieldExtractor - - if opts.simple { - simpleQuery = parseSimplePath(opts.expression) - } else { - var err error - compiled, err = jmespath.Compile(opts.expression) - if err != nil { - return fmt.Errorf("invalid JMESPath expression: %w", err) - } - } - - // Shared counters - var ( - totalOps int64 - matchCount int64 - bytesProcessed int64 - ) - - var progress *ui.ProgressBar - if !opts.noProgress { - progress = NewBundleProgressBar(mgr, opts.start, opts.end) - } - - jobs := make(chan int, opts.threads*2) - results := make(chan queryResult, opts.threads*2) - - var wg sync.WaitGroup - for w := 0; w < opts.threads; w++ { - wg.Add(1) - go func() { - defer wg.Done() - for bundleNum := range jobs { - select { - case <-ctx.Done(): - return - default: - } - - var res queryResult - if opts.simple { - res = processBundleQuerySimple(ctx, mgr, bundleNum, simpleQuery, opts.limit > 0, &matchCount, int64(opts.limit)) - } else { - res = processBundleQuery(ctx, mgr, bundleNum, compiled, opts.limit > 0, &matchCount, int64(opts.limit)) - } - results <- res - } - }() - } - - go func() { - wg.Wait() - close(results) - }() - - go func() { - defer close(jobs) - for bundleNum := opts.start; bundleNum <= opts.end; bundleNum++ { - select { - case jobs <- bundleNum: - case <-ctx.Done(): - return - } - } - }() - - processed := 0 - for res := range results { - processed++ - - if res.err != nil { - fmt.Fprintf(os.Stderr, "\nWarning: bundle %06d failed: %v\n", res.bundleNum, res.err) - } else { - atomic.AddInt64(&totalOps, int64(res.opsProcessed)) - atomic.AddInt64(&bytesProcessed, res.bytesProcessed) - - if opts.format != "count" { - for _, match := range res.matches { - if opts.limit > 0 && atomic.LoadInt64(&matchCount) >= int64(opts.limit) { - break - } - fmt.Println(match) - } - } - } - - if progress != nil { - progress.SetWithBytes(processed, atomic.LoadInt64(&bytesProcessed)) - } - - if opts.limit > 0 && atomic.LoadInt64(&matchCount) >= int64(opts.limit) { - break - } - } - - if progress != nil { - progress.Finish() - } - - finalMatchCount := atomic.LoadInt64(&matchCount) - finalTotalOps := atomic.LoadInt64(&totalOps) - finalBytes := atomic.LoadInt64(&bytesProcessed) - - fmt.Fprintf(os.Stderr, "\n") - if opts.format == "count" { - fmt.Println(finalMatchCount) - } - - fmt.Fprintf(os.Stderr, "✓ Query complete\n") - fmt.Fprintf(os.Stderr, " Total operations: %s\n", formatNumber(int(finalTotalOps))) - fmt.Fprintf(os.Stderr, " Matches: %s", formatNumber(int(finalMatchCount))) - if finalTotalOps > 0 { - fmt.Fprintf(os.Stderr, " (%.2f%%)", float64(finalMatchCount)/float64(finalTotalOps)*100) - } - fmt.Fprintf(os.Stderr, "\n") - if finalBytes > 0 { - fmt.Fprintf(os.Stderr, " Data processed: %s\n", formatBytes(finalBytes)) - } - - return nil -} - -type queryResult struct { - bundleNum int - matches []string - opsProcessed int - bytesProcessed int64 - err error -} - -func processBundleQuery( - ctx context.Context, - mgr BundleManager, - bundleNum int, - compiled jmespath.JMESPath, // FIXED: Interface, not pointer - checkLimit bool, - matchCount *int64, - limit int64, -) queryResult { - res := queryResult{bundleNum: bundleNum} - - bundle, err := mgr.LoadBundle(ctx, bundleNum) - if err != nil { - res.err = err - return res - } - - res.opsProcessed = len(bundle.Operations) - matches := make([]string, 0) - - for _, op := range bundle.Operations { - if checkLimit && atomic.LoadInt64(matchCount) >= limit { - break - } - - opSize := int64(len(op.RawJSON)) - if opSize == 0 { - data, _ := json.Marshal(op) - opSize = int64(len(data)) - } - res.bytesProcessed += opSize - - var opData map[string]interface{} - if len(op.RawJSON) > 0 { - if err := json.Unmarshal(op.RawJSON, &opData); err != nil { - continue - } - } else { - data, _ := json.Marshal(op) - json.Unmarshal(data, &opData) - } - - // Call Search on the interface - result, err := compiled.Search(opData) - if err != nil { - continue - } - - if result == nil { - continue - } - - atomic.AddInt64(matchCount, 1) - - resultJSON, err := json.Marshal(result) - if err != nil { - continue - } - - matches = append(matches, string(resultJSON)) - } - - res.matches = matches - return res -} - -// ============================================================================ -// SIMPLE DOT NOTATION QUERY (FAST PATH) -// ============================================================================ - -type simpleFieldExtractor struct { - path []pathSegment -} - -type pathSegment struct { - field string - arrayIndex int // -1 if not array access - isArray bool -} - -func parseSimplePath(path string) *simpleFieldExtractor { - segments := make([]pathSegment, 0) - current := "" - - for i := 0; i < len(path); i++ { - ch := path[i] - - switch ch { - case '.': - if current != "" { - segments = append(segments, pathSegment{field: current, arrayIndex: -1}) - current = "" - } - - case '[': - if current != "" { - end := i + 1 - for end < len(path) && path[end] != ']' { - end++ - } - if end < len(path) { - indexStr := path[i+1 : end] - index := 0 - fmt.Sscanf(indexStr, "%d", &index) - - segments = append(segments, pathSegment{ - field: current, - arrayIndex: index, - isArray: true, - }) - current = "" - i = end - } - } - - default: - current += string(ch) - } - } - - if current != "" { - segments = append(segments, pathSegment{field: current, arrayIndex: -1}) - } - - return &simpleFieldExtractor{path: segments} -} - -func (sfe *simpleFieldExtractor) extract(rawJSON []byte) (interface{}, bool) { - if len(sfe.path) == 0 { - return nil, false - } - - // ULTRA-FAST PATH: Single top-level field (no JSON parsing!) - if len(sfe.path) == 1 && !sfe.path[0].isArray { - field := sfe.path[0].field - return extractTopLevelField(rawJSON, field) - } - - // Nested paths: minimal parsing required - var data map[string]interface{} - if err := json.Unmarshal(rawJSON, &data); err != nil { - return nil, false - } - - return sfe.extractFromData(data, 0) -} - -// extractTopLevelField - NO JSON PARSING for simple fields (50-100x faster!) -func extractTopLevelField(rawJSON []byte, field string) (interface{}, bool) { - searchPattern := []byte(fmt.Sprintf(`"%s":`, field)) - - idx := bytes.Index(rawJSON, searchPattern) - if idx == -1 { - return nil, false - } - - valueStart := idx + len(searchPattern) - for valueStart < len(rawJSON) && (rawJSON[valueStart] == ' ' || rawJSON[valueStart] == '\t') { - valueStart++ - } - - if valueStart >= len(rawJSON) { - return nil, false - } - - switch rawJSON[valueStart] { - case '"': - // String: find closing quote - end := valueStart + 1 - for end < len(rawJSON) { - if rawJSON[end] == '"' { - if end > valueStart+1 && rawJSON[end-1] == '\\' { - end++ - continue - } - return string(rawJSON[valueStart+1 : end]), true - } - end++ - } - return nil, false - - case '{', '[': - // Complex type: need parsing - var temp map[string]interface{} - if err := json.Unmarshal(rawJSON, &temp); err != nil { - return nil, false - } - if val, ok := temp[field]; ok { - return val, true - } - return nil, false - - default: - // Primitives: number, boolean, null - end := valueStart - for end < len(rawJSON) { - ch := rawJSON[end] - if ch == ',' || ch == '}' || ch == ']' || ch == '\n' || ch == '\r' || ch == ' ' || ch == '\t' { - break - } - end++ - } - - valueStr := strings.TrimSpace(string(rawJSON[valueStart:end])) - - if valueStr == "null" { - return nil, false - } - if valueStr == "true" { - return true, true - } - if valueStr == "false" { - return false, true - } - - if num, err := strconv.ParseFloat(valueStr, 64); err == nil { - return num, true - } - - return valueStr, true - } -} - -func (sfe *simpleFieldExtractor) extractFromData(data interface{}, segmentIdx int) (interface{}, bool) { - if segmentIdx >= len(sfe.path) { - return data, true - } - - segment := sfe.path[segmentIdx] - - if m, ok := data.(map[string]interface{}); ok { - val, exists := m[segment.field] - if !exists { - return nil, false - } - - if segment.isArray { - if arr, ok := val.([]interface{}); ok { - if segment.arrayIndex >= 0 && segment.arrayIndex < len(arr) { - val = arr[segment.arrayIndex] - } else { - return nil, false - } - } else { - return nil, false - } - } - - if segmentIdx == len(sfe.path)-1 { - return val, true - } - return sfe.extractFromData(val, segmentIdx+1) - } - - if arr, ok := data.([]interface{}); ok { - if segment.isArray && segment.arrayIndex >= 0 && segment.arrayIndex < len(arr) { - val := arr[segment.arrayIndex] - if segmentIdx == len(sfe.path)-1 { - return val, true - } - return sfe.extractFromData(val, segmentIdx+1) - } - } - - return nil, false -} - -func processBundleQuerySimple( - ctx context.Context, - mgr BundleManager, - bundleNum int, - extractor *simpleFieldExtractor, - checkLimit bool, - matchCount *int64, - limit int64, -) queryResult { - res := queryResult{bundleNum: bundleNum} - - bundle, err := mgr.LoadBundle(ctx, bundleNum) - if err != nil { - res.err = err - return res - } - - res.opsProcessed = len(bundle.Operations) - matches := make([]string, 0) - - for _, op := range bundle.Operations { - if checkLimit && atomic.LoadInt64(matchCount) >= limit { - break - } - - opSize := int64(len(op.RawJSON)) - if opSize == 0 { - data, _ := json.Marshal(op) - opSize = int64(len(data)) - } - res.bytesProcessed += opSize - - var result interface{} - var found bool - - if len(op.RawJSON) > 0 { - result, found = extractor.extract(op.RawJSON) - } else { - data, _ := json.Marshal(op) - result, found = extractor.extract(data) - } - - if !found || result == nil { - continue - } - - atomic.AddInt64(matchCount, 1) - - var resultJSON []byte - if str, ok := result.(string); ok { - resultJSON = []byte(fmt.Sprintf(`"%s"`, str)) - } else { - resultJSON, _ = json.Marshal(result) - } - - matches = append(matches, string(resultJSON)) - } - - res.matches = matches - return res -} diff --git a/cmd/plcbundle/commands/rollback.go b/cmd/plcbundle/commands/rollback.go deleted file mode 100644 index 68596bf..0000000 --- a/cmd/plcbundle/commands/rollback.go +++ /dev/null @@ -1,529 +0,0 @@ -package commands - -import ( - "context" - "fmt" - "os" - "path/filepath" - "strings" - - "github.com/spf13/cobra" - "tangled.org/atscan.net/plcbundle/cmd/plcbundle/ui" - "tangled.org/atscan.net/plcbundle/internal/bundleindex" -) - -func NewRollbackCommand() *cobra.Command { - var ( - toBundle int - last int - force bool - rebuildDIDIndex bool - keepFiles bool - ) - - cmd := &cobra.Command{ - Use: "rollback [flags]", - Short: "Rollback repository to earlier state", - Long: `Rollback repository to an earlier bundle state - -Removes bundles from the repository and resets the state to a specific -bundle. This is useful for: - • Fixing corrupted bundles - • Testing and development - • Reverting unwanted changes - • Chain integrity issues - -IMPORTANT: This is a destructive operation that permanently deletes data. -Always verify your target bundle before proceeding. - -The rollback process: - 1. Validates target state - 2. Shows detailed rollback plan - 3. Requires confirmation (unless --force) - 4. Deletes bundle files - 5. Clears mempool (incompatible with new state) - 6. Updates bundle index - 7. Optionally rebuilds DID index`, - - Example: ` # Rollback TO bundle 100 (keeps 1-100, removes 101+) - plcbundle rollback --to 100 - - # Remove last 5 bundles - plcbundle rollback --last 5 - - # Rollback without confirmation - plcbundle rollback --to 50 --force - - # Rollback and rebuild DID index - plcbundle rollback --to 100 --rebuild-did-index - - # Rollback but keep bundle files (index-only) - plcbundle rollback --to 100 --keep-files`, - - RunE: func(cmd *cobra.Command, args []string) error { - verbose, _ := cmd.Root().PersistentFlags().GetBool("verbose") - - mgr, dir, err := getManager(&ManagerOptions{Cmd: cmd}) - if err != nil { - return err - } - defer mgr.Close() - - return executeRollback(cmd.Context(), mgr, dir, rollbackOptions{ - toBundle: toBundle, - last: last, - force: force, - rebuildDIDIndex: rebuildDIDIndex, - keepFiles: keepFiles, - verbose: verbose, - }) - }, - } - - // Flags - cmd.Flags().IntVar(&toBundle, "to", 0, "Rollback TO this bundle (keeps it)") - cmd.Flags().IntVar(&last, "last", 0, "Rollback last N bundles") - cmd.Flags().BoolVarP(&force, "force", "f", false, "Skip confirmation prompt") - cmd.Flags().BoolVar(&rebuildDIDIndex, "rebuild-did-index", false, "Rebuild DID index after rollback") - cmd.Flags().BoolVar(&keepFiles, "keep-files", false, "Update index only (don't delete bundle files)") - - return cmd -} - -type rollbackOptions struct { - toBundle int - last int - force bool - rebuildDIDIndex bool - keepFiles bool - verbose bool -} - -// rollbackPlan contains the calculated rollback plan -type rollbackPlan struct { - targetBundle int - toKeep []*bundleindex.BundleMetadata - toDelete []*bundleindex.BundleMetadata - deletedOps int - deletedSize int64 - hasMempool bool - hasDIDIndex bool - affectedPeriod string -} - -func executeRollback(ctx context.Context, mgr BundleManager, dir string, opts rollbackOptions) error { - // Step 1: Validate options and calculate plan - plan, err := calculateRollbackPlan(mgr, opts) - if err != nil { - return err - } - - // Step 2: Display plan and get confirmation - displayRollbackPlan(dir, plan) - - if !opts.force { - if !confirmRollback(opts.keepFiles) { - fmt.Println("Cancelled") - return nil - } - fmt.Println() - } - - // Step 3: Execute rollback - if err := performRollback(ctx, mgr, dir, plan, opts); err != nil { - return err - } - - // Step 4: Display success summary - displayRollbackSuccess(plan, opts) - - return nil -} - -// calculateRollbackPlan determines what will be affected -func calculateRollbackPlan(mgr BundleManager, opts rollbackOptions) (*rollbackPlan, error) { - // Validate options - if opts.toBundle == 0 && opts.last == 0 { - return nil, fmt.Errorf("either --to or --last must be specified") - } - - if opts.toBundle > 0 && opts.last > 0 { - return nil, fmt.Errorf("cannot use both --to and --last together") - } - - index := mgr.GetIndex() - bundles := index.GetBundles() - - if len(bundles) == 0 { - return nil, fmt.Errorf("no bundles to rollback") - } - - // Calculate target bundle - var targetBundle int - if opts.toBundle > 0 { - targetBundle = opts.toBundle - } else { - targetBundle = bundles[len(bundles)-1].BundleNumber - opts.last - } - - // Validate target - if targetBundle < 0 { - return nil, fmt.Errorf("invalid target: would result in negative bundle number") - } - - if targetBundle < 1 && len(bundles) > 0 { - return nil, fmt.Errorf("invalid rollback: would delete all bundles (use --to 0 explicitly if intended)") - } - - // Build plan - plan := &rollbackPlan{ - targetBundle: targetBundle, - toKeep: make([]*bundleindex.BundleMetadata, 0), - toDelete: make([]*bundleindex.BundleMetadata, 0), - } - - for _, meta := range bundles { - if meta.BundleNumber > targetBundle { - plan.toDelete = append(plan.toDelete, meta) - plan.deletedOps += meta.OperationCount - plan.deletedSize += meta.CompressedSize - } else { - plan.toKeep = append(plan.toKeep, meta) - } - } - - // Check if nothing to do - if len(plan.toDelete) == 0 { - return nil, fmt.Errorf("already at bundle %d (nothing to rollback)", targetBundle) - } - - // Check for mempool - mempoolStats := mgr.GetMempoolStats() - plan.hasMempool = mempoolStats["count"].(int) > 0 - - // Check for DID index - didStats := mgr.GetDIDIndexStats() - plan.hasDIDIndex = didStats["exists"].(bool) - - // Calculate affected time period - if len(plan.toDelete) > 0 { - start := plan.toDelete[0].StartTime - end := plan.toDelete[len(plan.toDelete)-1].EndTime - plan.affectedPeriod = fmt.Sprintf("%s to %s", - start.Format("2006-01-02 15:04"), - end.Format("2006-01-02 15:04")) - } - - return plan, nil -} - -// displayRollbackPlan shows what will happen -func displayRollbackPlan(dir string, plan *rollbackPlan) { - fmt.Printf("╔════════════════════════════════════════════════════════════════╗\n") - fmt.Printf("║ ROLLBACK PLAN ║\n") - fmt.Printf("╚════════════════════════════════════════════════════════════════╝\n\n") - - fmt.Printf("📁 Repository\n") - fmt.Printf(" Directory: %s\n", dir) - if len(plan.toKeep) > 0 { - fmt.Printf(" Current state: %d bundles (%06d → %06d)\n", - len(plan.toKeep)+len(plan.toDelete), - plan.toKeep[0].BundleNumber, - plan.toDelete[len(plan.toDelete)-1].BundleNumber) - } - fmt.Printf(" Target: bundle %06d\n\n", plan.targetBundle) - - fmt.Printf("🗑️ Will Delete\n") - fmt.Printf(" Bundles: %d\n", len(plan.toDelete)) - fmt.Printf(" Operations: %s\n", formatNumber(plan.deletedOps)) - fmt.Printf(" Data size: %s\n", formatBytes(plan.deletedSize)) - if plan.affectedPeriod != "" { - fmt.Printf(" Time period: %s\n", plan.affectedPeriod) - } - fmt.Printf("\n") - - // Show sample of deleted bundles - if len(plan.toDelete) > 0 { - fmt.Printf(" Bundles to delete:\n") - displayCount := min(10, len(plan.toDelete)) - for i := 0; i < displayCount; i++ { - meta := plan.toDelete[i] - fmt.Printf(" • %06d (%s, %s, %s ops)\n", - meta.BundleNumber, - meta.CreatedAt.Format("2006-01-02 15:04"), - formatBytes(meta.CompressedSize), - formatNumber(meta.OperationCount)) - } - if len(plan.toDelete) > displayCount { - fmt.Printf(" ... and %d more\n", len(plan.toDelete)-displayCount) - } - fmt.Printf("\n") - } - - // Show impacts - fmt.Printf("⚠️ Additional Impacts\n") - if plan.hasMempool { - fmt.Printf(" • Mempool will be cleared\n") - } - if plan.hasDIDIndex { - fmt.Printf(" • DID index will need rebuilding\n") - } - if len(plan.toKeep) == 0 { - fmt.Printf(" • Repository will be EMPTY after rollback\n") - } - fmt.Printf("\n") -} - -// confirmRollback prompts user for confirmation -func confirmRollback(keepFiles bool) bool { - if keepFiles { - fmt.Printf("Type 'rollback-index' to confirm (index-only mode): ") - } else { - fmt.Printf("⚠️ This will permanently DELETE data!\n") - fmt.Printf("Type 'rollback' to confirm: ") - } - - var response string - fmt.Scanln(&response) - - expectedResponse := "rollback" - if keepFiles { - expectedResponse = "rollback-index" - } - - return strings.TrimSpace(response) == expectedResponse -} - -// performRollback executes the rollback operations -func performRollback(ctx context.Context, mgr BundleManager, dir string, plan *rollbackPlan, opts rollbackOptions) error { - totalSteps := 4 - currentStep := 0 - - // Step 1: Delete bundle files (or skip if keepFiles) - currentStep++ - if !opts.keepFiles { - fmt.Printf("[%d/%d] Deleting bundle files...\n", currentStep, totalSteps) - if err := deleteBundleFiles(dir, plan.toDelete, opts.verbose); err != nil { - return fmt.Errorf("failed to delete bundles: %w", err) - } - fmt.Printf(" ✓ Deleted %d file(s)\n\n", len(plan.toDelete)) - } else { - fmt.Printf("[%d/%d] Skipping file deletion (--keep-files)...\n", currentStep, totalSteps) - fmt.Printf(" ℹ Bundle files remain on disk\n\n") - } - - // Step 2: Clear mempool - currentStep++ - fmt.Printf("[%d/%d] Clearing mempool...\n", currentStep, totalSteps) - if err := clearMempool(mgr, plan.hasMempool); err != nil { - return err - } - fmt.Printf(" ✓ Mempool cleared\n\n") - - // Step 3: Update index - currentStep++ - fmt.Printf("[%d/%d] Updating bundle index...\n", currentStep, totalSteps) - if err := updateBundleIndex(mgr, plan); err != nil { - return err - } - fmt.Printf(" ✓ Index updated (%d bundles)\n\n", len(plan.toKeep)) - - // Step 4: Handle DID index - currentStep++ - fmt.Printf("[%d/%d] DID index...\n", currentStep, totalSteps) - if err := handleDIDIndex(ctx, mgr, plan, opts); err != nil { - return err - } - - return nil -} - -// deleteBundleFiles removes bundle files from disk -func deleteBundleFiles(dir string, bundles []*bundleindex.BundleMetadata, verbose bool) error { - deletedCount := 0 - failedCount := 0 - var firstError error - - var progress *ui.ProgressBar - if !verbose && len(bundles) > 10 { - progress = ui.NewProgressBar(len(bundles)) - } - - for i, meta := range bundles { - bundlePath := filepath.Join(dir, fmt.Sprintf("%06d.jsonl.zst", meta.BundleNumber)) - - if err := os.Remove(bundlePath); err != nil { - if !os.IsNotExist(err) { - failedCount++ - if firstError == nil { - firstError = err - } - if verbose { - fmt.Printf(" ⚠️ Failed to delete %06d: %v\n", meta.BundleNumber, err) - } - continue - } - } - - deletedCount++ - - if verbose { - fmt.Printf(" ✓ Deleted %06d (%s)\n", - meta.BundleNumber, - formatBytes(meta.CompressedSize)) - } - - if progress != nil { - progress.Set(i + 1) - } - } - - if progress != nil { - progress.Finish() - } - - if failedCount > 0 { - return fmt.Errorf("failed to delete %d bundles (deleted %d successfully): %w", - failedCount, deletedCount, firstError) - } - - return nil -} - -// clearMempool clears the mempool -func clearMempool(mgr BundleManager, hasMempool bool) error { - if !hasMempool { - return nil - } - - if err := mgr.ClearMempool(); err != nil { - return fmt.Errorf("failed to clear mempool: %w", err) - } - - return nil -} - -// updateBundleIndex updates the index to reflect rollback -func updateBundleIndex(mgr BundleManager, plan *rollbackPlan) error { - index := mgr.GetIndex() - index.Rebuild(plan.toKeep) - - if err := mgr.SaveIndex(); err != nil { - return fmt.Errorf("failed to save index: %w", err) - } - - return nil -} - -// handleDIDIndex manages DID index state after rollback -func handleDIDIndex(ctx context.Context, mgr BundleManager, plan *rollbackPlan, opts rollbackOptions) error { - if !plan.hasDIDIndex { - fmt.Printf(" (no DID index)\n") - return nil - } - - if opts.rebuildDIDIndex { - fmt.Printf(" Rebuilding DID index...\n") - - bundleCount := len(plan.toKeep) - if bundleCount == 0 { - fmt.Printf(" ℹ No bundles to index\n") - return nil - } - - var progress *ui.ProgressBar - if !opts.verbose { - progress = ui.NewProgressBar(bundleCount) - } - - err := mgr.BuildDIDIndex(ctx, func(current, total int) { - if progress != nil { - progress.Set(current) - } else if current%100 == 0 || current == total { - fmt.Printf(" Progress: %d/%d (%.1f%%) \r", - current, total, float64(current)/float64(total)*100) - } - }) - - if progress != nil { - progress.Finish() - } - - if err != nil { - return fmt.Errorf("failed to rebuild DID index: %w", err) - } - - fmt.Printf(" ✓ DID index rebuilt (%d bundles)\n", bundleCount) - } else { - // Get DID index config to show which bundle it's at - didIndex := mgr.GetDIDIndex() - config := didIndex.GetConfig() - - fmt.Printf(" ⚠️ DID index is out of date\n") - if config.LastBundle > plan.targetBundle { - fmt.Printf(" Index has bundle %06d, but repository now at %06d\n", - config.LastBundle, plan.targetBundle) - } - fmt.Printf(" Run: plcbundle index build\n") - } - - return nil -} - -// displayRollbackSuccess shows the final state -func displayRollbackSuccess(plan *rollbackPlan, opts rollbackOptions) { - fmt.Printf("\n") - fmt.Printf("╔════════════════════════════════════════════════════════════════╗\n") - fmt.Printf("║ ROLLBACK COMPLETE ║\n") - fmt.Printf("╚════════════════════════════════════════════════════════════════╝\n\n") - - if len(plan.toKeep) > 0 { - lastBundle := plan.toKeep[len(plan.toKeep)-1] - firstBundle := plan.toKeep[0] - - fmt.Printf("📦 New State\n") - fmt.Printf(" Bundles: %d (%06d → %06d)\n", - len(plan.toKeep), - firstBundle.BundleNumber, - lastBundle.BundleNumber) - - totalOps := 0 - totalSize := int64(0) - for _, meta := range plan.toKeep { - totalOps += meta.OperationCount - totalSize += meta.CompressedSize - } - - fmt.Printf(" Operations: %s\n", formatNumber(totalOps)) - fmt.Printf(" Data size: %s\n", formatBytes(totalSize)) - fmt.Printf(" Chain head: %s...\n", lastBundle.Hash[:16]) - fmt.Printf(" Last updated: %s\n", lastBundle.EndTime.Format("2006-01-02 15:04:05")) - } else { - fmt.Printf("📦 New State\n") - fmt.Printf(" Repository: EMPTY (all bundles removed)\n") - if opts.keepFiles { - fmt.Printf(" Note: Bundle files remain on disk\n") - } - } - - fmt.Printf("\n") - - // Show what was removed - fmt.Printf("🗑️ Removed\n") - fmt.Printf(" Bundles: %d\n", len(plan.toDelete)) - fmt.Printf(" Operations: %s\n", formatNumber(plan.deletedOps)) - fmt.Printf(" Data freed: %s\n", formatBytes(plan.deletedSize)) - - if opts.keepFiles { - fmt.Printf(" Files: kept on disk\n") - } - - fmt.Printf("\n") - - // Next steps - if !opts.rebuildDIDIndex && plan.hasDIDIndex { - fmt.Printf("💡 Next Steps\n") - fmt.Printf(" DID index is out of date. Rebuild with:\n") - fmt.Printf(" plcbundle index build\n\n") - } -} diff --git a/cmd/plcbundle/commands/server.go b/cmd/plcbundle/commands/server.go deleted file mode 100644 index 5aa5873..0000000 --- a/cmd/plcbundle/commands/server.go +++ /dev/null @@ -1,273 +0,0 @@ -package commands - -import ( - "context" - "fmt" - "os" - "os/signal" - "syscall" - "time" - - "github.com/spf13/cobra" - "tangled.org/atscan.net/plcbundle/bundle" - internalsync "tangled.org/atscan.net/plcbundle/internal/sync" - "tangled.org/atscan.net/plcbundle/server" -) - -func NewServerCommand() *cobra.Command { - var ( - port string - host string - syncMode bool - plcURL string - syncInterval time.Duration - enableWebSocket bool - enableResolver bool - maxBundles int - ) - - cmd := &cobra.Command{ - Use: "server", - Aliases: []string{"serve"}, - Short: "Start HTTP server", - Long: `Start HTTP server to serve bundles over HTTP - -Serves bundle data over HTTP with optional live sync mode that continuously -fetches new bundles from PLC directory. - -The server provides: - - Bundle index (JSON) - - Individual bundle data (compressed or JSONL) - - WebSocket streaming (optional) - - DID resolution (optional) - - Live mempool (in sync mode) - -Sync mode (--sync) runs as a daemon, continuously fetching new bundles. -For one-time sync, use 'plcbundle sync' command instead.`, - - Example: ` # Basic server (read-only, current directory) - plcbundle server - - # Server with specific directory - plcbundle server --dir ./my-bundles - - # Live syncing server (daemon mode) - plcbundle server --sync - plcbundle server -s - - # Using alias - plcbundle serve -s - - # Custom port and host - plcbundle server --port 3000 --host 0.0.0.0 - - # Full featured server - plcbundle server -s --websocket --resolver - - # Fast sync interval - plcbundle server -s --interval 30s - - # Sync with limit (stop after 1000 bundles) - plcbundle server -s --max-bundles 1000 - - # Public server with all features - plcbundle serve -s --websocket --resolver --host 0.0.0.0 --port 80`, - - RunE: func(cmd *cobra.Command, args []string) error { - verbose, _ := cmd.Root().PersistentFlags().GetBool("verbose") - - // Server in sync mode can create repo, read-only mode cannot - mgr, dir, err := getManager(&ManagerOptions{ - Cmd: cmd, - PLCURL: plcURL, - AutoInit: syncMode, // Only auto-init if syncing - }) - if err != nil { - return err - } - defer mgr.Close() - - fmt.Printf("Starting plcbundle HTTP server...\n") - fmt.Printf(" Directory: %s\n", dir) - - // Build/verify DID index if resolver enabled - if enableResolver { - ctx := context.Background() - - built, err := mgr.EnsureDIDIndex(ctx, func(current, total int) { - if current%100 == 0 || current == total { - fmt.Printf(" Building DID index: %d/%d (%.1f%%) \r", - current, total, float64(current)/float64(total)*100) - if current == total { - fmt.Println() - } - } - }) - - if err != nil { - fmt.Fprintf(os.Stderr, "⚠️ DID index error: %v\n", err) - fmt.Fprintf(os.Stderr, " DID resolution will not be available\n\n") - } else if built { - stats := mgr.GetDIDIndexStats() - fmt.Printf(" ✓ DID index ready (%s DIDs)\n\n", - formatNumber(int(stats["total_dids"].(int64)))) - } else { - stats := mgr.GetDIDIndexStats() - fmt.Printf(" DID Index: ready (%s DIDs)\n\n", - formatNumber(int(stats["total_dids"].(int64)))) - } - } - - addr := fmt.Sprintf("%s:%s", host, port) - - // Display server info - displayServerInfo(mgr, addr, syncMode, enableWebSocket, enableResolver, plcURL, syncInterval, maxBundles) - - // Setup graceful shutdown - ctx, cancel := setupServerShutdown(mgr) - defer cancel() - - // Start sync loop if enabled (always continuous for server) - if syncMode { - go runServerSyncLoop(ctx, mgr, syncInterval, maxBundles, verbose) - } - mgr.SetQuiet(true) - - // Create and start HTTP server - serverConfig := &server.Config{ - Addr: addr, - SyncMode: syncMode, - SyncInterval: syncInterval, - EnableWebSocket: enableWebSocket, - EnableResolver: enableResolver, - Version: GetVersion(), - } - - srv := server.New(mgr, serverConfig) - - if err := srv.ListenAndServe(); err != nil { - return fmt.Errorf("server error: %w", err) - } - - return nil - }, - } - - // Server flags - cmd.Flags().StringVar(&port, "port", "8080", "HTTP server port") - cmd.Flags().StringVar(&host, "host", "127.0.0.1", "HTTP server host (use 0.0.0.0 for all interfaces)") - - // Sync mode flag (consolidated - always means daemon) - cmd.Flags().BoolVarP(&syncMode, "sync", "s", false, "Enable sync mode (run as daemon, continuously fetch from PLC)") - cmd.Flags().StringVar(&plcURL, "plc", "https://plc.directory", "PLC directory URL (for sync mode)") - cmd.Flags().DurationVar(&syncInterval, "interval", 1*time.Minute, "Sync interval (how often to check for new bundles)") - cmd.Flags().IntVar(&maxBundles, "max-bundles", 0, "Maximum bundles to fetch (0 = unlimited)") - - // Feature flags - cmd.Flags().BoolVar(&enableWebSocket, "websocket", false, "Enable WebSocket endpoint for streaming") - cmd.Flags().BoolVar(&enableResolver, "resolver", false, "Enable DID resolution endpoints") - - return cmd -} - -// setupServerShutdown sets up signal handling for server -func setupServerShutdown(mgr *bundle.Manager) (context.Context, context.CancelFunc) { - ctx, cancel := context.WithCancel(context.Background()) - - sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) - - go func() { - <-sigChan - fmt.Fprintf(os.Stderr, "\n\n⚠️ Shutdown signal received...\n") - fmt.Fprintf(os.Stderr, " Saving mempool...\n") - - if err := mgr.GetMempool().Save(); err != nil { - fmt.Fprintf(os.Stderr, " ✗ Failed to save mempool: %v\n", err) - } else { - fmt.Fprintf(os.Stderr, " ✓ Mempool saved\n") - } - - fmt.Fprintf(os.Stderr, " Closing DID index...\n") - if err := mgr.GetDIDIndex().Close(); err != nil { - fmt.Fprintf(os.Stderr, " ✗ Failed to close index: %v\n", err) - } else { - fmt.Fprintf(os.Stderr, " ✓ Index closed\n") - } - - fmt.Fprintf(os.Stderr, " ✓ Shutdown complete\n") - - cancel() - os.Exit(0) - }() - - return ctx, cancel -} - -// runServerSyncLoop runs the sync loop for server (always continuous) -func runServerSyncLoop(ctx context.Context, mgr *bundle.Manager, interval time.Duration, maxBundles int, verbose bool) { - logger := &serverLogger{} - - config := &internalsync.SyncLoopConfig{ - Interval: interval, - MaxBundles: maxBundles, - Verbose: verbose, - Logger: logger, - } - - // Server always runs continuous sync loop - if err := mgr.RunSyncLoop(ctx, config); err != nil { - if err != context.Canceled { - fmt.Fprintf(os.Stderr, "[Sync] Error: %v\n", err) - } - } -} - -// displayServerInfo shows server configuration -func displayServerInfo(mgr *bundle.Manager, addr string, syncMode, wsEnabled, resolverEnabled bool, plcURL string, syncInterval time.Duration, maxBundles int) { - fmt.Printf(" Listening: http://%s\n", addr) - - if syncMode { - fmt.Printf(" Sync: ENABLED (daemon mode)\n") - fmt.Printf(" PLC URL: %s\n", plcURL) - fmt.Printf(" Interval: %s\n", syncInterval) - if maxBundles > 0 { - fmt.Printf(" Max bundles: %d\n", maxBundles) - } - } else { - fmt.Printf(" Sync: disabled (read-only archive)\n") - fmt.Printf(" Tip: Use --sync to enable live syncing\n") - } - - if wsEnabled { - fmt.Printf(" WebSocket: ENABLED (ws://%s/ws)\n", addr) - } else { - fmt.Printf(" WebSocket: disabled\n") - } - - if resolverEnabled { - fmt.Printf(" Resolver: ENABLED (/ endpoints)\n") - stats := mgr.GetDIDIndexStats() - if stats["exists"].(bool) { - fmt.Printf(" Index: %s DIDs\n", formatNumber(int(stats["total_dids"].(int64)))) - } - } else { - fmt.Printf(" Resolver: disabled\n") - } - - bundleCount := mgr.GetIndex().Count() - fmt.Printf(" Bundles: %d available\n", bundleCount) - - fmt.Printf("\nPress Ctrl+C to stop\n\n") -} - -// serverLogger adapter -type serverLogger struct{} - -func (l *serverLogger) Printf(format string, v ...interface{}) { - fmt.Fprintf(os.Stderr, format+"\n", v...) -} - -func (l *serverLogger) Println(v ...interface{}) { - fmt.Fprintln(os.Stderr, v...) -} diff --git a/cmd/plcbundle/commands/status.go b/cmd/plcbundle/commands/status.go deleted file mode 100644 index 5972554..0000000 --- a/cmd/plcbundle/commands/status.go +++ /dev/null @@ -1,229 +0,0 @@ -package commands - -import ( - "fmt" - "strings" - "time" - - "github.com/spf13/cobra" - "tangled.org/atscan.net/plcbundle/bundle" - "tangled.org/atscan.net/plcbundle/internal/types" -) - -func NewStatusCommand() *cobra.Command { - cmd := &cobra.Command{ - Use: "status", - Aliases: []string{"info"}, - Short: "Show repository status", - Long: `Show repository status and statistics - -Displays overview of the bundle repository including bundles, -storage, timeline, mempool, and DID index status.`, - - Example: ` # Show status - plcbundle status - - # Using alias - plcbundle info`, - - RunE: func(cmd *cobra.Command, args []string) error { - mgr, dir, err := getManager(&ManagerOptions{Cmd: cmd}) - if err != nil { - return err - } - defer mgr.Close() - - return showStatus(mgr, dir) - }, - } - - return cmd -} - -func showStatus(mgr *bundle.Manager, dir string) error { - index := mgr.GetIndex() - stats := index.GetStats() - bundleCount := stats["bundle_count"].(int) - - // Header - fmt.Printf("═══════════════════════════════════════════════════════════════\n") - fmt.Printf(" plcbundle Repository Status\n") - fmt.Printf("═══════════════════════════════════════════════════════════════\n\n") - - // Location & Origin - fmt.Printf("📁 %s\n", dir) - if origin := index.Origin; origin != "" { - fmt.Printf("🌐 %s\n", origin) - } - fmt.Printf("\n") - - // Empty repository - if bundleCount == 0 { - fmt.Printf("⚠️ Empty repository (no bundles)\n\n") - fmt.Printf("Get started:\n") - fmt.Printf(" plcbundle clone Clone from remote\n") - fmt.Printf(" plcbundle sync Fetch from PLC directory\n\n") - return nil - } - - // Extract stats - firstBundle := stats["first_bundle"].(int) - lastBundle := stats["last_bundle"].(int) - totalCompressedSize := stats["total_size"].(int64) - totalUncompressedSize := stats["total_uncompressed_size"].(int64) - startTime := stats["start_time"].(time.Time) - endTime := stats["end_time"].(time.Time) - updatedAt := stats["updated_at"].(time.Time) - - // Bundles summary - ratio := float64(totalUncompressedSize) / float64(totalCompressedSize) - avgSize := totalCompressedSize / int64(bundleCount) - fmt.Printf("📦 Bundles\n") - fmt.Printf(" Count: %s (%06d → %06d)\n", - formatNumber(bundleCount), firstBundle, lastBundle) - fmt.Printf(" Compressed: %s (avg %s/bundle)\n", - formatBytes(totalCompressedSize), formatBytes(avgSize)) - fmt.Printf(" Uncompressed: %s (%.2fx compression)\n\n", - formatBytes(totalUncompressedSize), ratio) - - // Timeline - duration := endTime.Sub(startTime) - age := time.Since(updatedAt) - fmt.Printf("📅 Timeline\n") - fmt.Printf(" Coverage: %s → %s\n", - startTime.Format("2006-01-02 15:04"), endTime.Format("2006-01-02 15:04")) - fmt.Printf(" Timespan: %s\n", formatDuration(duration)) - fmt.Printf(" Last Updated: %s ago\n\n", formatDuration(age)) - - // Operations - mempoolStats := mgr.GetMempoolStats() - mempoolCount := mempoolStats["count"].(int) - bundleOpsCount := bundleCount * types.BUNDLE_SIZE - totalOps := bundleOpsCount + mempoolCount - - fmt.Printf("🔢 Operations\n") - if mempoolCount > 0 { - fmt.Printf(" Total: %s records (%s bundled + %s mempool)\n", - formatNumber(totalOps), formatNumber(bundleOpsCount), formatNumber(mempoolCount)) - } else { - fmt.Printf(" Total: %s records\n", formatNumber(bundleOpsCount)) - } - - if duration.Hours() > 0 { - opsPerHour := float64(bundleOpsCount) / duration.Hours() - fmt.Printf(" Rate: %.0f ops/hour\n", opsPerHour) - } - fmt.Printf("\n") - - // Mempool - if mempoolCount > 0 { - targetBundle := mempoolStats["target_bundle"].(int) - canCreate := mempoolStats["can_create_bundle"].(bool) - progress := float64(mempoolCount) / float64(types.BUNDLE_SIZE) * 100 - - fmt.Printf("🔄 Mempool (next bundle: %06d)\n", targetBundle) - fmt.Printf(" Operations: %s / %s (%.1f%%)\n", - formatNumber(mempoolCount), formatNumber(types.BUNDLE_SIZE), progress) - - // Progress bar - barWidth := 40 - filled := int(float64(barWidth) * progress / 100) - if filled > barWidth { - filled = barWidth - } - bar := strings.Repeat("█", filled) + strings.Repeat("░", barWidth-filled) - fmt.Printf(" [%s]\n", bar) - - if canCreate { - fmt.Printf(" ✓ Ready to create bundle\n") - } else { - remaining := types.BUNDLE_SIZE - mempoolCount - fmt.Printf(" Need %s more operations\n", formatNumber(remaining)) - } - fmt.Printf("\n") - } - - // DID Index - didStats := mgr.GetDIDIndexStats() - if didStats["exists"].(bool) { - indexedDIDs := didStats["indexed_dids"].(int64) - mempoolDIDs := didStats["mempool_dids"].(int64) - totalDIDs := didStats["total_dids"].(int64) - lastIndexedBundle := didStats["last_bundle"].(int) - shardCount := didStats["shard_count"].(int) - cachedShards := didStats["cached_shards"].(int) - - fmt.Printf("🔍 DID Index\n") - if mempoolDIDs > 0 { - fmt.Printf(" DIDs: %s (%s indexed + %s mempool)\n", - formatNumber(int(totalDIDs)), - formatNumber(int(indexedDIDs)), - formatNumber(int(mempoolDIDs))) - } else { - fmt.Printf(" DIDs: %s\n", formatNumber(int(totalDIDs))) - } - - fmt.Printf(" Shards: %d (%d cached)\n", shardCount, cachedShards) - - // Index health - if lastIndexedBundle < lastBundle { - behind := lastBundle - lastIndexedBundle - fmt.Printf(" Status: ⚠️ Behind by %d bundle", behind) - if behind > 1 { - fmt.Printf("s") - } - fmt.Printf(" (at %06d, need %06d)\n", lastIndexedBundle, lastBundle) - } else { - fmt.Printf(" Status: ✓ Up to date (bundle %06d)\n", lastIndexedBundle) - } - fmt.Printf("\n") - } - - // Gaps - gaps := index.FindGaps() - if len(gaps) > 0 { - fmt.Printf("⚠️ Missing Bundles: %d gap", len(gaps)) - if len(gaps) > 1 { - fmt.Printf("s") - } - fmt.Printf("\n") - - // Show gaps compactly - displayCount := len(gaps) - if displayCount > 15 { - displayCount = 15 - } - - fmt.Printf(" ") - for i := 0; i < displayCount; i++ { - if i > 0 { - fmt.Printf(", ") - } - if i > 0 && i%8 == 0 { - fmt.Printf("\n ") - } - fmt.Printf("%06d", gaps[i]) - } - if len(gaps) > displayCount { - fmt.Printf(", ... +%d more", len(gaps)-displayCount) - } - fmt.Printf("\n\n") - } - - // Chain Hashes - firstMeta, err := index.GetBundle(firstBundle) - if err == nil { - fmt.Printf("🔐 Chain Hashes\n") - fmt.Printf(" Root (bundle %06d):\n", firstBundle) - fmt.Printf(" %s\n", firstMeta.Hash) - - lastMeta, err := index.GetBundle(lastBundle) - if err == nil { - fmt.Printf(" Head (bundle %06d):\n", lastBundle) - fmt.Printf(" %s\n", lastMeta.Hash) - } - fmt.Printf("\n") - } - - return nil -} diff --git a/cmd/plcbundle/commands/sync.go b/cmd/plcbundle/commands/sync.go deleted file mode 100644 index 1d2670a..0000000 --- a/cmd/plcbundle/commands/sync.go +++ /dev/null @@ -1,180 +0,0 @@ -package commands - -import ( - "context" - "fmt" - "os" - "os/signal" - "syscall" - "time" - - "github.com/spf13/cobra" - "tangled.org/atscan.net/plcbundle/bundle" - internalsync "tangled.org/atscan.net/plcbundle/internal/sync" -) - -func NewSyncCommand() *cobra.Command { - var ( - plcURL string - continuous bool - interval time.Duration - maxBundles int - ) - - cmd := &cobra.Command{ - Use: "sync", - Aliases: []string{"fetch"}, - Short: "Fetch new bundles from PLC directory", - Long: `Fetch new bundles from PLC directory - -Download new operations from the PLC directory and create bundles. -Similar to 'git fetch' - updates your local repository with new data. - -By default, fetches until caught up. Use --continuous to run as a daemon -that continuously syncs at regular intervals.`, - - Example: ` # Fetch new bundles once - plcbundle sync - - # Fetch from specific directory - plcbundle sync --dir ./my-bundles - - # Run continuously (daemon mode) - plcbundle sync --continuous - - # Custom sync interval - plcbundle sync --continuous --interval 30s - - # Fetch maximum 10 bundles then stop - plcbundle sync --max-bundles 10 - - # Continuous with limit - plcbundle sync --continuous --max-bundles 100 --interval 1m - - # Verbose output - plcbundle sync --continuous -v`, - - RunE: func(cmd *cobra.Command, args []string) error { - verbose, _ := cmd.Root().PersistentFlags().GetBool("verbose") - quiet, _ := cmd.Root().PersistentFlags().GetBool("quiet") - - // Sync creates repository if missing - mgr, dir, err := getManager(&ManagerOptions{ - Cmd: cmd, - PLCURL: plcURL, - AutoInit: true, - }) - if err != nil { - return err - } - defer mgr.Close() - - if !quiet { - fmt.Printf("Syncing from: %s\n", dir) - fmt.Printf("PLC Directory: %s\n", plcURL) - if continuous { - fmt.Printf("Mode: continuous (interval: %s)\n", interval) - if maxBundles > 0 { - fmt.Printf("Max bundles: %d\n", maxBundles) - } - fmt.Printf("(Press Ctrl+C to stop)\n") - } else { - if maxBundles > 0 { - fmt.Printf("Max bundles: %d\n", maxBundles) - } - } - fmt.Printf("\n") - } - - if continuous { - return runContinuousSync(mgr, interval, maxBundles, verbose, quiet) - } - - return runSingleSync(cmd.Context(), mgr, maxBundles, verbose, quiet) - }, - } - - cmd.Flags().StringVar(&plcURL, "plc", "https://plc.directory", "PLC directory URL") - cmd.Flags().BoolVar(&continuous, "continuous", false, "Keep syncing (run as daemon)") - cmd.Flags().DurationVar(&interval, "interval", 1*time.Minute, "Sync interval for continuous mode") - cmd.Flags().IntVar(&maxBundles, "max-bundles", 0, "Maximum bundles to fetch (0 = all)") - - return cmd -} - -func runSingleSync(ctx context.Context, mgr *bundle.Manager, maxBundles int, verbose bool, quiet bool) error { - logger := &commandLogger{quiet: quiet} - - config := &internalsync.SyncLoopConfig{ - MaxBundles: maxBundles, - Verbose: verbose, - Logger: logger, - } - - // Call manager method (not internal directly) - synced, err := mgr.RunSyncOnce(ctx, config) - if err != nil { - return err - } - - if !quiet { - if synced == 0 { - fmt.Fprintf(os.Stderr, "\n✓ Already up to date\n") - } else { - fmt.Fprintf(os.Stderr, "\n✓ Sync complete: %d bundle(s) fetched\n", synced) - } - } - - return nil -} - -func runContinuousSync(mgr *bundle.Manager, interval time.Duration, maxBundles int, verbose bool, quiet bool) error { - // Setup graceful shutdown - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) - - go func() { - <-sigChan - if !quiet { - fmt.Fprintf(os.Stderr, "\n\n⚠️ Shutdown signal received...\n") - fmt.Fprintf(os.Stderr, " Saving mempool...\n") - } - - if err := mgr.GetMempool().Save(); err != nil { - fmt.Fprintf(os.Stderr, " ✗ Failed to save mempool: %v\n", err) - } else if !quiet { - fmt.Fprintf(os.Stderr, " ✓ Mempool saved\n") - } - - if !quiet { - fmt.Fprintf(os.Stderr, " Closing DID index...\n") - } - if err := mgr.GetDIDIndex().Close(); err != nil { - fmt.Fprintf(os.Stderr, " ✗ Failed to close index: %v\n", err) - } else if !quiet { - fmt.Fprintf(os.Stderr, " ✓ Index closed\n") - } - - if !quiet { - fmt.Fprintf(os.Stderr, " ✓ Shutdown complete\n") - } - - cancel() - os.Exit(0) - }() - - logger := &commandLogger{quiet: quiet} - - config := &internalsync.SyncLoopConfig{ - Interval: interval, - MaxBundles: maxBundles, - Verbose: verbose, - Logger: logger, - } - - // Call manager method (not internal directly) - return mgr.RunSyncLoop(ctx, config) -} diff --git a/cmd/plcbundle/commands/verify.go b/cmd/plcbundle/commands/verify.go deleted file mode 100644 index 4391a21..0000000 --- a/cmd/plcbundle/commands/verify.go +++ /dev/null @@ -1,450 +0,0 @@ -package commands - -import ( - "context" - "fmt" - "os" - "time" - - "github.com/spf13/cobra" - "tangled.org/atscan.net/plcbundle/bundle" - "tangled.org/atscan.net/plcbundle/cmd/plcbundle/ui" -) - -func NewVerifyCommand() *cobra.Command { - var ( - bundleNum int - rangeStr string - chain bool - parallel bool - workers int - ) - - cmd := &cobra.Command{ - Use: "verify [flags]", - Short: "Verify bundle integrity and chain", - Long: `Verify bundle integrity and cryptographic chain - -Verifies bundle file hashes, content integrity, and chain linkage. -By default, verifies the entire chain. You can verify specific bundles -or ranges using flags. - -Verification checks: - • Compressed file hash matches index - • Content hash matches index - • File sizes match metadata - • Chain parent references are correct - • Chain hash calculations are valid`, - - Example: ` # Verify entire chain - plcbundle verify - plcbundle verify --chain - - # Verify specific bundle - plcbundle verify --bundle 42 - - # Verify range of bundles - plcbundle verify --range 1-100 - - # Verbose output - plcbundle verify --chain -v - - # Parallel verification (faster for ranges) - plcbundle verify --range 1-1000 --parallel --workers 8`, - - RunE: func(cmd *cobra.Command, args []string) error { - verbose, _ := cmd.Root().PersistentFlags().GetBool("verbose") - - mgr, dir, err := getManager(&ManagerOptions{Cmd: cmd}) - if err != nil { - return err - } - defer mgr.Close() - - if !verbose { - fmt.Fprintf(os.Stderr, "Working in: %s\n\n", dir) - } - - ctx := context.Background() - - // Determine what to verify - if rangeStr != "" { - return verifyRange(ctx, mgr, rangeStr, verbose, parallel, workers) - } - - if bundleNum > 0 { - return verifySingleBundle(ctx, mgr, bundleNum, verbose) - } - - // Default: verify entire chain - return verifyChain(ctx, mgr, verbose) - }, - } - - cmd.Flags().IntVarP(&bundleNum, "bundle", "b", 0, "Verify specific bundle number") - cmd.Flags().StringVarP(&rangeStr, "range", "r", "", "Verify bundle range (e.g., '1-100')") - cmd.Flags().BoolVarP(&chain, "chain", "c", false, "Verify entire chain (default)") - cmd.Flags().BoolVar(¶llel, "parallel", false, "Use parallel verification for ranges") - cmd.Flags().IntVar(&workers, "workers", 4, "Number of parallel workers") - - return cmd -} - -func verifySingleBundle(ctx context.Context, mgr *bundle.Manager, bundleNum int, verbose bool) error { - fmt.Fprintf(os.Stderr, "Verifying bundle %06d...\n", bundleNum) - - start := time.Now() - result, err := mgr.VerifyBundle(ctx, bundleNum) - elapsed := time.Since(start) - - if err != nil { - return fmt.Errorf("verification failed: %w", err) - } - - if result.Valid { - fmt.Fprintf(os.Stderr, "✓ Bundle %06d is valid (%s)\n", bundleNum, elapsed.Round(time.Millisecond)) - if verbose { - fmt.Fprintf(os.Stderr, "\nDetails:\n") - fmt.Fprintf(os.Stderr, " File exists: %v\n", result.FileExists) - fmt.Fprintf(os.Stderr, " Hash match: %v\n", result.HashMatch) - fmt.Fprintf(os.Stderr, " Hash: %s...%s\n", result.LocalHash[:16], result.LocalHash[len(result.LocalHash)-16:]) - fmt.Fprintf(os.Stderr, " Verification time: %s\n", elapsed) - } - return nil - } - - fmt.Fprintf(os.Stderr, "✗ Bundle %06d is invalid (%s)\n", bundleNum, elapsed.Round(time.Millisecond)) - if result.Error != nil { - fmt.Fprintf(os.Stderr, " Error: %v\n", result.Error) - } - if !result.FileExists { - fmt.Fprintf(os.Stderr, " File not found\n") - } - if !result.HashMatch && result.FileExists { - fmt.Fprintf(os.Stderr, " Expected hash: %s...\n", result.ExpectedHash[:16]) - fmt.Fprintf(os.Stderr, " Actual hash: %s...\n", result.LocalHash[:16]) - } - return fmt.Errorf("bundle verification failed") -} - -func verifyChain(ctx context.Context, mgr *bundle.Manager, verbose bool) error { - index := mgr.GetIndex() - bundles := index.GetBundles() - - if len(bundles) == 0 { - fmt.Fprintf(os.Stderr, "No bundles to verify\n") - return nil - } - - fmt.Fprintf(os.Stderr, "Verifying chain of %d bundles...\n\n", len(bundles)) - - start := time.Now() - - var progress *ui.ProgressBar - if !verbose { - progress = ui.NewProgressBar(len(bundles)) - } - - verifiedCount := 0 - errorCount := 0 - var firstError error - - for i, meta := range bundles { - bundleNum := meta.BundleNumber - - if verbose { - fmt.Fprintf(os.Stderr, " Verifying bundle %06d... ", bundleNum) - } - - result, err := mgr.VerifyBundle(ctx, bundleNum) - if err != nil { - if verbose { - fmt.Fprintf(os.Stderr, "ERROR\n") - } - fmt.Fprintf(os.Stderr, "\n✗ Failed to verify bundle %06d: %v\n", bundleNum, err) - errorCount++ - if firstError == nil { - firstError = err - } - continue - } - - if !result.Valid { - if verbose { - fmt.Fprintf(os.Stderr, "INVALID\n") - } - fmt.Fprintf(os.Stderr, "\n✗ Bundle %06d hash verification failed\n", bundleNum) - if result.Error != nil { - fmt.Fprintf(os.Stderr, " Error: %v\n", result.Error) - if firstError == nil { - firstError = result.Error - } - } - errorCount++ - continue - } - - // Verify chain link - if i > 0 { - prevMeta := bundles[i-1] - if meta.Parent != prevMeta.Hash { - if verbose { - fmt.Fprintf(os.Stderr, "CHAIN BROKEN\n") - } - fmt.Fprintf(os.Stderr, "\n✗ Chain broken at bundle %06d\n", bundleNum) - fmt.Fprintf(os.Stderr, " Expected parent: %s...\n", prevMeta.Hash[:16]) - fmt.Fprintf(os.Stderr, " Actual parent: %s...\n", meta.Parent[:16]) - errorCount++ - if firstError == nil { - firstError = fmt.Errorf("chain broken at bundle %d", bundleNum) - } - continue - } - } - - if verbose { - fmt.Fprintf(os.Stderr, "✓\n") - } - verifiedCount++ - - if progress != nil { - progress.Set(i + 1) - } - } - - if progress != nil { - progress.Finish() - } - - elapsed := time.Since(start) - - fmt.Fprintf(os.Stderr, "\n") - if errorCount == 0 { - fmt.Fprintf(os.Stderr, "✓ Chain is valid (%d bundles verified)\n", verifiedCount) - fmt.Fprintf(os.Stderr, " First bundle: %06d\n", bundles[0].BundleNumber) - fmt.Fprintf(os.Stderr, " Last bundle: %06d\n", bundles[len(bundles)-1].BundleNumber) - fmt.Fprintf(os.Stderr, " Chain head: %s...%s\n", - bundles[len(bundles)-1].Hash[:16], - bundles[len(bundles)-1].Hash[len(bundles[len(bundles)-1].Hash)-16:]) - - // Timing information - fmt.Fprintf(os.Stderr, "\nPerformance:\n") - fmt.Fprintf(os.Stderr, " Time: %s\n", elapsed.Round(time.Millisecond)) - if elapsed.Seconds() > 0 { - bundlesPerSec := float64(verifiedCount) / elapsed.Seconds() - fmt.Fprintf(os.Stderr, " Throughput: %.1f bundles/sec\n", bundlesPerSec) - - // Calculate data throughput - index := mgr.GetIndex() - stats := index.GetStats() - if totalSize, ok := stats["total_size"].(int64); ok && totalSize > 0 { - mbPerSec := float64(totalSize) / elapsed.Seconds() / (1024 * 1024) - fmt.Fprintf(os.Stderr, " Data rate: %.1f MB/sec (compressed)\n", mbPerSec) - } - } - return nil - } - - fmt.Fprintf(os.Stderr, "✗ Chain verification failed\n") - fmt.Fprintf(os.Stderr, " Verified: %d/%d bundles\n", verifiedCount, len(bundles)) - fmt.Fprintf(os.Stderr, " Errors: %d\n", errorCount) - fmt.Fprintf(os.Stderr, " Time: %s\n", elapsed.Round(time.Millisecond)) - return fmt.Errorf("chain verification failed") -} - -func verifyRange(ctx context.Context, mgr *bundle.Manager, rangeStr string, verbose bool, parallel bool, workers int) error { - start, end, err := parseBundleRange(rangeStr) - if err != nil { - return err - } - - fmt.Fprintf(os.Stderr, "Verifying bundles %06d - %06d", start, end) - if parallel { - fmt.Fprintf(os.Stderr, " (parallel, %d workers)", workers) - } - fmt.Fprintf(os.Stderr, "\n\n") - - total := end - start + 1 - overallStart := time.Now() - - var verifyErr error - if parallel { - verifyErr = verifyRangeParallel(ctx, mgr, start, end, workers, verbose) - } else { - verifyErr = verifyRangeSequential(ctx, mgr, start, end, total, verbose) - } - - elapsed := time.Since(overallStart) - - // Add timing summary - fmt.Fprintf(os.Stderr, "\nPerformance:\n") - fmt.Fprintf(os.Stderr, " Time: %s\n", elapsed.Round(time.Millisecond)) - if elapsed.Seconds() > 0 { - bundlesPerSec := float64(total) / elapsed.Seconds() - fmt.Fprintf(os.Stderr, " Throughput: %.1f bundles/sec\n", bundlesPerSec) - - // Estimate average time per bundle - avgTime := elapsed / time.Duration(total) - fmt.Fprintf(os.Stderr, " Avg/bundle: %s\n", avgTime.Round(time.Millisecond)) - } - - return verifyErr -} - -func verifyRangeSequential(ctx context.Context, mgr *bundle.Manager, start, end, total int, verbose bool) error { - var progress *ui.ProgressBar - if !verbose { - progress = ui.NewProgressBar(total) - } - - verified := 0 - failed := 0 - failedBundles := []int{} - - for bundleNum := start; bundleNum <= end; bundleNum++ { - result, err := mgr.VerifyBundle(ctx, bundleNum) - - if verbose { - fmt.Fprintf(os.Stderr, "Bundle %06d: ", bundleNum) - } - - if err != nil { - if verbose { - fmt.Fprintf(os.Stderr, "ERROR - %v\n", err) - } - failed++ - failedBundles = append(failedBundles, bundleNum) - } else if !result.Valid { - if verbose { - fmt.Fprintf(os.Stderr, "INVALID - %v\n", result.Error) - } - failed++ - failedBundles = append(failedBundles, bundleNum) - } else { - if verbose { - fmt.Fprintf(os.Stderr, "✓\n") - } - verified++ - } - - if progress != nil { - progress.Set(bundleNum - start + 1) - } - } - - if progress != nil { - progress.Finish() - } - - fmt.Fprintf(os.Stderr, "\n") - if failed == 0 { - fmt.Fprintf(os.Stderr, "✓ All %d bundles verified successfully\n", verified) - return nil - } - - fmt.Fprintf(os.Stderr, "✗ Verification failed\n") - fmt.Fprintf(os.Stderr, " Verified: %d/%d\n", verified, total) - fmt.Fprintf(os.Stderr, " Failed: %d\n", failed) - - if len(failedBundles) > 0 && len(failedBundles) <= 20 { - fmt.Fprintf(os.Stderr, "\nFailed bundles: ") - for i, num := range failedBundles { - if i > 0 { - fmt.Fprintf(os.Stderr, ", ") - } - fmt.Fprintf(os.Stderr, "%06d", num) - } - fmt.Fprintf(os.Stderr, "\n") - } else if len(failedBundles) > 20 { - fmt.Fprintf(os.Stderr, "\nFailed bundles: %06d, %06d, ... and %d more\n", - failedBundles[0], failedBundles[1], len(failedBundles)-2) - } - - return fmt.Errorf("verification failed for %d bundles", failed) -} - -func verifyRangeParallel(ctx context.Context, mgr *bundle.Manager, start, end, workers int, verbose bool) error { - type job struct { - bundleNum int - } - - type result struct { - bundleNum int - valid bool - err error - } - - total := end - start + 1 - jobs := make(chan job, total) - results := make(chan result, total) - - // Start workers - for w := 0; w < workers; w++ { - go func() { - for j := range jobs { - vr, err := mgr.VerifyBundle(ctx, j.bundleNum) - results <- result{ - bundleNum: j.bundleNum, - valid: err == nil && vr.Valid, - err: err, - } - } - }() - } - - // Send jobs - for bundleNum := start; bundleNum <= end; bundleNum++ { - jobs <- job{bundleNum: bundleNum} - } - close(jobs) - - // Collect results with progress - progress := ui.NewProgressBar(total) - verified := 0 - failed := 0 - failedBundles := []int{} - - for i := 0; i < total; i++ { - res := <-results - - if res.valid { - verified++ - } else { - failed++ - failedBundles = append(failedBundles, res.bundleNum) - if verbose { - fmt.Fprintf(os.Stderr, "\nBundle %06d: FAILED - %v\n", res.bundleNum, res.err) - } - } - - progress.Set(i + 1) - } - - progress.Finish() - - fmt.Fprintf(os.Stderr, "\n") - if failed == 0 { - fmt.Fprintf(os.Stderr, "✓ All %d bundles verified successfully\n", verified) - return nil - } - - fmt.Fprintf(os.Stderr, "✗ Verification failed\n") - fmt.Fprintf(os.Stderr, " Verified: %d/%d\n", verified, total) - fmt.Fprintf(os.Stderr, " Failed: %d\n", failed) - - if len(failedBundles) > 0 && len(failedBundles) <= 20 { - fmt.Fprintf(os.Stderr, "\nFailed bundles: ") - for i, num := range failedBundles { - if i > 0 { - fmt.Fprintf(os.Stderr, ", ") - } - fmt.Fprintf(os.Stderr, "%06d", num) - } - fmt.Fprintf(os.Stderr, "\n") - } else if len(failedBundles) > 20 { - fmt.Fprintf(os.Stderr, "\nFailed bundles: %06d, %06d, ... and %d more\n", - failedBundles[0], failedBundles[1], len(failedBundles)-2) - } - - return fmt.Errorf("verification failed for %d bundles", failed) -} diff --git a/cmd/plcbundle/commands/version.go b/cmd/plcbundle/commands/version.go deleted file mode 100644 index 291ae11..0000000 --- a/cmd/plcbundle/commands/version.go +++ /dev/null @@ -1,49 +0,0 @@ -package commands - -import ( - "fmt" - "runtime/debug" -) - -var ( - version = "dev" - gitCommit = "unknown" - buildDate = "unknown" -) - -func init() { - if info, ok := debug.ReadBuildInfo(); ok { - if info.Main.Version != "" && info.Main.Version != "(devel)" { - version = info.Main.Version - } - - for _, setting := range info.Settings { - switch setting.Key { - case "vcs.revision": - if setting.Value != "" { - gitCommit = setting.Value - if len(gitCommit) > 7 { - gitCommit = gitCommit[:7] - } - } - case "vcs.time": - if setting.Value != "" { - buildDate = setting.Value - } - } - } - } -} - -// VersionCommand handles the version subcommand -func VersionCommand(args []string) error { - fmt.Printf("plcbundle version %s\n", version) - fmt.Printf(" commit: %s\n", gitCommit) - fmt.Printf(" built: %s\n", buildDate) - return nil -} - -// GetVersion returns the version string -func GetVersion() string { - return version -} diff --git a/cmd/plcbundle/main.go b/cmd/plcbundle/main.go deleted file mode 100644 index fbe2b1c..0000000 --- a/cmd/plcbundle/main.go +++ /dev/null @@ -1,170 +0,0 @@ -package main - -import ( - "fmt" - "os" - - "github.com/spf13/cobra" - "tangled.org/atscan.net/plcbundle/cmd/plcbundle/commands" -) - -func main() { - //debug.SetGCPercent(400) - - rootCmd := newRootCommand() - - if err := rootCmd.Execute(); err != nil { - os.Exit(1) - } -} - -func newRootCommand() *cobra.Command { - cmd := &cobra.Command{ - Use: "plcbundle", - Short: "DID PLC Bundle Management Tool", - Long: `plcbundle - DID PLC Bundle Management Tool - -Tool for archiving AT Protocol's DID PLC Directory operations -into immutable, cryptographically-chained bundles of 10,000 -operations each. - -Documentation: https://tangled.org/@atscan.net/plcbundle`, - - Version: GetVersion(), - - Run: func(cmd *cobra.Command, args []string) { - printRootHelp() - }, - } - - // GLOBAL FLAGS (available to all commands) - cmd.PersistentFlags().StringP("dir", "C", "", "Repository directory (default: current directory)") - cmd.PersistentFlags().BoolP("verbose", "v", false, "Show detailed output and progress") - cmd.PersistentFlags().BoolP("quiet", "q", false, "Suppress non-error output") - cmd.PersistentFlags().String("handle-resolver", "", - "Handle resolver URL (e.g., https://quickdid.smokesignal.tools)") - - // Bundle operations (root level - most common) - cmd.AddCommand(commands.NewSyncCommand()) - cmd.AddCommand(commands.NewCloneCommand()) - cmd.AddCommand(commands.NewExportCommand()) - cmd.AddCommand(commands.NewRollbackCommand()) - - // Status & info (root level) - cmd.AddCommand(commands.NewStatusCommand()) - cmd.AddCommand(commands.NewLogCommand()) - cmd.AddCommand(commands.NewLsCommand()) - cmd.AddCommand(commands.NewOpCommand()) - //cmd.AddCommand(commands.NewGapsCommand()) - cmd.AddCommand(commands.NewVerifyCommand()) - cmd.AddCommand(commands.NewDiffCommand()) - //cmd.AddCommand(commands.NewStatsCommand()) - cmd.AddCommand(commands.NewInspectCommand()) - cmd.AddCommand(commands.NewQueryCommand()) - - // Namespaced commands - cmd.AddCommand(commands.NewDIDCommand()) - cmd.AddCommand(commands.NewIndexCommand()) - cmd.AddCommand(commands.NewMempoolCommand()) - cmd.AddCommand(commands.NewDetectorCommand()) - - // Monitoring & maintenance - /*cmd.AddCommand(commands.NewWatchCommand()) - cmd.AddCommand(commands.NewHealCommand())*/ - cmd.AddCommand(commands.NewCleanCommand()) - cmd.AddCommand(commands.NewMigrateCommand()) - - // Server - cmd.AddCommand(commands.NewServerCommand()) - - // Utilities - cmd.AddCommand(newVersionCommand()) - cmd.AddCommand(newCompletionCommand()) - - return cmd -} - -func newVersionCommand() *cobra.Command { - return &cobra.Command{ - Use: "version", - Short: "Show version information", - Run: func(cmd *cobra.Command, args []string) { - cmd.Printf("plcbundle version %s\n", GetVersion()) - cmd.Printf(" commit: %s\n", getGitCommit()) - cmd.Printf(" built: %s\n", getBuildDate()) - }, - } -} - -func newCompletionCommand() *cobra.Command { - return &cobra.Command{ - Use: "completion [bash|zsh|fish|powershell]", - Short: "Generate shell completion script", - Long: `Generate shell completion script for your shell. - -To load completions: - -Bash: - $ source <(plcbundle completion bash) - - # To load automatically: - $ plcbundle completion bash > /etc/bash_completion.d/plcbundle - -Zsh: - $ plcbundle completion zsh > ~/.zsh/completion/_plcbundle - - # Add to ~/.zshrc: - fpath=(~/.zsh/completion $fpath) - -Fish: - $ plcbundle completion fish > ~/.config/fish/completions/plcbundle.fish`, - - Args: cobra.ExactArgs(1), - ValidArgs: []string{"bash", "zsh", "fish", "powershell"}, - DisableFlagsInUseLine: true, - - RunE: func(cmd *cobra.Command, args []string) error { - switch args[0] { - case "bash": - return cmd.Root().GenBashCompletion(os.Stdout) - case "zsh": - return cmd.Root().GenZshCompletion(os.Stdout) - case "fish": - return cmd.Root().GenFishCompletion(os.Stdout, true) - case "powershell": - return cmd.Root().GenPowerShellCompletion(os.Stdout) - } - return nil - }, - } -} - -func printRootHelp() { - fmt.Print(`plcbundle ` + GetVersion() + ` - DID PLC Bundle Management - -Usage: plcbundle [options] - -Main Commands: - sync Fetch new bundles from PLC - clone Clone from remote repository - status Show repository status - did resolve Resolve DID document - server Start HTTP server - -Command Groups: - Bundle: clone, sync, export, get, rollback - Status: status, log, gaps, verify, diff, stats, inspect - Ops: op - DID: did - Index: index - Tools: watch, heal, clean, mempool, detector - -Getting Started: - plcbundle clone https://plc.example.com - plcbundle sync - plcbundle status - -Run 'plcbundle help' for full documentation -Run 'plcbundle --help' for command help -`) -} diff --git a/cmd/plcbundle/ui/progress.go b/cmd/plcbundle/ui/progress.go deleted file mode 100644 index a2ddf56..0000000 --- a/cmd/plcbundle/ui/progress.go +++ /dev/null @@ -1,180 +0,0 @@ -package ui - -import ( - "fmt" - "os" - "strings" - "sync" - "time" -) - -// ProgressBar shows progress of an operation -type ProgressBar struct { - total int - current int - totalBytes int64 - currentBytes int64 - startTime time.Time - mu sync.Mutex - width int - lastPrint time.Time - showBytes bool - autoBytes bool // Auto-calculate bytes from items - bytesPerItem int64 -} - -// NewProgressBar creates a simple progress bar -func NewProgressBar(total int) *ProgressBar { - return &ProgressBar{ - total: total, - startTime: time.Now(), - width: 40, - lastPrint: time.Now(), - showBytes: false, - } -} - -// NewProgressBarWithBytes creates a progress bar that tracks bytes -func NewProgressBarWithBytes(total int, totalBytes int64) *ProgressBar { - return &ProgressBar{ - total: total, - totalBytes: totalBytes, - startTime: time.Now(), - width: 40, - lastPrint: time.Now(), - showBytes: true, - } -} - -// NewProgressBarWithBytesAuto creates a progress bar that auto-estimates bytes -// avgBytesPerItem is the estimated bytes per item (e.g., avg bundle size) -func NewProgressBarWithBytesAuto(total int, avgBytesPerItem int64) *ProgressBar { - return &ProgressBar{ - total: total, - totalBytes: int64(total) * avgBytesPerItem, - startTime: time.Now(), - width: 40, - lastPrint: time.Now(), - showBytes: true, - autoBytes: true, - bytesPerItem: avgBytesPerItem, - } -} - -// Set sets the current progress (auto-estimates bytes if enabled) -func (pb *ProgressBar) Set(current int) { - pb.mu.Lock() - defer pb.mu.Unlock() - pb.current = current - - // Auto-calculate bytes if enabled - if pb.autoBytes && pb.bytesPerItem > 0 { - pb.currentBytes = int64(current) * pb.bytesPerItem - } - - pb.print() -} - -// SetWithBytes sets progress with exact byte tracking -func (pb *ProgressBar) SetWithBytes(current int, bytesProcessed int64) { - pb.mu.Lock() - defer pb.mu.Unlock() - pb.current = current - pb.currentBytes = bytesProcessed - pb.showBytes = true - pb.print() -} - -// AddBytes increments current progress and adds bytes -func (pb *ProgressBar) AddBytes(increment int, bytes int64) { - pb.mu.Lock() - defer pb.mu.Unlock() - pb.current += increment - pb.currentBytes += bytes - pb.showBytes = true - pb.print() -} - -// Finish completes the progress bar -func (pb *ProgressBar) Finish() { - pb.mu.Lock() - defer pb.mu.Unlock() - pb.current = pb.total - pb.currentBytes = pb.totalBytes - pb.print() - fmt.Fprintf(os.Stderr, "\n") -} - -// print renders the progress bar -func (pb *ProgressBar) print() { - if time.Since(pb.lastPrint) < 100*time.Millisecond && pb.current < pb.total { - return - } - pb.lastPrint = time.Now() - - percent := 0.0 - if pb.total > 0 { - percent = float64(pb.current) / float64(pb.total) * 100 - } - - filled := 0 - if pb.total > 0 { - filled = int(float64(pb.width) * float64(pb.current) / float64(pb.total)) - if filled > pb.width { - filled = pb.width - } - } - - bar := strings.Repeat("█", filled) + strings.Repeat("░", pb.width-filled) - - elapsed := time.Since(pb.startTime) - speed := 0.0 - if elapsed.Seconds() > 0 { - speed = float64(pb.current) / elapsed.Seconds() - } - - remaining := pb.total - pb.current - var eta time.Duration - if speed > 0 && remaining > 0 { - eta = time.Duration(float64(remaining)/speed) * time.Second - } - - isComplete := pb.current >= pb.total - - if pb.showBytes && pb.currentBytes > 0 { - mbProcessed := float64(pb.currentBytes) / (1000 * 1000) - mbPerSec := 0.0 - if elapsed.Seconds() > 0 { - mbPerSec = mbProcessed / elapsed.Seconds() - } - - if isComplete { - fmt.Fprintf(os.Stderr, "\r [%s] %6.2f%% | %d/%d | %.1f/s | %.1f MB/s | Done ", - bar, percent, pb.current, pb.total, speed, mbPerSec) - } else { - fmt.Fprintf(os.Stderr, "\r [%s] %6.2f%% | %d/%d | %.1f/s | %.1f MB/s | ETA: %s ", - bar, percent, pb.current, pb.total, speed, mbPerSec, formatETA(eta)) - } - } else { - if isComplete { - fmt.Fprintf(os.Stderr, "\r [%s] %6.2f%% | %d/%d | %.1f/s | Done ", - bar, percent, pb.current, pb.total, speed) - } else { - fmt.Fprintf(os.Stderr, "\r [%s] %6.2f%% | %d/%d | %.1f/s | ETA: %s ", - bar, percent, pb.current, pb.total, speed, formatETA(eta)) - } - } -} - -func formatETA(d time.Duration) string { - if d == 0 { - return "0s" - } - if d < time.Minute { - return fmt.Sprintf("%ds", int(d.Seconds())) - } - if d < time.Hour { - return fmt.Sprintf("%dm %ds", int(d.Minutes()), int(d.Seconds())%60) - } - return fmt.Sprintf("%dh %dm", int(d.Hours()), int(d.Minutes())%60) -} diff --git a/cmd/plcbundle/version.go b/cmd/plcbundle/version.go deleted file mode 100644 index 8582e34..0000000 --- a/cmd/plcbundle/version.go +++ /dev/null @@ -1,46 +0,0 @@ -package main - -import "runtime/debug" - -var ( - version = "dev" - gitCommit = "unknown" - buildDate = "unknown" -) - -func init() { - if info, ok := debug.ReadBuildInfo(); ok { - if info.Main.Version != "" && info.Main.Version != "(devel)" { - version = info.Main.Version - } - - for _, setting := range info.Settings { - switch setting.Key { - case "vcs.revision": - if setting.Value != "" { - gitCommit = setting.Value - if len(gitCommit) > 7 { - gitCommit = gitCommit[:7] - } - } - case "vcs.time": - if setting.Value != "" { - buildDate = setting.Value - } - } - } - } -} - -// GetVersion returns the version string -func GetVersion() string { - return version -} - -func getGitCommit() string { - return gitCommit -} - -func getBuildDate() string { - return buildDate -} diff --git a/detector/builtin.go b/detector/builtin.go deleted file mode 100644 index 6656e99..0000000 --- a/detector/builtin.go +++ /dev/null @@ -1,562 +0,0 @@ -// detector/builtin.go -package detector - -import ( - "context" - "regexp" - "strings" - - "tangled.org/atscan.net/plcbundle/internal/plcclient" -) - -// NoOpDetector is an empty detector for speed testing -type NoOpDetector struct{} - -func NewNoOpDetector() *NoOpDetector { - return &NoOpDetector{} -} - -func (d *NoOpDetector) Name() string { return "noop" } -func (d *NoOpDetector) Description() string { - return "Empty detector for benchmarking (always returns no match)" -} -func (d *NoOpDetector) Version() string { return "1.0.0" } - -func (d *NoOpDetector) Detect(ctx context.Context, op plcclient.PLCOperation) (*Match, error) { - // Instant return - no work done - return nil, nil -} - -// InvalidHandleDetector detects operations with invalid handle patterns -type InvalidHandleDetector struct { - // Valid handle regex based on AT Protocol handle specification - validHandlePattern *regexp.Regexp -} - -func NewInvalidHandleDetector() *InvalidHandleDetector { - return &InvalidHandleDetector{ - // Valid handle pattern: domain segments + TLD - // Each segment: alphanumeric start/end, hyphens allowed in middle, max 63 chars per segment - // TLD must start with letter - validHandlePattern: regexp.MustCompile(`^([a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?$`), - } -} - -func (d *InvalidHandleDetector) Name() string { return "invalid_handle" } -func (d *InvalidHandleDetector) Description() string { - return "Detects operations with invalid handle patterns (underscores, invalid chars, malformed)" -} -func (d *InvalidHandleDetector) Version() string { return "1.0.0" } - -func (d *InvalidHandleDetector) Detect(ctx context.Context, op plcclient.PLCOperation) (*Match, error) { - // Parse Operation field on-demand - operation, err := op.GetOperationMap() - if err != nil { - return nil, err - } - if operation == nil { - return nil, nil - } - - if alsoKnownAs, ok := operation["alsoKnownAs"].([]interface{}); ok { - - for _, aka := range alsoKnownAs { - if str, ok := aka.(string); ok { - // Check if it's an at:// handle - if !strings.HasPrefix(str, "at://") { - continue - } - - // Extract handle (remove at:// prefix) - handle := strings.TrimPrefix(str, "at://") - - // Remove any path component (e.g., at://user.bsky.social/profile -> user.bsky.social) - if idx := strings.Index(handle, "/"); idx > 0 { - handle = handle[:idx] - } - - // Check for underscore (invalid in Bluesky handles) - if strings.Contains(handle, "_") { - return &Match{ - Reason: "underscore_in_handle", - Category: "invalid_handle", - Confidence: 0.99, - Note: "Handle contains underscore which is invalid in AT Protocol", - Metadata: map[string]interface{}{ - "invalid_handle": str, - "extracted": handle, - "violation": "underscore_character", - }, - }, nil - } - - // Check for other invalid characters (anything not alphanumeric, hyphen, or dot) - invalidChars := regexp.MustCompile(`[^a-zA-Z0-9.-]`) - if invalidChars.MatchString(handle) { - return &Match{ - Reason: "invalid_characters", - Category: "invalid_handle", - Confidence: 0.99, - Note: "Handle contains invalid characters", - Metadata: map[string]interface{}{ - "invalid_handle": str, - "extracted": handle, - "violation": "invalid_characters", - }, - }, nil - } - - // Check if handle matches valid AT Protocol pattern - if !d.validHandlePattern.MatchString(handle) { - return &Match{ - Reason: "invalid_handle_pattern", - Category: "invalid_handle", - Confidence: 0.95, - Note: "Handle does not match valid AT Protocol handle pattern", - Metadata: map[string]interface{}{ - "invalid_handle": str, - "extracted": handle, - "violation": "pattern_mismatch", - }, - }, nil - } - - // Additional checks: handle length - if len(handle) > 253 { // DNS maximum - return &Match{ - Reason: "handle_too_long", - Category: "invalid_handle", - Confidence: 0.98, - Note: "Handle exceeds maximum length (253 characters)", - Metadata: map[string]interface{}{ - "invalid_handle": str, - "extracted": handle, - "length": len(handle), - "violation": "exceeds_max_length", - }, - }, nil - } - - // Check segment lengths (each part between dots should be max 63 chars) - segments := strings.Split(handle, ".") - for i, segment := range segments { - if len(segment) == 0 { - return &Match{ - Reason: "empty_segment", - Category: "invalid_handle", - Confidence: 0.99, - Note: "Handle contains empty segment (consecutive dots)", - Metadata: map[string]interface{}{ - "invalid_handle": str, - "extracted": handle, - "violation": "empty_segment", - }, - }, nil - } - if len(segment) > 63 { - return &Match{ - Reason: "segment_too_long", - Category: "invalid_handle", - Confidence: 0.98, - Note: "Handle segment exceeds maximum length (63 characters)", - Metadata: map[string]interface{}{ - "invalid_handle": str, - "extracted": handle, - "segment": i, - "segment_value": segment, - "length": len(segment), - "violation": "segment_exceeds_max_length", - }, - }, nil - } - } - - // Check minimum segments (at least 2: subdomain.tld) - if len(segments) < 2 { - return &Match{ - Reason: "insufficient_segments", - Category: "invalid_handle", - Confidence: 0.99, - Note: "Handle must have at least 2 segments (subdomain.tld)", - Metadata: map[string]interface{}{ - "invalid_handle": str, - "extracted": handle, - "segments": len(segments), - "violation": "insufficient_segments", - }, - }, nil - } - } - } - } - - return nil, nil -} - -// AlsoKnownAsSpamDetector detects excessive/garbage alsoKnownAs entries -type AlsoKnownAsSpamDetector struct { - maxLegitimateEntries int - minGarbageLength int -} - -func NewAlsoKnownAsSpamDetector() *AlsoKnownAsSpamDetector { - return &AlsoKnownAsSpamDetector{ - maxLegitimateEntries: 3, // Normal operations have 1-3 entries - minGarbageLength: 100, // Garbage strings are very long - } -} - -func (d *AlsoKnownAsSpamDetector) Name() string { return "aka_spam" } -func (d *AlsoKnownAsSpamDetector) Description() string { - return "Detects spam through excessive or garbage alsoKnownAs entries" -} -func (d *AlsoKnownAsSpamDetector) Version() string { return "1.0.0" } - -func (d *AlsoKnownAsSpamDetector) Detect(ctx context.Context, op plcclient.PLCOperation) (*Match, error) { - // Parse Operation field on-demand - operation, err := op.GetOperationMap() - if err != nil { - return nil, err - } - if operation == nil { - return nil, nil - } - if alsoKnownAs, ok := operation["alsoKnownAs"].([]interface{}); ok { - entryCount := len(alsoKnownAs) - - // Count different types of entries - atURICount := 0 - garbageCount := 0 - var garbageExamples []string - - for _, aka := range alsoKnownAs { - if str, ok := aka.(string); ok { - if strings.HasPrefix(str, "at://") { - atURICount++ - } else if len(str) > d.minGarbageLength { - garbageCount++ - if len(garbageExamples) < 2 { - // Store first few for evidence - preview := str - if len(preview) > 50 { - preview = preview[:50] + "..." - } - garbageExamples = append(garbageExamples, preview) - } - } - } - } - - // Detection: Excessive entries - if entryCount > d.maxLegitimateEntries { - confidence := 0.80 - if garbageCount > 0 { - confidence = 0.95 // Higher confidence if garbage detected - } - - return &Match{ - Reason: "excessive_aka_entries", - Category: "spam", - Confidence: confidence, - Note: "Operation has excessive alsoKnownAs entries", - Metadata: map[string]interface{}{ - "total_entries": entryCount, - "at_uri_count": atURICount, - "garbage_count": garbageCount, - "garbage_examples": garbageExamples, - }, - }, nil - } - - // Detection: Garbage entries present (even if count is low) - if garbageCount > 0 { - return &Match{ - Reason: "garbage_aka_entries", - Category: "spam", - Confidence: 0.98, - Note: "Operation contains garbage/random strings in alsoKnownAs", - Metadata: map[string]interface{}{ - "total_entries": entryCount, - "garbage_count": garbageCount, - "garbage_examples": garbageExamples, - }, - }, nil - } - } - - return nil, nil -} - -// SpamPDSDetector detects known spam PDS endpoints -type SpamPDSDetector struct { - spamEndpoints map[string]bool - spamDomains map[string]bool -} - -func NewSpamPDSDetector() *SpamPDSDetector { - return &SpamPDSDetector{ - spamEndpoints: map[string]bool{ - "pds.trump.com": true, - // Add more as discovered - }, - spamDomains: map[string]bool{ - "trump.com": true, - "donald.trump.com": true, - // Add more as discovered - }, - } -} - -func (d *SpamPDSDetector) Name() string { return "spam_pds" } -func (d *SpamPDSDetector) Description() string { - return "Detects operations using known spam PDS endpoints and fake domain claims" -} -func (d *SpamPDSDetector) Version() string { return "1.0.0" } - -func (d *SpamPDSDetector) Detect(ctx context.Context, op plcclient.PLCOperation) (*Match, error) { - // Parse Operation field on-demand - operation, err := op.GetOperationMap() - if err != nil { - return nil, err - } - if operation == nil { - return nil, nil - } - // Check PDS endpoint - if services, ok := operation["services"].(map[string]interface{}); ok { - if pds, ok := services["atproto_pds"].(map[string]interface{}); ok { - if endpoint, ok := pds["endpoint"].(string); ok { - host := extractHost(endpoint) - - // Check if it's a known spam PDS - if d.spamEndpoints[host] { - return &Match{ - Reason: "spam_pds_endpoint", - Category: "spam", - Confidence: 0.99, - Note: "Operation uses known spam PDS endpoint", - Metadata: map[string]interface{}{ - "endpoint": endpoint, - "host": host, - }, - }, nil - } - } - } - } - - // Check for spam domain claims in alsoKnownAs - if alsoKnownAs, ok := operation["alsoKnownAs"].([]interface{}); ok { - for _, aka := range alsoKnownAs { - if str, ok := aka.(string); ok { - if !strings.HasPrefix(str, "at://") { - continue - } - - // Extract domain from at:// URI - domain := strings.TrimPrefix(str, "at://") - if idx := strings.Index(domain, "/"); idx > 0 { - domain = domain[:idx] - } - - // Check if claiming spam domain - if d.spamDomains[domain] { - return &Match{ - Reason: "fake_domain_claim", - Category: "impersonation", - Confidence: 0.99, - Note: "Operation claims known spam/fake domain", - Metadata: map[string]interface{}{ - "claimed_domain": domain, - "handle": str, - }, - }, nil - } - - // Check for subdomain patterns (like jr.donald.trump.com) - for spamDomain := range d.spamDomains { - if strings.HasSuffix(domain, "."+spamDomain) || domain == spamDomain { - return &Match{ - Reason: "fake_domain_claim", - Category: "impersonation", - Confidence: 0.99, - Note: "Operation claims domain related to known spam domain", - Metadata: map[string]interface{}{ - "claimed_domain": domain, - "spam_domain": spamDomain, - }, - }, nil - } - } - } - } - } - - return nil, nil -} - -// ServiceAbuseDetector detects operations with abused service structures -type ServiceAbuseDetector struct { - maxServiceTypeLength int - maxEndpointLength int - maxHandleLength int -} - -func NewServiceAbuseDetector() *ServiceAbuseDetector { - return &ServiceAbuseDetector{ - maxServiceTypeLength: 100, // Normal types are short (e.g., "AtprotoPersonalDataServer") - maxEndpointLength: 200, // Normal endpoints are reasonable URLs - maxHandleLength: 100, // Normal handles are short - } -} - -func (d *ServiceAbuseDetector) Name() string { return "service_abuse" } -func (d *ServiceAbuseDetector) Description() string { - return "Detects operations with abused service structures (random strings, numeric keys)" -} -func (d *ServiceAbuseDetector) Version() string { return "1.0.0" } - -func (d *ServiceAbuseDetector) Detect(ctx context.Context, op plcclient.PLCOperation) (*Match, error) { - // Parse Operation field on-demand - operation, err := op.GetOperationMap() - if err != nil { - return nil, err - } - if operation == nil { - return nil, nil - } - if services, ok := operation["services"].(map[string]interface{}); ok { - // Check for numeric service keys (spam uses "0", "1", "2" instead of proper names) - hasNumericKeys := false - numericKeyCount := 0 - - for key := range services { - // Check if key is a digit - if len(key) == 1 && key >= "0" && key <= "9" { - hasNumericKeys = true - numericKeyCount++ - } - } - - if hasNumericKeys && numericKeyCount > 1 { - return &Match{ - Reason: "numeric_service_keys", - Category: "service_abuse", - Confidence: 0.98, - Note: "Services use numeric keys instead of proper names", - Metadata: map[string]interface{}{ - "numeric_key_count": numericKeyCount, - }, - }, nil - } - - // Check each service for abuse patterns - for serviceName, serviceData := range services { - if serviceMap, ok := serviceData.(map[string]interface{}); ok { - // Check service type length - if serviceType, ok := serviceMap["type"].(string); ok { - if len(serviceType) > d.maxServiceTypeLength { - return &Match{ - Reason: "excessive_service_type_length", - Category: "service_abuse", - Confidence: 0.99, - Note: "Service type field contains excessively long random string", - Metadata: map[string]interface{}{ - "service_name": serviceName, - "type_length": len(serviceType), - "type_preview": serviceType[:50] + "...", - }, - }, nil - } - } - - // Check endpoint length - if endpoint, ok := serviceMap["endpoint"].(string); ok { - if len(endpoint) > d.maxEndpointLength { - return &Match{ - Reason: "excessive_endpoint_length", - Category: "service_abuse", - Confidence: 0.99, - Note: "Service endpoint contains excessively long random string", - Metadata: map[string]interface{}{ - "service_name": serviceName, - "endpoint_length": len(endpoint), - "endpoint_preview": endpoint[:min(100, len(endpoint))] + "...", - }, - }, nil - } - } - } - } - } - - // Check for excessively long handles in alsoKnownAs - if alsoKnownAs, ok := operation["alsoKnownAs"].([]interface{}); ok { - for _, aka := range alsoKnownAs { - if str, ok := aka.(string); ok { - if strings.HasPrefix(str, "at://") { - handle := strings.TrimPrefix(str, "at://") - if len(handle) > d.maxHandleLength { - return &Match{ - Reason: "excessive_handle_length", - Category: "service_abuse", - Confidence: 0.98, - Note: "Handle contains excessively long random string", - Metadata: map[string]interface{}{ - "handle_length": len(handle), - "handle_preview": handle[:min(50, len(handle))] + "...", - }, - }, nil - } - } - } - } - } - - // Check for empty verificationMethods (common in this spam) - if vm, ok := operation["verificationMethods"].(map[string]interface{}); ok { - if len(vm) == 0 { - // Empty verificationMethods alone isn't enough, but combined with other signals... - // Check if there are other suspicious signals - if services, ok := operation["services"].(map[string]interface{}); ok { - if len(services) > 2 { - // Multiple services + empty verificationMethods = suspicious - return &Match{ - Reason: "empty_verification_methods", - Category: "service_abuse", - Confidence: 0.85, - Note: "Empty verificationMethods with multiple services", - Metadata: map[string]interface{}{ - "service_count": len(services), - }, - }, nil - } - } - } - } - - return nil, nil -} - -// Helper function for min -func min(a, b int) int { - if a < b { - return a - } - return b -} - -// Helper functions - -func extractHost(endpoint string) string { - // Extract host from URL - endpoint = strings.TrimPrefix(endpoint, "http://") - endpoint = strings.TrimPrefix(endpoint, "https://") - if idx := strings.Index(endpoint, "/"); idx > 0 { - endpoint = endpoint[:idx] - } - if idx := strings.Index(endpoint, ":"); idx > 0 { - endpoint = endpoint[:idx] - } - return endpoint -} diff --git a/detector/detector.go b/detector/detector.go deleted file mode 100644 index 6333ecd..0000000 --- a/detector/detector.go +++ /dev/null @@ -1,63 +0,0 @@ -// detector/detector.go -package detector - -import ( - "context" - "time" - - "tangled.org/atscan.net/plcbundle/internal/plcclient" -) - -// Detector represents a spam detection algorithm -type Detector interface { - // Name returns the detector's unique identifier - Name() string - - // Description returns a human-readable description - Description() string - - // Detect analyzes an operation and returns a match result - Detect(ctx context.Context, op plcclient.PLCOperation) (*Match, error) - - // Version returns the detector version - Version() string -} - -// Match represents a positive spam detection -type Match struct { - Reason string // Short identifier (e.g., "nostr_crosspost") - Category string // Broader category (e.g., "cross_posting") - Confidence float64 // 0.0 to 1.0 - Note string // Optional human-readable explanation - Metadata map[string]interface{} // Additional context -} - -// Result represents the outcome of running a detector on an operation -type Result struct { - BundleNumber int - Position int - DID string - CID string - Match *Match // nil if no match - Error error - DetectorName string - DetectedAt time.Time -} - -// Config holds detector configuration -type Config struct { - MinConfidence float64 - Timeout time.Duration - Parallel bool - Workers int -} - -// DefaultConfig returns sensible defaults -func DefaultConfig() *Config { - return &Config{ - MinConfidence: 0.90, - Timeout: 5 * time.Second, - Parallel: true, - Workers: 4, - } -} diff --git a/detector/registry.go b/detector/registry.go deleted file mode 100644 index efaa761..0000000 --- a/detector/registry.go +++ /dev/null @@ -1,88 +0,0 @@ -// detector/registry.go -package detector - -import ( - "fmt" - "sync" -) - -// Registry manages available detectors -type Registry struct { - detectors map[string]Detector - mu sync.RWMutex -} - -// NewRegistry creates a new detector registry -func NewRegistry() *Registry { - return &Registry{ - detectors: make(map[string]Detector), - } -} - -// Register adds a detector to the registry -func (r *Registry) Register(d Detector) error { - r.mu.Lock() - defer r.mu.Unlock() - - name := d.Name() - if _, exists := r.detectors[name]; exists { - return fmt.Errorf("detector %q already registered", name) - } - - r.detectors[name] = d - return nil -} - -// Get retrieves a detector by name -func (r *Registry) Get(name string) (Detector, error) { - r.mu.RLock() - defer r.mu.RUnlock() - - d, ok := r.detectors[name] - if !ok { - return nil, fmt.Errorf("detector %q not found", name) - } - - return d, nil -} - -// List returns all registered detectors -func (r *Registry) List() []Detector { - r.mu.RLock() - defer r.mu.RUnlock() - - detectors := make([]Detector, 0, len(r.detectors)) - for _, d := range r.detectors { - detectors = append(detectors, d) - } - - return detectors -} - -// Names returns all detector names -func (r *Registry) Names() []string { - r.mu.RLock() - defer r.mu.RUnlock() - - names := make([]string, 0, len(r.detectors)) - for name := range r.detectors { - names = append(names, name) - } - - return names -} - -// DefaultRegistry returns a registry with built-in detectors -func DefaultRegistry() *Registry { - r := NewRegistry() - - r.Register(NewNoOpDetector()) - - // Register real spam detectors - r.Register(NewInvalidHandleDetector()) - r.Register(NewAlsoKnownAsSpamDetector()) - r.Register(NewSpamPDSDetector()) - r.Register(NewServiceAbuseDetector()) - - return r -} diff --git a/detector/runner.go b/detector/runner.go deleted file mode 100644 index b044f22..0000000 --- a/detector/runner.go +++ /dev/null @@ -1,216 +0,0 @@ -// detector/runner.go -package detector - -import ( - "context" - "fmt" - "sync" - "time" - - "tangled.org/atscan.net/plcbundle/bundle" - "tangled.org/atscan.net/plcbundle/internal/plcclient" -) - -// Runner executes detectors against operations -type Runner struct { - registry *Registry - config *Config - logger Logger -} - -type Logger interface { - Printf(format string, v ...interface{}) -} - -// NewRunner creates a new detector runner -func NewRunner(registry *Registry, config *Config, logger Logger) *Runner { - if config == nil { - config = DefaultConfig() - } - return &Runner{ - registry: registry, - config: config, - logger: logger, - } -} - -// RunOnBundle runs detector(s) on all operations in a bundle -func (r *Runner) RunOnBundle(ctx context.Context, detectorName string, b *bundle.Bundle) ([]*Result, error) { - detector, err := r.registry.Get(detectorName) - if err != nil { - return nil, err - } - - var results []*Result - - if r.config.Parallel { - results = r.runParallel(ctx, detector, b) - } else { - results = r.runSequential(ctx, detector, b) - } - - // Filter by minimum confidence - filtered := make([]*Result, 0) - for _, res := range results { - if res.Match != nil && res.Match.Confidence >= r.config.MinConfidence { - filtered = append(filtered, res) - } - } - - return filtered, nil -} - -func (r *Runner) runSequential(ctx context.Context, detector Detector, b *bundle.Bundle) []*Result { - results := make([]*Result, 0) - - for pos, op := range b.Operations { - select { - case <-ctx.Done(): - return results - default: - } - - result := r.detectOne(ctx, detector, b.BundleNumber, pos, op) - if result.Match != nil || result.Error != nil { - results = append(results, result) - } - } - - return results -} - -func (r *Runner) runParallel(ctx context.Context, detector Detector, b *bundle.Bundle) []*Result { - type job struct { - pos int - op plcclient.PLCOperation - } - - jobs := make(chan job, len(b.Operations)) - resultsChan := make(chan *Result, len(b.Operations)) - - // Start workers - var wg sync.WaitGroup - for i := 0; i < r.config.Workers; i++ { - wg.Add(1) - go func() { - defer wg.Done() - for j := range jobs { - select { - case <-ctx.Done(): - return - default: - } - - result := r.detectOne(ctx, detector, b.BundleNumber, j.pos, j.op) - if result.Match != nil || result.Error != nil { - resultsChan <- result - } - } - }() - } - - // Send jobs - for pos, op := range b.Operations { - jobs <- job{pos: pos, op: op} - } - close(jobs) - - // Wait for completion - go func() { - wg.Wait() - close(resultsChan) - }() - - // Collect results - results := make([]*Result, 0) - for result := range resultsChan { - results = append(results, result) - } - - return results -} - -func (r *Runner) detectOne(ctx context.Context, detector Detector, bundleNum, pos int, op plcclient.PLCOperation) *Result { - // Create timeout context - detectCtx, cancel := context.WithTimeout(ctx, r.config.Timeout) - defer cancel() - - result := &Result{ - BundleNumber: bundleNum, - Position: pos, - DID: op.DID, - CID: op.CID, - DetectorName: detector.Name(), - DetectedAt: time.Now(), - } - - match, err := detector.Detect(detectCtx, op) - result.Match = match - result.Error = err - - return result -} - -// RunMultipleDetectors runs multiple detectors on a bundle -func (r *Runner) RunMultipleDetectors(ctx context.Context, detectorNames []string, b *bundle.Bundle) (map[string][]*Result, error) { - allResults := make(map[string][]*Result) - - for _, name := range detectorNames { - results, err := r.RunOnBundle(ctx, name, b) - if err != nil { - return nil, fmt.Errorf("detector %s failed: %w", name, err) - } - allResults[name] = results - } - - return allResults, nil -} - -// Stats represents detection statistics -type Stats struct { - TotalOperations int - MatchedCount int - MatchRate float64 - ByReason map[string]int - ByCategory map[string]int - ByConfidence map[string]int // 0.9-1.0, 0.8-0.9, etc. -} - -// CalculateStats computes statistics from results -func CalculateStats(results []*Result, totalOps int) *Stats { - stats := &Stats{ - TotalOperations: totalOps, - MatchedCount: len(results), - ByReason: make(map[string]int), - ByCategory: make(map[string]int), - ByConfidence: make(map[string]int), - } - - if totalOps > 0 { - stats.MatchRate = float64(len(results)) / float64(totalOps) - } - - for _, res := range results { - if res.Match == nil { - continue - } - - stats.ByReason[res.Match.Reason]++ - stats.ByCategory[res.Match.Category]++ - - // Confidence buckets - conf := res.Match.Confidence - switch { - case conf >= 0.95: - stats.ByConfidence["0.95-1.00"]++ - case conf >= 0.90: - stats.ByConfidence["0.90-0.95"]++ - case conf >= 0.85: - stats.ByConfidence["0.85-0.90"]++ - default: - stats.ByConfidence["0.00-0.85"]++ - } - } - - return stats -} diff --git a/detector/script.go b/detector/script.go deleted file mode 100644 index 568635b..0000000 --- a/detector/script.go +++ /dev/null @@ -1,220 +0,0 @@ -// detector/script.go -package detector - -import ( - "bufio" - "context" - "fmt" - "net" - "os" - "os/exec" - "path/filepath" - "strings" - "sync" - "time" - - "github.com/goccy/go-json" - "tangled.org/atscan.net/plcbundle/internal/plcclient" -) - -// ScriptDetector runs a JavaScript detector via Unix socket -type ScriptDetector struct { - name string - scriptPath string - socketPath string - serverCmd *exec.Cmd - conn net.Conn - writer *bufio.Writer - reader *bufio.Reader - mu sync.Mutex -} - -func NewScriptDetector(scriptPath string) (*ScriptDetector, error) { - if _, err := exec.LookPath("bun"); err != nil { - return nil, fmt.Errorf("bun runtime not found in PATH") - } - - absPath, err := filepath.Abs(scriptPath) - if err != nil { - return nil, fmt.Errorf("invalid script path: %w", err) - } - - name := strings.TrimSuffix(filepath.Base(scriptPath), filepath.Ext(scriptPath)) - socketPath := filepath.Join(os.TempDir(), fmt.Sprintf("detector-%d-%s.sock", os.Getpid(), name)) - - sd := &ScriptDetector{ - name: name, - scriptPath: absPath, - socketPath: socketPath, - } - - if err := sd.startServer(); err != nil { - return nil, err - } - - return sd, nil -} - -func (d *ScriptDetector) Name() string { return d.name } -func (d *ScriptDetector) Description() string { return "JavaScript detector: " + d.name } -func (d *ScriptDetector) Version() string { return "1.0.0" } - -func (d *ScriptDetector) startServer() error { - userCode, err := os.ReadFile(d.scriptPath) - if err != nil { - return fmt.Errorf("failed to read script: %w", err) - } - - wrapperScript := d.createSocketWrapper(string(userCode)) - os.Remove(d.socketPath) - - d.serverCmd = exec.Command("bun", "run", "-", d.socketPath) - d.serverCmd.Stdout = os.Stderr - d.serverCmd.Stderr = os.Stderr - - stdin, err := d.serverCmd.StdinPipe() - if err != nil { - return fmt.Errorf("failed to create stdin pipe: %w", err) - } - - if err := d.serverCmd.Start(); err != nil { - return fmt.Errorf("failed to start server: %w", err) - } - - stdin.Write([]byte(wrapperScript)) - stdin.Close() - - if err := d.connectToServer(); err != nil { - d.serverCmd.Process.Kill() - os.Remove(d.socketPath) - return err - } - - return nil -} - -func (d *ScriptDetector) createSocketWrapper(userCode string) string { - return fmt.Sprintf(`// Auto-generated socket server wrapper -// User's detect function -%s - -// Unix socket server -const socketPath = process.argv[2]; - -try { - await Bun.file(socketPath).unlink(); -} catch {} - -const server = Bun.listen({ - unix: socketPath, - socket: { - data(socket, data) { - try { - const operation = JSON.parse(data.toString()); - const labels = detect({ op: operation }) || []; - socket.write(JSON.stringify({ labels }) + '\n'); - } catch (error) { - socket.write(JSON.stringify({ labels: [], error: error.message }) + '\n'); - } - }, - error(socket, error) {}, - close(socket) {} - } -}); - -console.error('Detector server ready on socket:', socketPath); -`, userCode) -} - -func (d *ScriptDetector) connectToServer() error { - maxRetries := 50 - for i := 0; i < maxRetries; i++ { - conn, err := net.Dial("unix", d.socketPath) - if err == nil { - d.conn = conn - d.writer = bufio.NewWriter(conn) - d.reader = bufio.NewReader(conn) - return nil - } - time.Sleep(50 * time.Millisecond) - } - return fmt.Errorf("failed to connect to socket within timeout") -} - -func (d *ScriptDetector) Detect(ctx context.Context, op plcclient.PLCOperation) (*Match, error) { - if d.conn == nil { - return nil, fmt.Errorf("not connected to server") - } - - // LOCK for entire socket communication - d.mu.Lock() - defer d.mu.Unlock() - - // Use RawJSON directly - data := op.RawJSON - if len(data) == 0 { - var err error - data, err = json.Marshal(op) - if err != nil { - return nil, fmt.Errorf("failed to serialize operation: %w", err) - } - } - - if _, err := d.writer.Write(data); err != nil { - return nil, fmt.Errorf("failed to write to socket: %w", err) - } - if _, err := d.writer.WriteString("\n"); err != nil { - return nil, fmt.Errorf("failed to write newline: %w", err) - } - if err := d.writer.Flush(); err != nil { - return nil, fmt.Errorf("failed to flush: %w", err) - } - - line, err := d.reader.ReadString('\n') - if err != nil { - return nil, fmt.Errorf("failed to read response: %w", err) - } - - var result struct { - Labels []string `json:"labels"` - Error string `json:"error,omitempty"` - } - - if err := json.Unmarshal([]byte(line), &result); err != nil { - return nil, fmt.Errorf("failed to parse response: %w", err) - } - - if result.Error != "" { - return nil, fmt.Errorf("detector error: %s", result.Error) - } - - if len(result.Labels) == 0 { - return nil, nil - } - - return &Match{ - Reason: strings.Join(result.Labels, "_"), - Category: "custom", - Confidence: 0.95, - Note: fmt.Sprintf("Labels: %s", strings.Join(result.Labels, ", ")), - Metadata: map[string]interface{}{ - "labels": result.Labels, - "detector": d.name, - }, - }, nil -} - -func (d *ScriptDetector) Close() error { - if d.conn != nil { - d.conn.Close() - d.conn = nil - } - - if d.serverCmd != nil && d.serverCmd.Process != nil { - d.serverCmd.Process.Kill() - d.serverCmd.Wait() - } - - os.Remove(d.socketPath) - return nil -} diff --git a/docker-compose.yaml b/docker-compose.yaml deleted file mode 100644 index d27cf7e..0000000 --- a/docker-compose.yaml +++ /dev/null @@ -1,65 +0,0 @@ -networks: - plcbundle: - -services: - plcbundle: - build: . - image: atscan/plcbundle:latest - container_name: plcbundle - restart: unless-stopped - - ports: - - "${HTTP_PORT:-8080}:8080" - - volumes: - - ${DATA_DIR:-./data}:/data - - environment: - - TZ=${TZ:-UTC} - - command: > - serve - --host 0.0.0.0 - --port 8080 - --sync - --websocket - - networks: - - plcbundle - - # watchtower for automatic updates - watchtower: - container_name: watchtower - image: containrrr/watchtower:latest - network_mode: host - volumes: - - type: bind - source: /var/run/docker.sock - target: /var/run/docker.sock - restart: unless-stopped - environment: - WATCHTOWER_CLEANUP: true - WATCHTOWER_SCHEDULE: "@midnight" - -# Optional: Caddy reverse proxy -# -# services: -# caddy: -# image: caddy:2-alpine -# restart: unless-stopped -# ports: -# - "80:80" -# - "443:443" -# - "443:443/udp" -# volumes: -# - caddy_data:/data -# environment: -# - DOMAIN=${DOMAIN:-localhost} -# command: caddy reverse-proxy --from ${DOMAIN:-localhost} --to plcbundle:8080 -# depends_on: -# - plcbundle -# networks: -# - plcbundle -# -# volumes: -# caddy_data: diff --git a/docs/cli.md b/docs/cli.md deleted file mode 100644 index a8d0651..0000000 --- a/docs/cli.md +++ /dev/null @@ -1,3 +0,0 @@ -# CLI Guide - -TODO \ No newline at end of file diff --git a/docs/contributing.md b/docs/contributing.md new file mode 100644 index 0000000..65e1498 --- /dev/null +++ b/docs/contributing.md @@ -0,0 +1,34 @@ +# Contributing to plcbundle + +Thank you for your interest in contributing to plcbundle! We welcome contributions from the community. + +## How to Contribute + +### Reporting Bugs + +- Check if the bug has already been reported in the [Issues](../../issues) +- If not, create a new issue with a clear title and description +- Include steps to reproduce, expected behavior, and actual behavior +- Add relevant labels and version information + +### Suggesting Features + +- Check if the feature has already been requested +- Create a new issue with the `enhancement` label +- Clearly describe the feature and its use case +- Explain why this feature would be useful + +## Questions? + +If you have questions, feel free to: +- Open an issue for discussion +- Reach out to maintainers +- Check existing documentation + +## License + +By contributing, you agree that your contributions will be licensed under the same license as the project (MIT License). + +--- + +Thank you for contributing! 🎉 \ No newline at end of file diff --git a/docs/getting-started.m b/docs/getting-started.m new file mode 100644 index 0000000..8315b43 --- /dev/null +++ b/docs/getting-started.m @@ -0,0 +1,3 @@ +# Getting Started with plcbundle + +TODO \ No newline at end of file diff --git a/docs/library.md b/docs/library.md deleted file mode 100644 index c433a8b..0000000 --- a/docs/library.md +++ /dev/null @@ -1,17 +0,0 @@ -# Library Guide - -A practical guide to using plcbundle as a Go library in your applications. - -## Table of Contents - -- [Getting Started](#getting-started) -- [Core Concepts](#core-concepts) -- [Common Patterns](#common-patterns) -- [Building Applications](#building-applications) -- [Advanced Usage](#advanced-usage) -- [Best Practices](#best-practices) -- [API Reference](#api-reference) - ---- - -TODO \ No newline at end of file diff --git a/go.mod b/go.mod deleted file mode 100644 index 46ebe56..0000000 --- a/go.mod +++ /dev/null @@ -1,19 +0,0 @@ -module tangled.org/atscan.net/plcbundle - -go 1.25 - -require ( - github.com/goccy/go-json v0.10.5 - github.com/gorilla/websocket v1.5.3 - github.com/jmespath-community/go-jmespath v1.1.1 - github.com/spf13/cobra v1.10.1 - github.com/valyala/gozstd v1.23.2 - golang.org/x/sys v0.38.0 - golang.org/x/term v0.36.0 -) - -require ( - github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/spf13/pflag v1.0.9 // indirect - golang.org/x/exp v0.0.0-20230314191032-db074128a8ec // indirect -) diff --git a/go.sum b/go.sum deleted file mode 100644 index 58f3ab0..0000000 --- a/go.sum +++ /dev/null @@ -1,31 +0,0 @@ -github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= -github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= -github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= -github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/jmespath-community/go-jmespath v1.1.1 h1:bFikPhsi/FdmlZhVgSCd2jj1e7G/rw+zyQfyg5UF+L4= -github.com/jmespath-community/go-jmespath v1.1.1/go.mod h1:4gOyFJsR/Gk+05RgTKYrifT7tBPWD8Lubtb5jRrfy9I= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= -github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0= -github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= -github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/valyala/gozstd v1.23.2 h1:S3rRsskaDvBCM2XJzQFYIDAO6txxmvTc1arA/9Wgi9o= -github.com/valyala/gozstd v1.23.2/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ= -golang.org/x/exp v0.0.0-20230314191032-db074128a8ec h1:pAv+d8BM2JNnNctsLJ6nnZ6NqXT8N4+eauvZSb3P0I0= -golang.org/x/exp v0.0.0-20230314191032-db074128a8ec/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q= -golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/bundleindex/index.go b/internal/bundleindex/index.go deleted file mode 100644 index 87397f2..0000000 --- a/internal/bundleindex/index.go +++ /dev/null @@ -1,350 +0,0 @@ -package bundleindex - -import ( - "fmt" - "os" - "sort" - "sync" - "time" - - "github.com/goccy/go-json" -) - -const ( - // INDEX_FILE is the default index filename - INDEX_FILE = "plc_bundles.json" - - // INDEX_VERSION is the current index format version - INDEX_VERSION = "1.0" -) - -// Index represents the JSON index file -type Index struct { - Version string `json:"version"` - Origin string `json:"origin"` - LastBundle int `json:"last_bundle"` - UpdatedAt time.Time `json:"updated_at"` - TotalSize int64 `json:"total_size_bytes"` - TotalUncompressedSize int64 `json:"total_uncompressed_size_bytes"` - Bundles []*BundleMetadata `json:"bundles"` - - mu sync.RWMutex `json:"-"` -} - -// NewIndex creates a new empty index -func NewIndex(origin string) *Index { - return &Index{ - Version: INDEX_VERSION, - Origin: origin, - Bundles: make([]*BundleMetadata, 0), - UpdatedAt: time.Now().UTC(), - } -} - -// LoadIndex loads an index from a file -func LoadIndex(path string) (*Index, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("failed to read index file: %w", err) - } - - var idx Index - if err := json.Unmarshal(data, &idx); err != nil { - return nil, fmt.Errorf("failed to parse index file: %w", err) - } - - // Validate version - if idx.Version != INDEX_VERSION { - return nil, fmt.Errorf("unsupported index version: %s (expected %s)", idx.Version, INDEX_VERSION) - } - - // Recalculate derived fields (handles new fields added to Index) - idx.recalculate() - - return &idx, nil -} - -// Save saves the index to a file -func (idx *Index) Save(path string) error { - idx.mu.Lock() - defer idx.mu.Unlock() - - idx.UpdatedAt = time.Now().UTC() - - data, err := json.MarshalIndent(idx, "", " ") - if err != nil { - return fmt.Errorf("failed to marshal index: %w", err) - } - - // Write atomically (write to temp file, then rename) - tempPath := path + ".tmp" - if err := os.WriteFile(tempPath, data, 0644); err != nil { - return fmt.Errorf("failed to write temp file: %w", err) - } - - if err := os.Rename(tempPath, path); err != nil { - os.Remove(tempPath) // Clean up temp file - return fmt.Errorf("failed to rename temp file: %w", err) - } - - return nil -} - -// AddBundle adds a bundle to the index -func (idx *Index) AddBundle(meta *BundleMetadata) { - idx.mu.Lock() - defer idx.mu.Unlock() - - // Check if bundle already exists - for i, existing := range idx.Bundles { - if existing.BundleNumber == meta.BundleNumber { - // Update existing - idx.Bundles[i] = meta - idx.recalculate() - return - } - } - - // Add new bundle - idx.Bundles = append(idx.Bundles, meta) - idx.sort() - idx.recalculate() -} - -// GetBundle retrieves a bundle metadata by number -func (idx *Index) GetBundle(bundleNumber int) (*BundleMetadata, error) { - idx.mu.RLock() - defer idx.mu.RUnlock() - - for _, meta := range idx.Bundles { - if meta.BundleNumber == bundleNumber { - return meta, nil - } - } - - return nil, fmt.Errorf("bundle %d not found in index", bundleNumber) -} - -// GetLastBundle returns the metadata of the last bundle -func (idx *Index) GetLastBundle() *BundleMetadata { - idx.mu.RLock() - defer idx.mu.RUnlock() - - if len(idx.Bundles) == 0 { - return nil - } - - return idx.Bundles[len(idx.Bundles)-1] -} - -// GetBundles returns all bundle metadata -func (idx *Index) GetBundles() []*BundleMetadata { - idx.mu.RLock() - defer idx.mu.RUnlock() - - // Return a copy - result := make([]*BundleMetadata, len(idx.Bundles)) - copy(result, idx.Bundles) - return result -} - -// GetBundleRange returns bundles in a specific range -func (idx *Index) GetBundleRange(start, end int) []*BundleMetadata { - idx.mu.RLock() - defer idx.mu.RUnlock() - - var result []*BundleMetadata - for _, meta := range idx.Bundles { - if meta.BundleNumber >= start && meta.BundleNumber <= end { - result = append(result, meta) - } - } - return result -} - -// Count returns the number of bundles in the index -func (idx *Index) Count() int { - idx.mu.RLock() - defer idx.mu.RUnlock() - return len(idx.Bundles) -} - -// GetStats returns statistics about the index -func (idx *Index) GetStats() map[string]interface{} { - idx.mu.RLock() - defer idx.mu.RUnlock() - - if len(idx.Bundles) == 0 { - return map[string]interface{}{ - "bundle_count": 0, - "total_size": 0, - "total_uncompressed_size": 0, - } - } - - first := idx.Bundles[0] - last := idx.Bundles[len(idx.Bundles)-1] - - return map[string]interface{}{ - "bundle_count": len(idx.Bundles), - "first_bundle": first.BundleNumber, - "last_bundle": last.BundleNumber, - "total_size": idx.TotalSize, - "total_uncompressed_size": idx.TotalUncompressedSize, - "start_time": first.StartTime, - "end_time": last.EndTime, - "updated_at": idx.UpdatedAt, - "gaps": len(idx.FindGaps()), - } -} - -// sort sorts bundles by bundle number -func (idx *Index) sort() { - sort.Slice(idx.Bundles, func(i, j int) bool { - return idx.Bundles[i].BundleNumber < idx.Bundles[j].BundleNumber - }) -} - -// recalculate recalculates derived fields (called after modifications) -func (idx *Index) recalculate() { - if len(idx.Bundles) == 0 { - idx.LastBundle = 0 - idx.TotalSize = 0 - idx.TotalUncompressedSize = 0 - return - } - - // Find last bundle - maxBundle := 0 - totalSize := int64(0) - totalUncompressed := int64(0) - - for _, meta := range idx.Bundles { - if meta.BundleNumber > maxBundle { - maxBundle = meta.BundleNumber - } - totalSize += meta.CompressedSize - totalUncompressed += meta.UncompressedSize - } - - idx.LastBundle = maxBundle - idx.TotalSize = totalSize - idx.TotalUncompressedSize = totalUncompressed -} - -// Rebuild rebuilds the index from bundle metadata -func (idx *Index) Rebuild(bundles []*BundleMetadata) { - idx.mu.Lock() - defer idx.mu.Unlock() - - idx.Bundles = bundles - idx.sort() - idx.recalculate() - idx.UpdatedAt = time.Now().UTC() -} - -// Clear clears all bundles from the index -func (idx *Index) Clear() { - idx.mu.Lock() - defer idx.mu.Unlock() - - idx.Bundles = make([]*BundleMetadata, 0) - idx.LastBundle = 0 - idx.TotalSize = 0 - idx.UpdatedAt = time.Now().UTC() -} - -// FindGaps finds missing bundle numbers in the sequence -func (idx *Index) FindGaps() []int { - idx.mu.RLock() - defer idx.mu.RUnlock() - - if len(idx.Bundles) == 0 { - return nil - } - - var gaps []int - first := idx.Bundles[0].BundleNumber - last := idx.Bundles[len(idx.Bundles)-1].BundleNumber - - bundleMap := make(map[int]bool) - for _, meta := range idx.Bundles { - bundleMap[meta.BundleNumber] = true - } - - for i := first; i <= last; i++ { - if !bundleMap[i] { - gaps = append(gaps, i) - } - } - - return gaps -} - -// Logger interface for bundleindex -type Logger interface { - Printf(format string, v ...interface{}) -} - -// UpdateFromRemote updates the index with metadata from remote bundles -// fileExists is a callback to check if bundle file exists locally -func (idx *Index) UpdateFromRemote( - bundleNumbers []int, - remoteMeta map[int]*BundleMetadata, - fileExists func(bundleNum int) bool, - verbose bool, - logger Logger, -) error { - if len(bundleNumbers) == 0 { - return nil - } - - idx.mu.Lock() - defer idx.mu.Unlock() - - addedCount := 0 - for _, num := range bundleNumbers { - meta, exists := remoteMeta[num] - if !exists { - continue - } - - // Verify the file exists locally - if !fileExists(num) { - if logger != nil && verbose { - logger.Printf("Warning: bundle %06d not found locally, skipping", num) - } - continue - } - - // Add to index (uses existing AddBundle logic but we're already locked) - // Check if bundle already exists - found := false - for i, existing := range idx.Bundles { - if existing.BundleNumber == num { - // Update existing - idx.Bundles[i] = meta - found = true - break - } - } - - if !found { - // Add new bundle - idx.Bundles = append(idx.Bundles, meta) - } - - addedCount++ - - if logger != nil && verbose { - logger.Printf("Added bundle %06d to index", num) - } - } - - if addedCount > 0 { - idx.sort() - idx.recalculate() - } - - return nil -} diff --git a/internal/bundleindex/index_test.go b/internal/bundleindex/index_test.go deleted file mode 100644 index 20a2b4c..0000000 --- a/internal/bundleindex/index_test.go +++ /dev/null @@ -1,844 +0,0 @@ -package bundleindex_test - -import ( - "fmt" - "os" - "path/filepath" - "sync" - "testing" - "time" - - "tangled.org/atscan.net/plcbundle/internal/bundleindex" - "tangled.org/atscan.net/plcbundle/internal/types" -) - -type testLogger struct { - t *testing.T -} - -func (l *testLogger) Printf(format string, v ...interface{}) { - l.t.Logf(format, v...) -} - -func (l *testLogger) Println(v ...interface{}) { - l.t.Log(v...) -} - -// ==================================================================================== -// INDEX CREATION & BASIC OPERATIONS -// ==================================================================================== - -func TestIndexCreation(t *testing.T) { - t.Run("NewIndex", func(t *testing.T) { - idx := bundleindex.NewIndex("https://plc.directory") - - if idx == nil { - t.Fatal("NewIndex returned nil") - } - - if idx.Version != types.INDEX_VERSION { - t.Errorf("version mismatch: got %s, want %s", idx.Version, types.INDEX_VERSION) - } - - if idx.Origin != "https://plc.directory" { - t.Errorf("origin mismatch: got %s", idx.Origin) - } - - if idx.Count() != 0 { - t.Error("new index should be empty") - } - }) - - t.Run("NewIndex_EmptyOrigin", func(t *testing.T) { - idx := bundleindex.NewIndex("") - - if idx.Origin != "" { - t.Error("should allow empty origin") - } - }) -} - -func TestIndexAddBundle(t *testing.T) { - t.Run("AddSingleBundle", func(t *testing.T) { - idx := bundleindex.NewIndex("test-origin") - - meta := &bundleindex.BundleMetadata{ - BundleNumber: 1, - StartTime: time.Now(), - EndTime: time.Now().Add(time.Hour), - OperationCount: types.BUNDLE_SIZE, - DIDCount: 1000, - Hash: "hash123", - ContentHash: "content123", - CompressedHash: "compressed123", - CompressedSize: 1024, - UncompressedSize: 5120, - } - - idx.AddBundle(meta) - - if idx.Count() != 1 { - t.Errorf("count should be 1, got %d", idx.Count()) - } - - retrieved, err := idx.GetBundle(1) - if err != nil { - t.Fatalf("GetBundle failed: %v", err) - } - - if retrieved.Hash != "hash123" { - t.Error("hash mismatch after retrieval") - } - }) - - t.Run("AddMultipleBundles_AutoSort", func(t *testing.T) { - idx := bundleindex.NewIndex("test-origin") - - // Add bundles out of order: 3, 1, 2 - for _, num := range []int{3, 1, 2} { - meta := &bundleindex.BundleMetadata{ - BundleNumber: num, - StartTime: time.Now(), - EndTime: time.Now().Add(time.Hour), - OperationCount: types.BUNDLE_SIZE, - } - idx.AddBundle(meta) - } - - bundles := idx.GetBundles() - - // Should be sorted: 1, 2, 3 - if bundles[0].BundleNumber != 1 { - t.Error("bundles not sorted") - } - if bundles[1].BundleNumber != 2 { - t.Error("bundles not sorted") - } - if bundles[2].BundleNumber != 3 { - t.Error("bundles not sorted") - } - }) - - t.Run("UpdateExistingBundle", func(t *testing.T) { - idx := bundleindex.NewIndex("test-origin") - - original := &bundleindex.BundleMetadata{ - BundleNumber: 1, - Hash: "original_hash", - StartTime: time.Now(), - EndTime: time.Now().Add(time.Hour), - OperationCount: types.BUNDLE_SIZE, - } - - idx.AddBundle(original) - - // Add again with different hash (update) - updated := &bundleindex.BundleMetadata{ - BundleNumber: 1, - Hash: "updated_hash", - StartTime: time.Now(), - EndTime: time.Now().Add(time.Hour), - OperationCount: types.BUNDLE_SIZE, - } - - idx.AddBundle(updated) - - // Should have only 1 bundle (updated, not duplicated) - if idx.Count() != 1 { - t.Errorf("should have 1 bundle after update, got %d", idx.Count()) - } - - retrieved, _ := idx.GetBundle(1) - if retrieved.Hash != "updated_hash" { - t.Error("bundle was not updated") - } - }) -} - -// ==================================================================================== -// SAVE & LOAD TESTS -// ==================================================================================== - -func TestIndexPersistence(t *testing.T) { - tmpDir := t.TempDir() - - t.Run("SaveAndLoad", func(t *testing.T) { - indexPath := filepath.Join(tmpDir, "test_index.json") - - // Create and populate index - idx := bundleindex.NewIndex("https://plc.directory") - - for i := 1; i <= 5; i++ { - meta := &bundleindex.BundleMetadata{ - BundleNumber: i, - StartTime: time.Now().Add(time.Duration(i-1) * time.Hour), - EndTime: time.Now().Add(time.Duration(i) * time.Hour), - OperationCount: types.BUNDLE_SIZE, - DIDCount: 1000 * i, - Hash: fmt.Sprintf("hash%d", i), - ContentHash: fmt.Sprintf("content%d", i), - CompressedHash: fmt.Sprintf("compressed%d", i), - CompressedSize: int64(1024 * i), - UncompressedSize: int64(5120 * i), - } - idx.AddBundle(meta) - } - - // Save - if err := idx.Save(indexPath); err != nil { - t.Fatalf("Save failed: %v", err) - } - - // Verify file exists - if _, err := os.Stat(indexPath); os.IsNotExist(err) { - t.Fatal("index file not created") - } - - // Load - loaded, err := bundleindex.LoadIndex(indexPath) - if err != nil { - t.Fatalf("LoadIndex failed: %v", err) - } - - // Verify data integrity - if loaded.Count() != 5 { - t.Errorf("loaded count mismatch: got %d, want 5", loaded.Count()) - } - - if loaded.Origin != "https://plc.directory" { - t.Error("origin not preserved") - } - - if loaded.LastBundle != 5 { - t.Error("LastBundle not calculated correctly") - } - - // Verify specific bundle - bundle3, err := loaded.GetBundle(3) - if err != nil { - t.Fatalf("GetBundle(3) failed: %v", err) - } - - if bundle3.Hash != "hash3" { - t.Error("bundle data not preserved") - } - }) - - t.Run("AtomicSave", func(t *testing.T) { - indexPath := filepath.Join(tmpDir, "atomic_test.json") - - idx := bundleindex.NewIndex("test") - idx.AddBundle(&bundleindex.BundleMetadata{ - BundleNumber: 1, - StartTime: time.Now(), - EndTime: time.Now(), - OperationCount: types.BUNDLE_SIZE, - }) - - idx.Save(indexPath) - - // Verify no .tmp file left behind - tmpPath := indexPath + ".tmp" - if _, err := os.Stat(tmpPath); !os.IsNotExist(err) { - t.Error("temporary file should not exist after successful save") - } - }) - - t.Run("LoadInvalidVersion", func(t *testing.T) { - indexPath := filepath.Join(tmpDir, "invalid_version.json") - - // Write index with wrong version - invalidData := `{"version":"99.99","origin":"test","bundles":[]}` - os.WriteFile(indexPath, []byte(invalidData), 0644) - - _, err := bundleindex.LoadIndex(indexPath) - if err == nil { - t.Error("should reject index with invalid version") - } - }) - - t.Run("LoadCorruptedJSON", func(t *testing.T) { - indexPath := filepath.Join(tmpDir, "corrupted.json") - - os.WriteFile(indexPath, []byte("{invalid json"), 0644) - - _, err := bundleindex.LoadIndex(indexPath) - if err == nil { - t.Error("should reject corrupted JSON") - } - }) -} - -// ==================================================================================== -// QUERY OPERATIONS -// ==================================================================================== - -func TestIndexQueries(t *testing.T) { - idx := bundleindex.NewIndex("test") - - // Populate with bundles - for i := 1; i <= 10; i++ { - meta := &bundleindex.BundleMetadata{ - BundleNumber: i, - StartTime: time.Now().Add(time.Duration(i-1) * time.Hour), - EndTime: time.Now().Add(time.Duration(i) * time.Hour), - OperationCount: types.BUNDLE_SIZE, - CompressedSize: int64(i * 1000), - } - idx.AddBundle(meta) - } - - t.Run("GetBundle", func(t *testing.T) { - meta, err := idx.GetBundle(5) - if err != nil { - t.Fatalf("GetBundle failed: %v", err) - } - - if meta.BundleNumber != 5 { - t.Error("wrong bundle returned") - } - }) - - t.Run("GetBundle_NotFound", func(t *testing.T) { - _, err := idx.GetBundle(999) - if err == nil { - t.Error("should return error for nonexistent bundle") - } - }) - - t.Run("GetLastBundle", func(t *testing.T) { - last := idx.GetLastBundle() - - if last == nil { - t.Fatal("GetLastBundle returned nil") - } - - if last.BundleNumber != 10 { - t.Errorf("last bundle should be 10, got %d", last.BundleNumber) - } - }) - - t.Run("GetLastBundle_Empty", func(t *testing.T) { - emptyIdx := bundleindex.NewIndex("test") - - last := emptyIdx.GetLastBundle() - - if last != nil { - t.Error("empty index should return nil for GetLastBundle") - } - }) - - t.Run("GetBundleRange", func(t *testing.T) { - bundles := idx.GetBundleRange(3, 7) - - if len(bundles) != 5 { - t.Errorf("expected 5 bundles, got %d", len(bundles)) - } - - if bundles[0].BundleNumber != 3 || bundles[4].BundleNumber != 7 { - t.Error("range boundaries incorrect") - } - }) - - t.Run("GetBundleRange_OutOfBounds", func(t *testing.T) { - bundles := idx.GetBundleRange(100, 200) - - if len(bundles) != 0 { - t.Errorf("expected 0 bundles for out-of-range query, got %d", len(bundles)) - } - }) - - t.Run("GetBundles_ReturnsShallowCopy", func(t *testing.T) { - bundles1 := idx.GetBundles() - bundles2 := idx.GetBundles() - - // Should be different slices - if &bundles1[0] == &bundles2[0] { - t.Error("GetBundles should return copy, not same slice") - } - - // But same data - if bundles1[0].BundleNumber != bundles2[0].BundleNumber { - t.Error("bundle data should be same") - } - }) -} - -// ==================================================================================== -// GAP DETECTION - CRITICAL FOR INTEGRITY -// ==================================================================================== - -func TestIndexFindGaps(t *testing.T) { - t.Run("NoGaps", func(t *testing.T) { - idx := bundleindex.NewIndex("test") - - for i := 1; i <= 10; i++ { - idx.AddBundle(createTestMetadata(i)) - } - - gaps := idx.FindGaps() - - if len(gaps) != 0 { - t.Errorf("expected no gaps, found %d: %v", len(gaps), gaps) - } - }) - - t.Run("SingleGap", func(t *testing.T) { - idx := bundleindex.NewIndex("test") - - // Add bundles 1, 2, 4, 5 (missing 3) - for _, num := range []int{1, 2, 4, 5} { - idx.AddBundle(createTestMetadata(num)) - } - - gaps := idx.FindGaps() - - if len(gaps) != 1 { - t.Errorf("expected 1 gap, got %d", len(gaps)) - } - - if len(gaps) > 0 && gaps[0] != 3 { - t.Errorf("expected gap at 3, got %d", gaps[0]) - } - }) - - t.Run("MultipleGaps", func(t *testing.T) { - idx := bundleindex.NewIndex("test") - - // Add bundles 1, 2, 5, 6, 9, 10 (missing 3, 4, 7, 8) - for _, num := range []int{1, 2, 5, 6, 9, 10} { - idx.AddBundle(createTestMetadata(num)) - } - - gaps := idx.FindGaps() - - expectedGaps := []int{3, 4, 7, 8} - if len(gaps) != len(expectedGaps) { - t.Errorf("expected %d gaps, got %d", len(expectedGaps), len(gaps)) - } - - for i, expected := range expectedGaps { - if gaps[i] != expected { - t.Errorf("gap %d: got %d, want %d", i, gaps[i], expected) - } - } - }) - - t.Run("FindGaps_EmptyIndex", func(t *testing.T) { - idx := bundleindex.NewIndex("test") - - gaps := idx.FindGaps() - - if len(gaps) > 0 { - t.Error("empty index should have no gaps") - } - }) - - t.Run("FindGaps_NonSequentialStart", func(t *testing.T) { - idx := bundleindex.NewIndex("test") - - // Start at bundle 100 - for i := 100; i <= 105; i++ { - idx.AddBundle(createTestMetadata(i)) - } - - gaps := idx.FindGaps() - - // No gaps between 100-105 - if len(gaps) != 0 { - t.Errorf("expected no gaps, got %d", len(gaps)) - } - }) -} - -// ==================================================================================== -// STATISTICS & DERIVED FIELDS -// ==================================================================================== - -func TestIndexStatistics(t *testing.T) { - idx := bundleindex.NewIndex("test") - - t.Run("StatsEmpty", func(t *testing.T) { - stats := idx.GetStats() - - if stats["bundle_count"].(int) != 0 { - t.Error("empty index should have count 0") - } - }) - - t.Run("StatsPopulated", func(t *testing.T) { - totalSize := int64(0) - totalUncompressed := int64(0) - - for i := 1; i <= 5; i++ { - meta := &bundleindex.BundleMetadata{ - BundleNumber: i, - StartTime: time.Now().Add(time.Duration(i-1) * time.Hour), - EndTime: time.Now().Add(time.Duration(i) * time.Hour), - OperationCount: types.BUNDLE_SIZE, - CompressedSize: int64(1000 * i), - UncompressedSize: int64(5000 * i), - } - idx.AddBundle(meta) - totalSize += meta.CompressedSize - totalUncompressed += meta.UncompressedSize - } - - stats := idx.GetStats() - - if stats["bundle_count"].(int) != 5 { - t.Error("bundle count mismatch") - } - - if stats["first_bundle"].(int) != 1 { - t.Error("first_bundle mismatch") - } - - if stats["last_bundle"].(int) != 5 { - t.Error("last_bundle mismatch") - } - - if stats["total_size"].(int64) != totalSize { - t.Errorf("total_size mismatch: got %d, want %d", stats["total_size"].(int64), totalSize) - } - - if stats["total_uncompressed_size"].(int64) != totalUncompressed { - t.Error("total_uncompressed_size mismatch") - } - - if _, ok := stats["start_time"]; !ok { - t.Error("stats missing start_time") - } - - if _, ok := stats["end_time"]; !ok { - t.Error("stats missing end_time") - } - - if stats["gaps"].(int) != 0 { - t.Error("should have no gaps") - } - }) - - t.Run("StatsRecalculateAfterAdd", func(t *testing.T) { - idx := bundleindex.NewIndex("test") - - idx.AddBundle(&bundleindex.BundleMetadata{ - BundleNumber: 1, - StartTime: time.Now(), - EndTime: time.Now(), - OperationCount: types.BUNDLE_SIZE, - CompressedSize: 1000, - }) - - stats1 := idx.GetStats() - size1 := stats1["total_size"].(int64) - - // Add another bundle - idx.AddBundle(&bundleindex.BundleMetadata{ - BundleNumber: 2, - StartTime: time.Now(), - EndTime: time.Now(), - OperationCount: types.BUNDLE_SIZE, - CompressedSize: 2000, - }) - - stats2 := idx.GetStats() - size2 := stats2["total_size"].(int64) - - if size2 != size1+2000 { - t.Errorf("total_size not recalculated: got %d, want %d", size2, size1+2000) - } - - if stats2["last_bundle"].(int) != 2 { - t.Error("last_bundle not recalculated") - } - }) -} - -// ==================================================================================== -// REBUILD OPERATION -// ==================================================================================== - -func TestIndexRebuild(t *testing.T) { - t.Run("RebuildFromMetadata", func(t *testing.T) { - idx := bundleindex.NewIndex("original") - - // Add some bundles - for i := 1; i <= 3; i++ { - idx.AddBundle(createTestMetadata(i)) - } - - if idx.Count() != 3 { - t.Fatal("setup failed") - } - - // Create new metadata for rebuild - newMetadata := []*bundleindex.BundleMetadata{ - createTestMetadata(1), - createTestMetadata(2), - createTestMetadata(5), - createTestMetadata(6), - } - - // Rebuild - idx.Rebuild(newMetadata) - - // Should now have 4 bundles - if idx.Count() != 4 { - t.Errorf("after rebuild, expected 4 bundles, got %d", idx.Count()) - } - - // Should have new bundles 5, 6 - if _, err := idx.GetBundle(5); err != nil { - t.Error("should have bundle 5 after rebuild") - } - - // Should not have bundle 3 - if _, err := idx.GetBundle(3); err == nil { - t.Error("should not have bundle 3 after rebuild") - } - - // Origin should be preserved - if idx.Origin != "original" { - t.Error("origin should be preserved during rebuild") - } - }) - - t.Run("RebuildAutoSorts", func(t *testing.T) { - idx := bundleindex.NewIndex("test") - - // Rebuild with unsorted data - unsorted := []*bundleindex.BundleMetadata{ - createTestMetadata(5), - createTestMetadata(2), - createTestMetadata(8), - createTestMetadata(1), - } - - idx.Rebuild(unsorted) - - bundles := idx.GetBundles() - - // Should be sorted - for i := 0; i < len(bundles)-1; i++ { - if bundles[i].BundleNumber >= bundles[i+1].BundleNumber { - t.Error("bundles not sorted after rebuild") - } - } - }) -} - -// ==================================================================================== -// CLEAR OPERATION -// ==================================================================================== - -func TestIndexClear(t *testing.T) { - idx := bundleindex.NewIndex("test") - - // Populate - for i := 1; i <= 10; i++ { - idx.AddBundle(createTestMetadata(i)) - } - - if idx.Count() != 10 { - t.Fatal("setup failed") - } - - // Clear - idx.Clear() - - if idx.Count() != 0 { - t.Error("count should be 0 after clear") - } - - if idx.LastBundle != 0 { - t.Error("LastBundle should be 0 after clear") - } - - if idx.TotalSize != 0 { - t.Error("TotalSize should be 0 after clear") - } - - // Should be able to add after clear - idx.AddBundle(createTestMetadata(1)) - - if idx.Count() != 1 { - t.Error("should be able to add after clear") - } -} - -// ==================================================================================== -// CONCURRENCY TESTS -// ==================================================================================== - -func TestIndexConcurrency(t *testing.T) { - t.Run("ConcurrentReads", func(t *testing.T) { - idx := bundleindex.NewIndex("test") - - // Populate - for i := 1; i <= 100; i++ { - idx.AddBundle(createTestMetadata(i)) - } - - // 100 concurrent readers - var wg sync.WaitGroup - errors := make(chan error, 100) - - for i := 0; i < 100; i++ { - wg.Add(1) - go func(id int) { - defer wg.Done() - - // Various read operations - idx.Count() - idx.GetLastBundle() - idx.GetBundles() - idx.FindGaps() - idx.GetStats() - - if _, err := idx.GetBundle(id%100 + 1); err != nil { - errors <- err - } - }(i) - } - - wg.Wait() - close(errors) - - for err := range errors { - t.Errorf("concurrent read error: %v", err) - } - }) - - t.Run("ConcurrentReadsDuringSave", func(t *testing.T) { - tmpDir := t.TempDir() - indexPath := filepath.Join(tmpDir, "concurrent.json") - - idx := bundleindex.NewIndex("test") - - for i := 1; i <= 50; i++ { - idx.AddBundle(createTestMetadata(i)) - } - - var wg sync.WaitGroup - - // Saver goroutine - wg.Add(1) - go func() { - defer wg.Done() - for i := 0; i < 10; i++ { - idx.Save(indexPath) - time.Sleep(10 * time.Millisecond) - } - }() - - // Reader goroutines - for i := 0; i < 10; i++ { - wg.Add(1) - go func() { - defer wg.Done() - for j := 0; j < 50; j++ { - idx.Count() - idx.GetBundles() - time.Sleep(5 * time.Millisecond) - } - }() - } - - wg.Wait() - }) -} - -// ==================================================================================== -// REMOTE UPDATE TESTS (FOR CLONING) -// ==================================================================================== - -func TestIndexUpdateFromRemote(t *testing.T) { - - t.Run("UpdateFromRemote_Basic", func(t *testing.T) { - idx := bundleindex.NewIndex("test") - - // Local has bundles 1-3 - for i := 1; i <= 3; i++ { - idx.AddBundle(createTestMetadata(i)) - } - - // Remote has bundles 1-5 - remoteMeta := make(map[int]*bundleindex.BundleMetadata) - for i := 1; i <= 5; i++ { - remoteMeta[i] = createTestMetadata(i) - } - - bundlesToUpdate := []int{4, 5} - - // Mock file existence (4 and 5 exist) - fileExists := func(bundleNum int) bool { - return bundleNum == 4 || bundleNum == 5 - } - - logger := &testLogger{t: &testing.T{}} - - err := idx.UpdateFromRemote(bundlesToUpdate, remoteMeta, fileExists, false, logger) - if err != nil { - t.Fatalf("UpdateFromRemote failed: %v", err) - } - - // Should now have 5 bundles - if idx.Count() != 5 { - t.Errorf("expected 5 bundles after update, got %d", idx.Count()) - } - }) - - t.Run("UpdateFromRemote_SkipsMissingFiles", func(t *testing.T) { - idx := bundleindex.NewIndex("test") - - remoteMeta := map[int]*bundleindex.BundleMetadata{ - 1: createTestMetadata(1), - 2: createTestMetadata(2), - } - - bundlesToUpdate := []int{1, 2} - - // Only bundle 1 exists locally - fileExists := func(bundleNum int) bool { - return bundleNum == 1 - } - - logger := &testLogger{t: &testing.T{}} - - err := idx.UpdateFromRemote(bundlesToUpdate, remoteMeta, fileExists, false, logger) - if err != nil { - t.Fatalf("UpdateFromRemote failed: %v", err) - } - - // Should only have bundle 1 - if idx.Count() != 1 { - t.Errorf("expected 1 bundle, got %d", idx.Count()) - } - - if _, err := idx.GetBundle(2); err == nil { - t.Error("should not have bundle 2 (file missing)") - } - }) -} - -// ==================================================================================== -// HELPER FUNCTIONS -// ==================================================================================== - -func createTestMetadata(bundleNum int) *bundleindex.BundleMetadata { - return &bundleindex.BundleMetadata{ - BundleNumber: bundleNum, - StartTime: time.Now().Add(time.Duration(bundleNum-1) * time.Hour), - EndTime: time.Now().Add(time.Duration(bundleNum) * time.Hour), - OperationCount: types.BUNDLE_SIZE, - DIDCount: 1000, - Hash: fmt.Sprintf("hash%d", bundleNum), - ContentHash: fmt.Sprintf("content%d", bundleNum), - Parent: fmt.Sprintf("parent%d", bundleNum-1), - CompressedHash: fmt.Sprintf("compressed%d", bundleNum), - CompressedSize: int64(1000 * bundleNum), - UncompressedSize: int64(5000 * bundleNum), - CreatedAt: time.Now(), - } -} diff --git a/internal/bundleindex/metadata.go b/internal/bundleindex/metadata.go deleted file mode 100644 index df19b4f..0000000 --- a/internal/bundleindex/metadata.go +++ /dev/null @@ -1,28 +0,0 @@ -package bundleindex - -import "time" - -// BundleMetadata represents metadata about a bundle -type BundleMetadata struct { - BundleNumber int `json:"bundle_number"` - StartTime time.Time `json:"start_time"` - EndTime time.Time `json:"end_time"` - OperationCount int `json:"operation_count"` - DIDCount int `json:"did_count"` - - // Primary hash - cumulative chain hash (includes all history) - Hash string `json:"hash"` - - // Content hash - SHA256 of bundle operations only - ContentHash string `json:"content_hash"` - - // Parent chain hash - links to previous bundle - Parent string `json:"parent,omitempty"` - - // File hashes and sizes - CompressedHash string `json:"compressed_hash"` - CompressedSize int64 `json:"compressed_size"` - UncompressedSize int64 `json:"uncompressed_size"` - Cursor string `json:"cursor"` - CreatedAt time.Time `json:"created_at"` -} diff --git a/internal/bundleindex/types.go b/internal/bundleindex/types.go deleted file mode 100644 index b812157..0000000 --- a/internal/bundleindex/types.go +++ /dev/null @@ -1,21 +0,0 @@ -package bundleindex - -// ChainVerificationResult contains the result of chain verification -type ChainVerificationResult struct { - Valid bool - ChainLength int - BrokenAt int - Error string - VerifiedBundles []int -} - -// VerificationResult contains the result of bundle verification -type VerificationResult struct { - BundleNumber int - Valid bool - HashMatch bool - FileExists bool - Error error - LocalHash string - ExpectedHash string -} diff --git a/internal/didindex/builder.go b/internal/didindex/builder.go deleted file mode 100644 index 2b68d74..0000000 --- a/internal/didindex/builder.go +++ /dev/null @@ -1,462 +0,0 @@ -package didindex - -import ( - "context" - "encoding/binary" - "fmt" - "os" - "path/filepath" - "runtime" - "sort" - "sync" - "time" -) - -// newShardBuilder creates a new shard builder -func newShardBuilder() *ShardBuilder { - return &ShardBuilder{ - entries: make(map[string][]OpLocation), - } -} - -// add adds a location to the shard -func (sb *ShardBuilder) add(identifier string, loc OpLocation) { - sb.mu.Lock() - defer sb.mu.Unlock() - - sb.entries[identifier] = append(sb.entries[identifier], loc) -} - -// updateAndSaveConfig updates config with new values and saves atomically -func (dim *Manager) updateAndSaveConfig(totalDIDs int64, lastBundle int) error { - dim.config.TotalDIDs = totalDIDs - dim.config.LastBundle = lastBundle - dim.config.Version = DIDINDEX_VERSION - dim.config.Format = "binary_v4" - dim.config.UpdatedAt = time.Now().UTC() - - return dim.saveIndexConfig() -} - -// BuildIndexFromScratch builds index with controlled memory usage -func (dim *Manager) BuildIndexFromScratch(ctx context.Context, mgr BundleProvider, progressCallback func(current, total int)) error { - dim.indexMu.Lock() - defer dim.indexMu.Unlock() - - dim.logger.Printf("Building DID index from scratch (memory-efficient mode)...") - - bundles := mgr.GetBundleIndex().GetBundles() - if len(bundles) == 0 { - return fmt.Errorf("no bundles to index") - } - - if err := os.MkdirAll(dim.shardDir, 0755); err != nil { - return fmt.Errorf("failed to create shard directory: %w", err) - } - - // Create temporary shard files - tempShards := make([]*os.File, DID_SHARD_COUNT) - for i := 0; i < DID_SHARD_COUNT; i++ { - tempPath := filepath.Join(dim.shardDir, fmt.Sprintf("%02x.tmp", i)) - f, err := os.Create(tempPath) - if err != nil { - for j := 0; j < i; j++ { - tempShards[j].Close() - os.Remove(filepath.Join(dim.shardDir, fmt.Sprintf("%02x.tmp", j))) - } - return fmt.Errorf("failed to create temp shard: %w", err) - } - tempShards[i] = f - } - - dim.logger.Printf("Pass 1/2: Scanning %d bundles...", len(bundles)) - - // Stream all operations to temp files - for i, meta := range bundles { - select { - case <-ctx.Done(): - for _, f := range tempShards { - f.Close() - os.Remove(f.Name()) - } - return ctx.Err() - default: - } - - if progressCallback != nil { - progressCallback(i+1, len(bundles)) - } - - // Load bundle - bundle, err := mgr.LoadBundleForDIDIndex(ctx, meta.BundleNumber) - if err != nil { - dim.logger.Printf("Warning: failed to load bundle %d: %v", meta.BundleNumber, err) - continue - } - - // Process each operation - for pos, op := range bundle.Operations { - identifier, err := extractDIDIdentifier(op.DID) - if err != nil { - continue - } - - shardNum := dim.calculateShard(identifier) - - // Write entry: [24 bytes ID][4 bytes packed OpLocation] - entry := make([]byte, 28) - copy(entry[0:24], identifier) - - // Create packed OpLocation (includes nullified bit) - loc := NewOpLocation(uint16(meta.BundleNumber), uint16(pos), op.IsNullified()) - binary.LittleEndian.PutUint32(entry[24:28], uint32(loc)) - - if _, err := tempShards[shardNum].Write(entry); err != nil { - dim.logger.Printf("Warning: failed to write to temp shard %02x: %v", shardNum, err) - } - } - } - - // Close temp files - for _, f := range tempShards { - f.Close() - } - - dim.logger.Printf("\n") - dim.logger.Printf("Pass 2/2: Consolidating %d shards...", DID_SHARD_COUNT) - - // Consolidate shards - totalDIDs := int64(0) - for i := 0; i < DID_SHARD_COUNT; i++ { - // Log every 32 shards - if i%32 == 0 || i == DID_SHARD_COUNT-1 { - dim.logger.Printf(" Consolidating shards: %d/%d (%.1f%%)", - i+1, DID_SHARD_COUNT, float64(i+1)/float64(DID_SHARD_COUNT)*100) - } - - count, err := dim.consolidateShard(uint8(i)) - if err != nil { - return fmt.Errorf("failed to consolidate shard %02x: %w", i, err) - } - totalDIDs += count - } - - if err := dim.updateAndSaveConfig(totalDIDs, bundles[len(bundles)-1].BundleNumber); err != nil { - return fmt.Errorf("failed to save config: %w", err) - } - - dim.logger.Printf("✓ Index built: %d DIDs indexed", totalDIDs) - - return nil -} - -// consolidateShard reads temp file, sorts, and writes final shard -func (dim *Manager) consolidateShard(shardNum uint8) (int64, error) { - tempPath := filepath.Join(dim.shardDir, fmt.Sprintf("%02x.tmp", shardNum)) - - // Read all entries from temp file - data, err := os.ReadFile(tempPath) - if err != nil { - if os.IsNotExist(err) { - return 0, nil - } - return 0, err - } - - if len(data) == 0 { - os.Remove(tempPath) - return 0, nil - } - - // Parse entries (28 bytes each) - entryCount := len(data) / 28 - if len(data)%28 != 0 { - return 0, fmt.Errorf("corrupted temp shard: size not multiple of 28") - } - - type tempEntry struct { - identifier string - location OpLocation - } - - entries := make([]tempEntry, entryCount) - for i := 0; i < entryCount; i++ { - offset := i * 28 - entries[i] = tempEntry{ - identifier: string(data[offset : offset+24]), - location: OpLocation(binary.LittleEndian.Uint32(data[offset+24 : offset+28])), - } - } - - // Free the data slice - data = nil - - // Sort by identifier - sort.Slice(entries, func(i, j int) bool { - return entries[i].identifier < entries[j].identifier - }) - - // Group by DID - builder := newShardBuilder() - for _, entry := range entries { - builder.add(entry.identifier, entry.location) - } - - // Free entries - entries = nil - - // Write final shard - if err := dim.writeShard(shardNum, builder); err != nil { - return 0, err - } - - // Clean up temp file - os.Remove(tempPath) - - return int64(len(builder.entries)), nil -} - -// UpdateIndexForBundle adds operations from a new bundle (incremental + ATOMIC + PARALLEL) -func (dim *Manager) UpdateIndexForBundle(ctx context.Context, bundle *BundleData) error { - dim.indexMu.Lock() - defer dim.indexMu.Unlock() - - totalStart := time.Now() - - // STEP 1: Group operations by shard - groupStart := time.Now() - shardOps := make(map[uint8]map[string][]OpLocation) - - for pos, op := range bundle.Operations { - identifier, err := extractDIDIdentifier(op.DID) - if err != nil { - continue - } - - shardNum := dim.calculateShard(identifier) - - if shardOps[shardNum] == nil { - shardOps[shardNum] = make(map[string][]OpLocation) - } - - loc := NewOpLocation(uint16(bundle.BundleNumber), uint16(pos), op.IsNullified()) - shardOps[shardNum][identifier] = append(shardOps[shardNum][identifier], loc) - } - - groupDuration := time.Since(groupStart) - if dim.verbose { - dim.logger.Printf(" [DID Index] Grouped operations into %d shards in %s", - len(shardOps), groupDuration) - } - - // STEP 2: Write ALL shards to .tmp files FIRST (PARALLEL) - writeStart := time.Now() - - tmpShards := make(map[uint8]string) - var tmpShardsMu sync.Mutex - var deltaCount int64 - var deltaCountMu sync.Mutex - - // Error handling - errChan := make(chan error, len(shardOps)) - - // Worker pool - workers := runtime.NumCPU() - if workers > len(shardOps) { - workers = len(shardOps) - } - if workers < 1 { - workers = 1 - } - - semaphore := make(chan struct{}, workers) - var wg sync.WaitGroup - - if dim.verbose { - dim.logger.Printf(" [DID Index] Updating %d shards in parallel (%d workers)...", - len(shardOps), workers) - } - - // Process each shard in parallel - for shardNum, newOps := range shardOps { - wg.Add(1) - go func(sNum uint8, ops map[string][]OpLocation) { - defer wg.Done() - - // Acquire semaphore (limit concurrency) - semaphore <- struct{}{} - defer func() { <-semaphore }() - - shardStart := time.Now() - tmpPath := filepath.Join(dim.shardDir, fmt.Sprintf("%02x.idx.tmp", sNum)) - - addedCount, err := dim.updateShardToTemp(sNum, ops, tmpPath) - if err != nil { - errChan <- fmt.Errorf("shard %02x: %w", sNum, err) - return - } - - shardDuration := time.Since(shardStart) - - // Update shared state - tmpShardsMu.Lock() - tmpShards[sNum] = tmpPath - tmpShardsMu.Unlock() - - deltaCountMu.Lock() - deltaCount += addedCount - deltaCountMu.Unlock() - - // Debug log for each shard - if dim.verbose { - dim.logger.Printf(" Shard %02x: +%d DIDs in %s (%d ops)", - sNum, addedCount, shardDuration, len(ops)) - } - }(shardNum, newOps) - } - - // Wait for all workers - wg.Wait() - close(errChan) - - writeDuration := time.Since(writeStart) - if dim.verbose { - dim.logger.Printf(" [DID Index] Wrote %d temp files in %s (%.1f shards/sec)", - len(tmpShards), writeDuration, float64(len(tmpShards))/writeDuration.Seconds()) - } - - // Check for errors - if err := <-errChan; err != nil { - dim.cleanupTempShards(tmpShards) - return err - } - - // STEP 3: Atomically commit ALL shards - commitStart := time.Now() - - for shardNum, tmpPath := range tmpShards { - finalPath := filepath.Join(dim.shardDir, fmt.Sprintf("%02x.idx", shardNum)) - - if err := os.Rename(tmpPath, finalPath); err != nil { - dim.logger.Printf("ERROR: Failed to commit shard %02x: %v", shardNum, err) - return fmt.Errorf("failed to commit shard %02x: %w", shardNum, err) - } - - // Invalidate cache - dim.invalidateShard(shardNum) - } - - commitDuration := time.Since(commitStart) - - // STEP 4: Update config - configStart := time.Now() - - newTotal := dim.config.TotalDIDs + deltaCount - if err := dim.updateAndSaveConfig(newTotal, bundle.BundleNumber); err != nil { - return fmt.Errorf("failed to save config: %w", err) - } - - configDuration := time.Since(configStart) - totalDuration := time.Since(totalStart) - - // Summary log - if dim.verbose { - dim.logger.Printf(" [DID Index] ✓ Bundle %06d indexed: +%d DIDs, %d shards updated in %s", - bundle.BundleNumber, deltaCount, len(tmpShards), totalDuration) - } - - if dim.verbose { - dim.logger.Printf(" Breakdown: group=%s write=%s commit=%s config=%s", - groupDuration, writeDuration, commitDuration, configDuration) - dim.logger.Printf(" Throughput: %.0f ops/sec", - float64(len(bundle.Operations))/totalDuration.Seconds()) - } - - return nil -} - -// updateShardToTemp updates a shard and writes to temp file -func (dim *Manager) updateShardToTemp(shardNum uint8, newOps map[string][]OpLocation, tmpPath string) (int64, error) { - shardPath := filepath.Join(dim.shardDir, fmt.Sprintf("%02x.idx", shardNum)) - - existingBuilder := newShardBuilder() - - // Read existing shard if it exists - if data, err := os.ReadFile(shardPath); err == nil && len(data) > 32 { - if err := dim.parseShardData(data, existingBuilder); err != nil { - return 0, fmt.Errorf("failed to parse existing shard: %w", err) - } - } - - beforeCount := len(existingBuilder.entries) - - // Merge new operations - for identifier, locations := range newOps { - existingBuilder.entries[identifier] = append(existingBuilder.entries[identifier], locations...) - } - - afterCount := len(existingBuilder.entries) - deltaCount := int64(afterCount - beforeCount) - - // Write to TEMP file - if err := dim.writeShardToPath(tmpPath, shardNum, existingBuilder); err != nil { - return 0, err - } - - return deltaCount, nil -} - -// cleanupTempShards removes all temporary shard files -func (dim *Manager) cleanupTempShards(tmpShards map[uint8]string) { - for shardNum, tmpPath := range tmpShards { - if err := os.Remove(tmpPath); err != nil && !os.IsNotExist(err) { - dim.logger.Printf("Warning: failed to cleanup temp shard %02x: %v", shardNum, err) - } - } -} - -// VerifyAndRepairIndex checks if index is consistent with bundles and repairs if needed -func (dim *Manager) VerifyAndRepairIndex(ctx context.Context, mgr BundleProvider) error { - bundles := mgr.GetBundleIndex().GetBundles() - if len(bundles) == 0 { - return nil - } - - lastBundleInRepo := bundles[len(bundles)-1].BundleNumber - lastBundleInIndex := dim.config.LastBundle - - if lastBundleInIndex == lastBundleInRepo { - return nil - } - - if lastBundleInIndex > lastBundleInRepo { - dim.logger.Printf("⚠️ Warning: Index claims bundle %d but only %d bundles exist", - lastBundleInIndex, lastBundleInRepo) - dim.logger.Printf(" Rebuilding index...") - return dim.BuildIndexFromScratch(ctx, mgr, nil) - } - - // Index is behind - update incrementally - dim.logger.Printf("Index is behind: has bundle %d, need %d", - lastBundleInIndex, lastBundleInRepo) - dim.logger.Printf("Updating index for %d missing bundles...", - lastBundleInRepo-lastBundleInIndex) - - for bundleNum := lastBundleInIndex + 1; bundleNum <= lastBundleInRepo; bundleNum++ { - bundle, err := mgr.LoadBundleForDIDIndex(ctx, bundleNum) - if err != nil { - return fmt.Errorf("failed to load bundle %d: %w", bundleNum, err) - } - - if err := dim.UpdateIndexForBundle(ctx, bundle); err != nil { - return fmt.Errorf("failed to update index for bundle %d: %w", bundleNum, err) - } - - if bundleNum%100 == 0 { - dim.logger.Printf(" Updated through bundle %d...", bundleNum) - } - } - - dim.logger.Printf("✓ Index repaired: now at bundle %d", lastBundleInRepo) - return nil -} diff --git a/internal/didindex/bundle.go b/internal/didindex/bundle.go deleted file mode 100644 index bc75312..0000000 --- a/internal/didindex/bundle.go +++ /dev/null @@ -1,43 +0,0 @@ -package didindex - -import ( - "context" - "time" - - "tangled.org/atscan.net/plcbundle/internal/plcclient" -) - -// BundleProvider is an interface to avoid circular dependencies -// The bundle.Manager implements this interface -type BundleProvider interface { - LoadBundleForDIDIndex(ctx context.Context, bundleNumber int) (*BundleData, error) - LoadOperation(ctx context.Context, bundleNumber int, position int) (*plcclient.PLCOperation, error) - LoadOperations(ctx context.Context, bundleNumber int, positions []int) (map[int]*plcclient.PLCOperation, error) - GetBundleIndex() BundleIndexProvider -} - -// BundleData contains the minimal bundle information needed by DID index -type BundleData struct { - BundleNumber int - Operations []plcclient.PLCOperation -} - -// BundleIndexProvider provides access to bundle metadata -type BundleIndexProvider interface { - GetBundles() []*BundleMetadata - GetBundle(bundleNumber int) (*BundleMetadata, error) - GetLastBundle() *BundleMetadata -} - -// BundleMetadata is the minimal metadata needed by DID index -type BundleMetadata struct { - BundleNumber int - StartTime time.Time - EndTime time.Time -} - -// Logger interface (shared) -type Logger interface { - Printf(format string, v ...interface{}) - Println(v ...interface{}) -} diff --git a/internal/didindex/lookup.go b/internal/didindex/lookup.go deleted file mode 100644 index 88b3478..0000000 --- a/internal/didindex/lookup.go +++ /dev/null @@ -1,193 +0,0 @@ -package didindex - -import ( - "context" - "fmt" - "sort" - "sync" - - "tangled.org/atscan.net/plcbundle/internal/plcclient" -) - -// GetDIDOperations retrieves all operations for a DID WITH location metadata -// Returns operations with bundle/position info (includes nullified operations) -func (dim *Manager) GetDIDOperations(ctx context.Context, did string, provider BundleProvider) ([]OpLocationWithOperation, error) { - if err := plcclient.ValidateDIDFormat(did); err != nil { - return nil, err - } - - if !dim.Exists() { - return nil, fmt.Errorf("DID index not available - run 'plcbundle index build' to enable DID lookups") - } - - locations, err := dim.GetDIDLocations(did) - if err != nil { - return nil, err - } - - if len(locations) == 0 { - return []OpLocationWithOperation{}, nil - } - - // Group by bundle - bundleMap := make(map[uint16][]OpLocation) - for _, loc := range locations { - bundleMap[loc.Bundle()] = append(bundleMap[loc.Bundle()], loc) - } - - var results []OpLocationWithOperation - for bundleNum, locs := range bundleMap { - positions := make([]int, len(locs)) - for i, l := range locs { - positions[i] = l.PositionInt() - } - opsMap, err := provider.LoadOperations(ctx, int(bundleNum), positions) - if err != nil { - dim.logger.Printf("Warning: failed to load bundle %d: %v", bundleNum, err) - continue - } - for i, l := range locs { - if op, ok := opsMap[positions[i]]; ok { - results = append(results, OpLocationWithOperation{ - Operation: *op, - Bundle: l.BundleInt(), - Position: l.PositionInt(), - }) - } - } - } - - // Sort by time - sort.Slice(results, func(i, j int) bool { - return results[i].Operation.CreatedAt.Before(results[j].Operation.CreatedAt) - }) - - return results, nil -} - -// GetLatestDIDOperation returns the most recent non-nullified operation -func (dim *Manager) GetLatestDIDOperation(ctx context.Context, did string, provider BundleProvider) (*plcclient.PLCOperation, error) { - if err := plcclient.ValidateDIDFormat(did); err != nil { - return nil, err - } - - if !dim.Exists() { - return nil, fmt.Errorf("DID index not available - run 'plcbundle index build' to enable DID lookups") - } - - locations, err := dim.GetDIDLocations(did) - if err != nil { - return nil, err - } - - if len(locations) == 0 { - return nil, fmt.Errorf("DID not found") - } - - // Find latest non-nullified location - var latestLoc *OpLocation - for i := range locations { - if locations[i].Nullified() { - continue - } - - if latestLoc == nil { - latestLoc = &locations[i] - } else { - if locations[i].Bundle() > latestLoc.Bundle() || - (locations[i].Bundle() == latestLoc.Bundle() && locations[i].Position() > latestLoc.Position()) { - latestLoc = &locations[i] - } - } - } - - if latestLoc == nil { - return nil, fmt.Errorf("no valid operations found (all nullified)") - } - - // Load ONLY the specific operation (efficient!) - return provider.LoadOperation(ctx, latestLoc.BundleInt(), latestLoc.PositionInt()) -} - -// BatchGetDIDLocations retrieves locations for multiple DIDs efficiently -// Returns map[did][]OpLocation - only locations, no operation loading -func (dim *Manager) BatchGetDIDLocations(dids []string) (map[string][]OpLocation, error) { - if !dim.Exists() { - return nil, fmt.Errorf("DID index not available") - } - - // Group DIDs by shard to minimize shard loads - type shardQuery struct { - shardNum uint8 - identifiers []string - didMap map[string]string // identifier -> original DID - } - - shardQueries := make(map[uint8]*shardQuery) - - for _, did := range dids { - identifier, err := extractDIDIdentifier(did) - if err != nil { - continue - } - - shardNum := dim.calculateShard(identifier) - - if shardQueries[shardNum] == nil { - shardQueries[shardNum] = &shardQuery{ - shardNum: shardNum, - identifiers: make([]string, 0), - didMap: make(map[string]string), - } - } - - sq := shardQueries[shardNum] - sq.identifiers = append(sq.identifiers, identifier) - sq.didMap[identifier] = did - } - - if dim.verbose { - dim.logger.Printf("DEBUG: Batch lookup: %d DIDs across %d shards", len(dids), len(shardQueries)) - } - - // Process each shard (load once, search multiple times) - results := make(map[string][]OpLocation) - var mu sync.Mutex - - var wg sync.WaitGroup - for _, sq := range shardQueries { - wg.Add(1) - go func(query *shardQuery) { - defer wg.Done() - - // Load shard once - shard, err := dim.loadShard(query.shardNum) - if err != nil { - if dim.verbose { - dim.logger.Printf("DEBUG: Failed to load shard %02x: %v", query.shardNum, err) - } - return - } - defer dim.releaseShard(shard) - - if shard.data == nil { - return - } - - // Search for all identifiers in this shard - for _, identifier := range query.identifiers { - locations := dim.searchShard(shard, identifier) - if len(locations) > 0 { - originalDID := query.didMap[identifier] - mu.Lock() - results[originalDID] = locations - mu.Unlock() - } - } - }(sq) - } - - wg.Wait() - - return results, nil -} diff --git a/internal/didindex/manager.go b/internal/didindex/manager.go deleted file mode 100644 index dee4b53..0000000 --- a/internal/didindex/manager.go +++ /dev/null @@ -1,1067 +0,0 @@ -package didindex - -import ( - "encoding/binary" - "fmt" - "hash/fnv" - "os" - "path/filepath" - "sort" - "sync/atomic" - "syscall" - "time" - - "github.com/goccy/go-json" - "golang.org/x/sys/unix" - "tangled.org/atscan.net/plcbundle/internal/plcclient" -) - -// NewManager creates a new DID index manager -func NewManager(baseDir string, logger Logger) *Manager { - indexDir := filepath.Join(baseDir, DID_INDEX_DIR) - shardDir := filepath.Join(indexDir, DID_INDEX_SHARDS) - configPath := filepath.Join(indexDir, DID_INDEX_CONFIG) - - // Load or create config - config, _ := loadIndexConfig(configPath) - if config == nil { - config = &Config{ - Version: DIDINDEX_VERSION, // Will be 4 - Format: "binary_v4", // Update format name - ShardCount: DID_SHARD_COUNT, - UpdatedAt: time.Now().UTC(), - } - } else if config.Version < DIDINDEX_VERSION { - // Auto-trigger rebuild on version mismatch - logger.Printf("DID index version outdated (v%d, need v%d) - rebuild required", - config.Version, DIDINDEX_VERSION) - } - - return &Manager{ - baseDir: baseDir, - indexDir: indexDir, - shardDir: shardDir, - configPath: configPath, - maxCache: 5, - evictionThreshold: 5, - config: config, - logger: logger, - recentLookupSize: 1000, // Track last 100 lookups - recentLookups: make([]int64, 1000), - } -} - -// Close unmaps all shards and cleans up -func (dim *Manager) Close() error { - // Mark all shards for eviction - var shards []*mmapShard - - dim.shardCache.Range(func(key, value interface{}) bool { - shard := value.(*mmapShard) - shards = append(shards, shard) - dim.shardCache.Delete(key) - return true - }) - - // Wait for refcounts to drop to 0 - for _, shard := range shards { - for atomic.LoadInt64(&shard.refCount) > 0 { - time.Sleep(1 * time.Millisecond) - } - dim.unmapShard(shard) - } - - return nil -} - -func (dim *Manager) SetVerbose(verbose bool) { - dim.verbose = verbose -} - -// GetDIDLocations returns all bundle+position locations for a DID (with timing) -func (dim *Manager) GetDIDLocations(did string) ([]OpLocation, error) { - // Start timing - lookupStart := time.Now() - defer func() { - dim.recordLookupTime(time.Since(lookupStart)) - }() - - identifier, err := extractDIDIdentifier(did) - if err != nil { - return nil, err - } - - shardNum := dim.calculateShard(identifier) - if dim.verbose { - dim.logger.Printf("DEBUG: DID %s -> identifier '%s' -> shard %02x", did, identifier, shardNum) - } - - shard, err := dim.loadShard(shardNum) - if err != nil { - if dim.verbose { - dim.logger.Printf("DEBUG: Failed to load shard: %v", err) - } - return nil, fmt.Errorf("failed to load shard %02x: %w", shardNum, err) - } - - defer dim.releaseShard(shard) - - if shard.data == nil { - if dim.verbose { - dim.logger.Printf("DEBUG: Shard %02x has no data (empty shard)", shardNum) - } - return nil, nil - } - - if dim.verbose { - dim.logger.Printf("DEBUG: Shard %02x loaded, size: %d bytes", shardNum, len(shard.data)) - } - - locations := dim.searchShard(shard, identifier) - - if dim.verbose { - dim.logger.Printf("DEBUG: Binary search found %d locations", len(locations)) - if len(locations) > 0 { - dim.logger.Printf("DEBUG: Locations: %v", locations) - } - } - - return locations, nil -} - -// calculateShard determines which shard a DID belongs to -func (dim *Manager) calculateShard(identifier string) uint8 { - h := fnv.New32a() - h.Write([]byte(identifier)) - hash := h.Sum32() - return uint8(hash % DID_SHARD_COUNT) -} - -// loadShard loads a shard from cache or disk (with madvise optimization) -func (dim *Manager) loadShard(shardNum uint8) (*mmapShard, error) { - // Fast path: cache hit - if val, ok := dim.shardCache.Load(shardNum); ok { - shard := val.(*mmapShard) - - // Increment refcount BEFORE returning - atomic.AddInt64(&shard.refCount, 1) - atomic.StoreInt64(&shard.lastUsed, time.Now().Unix()) - atomic.AddInt64(&shard.accessCount, 1) - atomic.AddInt64(&dim.cacheHits, 1) - - return shard, nil - } - atomic.AddInt64(&dim.cacheMisses, 1) - - // Cache miss - load from disk - shardPath := filepath.Join(dim.shardDir, fmt.Sprintf("%02x.idx", shardNum)) - - if _, err := os.Stat(shardPath); os.IsNotExist(err) { - // Empty shard - no refcount needed - return &mmapShard{ - shardNum: shardNum, - data: nil, - lastUsed: time.Now().Unix(), - refCount: 0, // Not in cache - }, nil - } - - file, err := os.Open(shardPath) - if err != nil { - return nil, err - } - - info, err := file.Stat() - if err != nil { - file.Close() - return nil, err - } - - if info.Size() == 0 { - file.Close() - return &mmapShard{ - shardNum: shardNum, - data: nil, - lastUsed: time.Now().Unix(), - refCount: 0, - }, nil - } - - // Memory-map the file - data, err := syscall.Mmap(int(file.Fd()), 0, int(info.Size()), - syscall.PROT_READ, syscall.MAP_SHARED) - if err != nil { - file.Close() - return nil, fmt.Errorf("mmap failed: %w", err) - } - - if err := dim.applyMadviseHints(data, info.Size()); err != nil { - if dim.verbose { - dim.logger.Printf("DEBUG: madvise failed (non-fatal): %v", err) - } - } - - shard := &mmapShard{ - shardNum: shardNum, - data: data, - file: file, - lastUsed: time.Now().Unix(), - accessCount: 1, - refCount: 1, - } - - // Try to store - actual, loaded := dim.shardCache.LoadOrStore(shardNum, shard) - - if loaded { - // Someone else loaded it - cleanup ours - dim.unmapShard(shard) - - actualShard := actual.(*mmapShard) - atomic.AddInt64(&actualShard.refCount, 1) // Increment their refcount - atomic.StoreInt64(&actualShard.lastUsed, time.Now().Unix()) - atomic.AddInt64(&actualShard.accessCount, 1) - return actualShard, nil - } - - // We stored it - maybe evict - go dim.evictIfNeeded() // Run async to avoid blocking - - return shard, nil -} - -// applyMadviseHints applies OS-level memory hints for optimal performance -func (dim *Manager) applyMadviseHints(data []byte, fileSize int64) error { - const headerPrefetchSize = 16 * 1024 // 16 KB for header + prefix + start of offset table - - // 1. Prefetch critical header section (prefix index + offset table start) - if len(data) >= headerPrefetchSize { - if err := unix.Madvise(data[:headerPrefetchSize], unix.MADV_WILLNEED); err != nil { - if dim.verbose { - dim.logger.Printf("DEBUG: [madvise] Header prefetch failed: %v", err) - } - } else if dim.verbose { - dim.logger.Printf("DEBUG: [madvise] Prefetching header (%d KB) + marking rest as RANDOM", - headerPrefetchSize/1024) - } - } - - // 2. Mark rest as random access (tells OS not to do read-ahead) - if err := unix.Madvise(data, unix.MADV_RANDOM); err != nil { - return err - } - - if dim.verbose { - dim.logger.Printf("DEBUG: [madvise] Shard size: %.1f MB → RANDOM access pattern", - float64(fileSize)/(1024*1024)) - } - - return nil -} - -// searchShard performs optimized binary search using prefix index -func (dim *Manager) searchShard(shard *mmapShard, identifier string) []OpLocation { - if len(shard.data) < 1056 { - return nil - } - - data := shard.data - - // Read header - if string(data[0:4]) != DIDINDEX_MAGIC { - dim.logger.Printf("Warning: invalid shard magic") - return nil - } - - version := binary.LittleEndian.Uint32(data[4:8]) - entryCount := binary.LittleEndian.Uint32(data[9:13]) - - if entryCount == 0 { - return nil - } - - // Determine search range using prefix index - left, right := 0, int(entryCount) - - // Use prefix index to narrow range (only for v3+) - if version >= 3 && len(identifier) > 0 { - prefixByte := identifier[0] - prefixIndexPos := 32 + (int(prefixByte) * 4) - - if prefixIndexPos+4 <= len(data) { - startIdx := binary.LittleEndian.Uint32(data[prefixIndexPos : prefixIndexPos+4]) - - if startIdx != 0xFFFFFFFF { - left = int(startIdx) - - // Find end of this prefix range - for nextPrefix := int(prefixByte) + 1; nextPrefix < 256; nextPrefix++ { - nextPos := 32 + (nextPrefix * 4) - if nextPos+4 > len(data) { - break - } - nextIdx := binary.LittleEndian.Uint32(data[nextPos : nextPos+4]) - if nextIdx != 0xFFFFFFFF { - right = int(nextIdx) - break - } - } - - if dim.verbose { - dim.logger.Printf("DEBUG: Prefix index narrowed search: %d entries → %d entries (%.1f%% reduction)", - entryCount, right-left, (1.0-float64(right-left)/float64(entryCount))*100) - } - } else { - // No entries with this prefix - if dim.verbose { - dim.logger.Printf("DEBUG: Prefix index: no entries with prefix 0x%02x", prefixByte) - } - return nil - } - } - } - - if dim.verbose { - dim.logger.Printf("DEBUG: Binary search range: [%d, %d) of %d entries", left, right, entryCount) - } - - // Binary search within narrowed range - attempts := 0 - offsetTableStart := 1056 // After header + prefix index - - for left < right { - attempts++ - mid := (left + right) / 2 - - // Get entry offset from offset table - offsetPos := offsetTableStart + (mid * 4) - if offsetPos+4 > len(data) { - if dim.verbose { - dim.logger.Printf("DEBUG: Offset position out of bounds") - } - return nil - } - - entryOffset := int(binary.LittleEndian.Uint32(data[offsetPos : offsetPos+4])) - - if entryOffset+DID_IDENTIFIER_LEN > len(data) { - if dim.verbose { - dim.logger.Printf("DEBUG: Entry offset out of bounds: %d + %d > %d", - entryOffset, DID_IDENTIFIER_LEN, len(data)) - } - return nil - } - - entryID := string(data[entryOffset : entryOffset+DID_IDENTIFIER_LEN]) - - if dim.verbose && attempts <= 5 { - dim.logger.Printf("DEBUG: Attempt %d: mid=%d, comparing '%s' vs '%s'", - attempts, mid, identifier, entryID) - } - - // Compare - if identifier == entryID { - if dim.verbose { - dim.logger.Printf("DEBUG: FOUND at mid=%d after %d attempts (vs ~%d without prefix index)", - mid, attempts, logBase2(int(entryCount))) - } - return dim.readLocations(data, entryOffset) - } else if identifier < entryID { - right = mid - } else { - left = mid + 1 - } - } - - if dim.verbose { - dim.logger.Printf("DEBUG: NOT FOUND after %d attempts", attempts) - } - - return nil -} - -// Helper function -func logBase2(n int) int { - if n <= 0 { - return 0 - } - count := 0 - for n > 1 { - n >>= 1 - count++ - } - return count -} - -// getEntryOffset reads entry offset from offset table - O(1) lookup -func (dim *Manager) getEntryOffset(data []byte, entryIndex int) int { - if len(data) < 1056 { - return -1 - } - - entryCount := binary.LittleEndian.Uint32(data[9:13]) - if entryIndex < 0 || entryIndex >= int(entryCount) { - return -1 - } - - // Offset table starts at 1056 (after header + prefix index) - offsetTableStart := 1056 - offsetPos := offsetTableStart + (entryIndex * 4) - - if offsetPos+4 > len(data) { - return -1 - } - - offset := int(binary.LittleEndian.Uint32(data[offsetPos : offsetPos+4])) - - if offset < 0 || offset >= len(data) { - return -1 - } - - return offset -} - -// readLocations reads location data at given offset -func (dim *Manager) readLocations(data []byte, offset int) []OpLocation { - // Skip identifier - offset += DID_IDENTIFIER_LEN - - // Read count - if offset+2 > len(data) { - return nil - } - count := binary.LittleEndian.Uint16(data[offset : offset+2]) - offset += 2 - - // Read locations - locations := make([]OpLocation, count) - for i := 0; i < int(count); i++ { - if offset+4 > len(data) { - return locations[:i] - } - - // Read packed uint32 - packed := binary.LittleEndian.Uint32(data[offset : offset+4]) - locations[i] = OpLocation(packed) - - offset += 4 - } - - return locations -} - -// unmapShard unmaps and closes a shard -func (dim *Manager) unmapShard(shard *mmapShard) { - if shard.data != nil { - unix.Madvise(shard.data, unix.MADV_DONTNEED) - - syscall.Munmap(shard.data) - } - if shard.file != nil { - if f, ok := shard.file.(*os.File); ok { - f.Close() - } - } -} - -// GetStats returns index statistics (updated) -func (dim *Manager) GetStats() map[string]interface{} { - cachedShards := make([]int, 0) - - dim.shardCache.Range(func(key, value interface{}) bool { - cachedShards = append(cachedShards, int(key.(uint8))) - return true - }) - - sort.Ints(cachedShards) - - // Calculate cache hit rate - hits := atomic.LoadInt64(&dim.cacheHits) - misses := atomic.LoadInt64(&dim.cacheMisses) - total := hits + misses - - cacheHitRate := 0.0 - if total > 0 { - cacheHitRate = float64(hits) / float64(total) - } - - baseStats := map[string]interface{}{ - "total_dids": dim.config.TotalDIDs, - "last_bundle": dim.config.LastBundle, - "shard_count": dim.config.ShardCount, - "cached_shards": len(cachedShards), - "cache_limit": dim.maxCache, - "cache_order": cachedShards, - "updated_at": dim.config.UpdatedAt, - "cache_hits": hits, - "cache_misses": misses, - "cache_hit_rate": cacheHitRate, - "total_lookups": total, - } - - // Merge with performance stats - perfStats := dim.calculateLookupStats() - for k, v := range perfStats { - baseStats[k] = v - } - - return baseStats -} - -// Exists checks if index exists -func (dim *Manager) Exists() bool { - _, err := os.Stat(dim.configPath) - return err == nil -} - -// TrimCache trims cache to keep only most recent shard -func (dim *Manager) TrimCache() { - // Count current size - size := 0 - dim.shardCache.Range(func(k, v interface{}) bool { - size++ - return true - }) - - if size <= 1 { - return - } - - // Find most recent shard - var newestTime int64 - var keepNum uint8 - - dim.shardCache.Range(func(key, value interface{}) bool { - shard := value.(*mmapShard) - lastUsed := atomic.LoadInt64(&shard.lastUsed) - if lastUsed > newestTime { - newestTime = lastUsed - keepNum = key.(uint8) - } - return true - }) - - // Evict all except newest - dim.shardCache.Range(func(key, value interface{}) bool { - num := key.(uint8) - if num != keepNum { - if val, ok := dim.shardCache.LoadAndDelete(key); ok { - shard := val.(*mmapShard) - dim.unmapShard(shard) - } - } - return true - }) -} - -// GetConfig returns the index configuration -func (dim *Manager) GetConfig() *Config { - return dim.config -} - -// DebugShard shows shard debugging information -func (dim *Manager) DebugShard(shardNum uint8) error { - shardPath := filepath.Join(dim.shardDir, fmt.Sprintf("%02x.idx", shardNum)) - data, err := os.ReadFile(shardPath) - if err != nil { - return err - } - - fmt.Printf("Shard %02x debug:\n", shardNum) - fmt.Printf(" File size: %d bytes\n", len(data)) - fmt.Printf(" Magic: %s\n", string(data[0:4])) - fmt.Printf(" Version: %d\n", binary.LittleEndian.Uint32(data[4:8])) - fmt.Printf(" Shard num: %d\n", data[8]) - - entryCount := binary.LittleEndian.Uint32(data[9:13]) - fmt.Printf(" Entry count: %d\n", entryCount) - - // Show offset table info - offsetTableSize := int(entryCount) * 4 - dataStartOffset := 32 + offsetTableSize - fmt.Printf(" Offset table size: %d bytes\n", offsetTableSize) - fmt.Printf(" Data starts at: %d\n", dataStartOffset) - - // Show first few entries - fmt.Printf("\n First 5 entries:\n") - - for i := 0; i < 5 && i < int(entryCount); i++ { - offset := dim.getEntryOffset(data, i) - if offset < 0 || offset+DID_IDENTIFIER_LEN+2 > len(data) { - break - } - - identifier := string(data[offset : offset+DID_IDENTIFIER_LEN]) - locCount := binary.LittleEndian.Uint16(data[offset+DID_IDENTIFIER_LEN : offset+DID_IDENTIFIER_LEN+2]) - - fmt.Printf(" %d. '%s' (%d locations) @ offset %d\n", i+1, identifier, locCount, offset) - } - - return nil -} - -func (dim *Manager) invalidateShard(shardNum uint8) { - if val, ok := dim.shardCache.LoadAndDelete(shardNum); ok { - shard := val.(*mmapShard) - - for atomic.LoadInt64(&shard.refCount) > 0 { - time.Sleep(1 * time.Millisecond) - } - - dim.unmapShard(shard) - } -} - -// writeShard writes a shard to disk in binary format with offset table -func (dim *Manager) writeShard(shardNum uint8, builder *ShardBuilder) error { - // Write to temp file first - shardPath := filepath.Join(dim.shardDir, fmt.Sprintf("%02x.idx", shardNum)) - tempPath := shardPath + ".tmp" - - if err := dim.writeShardToPath(tempPath, shardNum, builder); err != nil { - return err - } - - // Atomic rename - if err := os.Rename(tempPath, shardPath); err != nil { - os.Remove(tempPath) - return err - } - - // Invalidate cache for this shard - dim.invalidateShard(shardNum) - - return nil -} - -// writeShardToPath writes shard to a specific path with prefix index -func (dim *Manager) writeShardToPath(path string, shardNum uint8, builder *ShardBuilder) error { - // Sort identifiers for binary search - identifiers := make([]string, 0, len(builder.entries)) - for id := range builder.entries { - identifiers = append(identifiers, id) - } - sort.Strings(identifiers) - - if len(identifiers) == 0 { - // Write empty shard - return os.WriteFile(path, make([]byte, 0), 0644) - } - - // Build prefix index: map first byte → first entry index with that prefix - prefixIndex := make([]uint32, 256) - for i := range prefixIndex { - prefixIndex[i] = 0xFFFFFFFF // Marker for "no entries" - } - - for i, identifier := range identifiers { - if len(identifier) == 0 { - continue - } - prefixByte := identifier[0] - - // Set to first occurrence only - if prefixIndex[prefixByte] == 0xFFFFFFFF { - prefixIndex[prefixByte] = uint32(i) - } - } - - // Calculate entry offsets - offsetTableStart := 1056 // After header (32) + prefix index (1024) - dataStartOffset := offsetTableStart + (len(identifiers) * 4) - - offsetTable := make([]uint32, len(identifiers)) - currentOffset := dataStartOffset - - for i, id := range identifiers { - offsetTable[i] = uint32(currentOffset) - locations := builder.entries[id] - entrySize := DID_IDENTIFIER_LEN + 2 + (len(locations) * 4) - currentOffset += entrySize - } - - totalSize := currentOffset - - // Allocate buffer - buf := make([]byte, totalSize) - - // Write header (32 bytes) - copy(buf[0:4], DIDINDEX_MAGIC) - binary.LittleEndian.PutUint32(buf[4:8], DIDINDEX_VERSION) - buf[8] = shardNum - binary.LittleEndian.PutUint32(buf[9:13], uint32(len(identifiers))) - // bytes 13-31: reserved (zeros) - - // Write prefix index (32-1055: 256 × 4 bytes) - for i, entryIdx := range prefixIndex { - pos := 32 + (i * 4) - binary.LittleEndian.PutUint32(buf[pos:pos+4], entryIdx) - } - - // Write offset table (1056+) - for i, offset := range offsetTable { - pos := offsetTableStart + (i * 4) - binary.LittleEndian.PutUint32(buf[pos:pos+4], offset) - } - - // Write entries (same as before) - for i, identifier := range identifiers { - offset := int(offsetTable[i]) - locations := builder.entries[identifier] - - copy(buf[offset:offset+DID_IDENTIFIER_LEN], identifier) - offset += DID_IDENTIFIER_LEN - - binary.LittleEndian.PutUint16(buf[offset:offset+2], uint16(len(locations))) - offset += 2 - - for _, loc := range locations { - // Write packed uint32 (global position + nullified bit) - binary.LittleEndian.PutUint32(buf[offset:offset+4], uint32(loc)) - offset += 4 - } - } - - return os.WriteFile(path, buf, 0644) -} - -// parseShardData parses binary shard data into builder (supports v2 and v3) -func (dim *Manager) parseShardData(data []byte, builder *ShardBuilder) error { - if len(data) < 32 { - return nil - } - - entryCount := binary.LittleEndian.Uint32(data[9:13]) - - offsetTableStart := 1056 - - // Start reading entries after offset table - offset := offsetTableStart + (int(entryCount) * 4) - - for i := 0; i < int(entryCount); i++ { - if offset+DID_IDENTIFIER_LEN+2 > len(data) { - break - } - - // Read identifier - identifier := string(data[offset : offset+DID_IDENTIFIER_LEN]) - offset += DID_IDENTIFIER_LEN - - // Read location count - locCount := binary.LittleEndian.Uint16(data[offset : offset+2]) - offset += 2 - - // Read locations - locations := make([]OpLocation, locCount) - - // Check version to determine format - version := binary.LittleEndian.Uint32(data[4:8]) - - for j := 0; j < int(locCount); j++ { - if version >= 4 { - // New format: 4-byte packed uint32 - if offset+4 > len(data) { - break - } - packed := binary.LittleEndian.Uint32(data[offset : offset+4]) - locations[j] = OpLocation(packed) - offset += 4 - } else { - // Old format: 5-byte separate fields (for migration) - if offset+5 > len(data) { - break - } - bundle := binary.LittleEndian.Uint16(data[offset : offset+2]) - position := binary.LittleEndian.Uint16(data[offset+2 : offset+4]) - nullified := data[offset+4] != 0 - - // Convert to new format - locations[j] = NewOpLocation(bundle, position, nullified) - offset += 5 - } - } - - builder.entries[identifier] = locations - } - - return nil -} - -// saveIndexConfig saves index configuration -func (dim *Manager) saveIndexConfig() error { - dim.config.UpdatedAt = time.Now().UTC() - - data, err := json.MarshalIndent(dim.config, "", " ") - if err != nil { - return err - } - - return os.WriteFile(dim.configPath, data, 0644) -} - -// extractDIDIdentifier extracts the 24-char identifier from full DID -func extractDIDIdentifier(did string) (string, error) { - if err := plcclient.ValidateDIDFormat(did); err != nil { - return "", err - } - - // Remove "did:plc:" prefix - identifier := did[8:] - - if len(identifier) != DID_IDENTIFIER_LEN { - return "", fmt.Errorf("invalid identifier length: %d", len(identifier)) - } - - return identifier, nil -} - -// loadIndexConfig loads index configuration -func loadIndexConfig(path string) (*Config, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, err - } - - var config Config - if err := json.Unmarshal(data, &config); err != nil { - return nil, err - } - - return &config, nil -} - -func (dim *Manager) evictIfNeeded() { - size := 0 - dim.shardCache.Range(func(_, _ interface{}) bool { - size++ - return true - }) - - if size <= dim.evictionThreshold { - return - } - - type entry struct { - num uint8 - lastUsed int64 - refCount int64 - } - - var entries []entry - - dim.shardCache.Range(func(key, value interface{}) bool { - shard := value.(*mmapShard) - entries = append(entries, entry{ - num: key.(uint8), - lastUsed: atomic.LoadInt64(&shard.lastUsed), - refCount: atomic.LoadInt64(&shard.refCount), - }) - return true - }) - - // Sort by lastUsed (oldest first) - sort.Slice(entries, func(i, j int) bool { - return entries[i].lastUsed < entries[j].lastUsed - }) - - // Evict oldest shards that are NOT in use - toEvict := size - dim.maxCache - evicted := 0 - - for i := 0; i < len(entries) && evicted < toEvict; i++ { - // Only evict if refCount == 0 (not in use) - if entries[i].refCount == 0 { - if val, ok := dim.shardCache.LoadAndDelete(entries[i].num); ok { - shard := val.(*mmapShard) - - // Double-check refcount (race protection) - if atomic.LoadInt64(&shard.refCount) == 0 { - dim.unmapShard(shard) - evicted++ - } else { - // Someone started using it - put it back - dim.shardCache.Store(entries[i].num, shard) - } - } - } - } -} - -// releaseShard decrements reference count -func (dim *Manager) releaseShard(shard *mmapShard) { - if shard == nil || shard.data == nil { - return - } - - atomic.AddInt64(&shard.refCount, -1) -} - -// ResetCacheStats resets cache statistics (useful for monitoring) -func (dim *Manager) ResetCacheStats() { - atomic.StoreInt64(&dim.cacheHits, 0) - atomic.StoreInt64(&dim.cacheMisses, 0) -} - -// recordLookupTime records a lookup time (thread-safe) -func (dim *Manager) recordLookupTime(duration time.Duration) { - micros := duration.Microseconds() - - // Update totals (atomic) - atomic.AddInt64(&dim.totalLookups, 1) - atomic.AddInt64(&dim.totalLookupTime, micros) - - // Update circular buffer (with lock) - dim.lookupTimeLock.Lock() - dim.recentLookups[dim.recentLookupIdx] = micros - dim.recentLookupIdx = (dim.recentLookupIdx + 1) % dim.recentLookupSize - dim.lookupTimeLock.Unlock() -} - -// calculateLookupStats calculates performance statistics -func (dim *Manager) calculateLookupStats() map[string]interface{} { - totalLookups := atomic.LoadInt64(&dim.totalLookups) - totalTime := atomic.LoadInt64(&dim.totalLookupTime) - - stats := make(map[string]interface{}) - - if totalLookups == 0 { - return stats - } - - // Overall average (all time) - avgMicros := float64(totalTime) / float64(totalLookups) - stats["avg_lookup_time_ms"] = avgMicros / 1000.0 - stats["total_lookups"] = totalLookups - - // Recent statistics (last N lookups) - dim.lookupTimeLock.Lock() - recentCopy := make([]int64, dim.recentLookupSize) - copy(recentCopy, dim.recentLookups) - dim.lookupTimeLock.Unlock() - - // Find valid entries (non-zero) - validRecent := make([]int64, 0, dim.recentLookupSize) - for _, t := range recentCopy { - if t > 0 { - validRecent = append(validRecent, t) - } - } - - if len(validRecent) > 0 { - // Sort for percentiles - sortedRecent := make([]int64, len(validRecent)) - copy(sortedRecent, validRecent) - sort.Slice(sortedRecent, func(i, j int) bool { - return sortedRecent[i] < sortedRecent[j] - }) - - // Calculate recent average - var recentSum int64 - for _, t := range validRecent { - recentSum += t - } - recentAvg := float64(recentSum) / float64(len(validRecent)) - stats["recent_avg_lookup_time_ms"] = recentAvg / 1000.0 - stats["recent_sample_size"] = len(validRecent) - - // Min/Max - stats["min_lookup_time_ms"] = float64(sortedRecent[0]) / 1000.0 - stats["max_lookup_time_ms"] = float64(sortedRecent[len(sortedRecent)-1]) / 1000.0 - - // Percentiles (p50, p95, p99) - p50idx := len(sortedRecent) * 50 / 100 - p95idx := len(sortedRecent) * 95 / 100 - p99idx := len(sortedRecent) * 99 / 100 - - if p50idx < len(sortedRecent) { - stats["p50_lookup_time_ms"] = float64(sortedRecent[p50idx]) / 1000.0 - } - if p95idx < len(sortedRecent) { - stats["p95_lookup_time_ms"] = float64(sortedRecent[p95idx]) / 1000.0 - } - if p99idx < len(sortedRecent) { - stats["p99_lookup_time_ms"] = float64(sortedRecent[p99idx]) / 1000.0 - } - } - - return stats -} - -// ResetPerformanceStats resets performance statistics (useful for monitoring periods) -func (dim *Manager) ResetPerformanceStats() { - atomic.StoreInt64(&dim.cacheHits, 0) - atomic.StoreInt64(&dim.cacheMisses, 0) - atomic.StoreInt64(&dim.totalLookups, 0) - atomic.StoreInt64(&dim.totalLookupTime, 0) - - dim.lookupTimeLock.Lock() - dim.recentLookups = make([]int64, dim.recentLookupSize) - dim.recentLookupIdx = 0 - dim.lookupTimeLock.Unlock() -} - -// NeedsRebuild checks if index needs rebuilding and returns reason -func (dim *Manager) NeedsRebuild(bundleProvider BundleIndexProvider) (bool, string) { - // Check if index exists - if !dim.Exists() { - return true, "index does not exist" - } - - // Get repository state - bundles := bundleProvider.GetBundles() - if len(bundles) == 0 { - return false, "" // No bundles, no need to rebuild - } - - lastBundleInRepo := bundles[len(bundles)-1].BundleNumber - - // Check version - if dim.config.Version != DIDINDEX_VERSION { - return true, fmt.Sprintf("index version outdated (v%d, need v%d)", - dim.config.Version, DIDINDEX_VERSION) - } - - // Check if index is behind - if dim.config.LastBundle < lastBundleInRepo { - bundlesBehind := lastBundleInRepo - dim.config.LastBundle - - // Smart logic: only rebuild if significantly behind - // Otherwise can do incremental update - if bundlesBehind > 100 { - return true, fmt.Sprintf("index significantly behind (%d bundles)", bundlesBehind) - } - - return false, fmt.Sprintf("index slightly behind (%d bundles) - can update incrementally", bundlesBehind) - } - - // Check if index is ahead (corruption indicator) - if dim.config.LastBundle > lastBundleInRepo { - return true, fmt.Sprintf("index is ahead of repository (has %d, repo has %d) - likely corrupted", - dim.config.LastBundle, lastBundleInRepo) - } - - // Index is up to date - return false, "" -} - -// ShouldUpdateIncrementally checks if incremental update is appropriate -func (dim *Manager) ShouldUpdateIncrementally(bundleProvider BundleIndexProvider) (bool, int) { - if !dim.Exists() { - return false, 0 - } - - bundles := bundleProvider.GetBundles() - if len(bundles) == 0 { - return false, 0 - } - - lastBundleInRepo := bundles[len(bundles)-1].BundleNumber - bundlesBehind := lastBundleInRepo - dim.config.LastBundle - - // Only do incremental if behind by less than 100 bundles - if bundlesBehind > 0 && bundlesBehind <= 100 { - return true, bundlesBehind - } - - return false, 0 -} diff --git a/internal/didindex/manager_test.go b/internal/didindex/manager_test.go deleted file mode 100644 index 29b2642..0000000 --- a/internal/didindex/manager_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package didindex_test - -import ( - "testing" - - "tangled.org/atscan.net/plcbundle/internal/didindex" -) - -func TestOpLocationPacking(t *testing.T) { - tests := []struct { - bundle uint16 - position uint16 - nullified bool - }{ - {1, 0, false}, - {1, 9999, false}, - {100, 5000, true}, - {65535, 9999, true}, // Max values - } - - for _, tt := range tests { - loc := didindex.NewOpLocation(tt.bundle, tt.position, tt.nullified) - - // Test unpacking - if loc.Bundle() != tt.bundle { - t.Errorf("Bundle mismatch: got %d, want %d", loc.Bundle(), tt.bundle) - } - if loc.Position() != tt.position { - t.Errorf("Position mismatch: got %d, want %d", loc.Position(), tt.position) - } - if loc.Nullified() != tt.nullified { - t.Errorf("Nullified mismatch: got %v, want %v", loc.Nullified(), tt.nullified) - } - - // Test global position - expectedGlobal := uint32(tt.bundle)*10000 + uint32(tt.position) - if loc.GlobalPosition() != expectedGlobal { - t.Errorf("Global position mismatch: got %d, want %d", - loc.GlobalPosition(), expectedGlobal) - } - } -} - -func TestOpLocationComparison(t *testing.T) { - loc1 := didindex.NewOpLocation(100, 50, false) // 1,000,050 - loc2 := didindex.NewOpLocation(100, 51, false) // 1,000,051 - loc3 := didindex.NewOpLocation(200, 30, false) // 2,000,030 - - if !loc1.Less(loc2) { - t.Error("Expected loc1 < loc2") - } - if !loc2.Less(loc3) { - t.Error("Expected loc2 < loc3") - } - if loc3.Less(loc1) { - t.Error("Expected loc3 > loc1") - } -} diff --git a/internal/didindex/types.go b/internal/didindex/types.go deleted file mode 100644 index d4b00d2..0000000 --- a/internal/didindex/types.go +++ /dev/null @@ -1,153 +0,0 @@ -package didindex - -import ( - "sync" - "time" - - "tangled.org/atscan.net/plcbundle/internal/plcclient" -) - -const ( - DID_INDEX_DIR = ".plcbundle" - DID_INDEX_SHARDS = "shards" - DID_INDEX_CONFIG = "config.json" - DID_SHARD_COUNT = 256 - DID_PREFIX = "did:plc:" - DID_IDENTIFIER_LEN = 24 // Without "did:plc:" prefix - - // Binary format constants - DIDINDEX_MAGIC = "PLCD" - DIDINDEX_VERSION = 4 - - // Format sizes - LOCATION_SIZE_V3 = 5 // Old: 2+2+1 - LOCATION_SIZE_V4 = 4 // New: packed uint32 - - BUILD_BATCH_SIZE = 100 // Process 100 bundles at a time - - PREFIX_INDEX_SIZE = 256 // Index first byte of identifier (0x00-0xFF) -) - -// Manager manages sharded DID position indexes with mmap -type Manager struct { - baseDir string - indexDir string - shardDir string - configPath string - - // LRU cache for hot shards - shardCache sync.Map - maxCache int - evictionThreshold int - cacheHits int64 // atomic counter - cacheMisses int64 // atomic counter - - config *Config - logger Logger - verbose bool - - indexMu sync.RWMutex - - // Performance tracking - totalLookups int64 // Total number of lookups - totalLookupTime int64 // Total time in microseconds - lookupTimeLock sync.Mutex - recentLookups []int64 // Circular buffer for recent lookup times (microseconds) - recentLookupIdx int - recentLookupSize int -} - -// mmapShard represents a memory-mapped shard file -type mmapShard struct { - shardNum uint8 - data []byte - file interface{} // *os.File (avoid import) - lastUsed int64 - accessCount int64 - refCount int64 -} - -// Config stores index metadata -type Config struct { - Version int `json:"version"` - Format string `json:"format"` - ShardCount int `json:"shard_count"` - TotalDIDs int64 `json:"total_dids"` - UpdatedAt time.Time `json:"updated_at"` - LastBundle int `json:"last_bundle"` -} - -// ShardBuilder accumulates DID positions for a shard -type ShardBuilder struct { - entries map[string][]OpLocation - mu sync.Mutex -} - -// OpLocation represents exact location of an operation -type OpLocation uint32 - -// OpLocationWithOperation contains an operation with its bundle/position -type OpLocationWithOperation struct { - Operation plcclient.PLCOperation - Bundle int - Position int -} - -func NewOpLocation(bundle, position uint16, nullified bool) OpLocation { - globalPos := uint32(bundle)*10000 + uint32(position) - loc := globalPos << 1 - if nullified { - loc |= 1 - } - return OpLocation(loc) -} - -// Getters -func (loc OpLocation) GlobalPosition() uint32 { - return uint32(loc) >> 1 -} - -func (loc OpLocation) Bundle() uint16 { - return uint16(loc.GlobalPosition() / 10000) -} - -func (loc OpLocation) Position() uint16 { - return uint16(loc.GlobalPosition() % 10000) -} - -func (loc OpLocation) Nullified() bool { - return (loc & 1) == 1 -} - -func (loc OpLocation) IsAfter(other OpLocation) bool { - // Compare global positions directly - return loc.GlobalPosition() > other.GlobalPosition() -} - -func (loc OpLocation) IsBefore(other OpLocation) bool { - return loc.GlobalPosition() < other.GlobalPosition() -} - -func (loc OpLocation) Equals(other OpLocation) bool { - // Compare entire packed value (including nullified bit) - return loc == other -} - -func (loc OpLocation) PositionEquals(other OpLocation) bool { - // Compare only position (ignore nullified bit) - return loc.GlobalPosition() == other.GlobalPosition() -} - -// Convenience conversions -func (loc OpLocation) BundleInt() int { - return int(loc.Bundle()) -} - -func (loc OpLocation) PositionInt() int { - return int(loc.Position()) -} - -// For sorting/comparison -func (loc OpLocation) Less(other OpLocation) bool { - return loc.GlobalPosition() < other.GlobalPosition() -} diff --git a/internal/handleresolver/resolver.go b/internal/handleresolver/resolver.go deleted file mode 100644 index 2688cd2..0000000 --- a/internal/handleresolver/resolver.go +++ /dev/null @@ -1,130 +0,0 @@ -package handleresolver - -import ( - "context" - "fmt" - "io" - "net/http" - "net/url" - "regexp" - "strings" - "time" - - "github.com/goccy/go-json" -) - -// Client resolves AT Protocol handles to DIDs via XRPC -type Client struct { - baseURL string - httpClient *http.Client -} - -// NewClient creates a new handle resolver client -func NewClient(baseURL string) *Client { - return &Client{ - baseURL: strings.TrimSuffix(baseURL, "/"), - httpClient: &http.Client{ - Timeout: 10 * time.Second, - }, - } -} - -// ResolveHandle resolves a handle to a DID using com.atproto.identity.resolveHandle -func (c *Client) ResolveHandle(ctx context.Context, handle string) (string, error) { - // Validate handle format - if err := ValidateHandleFormat(handle); err != nil { - return "", err - } - - // Build XRPC URL - endpoint := fmt.Sprintf("%s/xrpc/com.atproto.identity.resolveHandle", c.baseURL) - - // Add query parameter - params := url.Values{} - params.Add("handle", handle) - fullURL := fmt.Sprintf("%s?%s", endpoint, params.Encode()) - - // Create request - req, err := http.NewRequestWithContext(ctx, "GET", fullURL, nil) - if err != nil { - return "", fmt.Errorf("failed to create request: %w", err) - } - - // Execute request - resp, err := c.httpClient.Do(req) - if err != nil { - return "", fmt.Errorf("failed to resolve handle: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return "", fmt.Errorf("resolver returned status %d: %s", resp.StatusCode, string(body)) - } - - // Parse response - var result struct { - DID string `json:"did"` - } - if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { - return "", fmt.Errorf("failed to parse response: %w", err) - } - - if result.DID == "" { - return "", fmt.Errorf("resolver returned empty DID") - } - - // Validate returned DID - if !strings.HasPrefix(result.DID, "did:plc:") && !strings.HasPrefix(result.DID, "did:web:") { - return "", fmt.Errorf("invalid DID format returned: %s", result.DID) - } - - return result.DID, nil -} - -// ValidateHandleFormat validates AT Protocol handle format -func ValidateHandleFormat(handle string) error { - if handle == "" { - return fmt.Errorf("handle cannot be empty") - } - - // Handle can't be a DID - if strings.HasPrefix(handle, "did:") { - return fmt.Errorf("input is already a DID, not a handle") - } - - // Basic length check - if len(handle) > 253 { - return fmt.Errorf("handle too long (max 253 chars)") - } - - // Must have at least one dot (domain.tld) - if !strings.Contains(handle, ".") { - return fmt.Errorf("handle must be a domain (e.g., user.bsky.social)") - } - - // Valid handle pattern (simplified - matches AT Protocol spec) - validPattern := regexp.MustCompile(`^([a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?$`) - if !validPattern.MatchString(handle) { - return fmt.Errorf("invalid handle format") - } - - return nil -} - -// IsHandle checks if a string looks like a handle (not a DID) -func IsHandle(input string) bool { - return !strings.HasPrefix(input, "did:") -} - -// NormalizeHandle normalizes handle format (removes at:// prefix if present) -func NormalizeHandle(handle string) string { - handle = strings.TrimPrefix(handle, "at://") - handle = strings.TrimPrefix(handle, "@") - return handle -} - -// GetBaseURL returns the PLC directory base URL -func (c *Client) GetBaseURL() string { - return c.baseURL -} diff --git a/internal/mempool/mempool.go b/internal/mempool/mempool.go deleted file mode 100644 index faab9ee..0000000 --- a/internal/mempool/mempool.go +++ /dev/null @@ -1,536 +0,0 @@ -package mempool - -import ( - "bufio" - "bytes" - "fmt" - "os" - "path/filepath" - "sync" - "time" - - "github.com/goccy/go-json" - "tangled.org/atscan.net/plcbundle/internal/plcclient" - "tangled.org/atscan.net/plcbundle/internal/types" -) - -const MEMPOOL_FILE_PREFIX = "plc_mempool_" - -// Mempool stores operations waiting to be bundled -// Operations must be strictly chronological -type Mempool struct { - operations []plcclient.PLCOperation - targetBundle int - minTimestamp time.Time - file string - mu sync.RWMutex - logger types.Logger - validated bool - dirty bool - verbose bool - - // Incremental save tracking - lastSavedLen int // How many ops are persisted - lastSaveTime time.Time // Last save timestamp - saveThreshold int // Save after N new ops - saveInterval time.Duration // Save after duration -} - -// NewMempool creates a new mempool for a specific bundle number -func NewMempool(bundleDir string, targetBundle int, minTimestamp time.Time, logger types.Logger, verbose bool) (*Mempool, error) { - filename := fmt.Sprintf("%s%06d.jsonl", MEMPOOL_FILE_PREFIX, targetBundle) - - m := &Mempool{ - file: filepath.Join(bundleDir, filename), - targetBundle: targetBundle, - minTimestamp: minTimestamp, - operations: make([]plcclient.PLCOperation, 0), - logger: logger, - validated: false, - verbose: verbose, - lastSavedLen: 0, - lastSaveTime: time.Now(), - saveThreshold: 100, - saveInterval: 15 * time.Second, - } - - if err := m.Load(); err != nil { - if !os.IsNotExist(err) { - return nil, fmt.Errorf("failed to load mempool: %w", err) - } - } - - return m, nil -} - -// Add adds operations to the mempool with strict validation -func (m *Mempool) Add(ops []plcclient.PLCOperation) (int, error) { - m.mu.Lock() - defer m.mu.Unlock() - - if len(ops) == 0 { - return 0, nil - } - - // Build existing CID set - existingCIDs := make(map[string]bool) - for _, op := range m.operations { - existingCIDs[op.CID] = true - } - - // Validate and add operations - var newOps []plcclient.PLCOperation - var lastTime time.Time - - // Start from last operation time if we have any - if len(m.operations) > 0 { - lastTime = m.operations[len(m.operations)-1].CreatedAt - } else { - lastTime = m.minTimestamp - } - - for _, op := range ops { - // Skip duplicates - if existingCIDs[op.CID] { - continue - } - - // CRITICAL: Validate chronological order - if !op.CreatedAt.After(lastTime) && !op.CreatedAt.Equal(lastTime) { - return len(newOps), fmt.Errorf( - "chronological violation: operation %s at %s is not after %s", - op.CID, op.CreatedAt.Format(time.RFC3339Nano), lastTime.Format(time.RFC3339Nano), - ) - } - - // Validate operation is after minimum timestamp - if op.CreatedAt.Before(m.minTimestamp) { - return len(newOps), fmt.Errorf( - "operation %s at %s is before minimum timestamp %s (belongs in earlier bundle)", - op.CID, op.CreatedAt.Format(time.RFC3339Nano), m.minTimestamp.Format(time.RFC3339Nano), - ) - } - - newOps = append(newOps, op) - existingCIDs[op.CID] = true - lastTime = op.CreatedAt - } - - // Add new operations - m.operations = append(m.operations, newOps...) - m.validated = true - m.dirty = true - - return len(newOps), nil -} - -// Validate performs a full chronological validation of all operations -func (m *Mempool) Validate() error { - m.mu.RLock() - defer m.mu.RUnlock() - - if len(m.operations) == 0 { - return nil - } - - // Check all operations are after minimum timestamp - for i, op := range m.operations { - if op.CreatedAt.Before(m.minTimestamp) { - return fmt.Errorf( - "operation %d (CID: %s) at %s is before minimum timestamp %s", - i, op.CID, op.CreatedAt.Format(time.RFC3339Nano), m.minTimestamp.Format(time.RFC3339Nano), - ) - } - } - - // Check chronological order - for i := 1; i < len(m.operations); i++ { - prev := m.operations[i-1] - curr := m.operations[i] - - if curr.CreatedAt.Before(prev.CreatedAt) { - return fmt.Errorf( - "chronological violation at index %d: %s (%s) is before %s (%s)", - i, curr.CID, curr.CreatedAt.Format(time.RFC3339Nano), - prev.CID, prev.CreatedAt.Format(time.RFC3339Nano), - ) - } - } - - // Check for duplicate CIDs - cidSet := make(map[string]int) - for i, op := range m.operations { - if prevIdx, exists := cidSet[op.CID]; exists { - return fmt.Errorf( - "duplicate CID %s at indices %d and %d", - op.CID, prevIdx, i, - ) - } - cidSet[op.CID] = i - } - - return nil -} - -// Count returns the number of operations in mempool -func (m *Mempool) Count() int { - m.mu.RLock() - defer m.mu.RUnlock() - return len(m.operations) -} - -// Take removes and returns up to n operations from the front -func (m *Mempool) Take(n int) ([]plcclient.PLCOperation, error) { - m.mu.Lock() - defer m.mu.Unlock() - - // Validate before taking - if err := m.validateLocked(); err != nil { - return nil, fmt.Errorf("mempool validation failed: %w", err) - } - - if n > len(m.operations) { - n = len(m.operations) - } - - result := make([]plcclient.PLCOperation, n) - copy(result, m.operations[:n]) - - // Remove taken operations - m.operations = m.operations[n:] - - // ALWAYS reset tracking after Take - // Take() means we're consuming these ops for a bundle - // Any remaining ops are "new" and unsaved - m.lastSavedLen = 0 - m.lastSaveTime = time.Now() - - // Mark dirty only if ops remain - m.dirty = len(m.operations) > 0 - m.validated = false - - return result, nil -} - -// validateLocked performs validation with lock already held -func (m *Mempool) validateLocked() error { - if m.validated { - return nil - } - - if len(m.operations) == 0 { - return nil - } - - // Check chronological order - lastTime := m.minTimestamp - for i, op := range m.operations { - if op.CreatedAt.Before(lastTime) { - return fmt.Errorf( - "chronological violation at index %d: %s is before %s", - i, op.CreatedAt.Format(time.RFC3339Nano), lastTime.Format(time.RFC3339Nano), - ) - } - lastTime = op.CreatedAt - } - - m.validated = true - return nil -} - -// Peek returns up to n operations without removing them -func (m *Mempool) Peek(n int) []plcclient.PLCOperation { - m.mu.RLock() - defer m.mu.RUnlock() - - if n > len(m.operations) { - n = len(m.operations) - } - - result := make([]plcclient.PLCOperation, n) - copy(result, m.operations[:n]) - - return result -} - -// Clear removes all operations -func (m *Mempool) Clear() { - m.mu.Lock() - defer m.mu.Unlock() - m.operations = make([]plcclient.PLCOperation, 0) - m.validated = false -} - -// ShouldSave checks if threshold/interval is met -func (m *Mempool) ShouldSave() bool { - m.mu.RLock() - defer m.mu.RUnlock() - - if !m.dirty { - return false - } - - newOps := len(m.operations) - m.lastSavedLen - timeSinceLastSave := time.Since(m.lastSaveTime) - - return newOps >= m.saveThreshold || timeSinceLastSave >= m.saveInterval -} - -// SaveIfNeeded saves only if threshold is met -func (m *Mempool) SaveIfNeeded() error { - if !m.ShouldSave() { - return nil - } - return m.Save() -} - -// Save - always append-only since mempool only grows -func (m *Mempool) Save() error { - m.mu.Lock() - defer m.mu.Unlock() - - if !m.dirty { - return nil - } - - if len(m.operations) == 0 { - os.Remove(m.file) - m.lastSavedLen = 0 - m.lastSaveTime = time.Now() - m.dirty = false - return nil - } - - // Validate before saving - if err := m.validateLocked(); err != nil { - return fmt.Errorf("mempool validation failed, refusing to save: %w", err) - } - - // Bounds check to prevent panic - if m.lastSavedLen > len(m.operations) { - // This shouldn't happen, but if it does, log and reset - if m.verbose { - m.logger.Printf("Warning: lastSavedLen (%d) > operations (%d), resetting to 0", - m.lastSavedLen, len(m.operations)) - } - m.lastSavedLen = 0 - } - - // Get only new operations since last save - newOps := m.operations[m.lastSavedLen:] - - if len(newOps) == 0 { - // Nothing new to save - m.dirty = false - return nil - } - - // Open for append (or create if first save) - file, err := os.OpenFile(m.file, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - return fmt.Errorf("failed to open mempool: %w", err) - } - defer file.Close() - - // Write only new operations - writer := bufio.NewWriter(file) - for _, op := range newOps { - if len(op.RawJSON) > 0 { - writer.Write(op.RawJSON) - } else { - data, _ := json.Marshal(op) - writer.Write(data) - } - writer.WriteByte('\n') - } - - if err := writer.Flush(); err != nil { - return fmt.Errorf("failed to flush mempool: %w", err) - } - - // Sync to disk for durability - if err := file.Sync(); err != nil { - return fmt.Errorf("failed to sync mempool: %w", err) - } - - m.lastSavedLen = len(m.operations) - m.lastSaveTime = time.Now() - m.dirty = false - - return nil -} - -// Load reads mempool from disk and validates it -func (m *Mempool) Load() error { - data, err := os.ReadFile(m.file) - if err != nil { - return err - } - - m.mu.Lock() - defer m.mu.Unlock() - - // Parse JSONL - scanner := bufio.NewScanner(bytes.NewReader(data)) - buf := make([]byte, 0, 64*1024) - scanner.Buffer(buf, 1024*1024) - - m.operations = make([]plcclient.PLCOperation, 0) - - for scanner.Scan() { - line := scanner.Bytes() - if len(line) == 0 { - continue - } - - var op plcclient.PLCOperation - if err := json.Unmarshal(line, &op); err != nil { - return fmt.Errorf("failed to parse mempool operation: %w", err) - } - - op.RawJSON = make([]byte, len(line)) - copy(op.RawJSON, line) - - m.operations = append(m.operations, op) - } - - if err := scanner.Err(); err != nil { - return fmt.Errorf("scanner error: %w", err) - } - - // Validate loaded data - if err := m.validateLocked(); err != nil { - return fmt.Errorf("loaded mempool failed validation: %w", err) - } - - // Mark as saved (just loaded from disk) - m.lastSavedLen = len(m.operations) - m.lastSaveTime = time.Now() - m.dirty = false - - if m.verbose && len(m.operations) > 0 { - m.logger.Printf("Loaded %d operations from mempool for bundle %06d", len(m.operations), m.targetBundle) - } - - return nil -} - -// GetFirstTime returns the created_at of the first operation -func (m *Mempool) GetFirstTime() string { - m.mu.RLock() - defer m.mu.RUnlock() - - if len(m.operations) == 0 { - return "" - } - - return m.operations[0].CreatedAt.Format(time.RFC3339Nano) -} - -// GetLastTime returns the created_at of the last operation -func (m *Mempool) GetLastTime() string { - m.mu.RLock() - defer m.mu.RUnlock() - - if len(m.operations) == 0 { - return "" - } - - return m.operations[len(m.operations)-1].CreatedAt.Format(time.RFC3339Nano) -} - -// GetTargetBundle returns the bundle number this mempool is for -func (m *Mempool) GetTargetBundle() int { - return m.targetBundle -} - -// GetMinTimestamp returns the minimum timestamp for operations -func (m *Mempool) GetMinTimestamp() time.Time { - return m.minTimestamp -} - -// Stats returns mempool statistics -func (m *Mempool) Stats() map[string]interface{} { - m.mu.RLock() - defer m.mu.RUnlock() - - count := len(m.operations) - - stats := map[string]interface{}{ - "count": count, - "can_create_bundle": count >= types.BUNDLE_SIZE, - "target_bundle": m.targetBundle, - "min_timestamp": m.minTimestamp, - "validated": m.validated, - } - - if count > 0 { - stats["first_time"] = m.operations[0].CreatedAt - stats["last_time"] = m.operations[len(m.operations)-1].CreatedAt - - // Calculate size and unique DIDs - totalSize := 0 - didSet := make(map[string]bool) - for _, op := range m.operations { - totalSize += len(op.RawJSON) - didSet[op.DID] = true - } - stats["size_bytes"] = totalSize - stats["did_count"] = len(didSet) - } - - return stats -} - -// Delete removes the mempool file -func (m *Mempool) Delete() error { - if err := os.Remove(m.file); err != nil && !os.IsNotExist(err) { - return fmt.Errorf("failed to delete mempool file: %w", err) - } - return nil -} - -// GetFilename returns the mempool filename -func (m *Mempool) GetFilename() string { - return filepath.Base(m.file) -} - -// FindDIDOperations searches for operations matching a DID (no full copy) -func (m *Mempool) FindDIDOperations(did string) []plcclient.PLCOperation { - m.mu.RLock() - defer m.mu.RUnlock() - - if len(m.operations) == 0 { - return []plcclient.PLCOperation{} - } - - // Pre-allocate with small capacity (most DIDs have 1-5 ops) - matching := make([]plcclient.PLCOperation, 0, 4) - - for _, op := range m.operations { - if op.DID == did { - matching = append(matching, op) - } - } - - return matching -} - -// FindLatestDIDOperation finds the most recent non-nullified operation for a DID -// Returns nil if not found. Searches backwards for early exit. -func (m *Mempool) FindLatestDIDOperation(did string) *plcclient.PLCOperation { - m.mu.RLock() - defer m.mu.RUnlock() - - // Search backwards from most recent - for i := len(m.operations) - 1; i >= 0; i-- { - if m.operations[i].DID == did { - if !m.operations[i].IsNullified() { - // Return pointer to avoid copy - return &m.operations[i] - } - } - } - - return nil -} diff --git a/internal/mempool/mempool_test.go b/internal/mempool/mempool_test.go deleted file mode 100644 index 445bcf2..0000000 --- a/internal/mempool/mempool_test.go +++ /dev/null @@ -1,907 +0,0 @@ -package mempool_test - -import ( - "fmt" - "os" - "path/filepath" - "sync" - "testing" - "time" - - "tangled.org/atscan.net/plcbundle/internal/mempool" - "tangled.org/atscan.net/plcbundle/internal/plcclient" - "tangled.org/atscan.net/plcbundle/internal/types" -) - -type testLogger struct { - t *testing.T -} - -func (l *testLogger) Printf(format string, v ...interface{}) { - l.t.Logf(format, v...) -} - -func (l *testLogger) Println(v ...interface{}) { - l.t.Log(v...) -} - -// ==================================================================================== -// CHRONOLOGICAL VALIDATION - MOST CRITICAL -// ==================================================================================== - -func TestMempoolChronologicalStrict(t *testing.T) { - tmpDir := t.TempDir() - logger := &testLogger{t: t} - baseTime := time.Now().Add(-time.Hour) - - t.Run("RejectOutOfOrder", func(t *testing.T) { - minTime := baseTime - m, err := mempool.NewMempool(tmpDir, 1, minTime, logger, false) - if err != nil { - t.Fatalf("NewMempool failed: %v", err) - } - - // Add operations in order: 1, 2, 4 - ops := []plcclient.PLCOperation{ - {CID: "cid1", CreatedAt: baseTime.Add(1 * time.Second)}, - {CID: "cid2", CreatedAt: baseTime.Add(2 * time.Second)}, - {CID: "cid4", CreatedAt: baseTime.Add(4 * time.Second)}, - } - - _, err = m.Add(ops) - if err != nil { - t.Fatalf("Add failed: %v", err) - } - - // Now try to add operation 3 (out of order) - outOfOrder := []plcclient.PLCOperation{ - {CID: "cid3", CreatedAt: baseTime.Add(3 * time.Second)}, - } - - _, err = m.Add(outOfOrder) - if err == nil { - t.Error("expected chronological validation error, got nil") - } - - if m.Count() != 3 { - t.Errorf("count should still be 3, got %d", m.Count()) - } - }) - - t.Run("RejectBeforeMinTimestamp", func(t *testing.T) { - minTime := baseTime.Add(10 * time.Second) - m, err := mempool.NewMempool(tmpDir, 2, minTime, logger, false) - if err != nil { - t.Fatalf("NewMempool failed: %v", err) - } - - // Try to add operation before min timestamp - tooEarly := []plcclient.PLCOperation{ - {CID: "cid1", CreatedAt: baseTime}, // Before minTime - } - - _, err = m.Add(tooEarly) - if err == nil { - t.Error("expected error for operation before min timestamp") - } - }) - - t.Run("AllowEqualTimestamps", func(t *testing.T) { - minTime := baseTime - m, err := mempool.NewMempool(tmpDir, 3, minTime, logger, false) - if err != nil { - t.Fatalf("NewMempool failed: %v", err) - } - - // Multiple operations with same timestamp (happens in real PLC data) - sameTime := baseTime.Add(5 * time.Second) - ops := []plcclient.PLCOperation{ - {CID: "cid1", CreatedAt: sameTime}, - {CID: "cid2", CreatedAt: sameTime}, - {CID: "cid3", CreatedAt: sameTime}, - } - - added, err := m.Add(ops) - if err != nil { - t.Fatalf("should allow equal timestamps: %v", err) - } - - if added != 3 { - t.Errorf("expected 3 added, got %d", added) - } - }) - - t.Run("ChronologicalAfterReload", func(t *testing.T) { - minTime := baseTime - m, err := mempool.NewMempool(tmpDir, 4, minTime, logger, false) - if err != nil { - t.Fatalf("NewMempool failed: %v", err) - } - - // Add some operations - ops1 := []plcclient.PLCOperation{ - {CID: "cid1", CreatedAt: baseTime.Add(1 * time.Second)}, - {CID: "cid2", CreatedAt: baseTime.Add(2 * time.Second)}, - } - m.Add(ops1) - m.Save() - - // Reload mempool - m2, err := mempool.NewMempool(tmpDir, 4, minTime, logger, false) - if err != nil { - t.Fatalf("NewMempool reload failed: %v", err) - } - - // Try to add out-of-order operation - outOfOrder := []plcclient.PLCOperation{ - {CID: "cid0", CreatedAt: baseTime}, // Before loaded ops - } - - _, err = m2.Add(outOfOrder) - if err == nil { - t.Error("should reject out-of-order after reload") - } - - // Add valid operation after loaded ones - validOps := []plcclient.PLCOperation{ - {CID: "cid3", CreatedAt: baseTime.Add(3 * time.Second)}, - } - - added, err := m2.Add(validOps) - if err != nil { - t.Fatalf("should accept in-order operation: %v", err) - } - - if added != 1 { - t.Error("should have added 1 operation") - } - }) - - t.Run("StrictIncreasingOrder", func(t *testing.T) { - minTime := baseTime - m, err := mempool.NewMempool(tmpDir, 5, minTime, logger, false) - if err != nil { - t.Fatalf("NewMempool failed: %v", err) - } - - // Each operation must be >= previous timestamp - ops := []plcclient.PLCOperation{ - {CID: "cid1", CreatedAt: baseTime.Add(1 * time.Second)}, - {CID: "cid2", CreatedAt: baseTime.Add(2 * time.Second)}, - {CID: "cid3", CreatedAt: baseTime.Add(2 * time.Second)}, // Equal - OK - {CID: "cid4", CreatedAt: baseTime.Add(3 * time.Second)}, - } - - added, err := m.Add(ops) - if err != nil { - t.Fatalf("should allow non-decreasing timestamps: %v", err) - } - - if added != 4 { - t.Errorf("expected 4 added, got %d", added) - } - }) -} - -// ==================================================================================== -// DUPLICATE PREVENTION -// ==================================================================================== - -func TestMempoolDuplicatePrevention(t *testing.T) { - tmpDir := t.TempDir() - logger := &testLogger{t: t} - baseTime := time.Now().Add(-time.Hour) - - t.Run("SameCIDTwice", func(t *testing.T) { - minTime := baseTime - m, err := mempool.NewMempool(tmpDir, 6, minTime, logger, false) - if err != nil { - t.Fatalf("NewMempool failed: %v", err) - } - - op := plcclient.PLCOperation{ - CID: "duplicate_cid", - DID: "did:plc:test", - CreatedAt: baseTime.Add(1 * time.Second), - } - - // Add first time - added, err := m.Add([]plcclient.PLCOperation{op}) - if err != nil { - t.Fatalf("first add failed: %v", err) - } - if added != 1 { - t.Error("first add should succeed") - } - - // Add same CID again (should be silently skipped) - added, err = m.Add([]plcclient.PLCOperation{op}) - if err != nil { - t.Fatalf("duplicate add should not error: %v", err) - } - if added != 0 { - t.Errorf("duplicate should be skipped, but added=%d", added) - } - - if m.Count() != 1 { - t.Errorf("count should be 1, got %d", m.Count()) - } - }) - - t.Run("DuplicateAcrossSaveLoad", func(t *testing.T) { - minTime := baseTime - m, err := mempool.NewMempool(tmpDir, 7, minTime, logger, false) - if err != nil { - t.Fatalf("NewMempool failed: %v", err) - } - - op := plcclient.PLCOperation{ - CID: "persistent_cid", - DID: "did:plc:test", - CreatedAt: baseTime.Add(1 * time.Second), - } - - // Add and save - m.Add([]plcclient.PLCOperation{op}) - m.Save() - - // Reload - m2, err := mempool.NewMempool(tmpDir, 7, minTime, logger, false) - if err != nil { - t.Fatalf("reload failed: %v", err) - } - - // Try to add same operation - added, err := m2.Add([]plcclient.PLCOperation{op}) - if err != nil { - t.Fatalf("add after reload failed: %v", err) - } - - if added != 0 { - t.Errorf("duplicate should be skipped after reload, added=%d", added) - } - - if m2.Count() != 1 { - t.Errorf("count should be 1, got %d", m2.Count()) - } - }) - - t.Run("DuplicatesInBatch", func(t *testing.T) { - minTime := baseTime - m, err := mempool.NewMempool(tmpDir, 8, minTime, logger, false) - if err != nil { - t.Fatalf("NewMempool failed: %v", err) - } - - // Batch contains duplicates - ops := []plcclient.PLCOperation{ - {CID: "cid1", DID: "did:plc:001", CreatedAt: baseTime.Add(1 * time.Second)}, - {CID: "cid2", DID: "did:plc:002", CreatedAt: baseTime.Add(2 * time.Second)}, - {CID: "cid1", DID: "did:plc:001", CreatedAt: baseTime.Add(3 * time.Second)}, // Duplicate CID - } - - added, err := m.Add(ops) - if err != nil { - t.Fatalf("Add failed: %v", err) - } - - // Should only add 2 (skip duplicate) - if added != 2 { - t.Errorf("expected 2 unique operations, added %d", added) - } - - if m.Count() != 2 { - t.Errorf("count should be 2, got %d", m.Count()) - } - }) -} - -// ==================================================================================== -// PERSISTENCE & CORRUPTION HANDLING -// ==================================================================================== - -func TestMempoolPersistence(t *testing.T) { - tmpDir := t.TempDir() - logger := &testLogger{t: t} - baseTime := time.Now().Add(-time.Hour) - - t.Run("SaveAndLoad", func(t *testing.T) { - minTime := baseTime - m, err := mempool.NewMempool(tmpDir, 9, minTime, logger, false) - if err != nil { - t.Fatalf("NewMempool failed: %v", err) - } - - ops := makeTestOperations(50) - m.Add(ops) - - if err := m.Save(); err != nil { - t.Fatalf("Save failed: %v", err) - } - - // Reload - m2, err := mempool.NewMempool(tmpDir, 9, minTime, logger, false) - if err != nil { - t.Fatalf("reload failed: %v", err) - } - - if m2.Count() != 50 { - t.Errorf("after reload, expected 50 ops, got %d", m2.Count()) - } - - // Verify data integrity - loaded := m2.Peek(50) - for i := 0; i < 50; i++ { - if loaded[i].CID != ops[i].CID { - t.Errorf("op %d CID mismatch after reload", i) - } - } - }) - - // Fix the IncrementalSave test - line ~353 - t.Run("IncrementalSave", func(t *testing.T) { - minTime := baseTime - m, err := mempool.NewMempool(tmpDir, 10, minTime, logger, false) - if err != nil { - t.Fatalf("NewMempool failed: %v", err) - } - - // Add 10 ops and save - ops1 := makeTestOperations(10) - m.Add(ops1) - m.Save() - - // Add 10 more and save - ops2 := makeTestOperationsFrom(10, 10) - m.Add(ops2) - m.Save() - - // Reload - should have all 20 - m2, err := mempool.NewMempool(tmpDir, 10, minTime, logger, false) - if err != nil { - t.Fatalf("reload failed: %v", err) - } - - if m2.Count() != 20 { - t.Errorf("expected 20 ops after incremental saves, got %d", m2.Count()) - } - }) - - t.Run("CorruptedMempoolFile", func(t *testing.T) { - minTime := baseTime - mempoolFile := filepath.Join(tmpDir, "plc_mempool_000011.jsonl") - - // Write corrupted data - os.WriteFile(mempoolFile, []byte("{invalid json\n{also bad"), 0644) - - // Should error on load - _, err := mempool.NewMempool(tmpDir, 11, minTime, logger, false) - if err == nil { - t.Error("expected error loading corrupted mempool") - } - }) - - t.Run("DeleteMempool", func(t *testing.T) { - minTime := baseTime - m, err := mempool.NewMempool(tmpDir, 12, minTime, logger, false) - if err != nil { - t.Fatalf("NewMempool failed: %v", err) - } - - ops := makeTestOperations(10) - m.Add(ops) - m.Save() - - // Verify file exists - mempoolFile := filepath.Join(tmpDir, "plc_mempool_000012.jsonl") - if _, err := os.Stat(mempoolFile); os.IsNotExist(err) { - t.Fatal("mempool file should exist after save") - } - - // Delete - if err := m.Delete(); err != nil { - t.Fatalf("Delete failed: %v", err) - } - - // Verify file gone - if _, err := os.Stat(mempoolFile); !os.IsNotExist(err) { - t.Error("mempool file should be deleted") - } - }) -} - -// ==================================================================================== -// TAKE OPERATIONS - CRITICAL FOR BUNDLING -// ==================================================================================== - -func TestMempoolTakeOperations(t *testing.T) { - tmpDir := t.TempDir() - logger := &testLogger{t: t} - baseTime := time.Now().Add(-time.Hour) - - t.Run("TakeExact", func(t *testing.T) { - minTime := baseTime - m, err := mempool.NewMempool(tmpDir, 13, minTime, logger, false) - if err != nil { - t.Fatalf("NewMempool failed: %v", err) - } - - m.Add(makeTestOperations(100)) - - taken, err := m.Take(50) - if err != nil { - t.Fatalf("Take failed: %v", err) - } - - if len(taken) != 50 { - t.Errorf("expected 50 operations, got %d", len(taken)) - } - - if m.Count() != 50 { - t.Errorf("expected 50 remaining, got %d", m.Count()) - } - }) - - t.Run("TakeMoreThanAvailable", func(t *testing.T) { - minTime := baseTime - m, err := mempool.NewMempool(tmpDir, 14, minTime, logger, false) - if err != nil { - t.Fatalf("NewMempool failed: %v", err) - } - - m.Add(makeTestOperations(30)) - - // Try to take 100 (only 30 available) - taken, err := m.Take(100) - if err != nil { - t.Fatalf("Take failed: %v", err) - } - - if len(taken) != 30 { - t.Errorf("expected 30 operations (all available), got %d", len(taken)) - } - - if m.Count() != 0 { - t.Errorf("mempool should be empty, got %d", m.Count()) - } - }) - - t.Run("TakePreservesOrder", func(t *testing.T) { - minTime := baseTime - m, err := mempool.NewMempool(tmpDir, 15, minTime, logger, false) - if err != nil { - t.Fatalf("NewMempool failed: %v", err) - } - - ops := makeTestOperations(100) - m.Add(ops) - - taken, err := m.Take(50) - if err != nil { - t.Fatalf("Take failed: %v", err) - } - - // Verify first 50 match - for i := 0; i < 50; i++ { - if taken[i].CID != ops[i].CID { - t.Errorf("operation %d mismatch: got %s, want %s", i, taken[i].CID, ops[i].CID) - } - } - - // Remaining should be ops[50:100] - remaining := m.Peek(50) - for i := 0; i < 50; i++ { - if remaining[i].CID != ops[50+i].CID { - t.Errorf("remaining op %d mismatch", i) - } - } - }) - - t.Run("TakeFromEmpty", func(t *testing.T) { - minTime := baseTime - m, err := mempool.NewMempool(tmpDir, 16, minTime, logger, false) - if err != nil { - t.Fatalf("NewMempool failed: %v", err) - } - - taken, err := m.Take(10) - if err != nil { - t.Fatalf("Take from empty failed: %v", err) - } - - if len(taken) != 0 { - t.Errorf("expected 0 operations from empty mempool, got %d", len(taken)) - } - }) -} - -// ==================================================================================== -// VALIDATION TESTS -// ==================================================================================== - -func TestMempoolValidation(t *testing.T) { - tmpDir := t.TempDir() - logger := &testLogger{t: t} - baseTime := time.Now().Add(-time.Hour) - - t.Run("ValidateChronological", func(t *testing.T) { - minTime := baseTime - m, err := mempool.NewMempool(tmpDir, 17, minTime, logger, false) - if err != nil { - t.Fatalf("NewMempool failed: %v", err) - } - - ops := makeTestOperations(100) - m.Add(ops) - - if err := m.Validate(); err != nil { - t.Errorf("Validate failed on valid mempool: %v", err) - } - }) - - t.Run("ValidateDetectsMinTimestampViolation", func(t *testing.T) { - minTime := baseTime.Add(10 * time.Second) - _, err := mempool.NewMempool(tmpDir, 18, minTime, logger, false) - if err != nil { - t.Fatalf("NewMempool failed: %v", err) - } - - // Manually add operation before min (bypassing Add validation) - // This simulates corrupted state - ops := makeTestOperations(10) - ops[0].CreatedAt = baseTime // Before minTime - - // Note: This is hard to test since Add enforces validation - // Better to test through file corruption - }) - - t.Run("ValidateDetectsDuplicateCIDs", func(t *testing.T) { - // Test for duplicate CID detection - // Similar challenge - Add prevents duplicates - // Would need to manually construct corrupted state - }) -} - -// ==================================================================================== -// CONCURRENCY TESTS -// ==================================================================================== - -func TestMempoolConcurrency(t *testing.T) { - tmpDir := t.TempDir() - logger := &testLogger{t: t} - baseTime := time.Now().Add(-time.Hour) - - t.Run("ConcurrentReads", func(t *testing.T) { - minTime := baseTime - m, err := mempool.NewMempool(tmpDir, 19, minTime, logger, false) - if err != nil { - t.Fatalf("NewMempool failed: %v", err) - } - - m.Add(makeTestOperations(1000)) - - // 100 concurrent readers - var wg sync.WaitGroup - for i := 0; i < 100; i++ { - wg.Add(1) - go func() { - defer wg.Done() - count := m.Count() - if count != 1000 { - t.Errorf("count mismatch: got %d", count) - } - - peek := m.Peek(10) - if len(peek) != 10 { - t.Errorf("peek mismatch: got %d", len(peek)) - } - }() - } - wg.Wait() - }) - - t.Run("ConcurrentAddAndRead", func(t *testing.T) { - minTime := baseTime - m, err := mempool.NewMempool(tmpDir, 20, minTime, logger, false) - if err != nil { - t.Fatalf("NewMempool failed: %v", err) - } - - var wg sync.WaitGroup - errors := make(chan error, 100) - - // Writer goroutine - wg.Add(1) - go func() { - defer wg.Done() - for i := 0; i < 10; i++ { - ops := []plcclient.PLCOperation{ - {CID: fmt.Sprintf("cid%d", i*100), CreatedAt: baseTime.Add(time.Duration(i*100) * time.Second)}, - } - if _, err := m.Add(ops); err != nil { - errors <- err - } - time.Sleep(10 * time.Millisecond) - } - }() - - // Reader goroutines - for i := 0; i < 10; i++ { - wg.Add(1) - go func() { - defer wg.Done() - for j := 0; j < 20; j++ { - m.Count() - m.Peek(5) - time.Sleep(5 * time.Millisecond) - } - }() - } - - wg.Wait() - close(errors) - - for err := range errors { - t.Errorf("concurrent operation error: %v", err) - } - }) -} - -// ==================================================================================== -// STATS & METADATA TESTS -// ==================================================================================== - -func TestMempoolStats(t *testing.T) { - tmpDir := t.TempDir() - logger := &testLogger{t: t} - baseTime := time.Now().Add(-time.Hour) - - t.Run("StatsEmpty", func(t *testing.T) { - minTime := baseTime - m, err := mempool.NewMempool(tmpDir, 21, minTime, logger, false) - if err != nil { - t.Fatalf("NewMempool failed: %v", err) - } - - stats := m.Stats() - - if stats["count"].(int) != 0 { - t.Error("empty mempool should have count 0") - } - - if stats["can_create_bundle"].(bool) { - t.Error("empty mempool cannot create bundle") - } - - if stats["target_bundle"].(int) != 21 { - t.Error("target bundle mismatch") - } - }) - - t.Run("StatsPopulated", func(t *testing.T) { - minTime := baseTime - m, err := mempool.NewMempool(tmpDir, 22, minTime, logger, false) - if err != nil { - t.Fatalf("NewMempool failed: %v", err) - } - - ops := makeTestOperations(100) - m.Add(ops) - - stats := m.Stats() - - if stats["count"].(int) != 100 { - t.Error("count mismatch in stats") - } - - if _, ok := stats["first_time"]; !ok { - t.Error("stats missing first_time") - } - - if _, ok := stats["last_time"]; !ok { - t.Error("stats missing last_time") - } - - if _, ok := stats["size_bytes"]; !ok { - t.Error("stats missing size_bytes") - } - - if stats["did_count"].(int) != 100 { - t.Error("did_count should match operation count for unique DIDs") - } - }) - - t.Run("StatsCanCreateBundle", func(t *testing.T) { - minTime := baseTime - m, err := mempool.NewMempool(tmpDir, 23, minTime, logger, false) - if err != nil { - t.Fatalf("NewMempool failed: %v", err) - } - - // Add exactly BUNDLE_SIZE operations - m.Add(makeTestOperations(types.BUNDLE_SIZE)) - - stats := m.Stats() - - if !stats["can_create_bundle"].(bool) { - t.Error("should be able to create bundle with BUNDLE_SIZE operations") - } - }) -} - -// ==================================================================================== -// DID SEARCH TESTS -// ==================================================================================== - -func TestMempoolDIDSearch(t *testing.T) { - tmpDir := t.TempDir() - logger := &testLogger{t: t} - baseTime := time.Now().Add(-time.Hour) - - t.Run("FindDIDOperations", func(t *testing.T) { - minTime := baseTime - m, err := mempool.NewMempool(tmpDir, 24, minTime, logger, false) - if err != nil { - t.Fatalf("NewMempool failed: %v", err) - } - - targetDID := "did:plc:target" - - ops := []plcclient.PLCOperation{ - {DID: "did:plc:other1", CID: "cid1", CreatedAt: baseTime.Add(1 * time.Second)}, - {DID: targetDID, CID: "cid2", CreatedAt: baseTime.Add(2 * time.Second)}, - {DID: "did:plc:other2", CID: "cid3", CreatedAt: baseTime.Add(3 * time.Second)}, - {DID: targetDID, CID: "cid4", CreatedAt: baseTime.Add(4 * time.Second)}, - {DID: "did:plc:other3", CID: "cid5", CreatedAt: baseTime.Add(5 * time.Second)}, - } - - m.Add(ops) - - // Search - found := m.FindDIDOperations(targetDID) - - if len(found) != 2 { - t.Errorf("expected 2 operations for %s, got %d", targetDID, len(found)) - } - - if found[0].CID != "cid2" || found[1].CID != "cid4" { - t.Error("wrong operations returned") - } - }) - - t.Run("FindLatestDIDOperation", func(t *testing.T) { - minTime := baseTime - m, err := mempool.NewMempool(tmpDir, 25, minTime, logger, false) - if err != nil { - t.Fatalf("NewMempool failed: %v", err) - } - - targetDID := "did:plc:target" - - ops := []plcclient.PLCOperation{ - {DID: targetDID, CID: "cid1", CreatedAt: baseTime.Add(1 * time.Second), Nullified: false}, - {DID: targetDID, CID: "cid2", CreatedAt: baseTime.Add(2 * time.Second), Nullified: false}, - {DID: targetDID, CID: "cid3", CreatedAt: baseTime.Add(3 * time.Second), Nullified: true}, // Nullified - } - - m.Add(ops) - - // Should return cid2 (latest non-nullified) - latest := m.FindLatestDIDOperation(targetDID) - - if latest == nil { - t.Fatal("expected to find operation, got nil") - } - - if latest.CID != "cid2" { - t.Errorf("expected cid2 (latest non-nullified), got %s", latest.CID) - } - }) - - t.Run("FindLatestDIDOperation_AllNullified", func(t *testing.T) { - minTime := baseTime - m, err := mempool.NewMempool(tmpDir, 26, minTime, logger, false) - if err != nil { - t.Fatalf("NewMempool failed: %v", err) - } - - targetDID := "did:plc:target" - - ops := []plcclient.PLCOperation{ - {DID: targetDID, CID: "cid1", CreatedAt: baseTime.Add(1 * time.Second), Nullified: true}, - {DID: targetDID, CID: "cid2", CreatedAt: baseTime.Add(2 * time.Second), Nullified: true}, - } - - m.Add(ops) - - latest := m.FindLatestDIDOperation(targetDID) - - if latest != nil { - t.Error("should return nil when all operations are nullified") - } - }) - - t.Run("FindDIDOperations_NotFound", func(t *testing.T) { - minTime := baseTime - m, err := mempool.NewMempool(tmpDir, 27, minTime, logger, false) - if err != nil { - t.Fatalf("NewMempool failed: %v", err) - } - - m.Add(makeTestOperations(100)) - - found := m.FindDIDOperations("did:plc:nonexistent") - - if len(found) != 0 { - t.Errorf("expected empty result, got %d operations", len(found)) - } - }) -} - -// ==================================================================================== -// CLEAR OPERATION TESTS -// ==================================================================================== - -func TestMempoolClear(t *testing.T) { - tmpDir := t.TempDir() - logger := &testLogger{t: t} - baseTime := time.Now().Add(-time.Hour) - - t.Run("ClearPopulated", func(t *testing.T) { - minTime := baseTime - m, err := mempool.NewMempool(tmpDir, 28, minTime, logger, false) - if err != nil { - t.Fatalf("NewMempool failed: %v", err) - } - - m.Add(makeTestOperations(100)) - - if m.Count() != 100 { - t.Fatal("setup failed") - } - - m.Clear() - - if m.Count() != 0 { - t.Errorf("after clear, count should be 0, got %d", m.Count()) - } - - // Should be able to add new operations - newOps := []plcclient.PLCOperation{ - {CID: "new1", CreatedAt: baseTime.Add(200 * time.Second)}, - } - - added, err := m.Add(newOps) - if err != nil { - t.Fatalf("Add after clear failed: %v", err) - } - - if added != 1 { - t.Error("should be able to add after clear") - } - }) -} - -// ==================================================================================== -// HELPER FUNCTIONS -// ==================================================================================== - -func makeTestOperations(count int) []plcclient.PLCOperation { - return makeTestOperationsFrom(0, count) -} - -func makeTestOperationsFrom(start, count int) []plcclient.PLCOperation { - ops := make([]plcclient.PLCOperation, count) - baseTime := time.Now().Add(-time.Hour) - - for i := 0; i < count; i++ { - idx := start + i - ops[i] = plcclient.PLCOperation{ - DID: fmt.Sprintf("did:plc:test%06d", idx), - CID: fmt.Sprintf("bafy%06d", idx), - CreatedAt: baseTime.Add(time.Duration(idx) * time.Second), - } - } - - return ops -} diff --git a/internal/plcclient/client.go b/internal/plcclient/client.go deleted file mode 100644 index 533b3c4..0000000 --- a/internal/plcclient/client.go +++ /dev/null @@ -1,302 +0,0 @@ -package plcclient - -import ( - "bufio" - "context" - "fmt" - "io" - "log" - "net/http" - "strconv" - "time" - - "github.com/goccy/go-json" -) - -// Client is a client for the PLC directory -type Client struct { - baseURL string - httpClient *http.Client - rateLimiter *RateLimiter - logger Logger - userAgent string -} - -// Logger is a simple logging interface -type Logger interface { - Printf(format string, v ...interface{}) - Println(v ...interface{}) -} - -// defaultLogger uses standard log package -type defaultLogger struct{} - -func (d defaultLogger) Printf(format string, v ...interface{}) { - log.Printf(format, v...) -} - -func (d defaultLogger) Println(v ...interface{}) { - log.Println(v...) -} - -// ClientOption is a functional option for configuring the Client -type ClientOption func(*Client) - -// WithLogger sets a custom logger -func WithLogger(logger Logger) ClientOption { - return func(c *Client) { - c.logger = logger - } -} - -// WithUserAgent sets a custom user agent string -func WithUserAgent(userAgent string) ClientOption { - return func(c *Client) { - c.userAgent = userAgent - } -} - -// WithTimeout sets a custom HTTP timeout -func WithTimeout(timeout time.Duration) ClientOption { - return func(c *Client) { - c.httpClient.Timeout = timeout - } -} - -// WithRateLimit sets a custom rate limit (requests per period) -func WithRateLimit(requestsPerPeriod int, period time.Duration) ClientOption { - return func(c *Client) { - if c.rateLimiter != nil { - c.rateLimiter.Stop() - } - c.rateLimiter = NewRateLimiter(requestsPerPeriod, period) - } -} - -// NewClient creates a new PLC directory client -// Default: 90 requests per minute, 60 second timeout -func NewClient(baseURL string, opts ...ClientOption) *Client { - c := &Client{ - baseURL: baseURL, - httpClient: &http.Client{ - Timeout: 60 * time.Second, - }, - rateLimiter: NewRateLimiter(90, time.Minute), - logger: defaultLogger{}, - userAgent: "plcbundle/dev", - } - - for _, opt := range opts { - opt(c) - } - - return c -} - -// Close closes the client and cleans up resources -func (c *Client) Close() { - if c.rateLimiter != nil { - c.rateLimiter.Stop() - } -} - -// Export fetches export data from PLC directory with rate limiting and retry -func (c *Client) Export(ctx context.Context, opts ExportOptions) ([]PLCOperation, error) { - return c.exportWithRetry(ctx, opts, 5) -} - -// exportWithRetry implements retry logic with exponential backoff for rate limits -func (c *Client) exportWithRetry(ctx context.Context, opts ExportOptions, maxRetries int) ([]PLCOperation, error) { - var lastErr error - backoff := 1 * time.Second - - for attempt := 1; attempt <= maxRetries; attempt++ { - // Wait for rate limiter token - if err := c.rateLimiter.Wait(ctx); err != nil { - return nil, err - } - - operations, retryAfter, err := c.doExport(ctx, opts) - - if err == nil { - return operations, nil - } - - lastErr = err - - // Check if it's a rate limit error (429) - if retryAfter > 0 { - c.logger.Printf("Rate limited by PLC directory, waiting %v before retry %d/%d", - retryAfter, attempt, maxRetries) - - select { - case <-time.After(retryAfter): - continue - case <-ctx.Done(): - return nil, ctx.Err() - } - } - - // Other errors - exponential backoff - if attempt < maxRetries { - c.logger.Printf("Request failed (attempt %d/%d): %v, retrying in %v", - attempt, maxRetries, err, backoff) - - select { - case <-time.After(backoff): - backoff *= 2 // Exponential backoff - case <-ctx.Done(): - return nil, ctx.Err() - } - } - } - - return nil, fmt.Errorf("failed after %d attempts: %w", maxRetries, lastErr) -} - -// doExport performs the actual HTTP request -func (c *Client) doExport(ctx context.Context, opts ExportOptions) ([]PLCOperation, time.Duration, error) { - url := fmt.Sprintf("%s/export", c.baseURL) - - req, err := http.NewRequestWithContext(ctx, "GET", url, nil) - if err != nil { - return nil, 0, err - } - req.Header.Set("User-Agent", c.userAgent) - - // Add query parameters - q := req.URL.Query() - if opts.Count > 0 { - q.Add("count", fmt.Sprintf("%d", opts.Count)) - } - if opts.After != "" { - q.Add("after", opts.After) - } - req.URL.RawQuery = q.Encode() - - resp, err := c.httpClient.Do(req) - if err != nil { - return nil, 0, fmt.Errorf("request failed: %w", err) - } - defer resp.Body.Close() - - // Handle rate limiting (429) - if resp.StatusCode == http.StatusTooManyRequests { - retryAfter := parseRetryAfter(resp) - return nil, retryAfter, fmt.Errorf("rate limited (429)") - } - - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return nil, 0, fmt.Errorf("unexpected status code: %d, body: %s", resp.StatusCode, string(body)) - } - - var operations []PLCOperation - - // PLC export returns newline-delimited JSON - scanner := bufio.NewScanner(resp.Body) - buf := make([]byte, 0, 64*1024) - scanner.Buffer(buf, 1024*1024) - - lineCount := 0 - for scanner.Scan() { - lineCount++ - line := scanner.Bytes() - - if len(line) == 0 { - continue - } - - var op PLCOperation - if err := json.Unmarshal(line, &op); err != nil { - c.logger.Printf("Warning: failed to parse operation on line %d: %v", lineCount, err) - continue - } - - // CRITICAL: Store the original raw JSON bytes - op.RawJSON = make([]byte, len(line)) - copy(op.RawJSON, line) - - operations = append(operations, op) - } - - if err := scanner.Err(); err != nil { - return nil, 0, fmt.Errorf("error reading response: %w", err) - } - - return operations, 0, nil -} - -// parseRetryAfter parses the Retry-After header -func parseRetryAfter(resp *http.Response) time.Duration { - retryAfter := resp.Header.Get("Retry-After") - if retryAfter == "" { - // Default to 5 minutes if no header - return 5 * time.Minute - } - - // Try parsing as seconds - if seconds, err := strconv.Atoi(retryAfter); err == nil { - return time.Duration(seconds) * time.Second - } - - // Try parsing as HTTP date - if t, err := http.ParseTime(retryAfter); err == nil { - return time.Until(t) - } - - // Default - return 5 * time.Minute -} - -// GetDID fetches a specific DID document from PLC -func (c *Client) GetDID(ctx context.Context, did string) (*DIDDocument, error) { - // Wait for rate limiter - if err := c.rateLimiter.Wait(ctx); err != nil { - return nil, err - } - - url := fmt.Sprintf("%s/%s", c.baseURL, did) - - req, err := http.NewRequestWithContext(ctx, "GET", url, nil) - if err != nil { - return nil, err - } - req.Header.Set("User-Agent", c.userAgent) - - resp, err := c.httpClient.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode == http.StatusTooManyRequests { - retryAfter := parseRetryAfter(resp) - return nil, fmt.Errorf("rate limited, retry after %v", retryAfter) - } - - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return nil, fmt.Errorf("unexpected status code: %d, body: %s", resp.StatusCode, string(body)) - } - - var doc DIDDocument - if err := json.NewDecoder(resp.Body).Decode(&doc); err != nil { - return nil, err - } - - return &doc, nil -} - -// GetStats returns basic stats about the client -func (c *Client) GetStats() map[string]interface{} { - return map[string]interface{}{ - "base_url": c.baseURL, - "timeout": c.httpClient.Timeout, - } -} - -// GetBaseURL returns the PLC directory base URL -func (c *Client) GetBaseURL() string { - return c.baseURL -} diff --git a/internal/plcclient/plc_test.go b/internal/plcclient/plc_test.go deleted file mode 100644 index 8d4d562..0000000 --- a/internal/plcclient/plc_test.go +++ /dev/null @@ -1,259 +0,0 @@ -package plcclient_test - -import ( - "context" - "encoding/json" - "net/http" - "net/http/httptest" - "testing" - "time" - - "tangled.org/atscan.net/plcbundle/internal/plcclient" - "tangled.org/atscan.net/plcbundle/internal/storage" -) - -// TestPLCOperation tests operation parsing and methods -func TestPLCOperation(t *testing.T) { - t.Run("IsNullified", func(t *testing.T) { - tests := []struct { - name string - nullified interface{} - want bool - }{ - {"nil", nil, false}, - {"false", false, false}, - {"true", true, true}, - {"empty string", "", false}, - {"non-empty string", "cid123", true}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - op := plcclient.PLCOperation{Nullified: tt.nullified} - if got := op.IsNullified(); got != tt.want { - t.Errorf("IsNullified() = %v, want %v", got, tt.want) - } - }) - } - }) - - t.Run("GetNullifyingCID", func(t *testing.T) { - op := plcclient.PLCOperation{Nullified: "bafytest123"} - if cid := op.GetNullifyingCID(); cid != "bafytest123" { - t.Errorf("expected 'bafytest123', got '%s'", cid) - } - - op2 := plcclient.PLCOperation{Nullified: true} - if cid := op2.GetNullifyingCID(); cid != "" { - t.Errorf("expected empty string, got '%s'", cid) - } - }) - - t.Run("JSONParsing", func(t *testing.T) { - jsonData := `{ - "did": "did:plc:test123", - "cid": "bafytest", - "createdAt": "2024-01-01T12:00:00.000Z", - "operation": {"type": "create"}, - "nullified": false - }` - - var op plcclient.PLCOperation - if err := json.Unmarshal([]byte(jsonData), &op); err != nil { - t.Fatalf("failed to parse operation: %v", err) - } - - if op.DID != "did:plc:test123" { - t.Errorf("unexpected DID: %s", op.DID) - } - if op.CID != "bafytest" { - t.Errorf("unexpected CID: %s", op.CID) - } - }) -} - -// TestClient tests PLC client operations -func TestClient(t *testing.T) { - t.Run("Export", func(t *testing.T) { - // Create mock server - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/export" { - t.Errorf("unexpected path: %s", r.URL.Path) - } - - // Check query parameters - count := r.URL.Query().Get("count") - if count != "100" { - t.Errorf("unexpected count: %s", count) - } - - // Return mock JSONL data - w.Header().Set("Content-Type", "application/x-ndjson") - for i := 0; i < 10; i++ { - op := plcclient.PLCOperation{ - DID: "did:plc:test" + string(rune(i)), - CID: "bafytest" + string(rune(i)), - CreatedAt: time.Now(), - //Operation: map[string]interface{}{"type": "create"}, - } - json.NewEncoder(w).Encode(op) - } - })) - defer server.Close() - - // Create client - client := plcclient.NewClient(server.URL) - defer client.Close() - - // Test export - ctx := context.Background() - ops, err := client.Export(ctx, plcclient.ExportOptions{ - Count: 100, - }) - if err != nil { - t.Fatalf("Export failed: %v", err) - } - - if len(ops) != 10 { - t.Errorf("expected 10 operations, got %d", len(ops)) - } - - // Check that RawJSON is preserved - if len(ops[0].RawJSON) == 0 { - t.Error("RawJSON not preserved") - } - }) - - t.Run("RateLimitRetry", func(t *testing.T) { - attempts := 0 - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - attempts++ - if attempts < 2 { - // Return 429 on first attempt - w.Header().Set("Retry-After", "1") - w.WriteHeader(http.StatusTooManyRequests) - return - } - // Success on second attempt - w.Header().Set("Content-Type", "application/x-ndjson") - op := plcclient.PLCOperation{DID: "did:plc:test", CID: "bafytest", CreatedAt: time.Now()} - json.NewEncoder(w).Encode(op) - })) - defer server.Close() - - client := plcclient.NewClient(server.URL) - defer client.Close() - - ctx := context.Background() - ops, err := client.Export(ctx, plcclient.ExportOptions{Count: 1}) - if err != nil { - t.Fatalf("Export failed after retry: %v", err) - } - - if len(ops) != 1 { - t.Errorf("expected 1 operation, got %d", len(ops)) - } - - if attempts < 2 { - t.Errorf("expected at least 2 attempts, got %d", attempts) - } - }) - - t.Run("GetDID", func(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/did:plc:test123" { - t.Errorf("unexpected path: %s", r.URL.Path) - } - - doc := plcclient.DIDDocument{ - Context: []string{"https://www.w3.org/ns/did/v1"}, - ID: "did:plc:test123", - } - json.NewEncoder(w).Encode(doc) - })) - defer server.Close() - - client := plcclient.NewClient(server.URL) - defer client.Close() - - ctx := context.Background() - doc, err := client.GetDID(ctx, "did:plc:test123") - if err != nil { - t.Fatalf("GetDID failed: %v", err) - } - - if doc.ID != "did:plc:test123" { - t.Errorf("unexpected DID: %s", doc.ID) - } - }) -} - -// TestRateLimiter tests rate limiting functionality -func TestRateLimiter(t *testing.T) { - t.Run("BasicRateLimit", func(t *testing.T) { - // 10 requests per second - rl := plcclient.NewRateLimiter(10, time.Second) - defer rl.Stop() - - ctx := context.Background() - - // First 10 should be fast - start := time.Now() - for i := 0; i < 10; i++ { - if err := rl.Wait(ctx); err != nil { - t.Fatalf("Wait failed: %v", err) - } - } - elapsed := time.Since(start) - - // Should be very fast (less than 100ms) - if elapsed > 100*time.Millisecond { - t.Errorf("expected fast execution, took %v", elapsed) - } - }) - - t.Run("ContextCancellation", func(t *testing.T) { - rl := plcclient.NewRateLimiter(1, time.Minute) // Very slow rate - defer rl.Stop() - - // Consume the one available token - ctx := context.Background() - rl.Wait(ctx) - - // Try to wait with cancelled context - cancelCtx, cancel := context.WithCancel(context.Background()) - cancel() // Cancel immediately - - err := rl.Wait(cancelCtx) - if err != context.Canceled { - t.Errorf("expected context.Canceled, got %v", err) - } - }) -} - -// Benchmark tests -func BenchmarkSerializeJSONL(b *testing.B) { - ops := make([]plcclient.PLCOperation, 10000) - for i := 0; i < 10000; i++ { - ops[i] = plcclient.PLCOperation{ - DID: "did:plc:test", - CID: "bafytest", - CreatedAt: time.Now(), - //Operation: map[string]interface{}{"type": "create"}, - } - } - - logger := &benchLogger{} - operations, _ := storage.NewOperations(logger, false) - defer operations.Close() - - b.ResetTimer() - for i := 0; i < b.N; i++ { - _ = operations.SerializeJSONL(ops) - } -} - -type benchLogger struct{} - -func (l *benchLogger) Printf(format string, v ...interface{}) {} -func (l *benchLogger) Println(v ...interface{}) {} diff --git a/internal/plcclient/ratelimiter.go b/internal/plcclient/ratelimiter.go deleted file mode 100644 index 877e21f..0000000 --- a/internal/plcclient/ratelimiter.go +++ /dev/null @@ -1,70 +0,0 @@ -package plcclient - -import ( - "context" - "time" -) - -// RateLimiter implements a token bucket rate limiter -type RateLimiter struct { - tokens chan struct{} - refillRate time.Duration - maxTokens int - stopRefill chan struct{} -} - -// NewRateLimiter creates a new rate limiter -// Example: NewRateLimiter(90, time.Minute) = 90 requests per minute -func NewRateLimiter(requestsPerPeriod int, period time.Duration) *RateLimiter { - rl := &RateLimiter{ - tokens: make(chan struct{}, requestsPerPeriod), - refillRate: period / time.Duration(requestsPerPeriod), - maxTokens: requestsPerPeriod, - stopRefill: make(chan struct{}), - } - - // Fill initially - for i := 0; i < requestsPerPeriod; i++ { - rl.tokens <- struct{}{} - } - - // Start refill goroutine - go rl.refill() - - return rl -} - -// refill adds tokens at the specified rate -func (rl *RateLimiter) refill() { - ticker := time.NewTicker(rl.refillRate) - defer ticker.Stop() - - for { - select { - case <-ticker.C: - select { - case rl.tokens <- struct{}{}: - // Token added - default: - // Buffer full, skip - } - case <-rl.stopRefill: - return - } - } -} - -// Wait blocks until a token is available or context is cancelled -func (rl *RateLimiter) Wait(ctx context.Context) error { - select { - case <-rl.tokens: - return nil - case <-ctx.Done(): - return ctx.Err() - } -} - -// Stop stops the rate limiter and cleans up resources -func (rl *RateLimiter) Stop() { - close(rl.stopRefill) -} diff --git a/internal/plcclient/resolver.go b/internal/plcclient/resolver.go deleted file mode 100644 index 706be9e..0000000 --- a/internal/plcclient/resolver.go +++ /dev/null @@ -1,322 +0,0 @@ -package plcclient - -import ( - "fmt" - "strings" -) - -// DIDState represents the current state of a DID (PLC-specific format) -type DIDState struct { - DID string `json:"did"` - RotationKeys []string `json:"rotationKeys"` - VerificationMethods map[string]string `json:"verificationMethods"` - AlsoKnownAs []string `json:"alsoKnownAs"` - Services map[string]ServiceDefinition `json:"services"` -} - -// ServiceDefinition represents a service endpoint -type ServiceDefinition struct { - Type string `json:"type"` - Endpoint string `json:"endpoint"` -} - -// AuditLogEntry represents a single entry in the audit log -type AuditLogEntry struct { - DID string `json:"did"` - Operation interface{} `json:"operation"` // The parsed operation data - CID string `json:"cid"` - Nullified interface{} `json:"nullified,omitempty"` - CreatedAt string `json:"createdAt"` -} - -// ResolveDIDDocument constructs a DID document from operation log -// This is the main entry point for DID resolution -func ResolveDIDDocument(did string, operations []PLCOperation) (*DIDDocument, error) { - if len(operations) == 0 { - return nil, fmt.Errorf("no operations found for DID") - } - - // Build current state from operations - state, err := BuildDIDState(did, operations) - if err != nil { - return nil, err - } - - // Convert to DID document format - return StateToDIDDocument(state), nil -} - -// BuildDIDState applies operations in order to build current DID state -func BuildDIDState(did string, operations []PLCOperation) (*DIDState, error) { - var state *DIDState - - for _, op := range operations { - // Skip nullified operations - if op.IsNullified() { - continue - } - - // Parse operation data - opData, err := op.GetOperationData() - if err != nil { - return nil, fmt.Errorf("failed to parse operation: %w", err) - } - - if opData == nil { - continue - } - - // Check operation type - opType, _ := opData["type"].(string) - - // Handle tombstone (deactivated DID) - if opType == "plc_tombstone" { - return nil, fmt.Errorf("DID has been deactivated") - } - - // Initialize state on first operation - if state == nil { - state = &DIDState{ - DID: did, - RotationKeys: []string{}, - VerificationMethods: make(map[string]string), - AlsoKnownAs: []string{}, - Services: make(map[string]ServiceDefinition), - } - } - - // Apply operation to state - applyOperationToState(state, opData) - } - - if state == nil { - return nil, fmt.Errorf("no valid operations found") - } - - return state, nil -} - -// applyOperationToState updates state with data from an operation -func applyOperationToState(state *DIDState, opData map[string]interface{}) { - // Update rotation keys - if rotKeys, ok := opData["rotationKeys"].([]interface{}); ok { - state.RotationKeys = make([]string, 0, len(rotKeys)) - for _, k := range rotKeys { - if keyStr, ok := k.(string); ok { - state.RotationKeys = append(state.RotationKeys, keyStr) - } - } - } - - // Update verification methods - if vm, ok := opData["verificationMethods"].(map[string]interface{}); ok { - state.VerificationMethods = make(map[string]string) - for key, val := range vm { - if valStr, ok := val.(string); ok { - state.VerificationMethods[key] = valStr - } - } - } - - // Handle legacy signingKey format - if signingKey, ok := opData["signingKey"].(string); ok { - if state.VerificationMethods == nil { - state.VerificationMethods = make(map[string]string) - } - state.VerificationMethods["atproto"] = signingKey - } - - // Update alsoKnownAs - if aka, ok := opData["alsoKnownAs"].([]interface{}); ok { - state.AlsoKnownAs = make([]string, 0, len(aka)) - for _, a := range aka { - if akaStr, ok := a.(string); ok { - state.AlsoKnownAs = append(state.AlsoKnownAs, akaStr) - } - } - } - - // Handle legacy handle format - if handle, ok := opData["handle"].(string); ok { - // Only set if alsoKnownAs is empty - if len(state.AlsoKnownAs) == 0 { - state.AlsoKnownAs = []string{"at://" + handle} - } - } - - // Update services - if services, ok := opData["services"].(map[string]interface{}); ok { - state.Services = make(map[string]ServiceDefinition) - for key, svc := range services { - if svcMap, ok := svc.(map[string]interface{}); ok { - svcType, _ := svcMap["type"].(string) - endpoint, _ := svcMap["endpoint"].(string) - state.Services[key] = ServiceDefinition{ - Type: svcType, - Endpoint: normalizeServiceEndpoint(endpoint), - } - } - } - } - - // Handle legacy service format - if service, ok := opData["service"].(string); ok { - if state.Services == nil { - state.Services = make(map[string]ServiceDefinition) - } - state.Services["atproto_pds"] = ServiceDefinition{ - Type: "AtprotoPersonalDataServer", - Endpoint: normalizeServiceEndpoint(service), - } - } -} - -// StateToDIDDocument converts internal PLC state to W3C DID document format -func StateToDIDDocument(state *DIDState) *DIDDocument { - // Base contexts - ALWAYS include multikey (matches PLC directory behavior) - contexts := []string{ - "https://www.w3.org/ns/did/v1", - "https://w3id.org/security/multikey/v1", - } - - hasSecp256k1 := false - hasP256 := false - - // Check verification method key types for additional contexts - for _, didKey := range state.VerificationMethods { - keyType := detectKeyType(didKey) - switch keyType { - case "secp256k1": - hasSecp256k1 = true - case "p256": - hasP256 = true - } - } - - // Add suite-specific contexts only if those key types are present - if hasSecp256k1 { - contexts = append(contexts, "https://w3id.org/security/suites/secp256k1-2019/v1") - } - if hasP256 { - contexts = append(contexts, "https://w3id.org/security/suites/ecdsa-2019/v1") - } - - doc := &DIDDocument{ - Context: contexts, - ID: state.DID, - AlsoKnownAs: []string{}, - VerificationMethod: []VerificationMethod{}, - Service: []Service{}, - } - - // Copy alsoKnownAs if present - if len(state.AlsoKnownAs) > 0 { - doc.AlsoKnownAs = state.AlsoKnownAs - } - - // Convert services - for id, svc := range state.Services { - doc.Service = append(doc.Service, Service{ - ID: "#" + id, - Type: svc.Type, - ServiceEndpoint: svc.Endpoint, - }) - } - - // Keep verification methods with full DID - for id, didKey := range state.VerificationMethods { - doc.VerificationMethod = append(doc.VerificationMethod, VerificationMethod{ - ID: state.DID + "#" + id, - Type: "Multikey", - Controller: state.DID, - PublicKeyMultibase: ExtractMultibaseFromDIDKey(didKey), - }) - } - - return doc -} - -// detectKeyType detects the key type from did:key encoding -func detectKeyType(didKey string) string { - multibase := ExtractMultibaseFromDIDKey(didKey) - - if len(multibase) < 3 { - return "unknown" - } - - // The 'z' is the base58btc multibase prefix - // Actual key starts at position 1 - switch { - case multibase[1] == 'Q' && multibase[2] == '3': - return "secp256k1" // Starts with zQ3s - case multibase[1] == 'D' && multibase[2] == 'n': - return "p256" // Starts with zDn - case multibase[1] == '6' && multibase[2] == 'M': - return "ed25519" // Starts with z6Mk - default: - return "unknown" - } -} - -// ExtractMultibaseFromDIDKey extracts the multibase string from did:key: format -func ExtractMultibaseFromDIDKey(didKey string) string { - return strings.TrimPrefix(didKey, "did:key:") -} - -// ValidateDIDFormat validates did:plc format -func ValidateDIDFormat(did string) error { - if !strings.HasPrefix(did, "did:plc:") { - return fmt.Errorf("invalid DID method: must start with 'did:plc:'") - } - - if len(did) != 32 { - return fmt.Errorf("invalid DID length: expected 32 chars, got %d", len(did)) - } - - // Validate identifier part (24 chars, base32 alphabet) - identifier := strings.TrimPrefix(did, "did:plc:") - if len(identifier) != 24 { - return fmt.Errorf("invalid identifier length: expected 24 chars, got %d", len(identifier)) - } - - // Check base32 alphabet (a-z, 2-7, no 0189) - for _, c := range identifier { - if !((c >= 'a' && c <= 'z') || (c >= '2' && c <= '7')) { - return fmt.Errorf("invalid character in identifier: %c (must be base32: a-z, 2-7)", c) - } - } - - return nil -} - -// FormatAuditLog formats operations as an audit log -func FormatAuditLog(operations []PLCOperation) []AuditLogEntry { - log := make([]AuditLogEntry, 0, len(operations)) - - for _, op := range operations { - // Parse operation for the log - opData, _ := op.GetOperationData() - - entry := AuditLogEntry{ - DID: op.DID, - Operation: opData, - CID: op.CID, - Nullified: op.Nullified, - CreatedAt: op.CreatedAt.Format("2006-01-02T15:04:05.000Z"), - } - - log = append(log, entry) - } - - return log -} - -func normalizeServiceEndpoint(endpoint string) string { - // If already has protocol, return as-is - if strings.HasPrefix(endpoint, "http://") || strings.HasPrefix(endpoint, "https://") { - return endpoint - } - - // Legacy format: add https:// prefix - return "https://" + endpoint -} diff --git a/internal/plcclient/types.go b/internal/plcclient/types.go deleted file mode 100644 index adbf582..0000000 --- a/internal/plcclient/types.go +++ /dev/null @@ -1,120 +0,0 @@ -package plcclient - -import ( - "time" - - "github.com/goccy/go-json" -) - -// PLCOperation represents a single operation from the PLC directory -type PLCOperation struct { - DID string `json:"did"` - Operation json.RawMessage `json:"operation"` // Lazy - CID string `json:"cid"` - Nullified interface{} `json:"nullified,omitempty"` - CreatedAt time.Time `json:"createdAt"` - - RawJSON []byte `json:"-"` - ParsedOperation map[string]interface{} `json:"-"` // Pre-parsed cache -} - -// IsNullified checks if this operation has been nullified -func (op *PLCOperation) IsNullified() bool { - if op.Nullified == nil { - return false - } - - switch v := op.Nullified.(type) { - case bool: - return v - case string: - return v != "" - default: - return false - } -} - -// GetNullifyingCID returns the CID that nullified this operation -func (op *PLCOperation) GetNullifyingCID() string { - if s, ok := op.Nullified.(string); ok { - return s - } - return "" -} - -// DIDDocument represents a DID document from PLC -type DIDDocument struct { - Context []string `json:"@context"` - ID string `json:"id"` - AlsoKnownAs []string `json:"alsoKnownAs"` - VerificationMethod []VerificationMethod `json:"verificationMethod"` - Service []Service `json:"service"` -} - -// VerificationMethod represents a verification method in a DID document -type VerificationMethod struct { - ID string `json:"id"` - Type string `json:"type"` - Controller string `json:"controller"` - PublicKeyMultibase string `json:"publicKeyMultibase"` -} - -// Service represents a service endpoint in a DID document -type Service struct { - ID string `json:"id"` - Type string `json:"type"` - ServiceEndpoint string `json:"serviceEndpoint"` -} - -// ExportOptions contains options for exporting PLC operations -type ExportOptions struct { - Count int // Number of operations to fetch - After string // ISO 8601 datetime string to start after -} - -// DIDHistoryEntry represents a single operation in DID history -type DIDHistoryEntry struct { - Operation PLCOperation `json:"operation"` - PLCBundle string `json:"plc_bundle,omitempty"` -} - -// DIDHistory represents the full history of a DID -type DIDHistory struct { - DID string `json:"did"` - Current *PLCOperation `json:"current"` - Operations []DIDHistoryEntry `json:"operations"` -} - -// EndpointInfo contains extracted endpoint information from an operation -type EndpointInfo struct { - Type string // "pds", "labeler", etc. - Endpoint string -} - -// GetOperationData parses Operation into map (with caching) -func (op *PLCOperation) GetOperationData() (map[string]interface{}, error) { - // Return cached if already parsed - if op.ParsedOperation != nil { - return op.ParsedOperation, nil - } - - // Parse on first call - if len(op.Operation) == 0 { - return nil, nil - } - - var data map[string]interface{} - if err := json.UnmarshalNoEscape(op.Operation, &data); err != nil { - return nil, err - } - - // Cache it - op.ParsedOperation = data - - return data, nil -} - -// GetOperationMap is an alias for compatibility -func (op *PLCOperation) GetOperationMap() (map[string]interface{}, error) { - return op.GetOperationData() -} diff --git a/internal/storage/storage.go b/internal/storage/storage.go deleted file mode 100644 index 125d5b2..0000000 --- a/internal/storage/storage.go +++ /dev/null @@ -1,841 +0,0 @@ -package storage - -import ( - "bufio" - "bytes" - "crypto/sha256" - "encoding/hex" - "fmt" - "io" - "os" - "path/filepath" - "sync" - "time" - - "github.com/goccy/go-json" - "tangled.org/atscan.net/plcbundle/internal/plcclient" -) - -const ( - MetadataFormatVersion = 1 -) - -// BundleMetadata - Self-describing bundle (content-focused, not container) -type BundleMetadata struct { - // === Format Info === - Format string `json:"format"` // "plcbundle-v1" - - // === Bundle Identity === - Origin string `json:"origin"` // Source PLC directory URL - BundleNumber int `json:"bundle_number"` // Sequential bundle number - - // === Content Integrity === - ContentHash string `json:"content_hash"` // SHA256 of uncompressed JSONL content - ParentHash string `json:"parent_hash,omitempty"` // Hash of previous bundle (chain) - - // === Content Description === - OperationCount int `json:"operation_count"` // Always 10000 for complete bundles - DIDCount int `json:"did_count"` // Unique DIDs in this bundle - StartTime time.Time `json:"start_time"` // First operation timestamp - EndTime time.Time `json:"end_time"` // Last operation timestamp - - // === Creation Provenance === - CreatedAt time.Time `json:"created_at"` // When bundle was created - CreatedBy string `json:"created_by"` // "plcbundle/v1.2.3" - CreatedByHost string `json:"created_by_host,omitempty"` // Optional: hostname that created it - - // === Optional Context === - Cursor string `json:"cursor,omitempty"` // PLC export cursor for this bundle - Notes string `json:"notes,omitempty"` // Optional description - - // === Frame Structure (for random access) === - FrameCount int `json:"frame_count"` // Number of zstd frames (usually 100) - FrameSize int `json:"frame_size"` // Operations per frame (100) - FrameOffsets []int64 `json:"frame_offsets"` // Byte offsets of each frame -} - -// Operations handles low-level bundle file operations -type Operations struct { - logger Logger - verbose bool -} - -// Logger interface -type Logger interface { - Printf(format string, v ...interface{}) - Println(v ...interface{}) -} - -func NewOperations(logger Logger, verbose bool) (*Operations, error) { - return &Operations{ - logger: logger, - verbose: verbose, - }, nil -} - -func (op *Operations) Close() { - // Nothing to close -} - -// BundleInfo contains info needed to create metadata -type BundleInfo struct { - BundleNumber int - Origin string - ParentHash string - Cursor string - CreatedBy string // "plcbundle/v1.2.3" - Hostname string // Optional -} - -// ======================================== -// CORE SERIALIZATION (JSONL) -// ======================================== - -// SerializeJSONL serializes operations to newline-delimited JSON -func (op *Operations) SerializeJSONL(operations []plcclient.PLCOperation) []byte { - var buf bytes.Buffer - - for _, operation := range operations { - if len(operation.RawJSON) > 0 { - buf.Write(operation.RawJSON) - } else { - data, _ := json.Marshal(operation) - buf.Write(data) - } - buf.WriteByte('\n') - } - - return buf.Bytes() -} - -// ParseJSONL parses newline-delimited JSON into operations -func (op *Operations) ParseJSONL(data []byte) ([]plcclient.PLCOperation, error) { - var operations []plcclient.PLCOperation - scanner := bufio.NewScanner(bytes.NewReader(data)) - buf := make([]byte, 0, 64*1024) - scanner.Buffer(buf, 1024*1024) - - for scanner.Scan() { - line := scanner.Bytes() - if len(line) == 0 { - continue - } - - var operation plcclient.PLCOperation - if err := json.UnmarshalNoEscape(line, &operation); err != nil { - return nil, fmt.Errorf("failed to parse line: %w", err) - } - - operation.RawJSON = make([]byte, len(line)) - copy(operation.RawJSON, line) - operations = append(operations, operation) - } - - return operations, nil -} - -// ======================================== -// FILE OPERATIONS (using zstd abstraction) -// ======================================== - -// SaveBundle saves operations with metadata containing RELATIVE frame offsets -func (op *Operations) SaveBundle(path string, operations []plcclient.PLCOperation, bundleInfo *BundleInfo) (string, string, int64, int64, error) { - if bundleInfo == nil { - return "", "", 0, 0, fmt.Errorf("bundleInfo cannot be nil") - } - - // 1. Calculate content - jsonlData := op.SerializeJSONL(operations) - contentSize := int64(len(jsonlData)) - contentHash := op.Hash(jsonlData) - dids := op.ExtractUniqueDIDs(operations) - - hostnameHash := "" - if bundleInfo.Hostname != "" { - hostnameHash = op.Hash([]byte(bundleInfo.Hostname))[:16] // First 16 chars (64 bits) - } - - // 2. Compress all frames - compressedFrames := make([][]byte, 0) - - for i := 0; i < len(operations); i += FrameSize { - end := i + FrameSize - if end > len(operations) { - end = len(operations) - } - opChunk := operations[i:end] - chunkJsonlData := op.SerializeJSONL(opChunk) - - compressedChunk, err := CompressFrame(chunkJsonlData) - if err != nil { - return "", "", 0, 0, fmt.Errorf("failed to compress frame: %w", err) - } - - compressedFrames = append(compressedFrames, compressedChunk) - } - - // 3. Calculate RELATIVE offsets (relative to first data frame) - relativeOffsets := make([]int64, len(compressedFrames)+1) - relativeOffsets[0] = 0 - - cumulative := int64(0) - for i, frame := range compressedFrames { - cumulative += int64(len(frame)) - relativeOffsets[i+1] = cumulative - } - - // 4. Build metadata with RELATIVE offsets - metadata := &BundleMetadata{ - Format: fmt.Sprintf("plcbundle-v%d", MetadataFormatVersion), - BundleNumber: bundleInfo.BundleNumber, - Origin: bundleInfo.Origin, - CreatedAt: time.Now().UTC(), - CreatedBy: bundleInfo.CreatedBy, - CreatedByHost: hostnameHash, - ContentHash: contentHash, - ParentHash: bundleInfo.ParentHash, - OperationCount: len(operations), - DIDCount: len(dids), - FrameCount: len(compressedFrames), - FrameSize: FrameSize, - Cursor: bundleInfo.Cursor, - FrameOffsets: relativeOffsets, // RELATIVE to data start! - } - - if len(operations) > 0 { - metadata.StartTime = operations[0].CreatedAt - metadata.EndTime = operations[len(operations)-1].CreatedAt - } - - // 5. Write final file - finalFile, err := os.Create(path) - if err != nil { - return "", "", 0, 0, fmt.Errorf("failed to create file: %w", err) - } - defer func() { - finalFile.Close() - if err != nil { - os.Remove(path) - } - }() - - // Write metadata frame - if _, err := op.WriteMetadataFrame(finalFile, metadata); err != nil { - return "", "", 0, 0, fmt.Errorf("failed to write metadata: %w", err) - } - - // Write all data frames - for _, frame := range compressedFrames { - if _, err := finalFile.Write(frame); err != nil { - return "", "", 0, 0, fmt.Errorf("failed to write frame: %w", err) - } - } - - finalFile.Sync() - finalFile.Close() - - // 6. Hash - compressedData, err := os.ReadFile(path) - if err != nil { - return "", "", 0, 0, err - } - compressedHash := op.Hash(compressedData) - - os.Remove(path + ".idx") - - return contentHash, compressedHash, contentSize, int64(len(compressedData)), nil -} - -// LoadBundle loads a compressed bundle -func (op *Operations) LoadBundle(path string) ([]plcclient.PLCOperation, error) { - file, err := os.Open(path) - if err != nil { - return nil, fmt.Errorf("failed to open file: %w", err) - } - defer file.Close() - - // Use abstracted streaming reader - reader, err := NewStreamingReader(file) - if err != nil { - return nil, fmt.Errorf("failed to create reader: %w", err) - } - defer reader.Release() - - // Read all decompressed data from all frames - decompressed, err := io.ReadAll(reader) - if err != nil { - return nil, fmt.Errorf("failed to decompress: %w", err) - } - - // DEFENSIVE: Validate we got actual data - if len(decompressed) == 0 { - return nil, fmt.Errorf("decompression produced empty result") - } - - // Parse JSONL - operations, err := op.ParseJSONL(decompressed) - if err != nil { - return nil, fmt.Errorf("failed to parse JSONL: %w", err) - } - - // DEFENSIVE: Additional validation - if len(operations) == 0 { - return nil, fmt.Errorf("bundle contains no valid operations") - } - - return operations, nil -} - -// ======================================== -// STREAMING -// ======================================== - -// StreamRaw returns a reader for the raw compressed bundle file -func (op *Operations) StreamRaw(path string) (io.ReadCloser, error) { - file, err := os.Open(path) - if err != nil { - return nil, fmt.Errorf("failed to open bundle: %w", err) - } - return file, nil -} - -// StreamDecompressed returns a reader for decompressed bundle data -func (op *Operations) StreamDecompressed(path string) (io.ReadCloser, error) { - file, err := os.Open(path) - if err != nil { - return nil, fmt.Errorf("failed to open bundle: %w", err) - } - - // Use abstracted reader - reader, err := NewStreamingReader(file) - if err != nil { - file.Close() - return nil, fmt.Errorf("failed to create reader: %w", err) - } - - return &decompressedReader{ - reader: reader, - file: file, - }, nil -} - -// decompressedReader wraps a zstd decoder and underlying file -type decompressedReader struct { - reader StreamReader - file *os.File -} - -func (dr *decompressedReader) Read(p []byte) (int, error) { - return dr.reader.Read(p) -} - -func (dr *decompressedReader) Close() error { - dr.reader.Release() - return dr.file.Close() -} - -// ======================================== -// HASHING -// ======================================== - -// Hash computes SHA256 hash of data -func (op *Operations) Hash(data []byte) string { - h := sha256.Sum256(data) - return hex.EncodeToString(h[:]) -} - -// CalculateChainHash calculates the cumulative chain hash -func (op *Operations) CalculateChainHash(parent string, contentHash string) string { - var data string - if parent == "" { - data = "plcbundle:genesis:" + contentHash - } else { - data = parent + ":" + contentHash - } - return op.Hash([]byte(data)) -} - -// CalculateFileHashes calculates both content and compressed hashes efficiently -func (op *Operations) CalculateFileHashes(path string) (compressedHash string, compressedSize int64, contentHash string, contentSize int64, err error) { - // Read compressed file - compressedData, err := os.ReadFile(path) - if err != nil { - return "", 0, "", 0, fmt.Errorf("failed to read file: %w", err) - } - - compressedHash = op.Hash(compressedData) - compressedSize = int64(len(compressedData)) - - // Use abstracted decompression - decompressed, err := DecompressAll(compressedData) - if err != nil { - return "", 0, "", 0, fmt.Errorf("failed to decompress: %w", err) - } - - contentHash = op.Hash(decompressed) - contentSize = int64(len(decompressed)) - - return compressedHash, compressedSize, contentHash, contentSize, nil -} - -// VerifyHash verifies the hash of a bundle file -func (op *Operations) VerifyHash(path string, expectedHash string) (bool, string, error) { - data, err := os.ReadFile(path) - if err != nil { - return false, "", fmt.Errorf("failed to read file: %w", err) - } - - actualHash := op.Hash(data) - return actualHash == expectedHash, actualHash, nil -} - -// ======================================== -// UTILITY FUNCTIONS -// ======================================== - -// FileExists checks if a file exists -func (op *Operations) FileExists(path string) bool { - _, err := os.Stat(path) - return err == nil -} - -// GetFileSize returns the size of a file -func (op *Operations) GetFileSize(path string) (int64, error) { - info, err := os.Stat(path) - if err != nil { - return 0, err - } - return info.Size(), nil -} - -// ExtractUniqueDIDs extracts unique DIDs from operations -func (op *Operations) ExtractUniqueDIDs(operations []plcclient.PLCOperation) []string { - didSet := make(map[string]bool) - for _, operation := range operations { - didSet[operation.DID] = true - } - - dids := make([]string, 0, len(didSet)) - for did := range didSet { - dids = append(dids, did) - } - - return dids -} - -// GetBoundaryCIDs returns CIDs that share the same timestamp as the last operation -func (op *Operations) GetBoundaryCIDs(operations []plcclient.PLCOperation) (time.Time, map[string]bool) { - if len(operations) == 0 { - return time.Time{}, nil - } - - lastOp := operations[len(operations)-1] - boundaryTime := lastOp.CreatedAt - cidSet := make(map[string]bool) - - // Walk backwards from the end - for i := len(operations) - 1; i >= 0; i-- { - op := operations[i] - if op.CreatedAt.Equal(boundaryTime) { - cidSet[op.CID] = true - } else { - break - } - } - - return boundaryTime, cidSet -} - -// StripBoundaryDuplicates removes operations that are in prevBoundaryCIDs -func (op *Operations) StripBoundaryDuplicates(operations []plcclient.PLCOperation, boundaryTimestamp string, prevBoundaryCIDs map[string]bool) []plcclient.PLCOperation { - if len(operations) == 0 || len(prevBoundaryCIDs) == 0 { - return operations - } - - boundaryTime, err := time.Parse(time.RFC3339Nano, boundaryTimestamp) - if err != nil { - return operations - } - - startIdx := 0 - for startIdx < len(operations) { - op := operations[startIdx] - - if op.CreatedAt.After(boundaryTime) { - break - } - - if op.CreatedAt.Equal(boundaryTime) && prevBoundaryCIDs[op.CID] { - startIdx++ - continue - } - - break - } - - return operations[startIdx:] -} - -// Pool for scanner buffers -var scannerBufPool = sync.Pool{ - New: func() interface{} { - buf := make([]byte, 64*1024) - return &buf - }, -} - -// ======================================== -// POSITION-BASED LOADING (with frame index) -// ======================================== - -// LoadOperationAtPosition loads a single operation from a bundle -func (op *Operations) LoadOperationAtPosition(path string, position int) (*plcclient.PLCOperation, error) { - if position < 0 { - return nil, fmt.Errorf("invalid position: %d", position) - } - - // Try multiple sources for frame index (no goto!) - frameOffsets, err := op.loadFrameIndex(path) - if err != nil { - // No frame index available - use legacy full scan - if op.logger != nil { - op.logger.Printf("No frame index found for %s, using legacy scan", filepath.Base(path)) - } - return op.loadOperationAtPositionLegacy(path, position) - } - - // We have frame index - use it for fast random access - return op.loadOperationFromFrame(path, position, frameOffsets) -} - -// loadFrameIndex loads frame offsets and converts to absolute positions -func (op *Operations) loadFrameIndex(path string) ([]int64, error) { - // Try embedded metadata first - meta, err := op.ExtractMetadataFromFile(path) - if err == nil && len(meta.FrameOffsets) > 0 { - // Convert relative offsets to absolute - // First, get metadata frame size by re-reading - file, _ := os.Open(path) - if file != nil { - defer file.Close() - - // Read metadata frame to find where data starts - magic, data, readErr := op.ReadSkippableFrame(file) - if readErr == nil && magic == SkippableMagicMetadata { - // Metadata frame size = 4 (magic) + 4 (size) + len(data) - metadataFrameSize := int64(8 + len(data)) - - // Convert relative to absolute - absoluteOffsets := make([]int64, len(meta.FrameOffsets)) - for i, relOffset := range meta.FrameOffsets { - absoluteOffsets[i] = metadataFrameSize + relOffset - } - - return absoluteOffsets, nil - } - } - } - - // Fallback to external .idx file - indexPath := path + ".idx" - indexData, err := os.ReadFile(indexPath) - if err != nil { - return nil, fmt.Errorf("no frame index available: %w", err) - } - - var offsets []int64 - if err := json.Unmarshal(indexData, &offsets); err != nil { - return nil, fmt.Errorf("invalid frame index: %w", err) - } - - return offsets, nil -} - -// loadOperationFromFrame loads operation using frame index -func (op *Operations) loadOperationFromFrame(path string, position int, frameOffsets []int64) (*plcclient.PLCOperation, error) { - frameIndex := position / FrameSize - lineInFrame := position % FrameSize - - if frameIndex >= len(frameOffsets)-1 { - return nil, fmt.Errorf("position %d out of bounds (frame %d, total frames %d)", - position, frameIndex, len(frameOffsets)-1) - } - - startOffset := frameOffsets[frameIndex] - endOffset := frameOffsets[frameIndex+1] - frameLength := endOffset - startOffset - - if frameLength <= 0 || frameLength > 10*1024*1024 { - return nil, fmt.Errorf("invalid frame length: %d (offsets: %d-%d)", - frameLength, startOffset, endOffset) - } - - bundleFile, err := os.Open(path) - if err != nil { - return nil, fmt.Errorf("failed to open bundle: %w", err) - } - defer bundleFile.Close() - - compressedFrame := make([]byte, frameLength) - _, err = bundleFile.ReadAt(compressedFrame, startOffset) - if err != nil { - return nil, fmt.Errorf("failed to read frame %d (offset %d, length %d): %w", - frameIndex, startOffset, frameLength, err) - } - - // Decompress - decompressed, err := DecompressFrame(compressedFrame) - if err != nil { - if op.logger != nil { - preview := compressedFrame - if len(preview) > 16 { - preview = preview[:16] - } - if op.verbose { - op.logger.Printf("DEBUG: Failed frame data (first 16 bytes): % x", preview) - } - } - return nil, fmt.Errorf("failed to decompress frame %d: %w", frameIndex, err) - } - - // Scan to find the line - scanner := bufio.NewScanner(bytes.NewReader(decompressed)) - lineNum := 0 - - for scanner.Scan() { - if lineNum == lineInFrame { - line := scanner.Bytes() - var operation plcclient.PLCOperation - if err := json.UnmarshalNoEscape(line, &operation); err != nil { - return nil, fmt.Errorf("failed to parse operation at position %d: %w", position, err) - } - operation.RawJSON = make([]byte, len(line)) - copy(operation.RawJSON, line) - return &operation, nil - } - lineNum++ - } - - if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("scanner error on frame %d: %w", frameIndex, err) - } - - return nil, fmt.Errorf("position %d not found in frame %d", position, frameIndex) -} - -// loadOperationAtPositionLegacy loads operation from old single-frame bundles -func (op *Operations) loadOperationAtPositionLegacy(path string, position int) (*plcclient.PLCOperation, error) { - file, err := os.Open(path) - if err != nil { - return nil, fmt.Errorf("failed to open file: %w", err) - } - defer file.Close() - - // Use abstracted streaming reader - reader, err := NewStreamingReader(file) - if err != nil { - return nil, fmt.Errorf("failed to create reader: %w", err) - } - defer reader.Release() - - scanner := bufio.NewScanner(reader) - buf := make([]byte, 512*1024) - scanner.Buffer(buf, 1024*1024) - - lineNum := 0 - for scanner.Scan() { - if lineNum == position { - line := scanner.Bytes() - var operation plcclient.PLCOperation - if err := json.UnmarshalNoEscape(line, &operation); err != nil { - return nil, fmt.Errorf("failed to parse operation at position %d: %w", position, err) - } - operation.RawJSON = make([]byte, len(line)) - copy(operation.RawJSON, line) - return &operation, nil - } - lineNum++ - } - - if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("scanner error: %w", err) - } - - return nil, fmt.Errorf("position %d not found in bundle", position) -} - -// LoadOperationsAtPositions loads multiple operations from a bundle in one pass -func (op *Operations) LoadOperationsAtPositions(path string, positions []int) (map[int]*plcclient.PLCOperation, error) { - if len(positions) == 0 { - return make(map[int]*plcclient.PLCOperation), nil - } - - // Create position set for fast lookup - posSet := make(map[int]bool) - maxPos := 0 - for _, pos := range positions { - if pos < 0 { - continue - } - posSet[pos] = true - if pos > maxPos { - maxPos = pos - } - } - - file, err := os.Open(path) - if err != nil { - return nil, fmt.Errorf("failed to open file: %w", err) - } - defer file.Close() - - // Use abstracted streaming reader - reader, err := NewStreamingReader(file) - if err != nil { - return nil, fmt.Errorf("failed to create reader: %w", err) - } - defer reader.Release() - - bufPtr := scannerBufPool.Get().(*[]byte) - defer scannerBufPool.Put(bufPtr) - - scanner := bufio.NewScanner(reader) - scanner.Buffer(*bufPtr, 512*1024) - - results := make(map[int]*plcclient.PLCOperation) - lineNum := 0 - - for scanner.Scan() { - // Early exit if we found everything - if len(results) == len(posSet) { - break - } - - // Only parse if this position is requested - if posSet[lineNum] { - line := scanner.Bytes() - var operation plcclient.PLCOperation - if err := json.UnmarshalNoEscape(line, &operation); err != nil { - return nil, fmt.Errorf("failed to parse operation at position %d: %w", lineNum, err) - } - - operation.RawJSON = make([]byte, len(line)) - copy(operation.RawJSON, line) - results[lineNum] = &operation - } - - lineNum++ - - // Early exit if we passed the max position - if lineNum > maxPos { - break - } - } - - if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("scanner error: %w", err) - } - - return results, nil -} - -// CalculateMetadataWithoutLoading calculates metadata by streaming (no full load) -func (op *Operations) CalculateMetadataWithoutLoading(path string) (opCount int, didCount int, startTime, endTime time.Time, err error) { - file, err := os.Open(path) - if err != nil { - return 0, 0, time.Time{}, time.Time{}, err - } - defer file.Close() - - // Use abstracted reader - reader, err := NewStreamingReader(file) - if err != nil { - return 0, 0, time.Time{}, time.Time{}, fmt.Errorf("failed to create reader: %w", err) - } - defer reader.Release() - - scanner := bufio.NewScanner(reader) - buf := make([]byte, 64*1024) - scanner.Buffer(buf, 1024*1024) - - didSet := make(map[string]bool) - lineNum := 0 - - for scanner.Scan() { - line := scanner.Bytes() - if len(line) == 0 { - continue - } - - // Only parse minimal fields needed for metadata - var op struct { - DID string `json:"did"` - CreatedAt time.Time `json:"createdAt"` - } - - if err := json.Unmarshal(line, &op); err != nil { - continue - } - - if lineNum == 0 { - startTime = op.CreatedAt - } - endTime = op.CreatedAt - - didSet[op.DID] = true - lineNum++ - } - - return lineNum, len(didSet), startTime, endTime, scanner.Err() -} - -// ExtractBundleMetadata extracts metadata from bundle file without decompressing -func (op *Operations) ExtractBundleMetadata(path string) (*BundleMetadata, error) { - meta, err := op.ExtractMetadataFromFile(path) - if err != nil { - return nil, fmt.Errorf("failed to extract metadata: %w", err) - } - return meta, nil -} - -// LoadBundleWithMetadata loads bundle and returns both data and embedded metadata -func (op *Operations) LoadBundleWithMetadata(path string) ([]plcclient.PLCOperation, *BundleMetadata, error) { - file, err := os.Open(path) - if err != nil { - return nil, nil, fmt.Errorf("failed to open file: %w", err) - } - defer file.Close() - - // 1. Try to read metadata frame first - meta, err := op.ReadMetadataFrame(file) - if err != nil { - // No metadata frame - fall back to regular load - file.Seek(0, io.SeekStart) // Reset to beginning - ops, err := op.loadFromReader(file) - return ops, nil, err - } - - // 2. Read compressed data (file position is now after metadata frame) - ops, err := op.loadFromReader(file) - if err != nil { - return nil, nil, err - } - - return ops, meta, nil -} - -// loadFromReader loads operations from a reader (internal helper) -func (op *Operations) loadFromReader(r io.Reader) ([]plcclient.PLCOperation, error) { - reader, err := NewStreamingReader(r) - if err != nil { - return nil, fmt.Errorf("failed to create reader: %w", err) - } - defer reader.Release() - - decompressed, err := io.ReadAll(reader) - if err != nil { - return nil, fmt.Errorf("failed to decompress: %w", err) - } - - return op.ParseJSONL(decompressed) -} diff --git a/internal/storage/storage_test.go b/internal/storage/storage_test.go deleted file mode 100644 index f3800af..0000000 --- a/internal/storage/storage_test.go +++ /dev/null @@ -1,878 +0,0 @@ -package storage_test - -import ( - "bufio" - "bytes" - "fmt" - "os" - "path/filepath" - "sync" - "testing" - "time" - - "tangled.org/atscan.net/plcbundle/internal/plcclient" - "tangled.org/atscan.net/plcbundle/internal/storage" -) - -type testLogger struct { - t *testing.T -} - -func (l *testLogger) Printf(format string, v ...interface{}) { - l.t.Logf(format, v...) -} - -func (l *testLogger) Println(v ...interface{}) { - l.t.Log(v...) -} - -var ( - bundleInfo = &storage.BundleInfo{ - BundleNumber: 1, - Origin: "test-origin", - ParentHash: "", - Cursor: "", - CreatedBy: "test", - Hostname: "test-host", - } -) - -// ==================================================================================== -// COMPRESSION TESTS -// ==================================================================================== - -func TestStorageCompression(t *testing.T) { - tmpDir := t.TempDir() - logger := &testLogger{t: t} - ops, err := storage.NewOperations(logger, false) - if err != nil { - t.Fatalf("NewOperations failed: %v", err) - } - defer ops.Close() - - t.Run("RoundTripCompression", func(t *testing.T) { - tests := []struct { - name string - count int - }{ - {"Empty", 0}, - {"Single", 1}, - {"Small", 10}, - {"Medium", 100}, - {"Large", 1000}, - {"FullBundle", 10000}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if tt.count == 0 { - return // Skip empty for now - } - - original := makeTestOperations(tt.count) - path := filepath.Join(tmpDir, tt.name+".jsonl.zst") - - // Save - _, _, _, _, err := ops.SaveBundle(path, original, bundleInfo) - if err != nil { - t.Fatalf("SaveBundle failed: %v", err) - } - - // Load - loaded, err := ops.LoadBundle(path) - if err != nil { - t.Fatalf("LoadBundle failed: %v", err) - } - - // Verify count - if len(loaded) != len(original) { - t.Errorf("count mismatch: got %d, want %d", len(loaded), len(original)) - } - - // Verify each operation - for i := range original { - if loaded[i].DID != original[i].DID { - t.Errorf("op %d DID mismatch: got %s, want %s", i, loaded[i].DID, original[i].DID) - } - if loaded[i].CID != original[i].CID { - t.Errorf("op %d CID mismatch: got %s, want %s", i, loaded[i].CID, original[i].CID) - } - if !loaded[i].CreatedAt.Equal(original[i].CreatedAt) { - t.Errorf("op %d timestamp mismatch", i) - } - } - }) - } - }) - - t.Run("CompressionRatio", func(t *testing.T) { - operations := makeTestOperations(10000) - path := filepath.Join(tmpDir, "compression_test.jsonl.zst") - - _, _, uncompSize, compSize, err := ops.SaveBundle(path, operations, bundleInfo) - if err != nil { - t.Fatalf("SaveBundle failed: %v", err) - } - - if compSize >= uncompSize { - t.Errorf("compression failed: compressed=%d >= uncompressed=%d", compSize, uncompSize) - } - - ratio := float64(uncompSize) / float64(compSize) - if ratio < 2.0 { - t.Errorf("poor compression ratio: %.2fx (expected > 2.0x)", ratio) - } - - t.Logf("Compression ratio: %.2fx (%d → %d bytes)", ratio, uncompSize, compSize) - }) - - t.Run("CompressedDataIntegrity", func(t *testing.T) { - operations := makeTestOperations(100) - path := filepath.Join(tmpDir, "integrity_test.jsonl.zst") - - contentHash, compHash, _, _, err := ops.SaveBundle(path, operations, bundleInfo) - if err != nil { - t.Fatalf("SaveBundle failed: %v", err) - } - - // Recalculate hashes - calcCompHash, _, calcContentHash, _, err := ops.CalculateFileHashes(path) - if err != nil { - t.Fatalf("CalculateFileHashes failed: %v", err) - } - - if calcCompHash != compHash { - t.Errorf("compressed hash mismatch: got %s, want %s", calcCompHash, compHash) - } - - if calcContentHash != contentHash { - t.Errorf("content hash mismatch: got %s, want %s", calcContentHash, contentHash) - } - }) -} - -// ==================================================================================== -// HASHING TESTS - CRITICAL FOR CHAIN INTEGRITY -// ==================================================================================== - -func TestStorageHashing(t *testing.T) { - logger := &testLogger{t: t} - ops, err := storage.NewOperations(logger, false) - if err != nil { - t.Fatalf("NewOperations failed: %v", err) - } - defer ops.Close() - - t.Run("HashDeterminism", func(t *testing.T) { - data := []byte("test data for hashing") - - // Calculate hash multiple times - hashes := make([]string, 100) - for i := 0; i < 100; i++ { - hashes[i] = ops.Hash(data) - } - - // All should be identical - firstHash := hashes[0] - for i, h := range hashes { - if h != firstHash { - t.Errorf("hash %d differs: got %s, want %s", i, h, firstHash) - } - } - - // Verify it's actually a valid SHA256 hex (64 chars) - if len(firstHash) != 64 { - t.Errorf("invalid hash length: got %d, want 64", len(firstHash)) - } - }) - - t.Run("ChainHashCalculation", func(t *testing.T) { - contentHash := "abc123def456" - - // Genesis bundle (no parent) - genesisHash := ops.CalculateChainHash("", contentHash) - expectedGenesis := ops.Hash([]byte("plcbundle:genesis:" + contentHash)) - if genesisHash != expectedGenesis { - t.Errorf("genesis hash mismatch: got %s, want %s", genesisHash, expectedGenesis) - } - - // Second bundle (has parent) - parentHash := genesisHash - childHash := ops.CalculateChainHash(parentHash, contentHash) - expectedChild := ops.Hash([]byte(parentHash + ":" + contentHash)) - if childHash != expectedChild { - t.Errorf("child hash mismatch: got %s, want %s", childHash, expectedChild) - } - - // Chain continues - grandchildHash := ops.CalculateChainHash(childHash, contentHash) - expectedGrandchild := ops.Hash([]byte(childHash + ":" + contentHash)) - if grandchildHash != expectedGrandchild { - t.Errorf("grandchild hash mismatch") - } - }) - - t.Run("HashSensitivity", func(t *testing.T) { - // Small changes should produce completely different hashes - data1 := []byte("test data") - data2 := []byte("test datb") // Changed one char - data3 := []byte("test data ") // Added space - - hash1 := ops.Hash(data1) - hash2 := ops.Hash(data2) - hash3 := ops.Hash(data3) - - if hash1 == hash2 { - t.Error("different data produced same hash (collision!)") - } - if hash1 == hash3 { - t.Error("different data produced same hash (collision!)") - } - }) - - t.Run("EmptyDataHash", func(t *testing.T) { - hash := ops.Hash([]byte{}) - if len(hash) != 64 { - t.Errorf("empty data hash invalid length: %d", len(hash)) - } - // SHA256 of empty string is known constant - // e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 - expected := "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" - if hash != expected { - t.Errorf("empty data hash mismatch: got %s, want %s", hash, expected) - } - }) -} - -// ==================================================================================== -// CONCURRENCY TESTS - CRITICAL FOR PRODUCTION -// ==================================================================================== - -func TestStorageConcurrency(t *testing.T) { - tmpDir := t.TempDir() - logger := &testLogger{t: t} - ops, err := storage.NewOperations(logger, false) - if err != nil { - t.Fatalf("NewOperations failed: %v", err) - } - defer ops.Close() - - t.Run("ParallelBundleReads", func(t *testing.T) { - // Create test bundle - operations := makeTestOperations(10000) - path := filepath.Join(tmpDir, "parallel_test.jsonl.zst") - _, _, _, _, err := ops.SaveBundle(path, operations, bundleInfo) - if err != nil { - t.Fatalf("SaveBundle failed: %v", err) - } - - // Read from 100 goroutines simultaneously - var wg sync.WaitGroup - errors := make(chan error, 100) - - for i := 0; i < 100; i++ { - wg.Add(1) - go func(id int) { - defer wg.Done() - loaded, err := ops.LoadBundle(path) - if err != nil { - errors <- err - return - } - if len(loaded) != 10000 { - errors <- err - } - }(i) - } - - wg.Wait() - close(errors) - - for err := range errors { - t.Errorf("concurrent read error: %v", err) - } - }) - - t.Run("LoadOperationAtPositionConcurrency", func(t *testing.T) { - // Critical test - this is heavily used by DID lookups - operations := makeTestOperations(10000) - path := filepath.Join(tmpDir, "position_test.jsonl.zst") - _, _, _, _, err := ops.SaveBundle(path, operations, bundleInfo) - if err != nil { - t.Fatalf("SaveBundle failed: %v", err) - } - - // 200 concurrent position reads - var wg sync.WaitGroup - errors := make(chan error, 200) - - for i := 0; i < 200; i++ { - wg.Add(1) - go func(position int) { - defer wg.Done() - op, err := ops.LoadOperationAtPosition(path, position%10000) - if err != nil { - errors <- err - return - } - if op == nil { - errors <- err - } - }(i) - } - - wg.Wait() - close(errors) - - for err := range errors { - t.Errorf("concurrent position read error: %v", err) - } - }) - - t.Run("ConcurrentHashVerification", func(t *testing.T) { - operations := makeTestOperations(1000) - path := filepath.Join(tmpDir, "verify_test.jsonl.zst") - _, compHash, _, _, err := ops.SaveBundle(path, operations, bundleInfo) - if err != nil { - t.Fatalf("SaveBundle failed: %v", err) - } - - var wg sync.WaitGroup - for i := 0; i < 50; i++ { - wg.Add(1) - go func() { - defer wg.Done() - valid, _, err := ops.VerifyHash(path, compHash) - if err != nil { - t.Errorf("VerifyHash failed: %v", err) - } - if !valid { - t.Error("hash verification failed") - } - }() - } - wg.Wait() - }) -} - -// ==================================================================================== -// EDGE CASES & ERROR HANDLING -// ==================================================================================== - -func TestStorageEdgeCases(t *testing.T) { - tmpDir := t.TempDir() - logger := &testLogger{t: t} - ops, err := storage.NewOperations(logger, false) - if err != nil { - t.Fatalf("NewOperations failed: %v", err) - } - defer ops.Close() - - t.Run("CorruptedZstdFile", func(t *testing.T) { - path := filepath.Join(tmpDir, "corrupted.jsonl.zst") - // Write invalid zstd data - os.WriteFile(path, []byte("this is not valid zstd data"), 0644) - - _, err := ops.LoadBundle(path) - if err == nil { - t.Error("expected error loading corrupted file, got nil") - } - }) - - t.Run("TruncatedFile", func(t *testing.T) { - operations := makeTestOperations(100) - path := filepath.Join(tmpDir, "truncated.jsonl.zst") - ops.SaveBundle(path, operations, bundleInfo) - - // Read and truncate - data, _ := os.ReadFile(path) - os.WriteFile(path, data[:len(data)/2], 0644) - - _, err := ops.LoadBundle(path) - if err == nil { - t.Error("expected error loading truncated file, got nil") - } - }) - - t.Run("InvalidJSONL", func(t *testing.T) { - path := filepath.Join(tmpDir, "invalid.jsonl.zst") - invalidData := []byte("{invalid json}\n{also invalid}") - - // Manually compress invalid data - operations := makeTestOperations(10) - ops.SaveBundle(path, operations, bundleInfo) // Create valid file first - - // Now corrupt it with invalid JSON - // This is hard to test properly since SaveBundle enforces valid data - // Better to test ParseJSONL directly - _, err := ops.ParseJSONL(invalidData) - if err == nil { - t.Error("expected error parsing invalid JSONL, got nil") - } - }) - - t.Run("NonExistentFile", func(t *testing.T) { - _, err := ops.LoadBundle("/nonexistent/path/file.jsonl.zst") - if err == nil { - t.Error("expected error loading nonexistent file, got nil") - } - }) - - t.Run("InvalidPosition", func(t *testing.T) { - operations := makeTestOperations(100) - path := filepath.Join(tmpDir, "position_test.jsonl.zst") - ops.SaveBundle(path, operations, bundleInfo) - - // Negative position - _, err := ops.LoadOperationAtPosition(path, -1) - if err == nil { - t.Error("expected error for negative position") - } - - // Position beyond file - _, err = ops.LoadOperationAtPosition(path, 10000) - if err == nil { - t.Error("expected error for position beyond file") - } - }) -} - -// ==================================================================================== -// BOUNDARY CONDITIONS - CRITICAL FOR BUNDLE CHAINING -// ==================================================================================== - -func TestStorageBoundaryConditions(t *testing.T) { - logger := &testLogger{t: t} - ops, err := storage.NewOperations(logger, false) - if err != nil { - t.Fatalf("NewOperations failed: %v", err) - } - defer ops.Close() - - t.Run("GetBoundaryCIDs_SingleOperation", func(t *testing.T) { - baseTime := time.Now() - operations := []plcclient.PLCOperation{ - {CID: "cid1", CreatedAt: baseTime}, - } - - boundaryTime, cids := ops.GetBoundaryCIDs(operations) - - if !boundaryTime.Equal(baseTime) { - t.Error("boundary time mismatch") - } - if len(cids) != 1 { - t.Errorf("expected 1 boundary CID, got %d", len(cids)) - } - if !cids["cid1"] { - t.Error("expected cid1 in boundary set") - } - }) - - t.Run("GetBoundaryCIDs_MultipleSameTimestamp", func(t *testing.T) { - // CRITICAL: Operations with identical timestamps (happens in real data) - baseTime := time.Now() - operations := []plcclient.PLCOperation{ - {CID: "cid1", CreatedAt: baseTime.Add(-2 * time.Second)}, - {CID: "cid2", CreatedAt: baseTime.Add(-1 * time.Second)}, - {CID: "cid3", CreatedAt: baseTime}, // Last timestamp - {CID: "cid4", CreatedAt: baseTime}, // Same as cid3 - {CID: "cid5", CreatedAt: baseTime}, // Same as cid3 - } - - boundaryTime, cids := ops.GetBoundaryCIDs(operations) - - if !boundaryTime.Equal(baseTime) { - t.Error("boundary time should be last operation time") - } - - // Should return ALL CIDs with the last timestamp - if len(cids) != 3 { - t.Errorf("expected 3 boundary CIDs, got %d", len(cids)) - } - - for _, expectedCID := range []string{"cid3", "cid4", "cid5"} { - if !cids[expectedCID] { - t.Errorf("expected %s in boundary set", expectedCID) - } - } - - // Earlier CIDs should NOT be in set - if cids["cid1"] || cids["cid2"] { - t.Error("earlier CIDs should not be in boundary set") - } - }) - - t.Run("GetBoundaryCIDs_AllSameTimestamp", func(t *testing.T) { - baseTime := time.Now() - operations := []plcclient.PLCOperation{ - {CID: "cid1", CreatedAt: baseTime}, - {CID: "cid2", CreatedAt: baseTime}, - {CID: "cid3", CreatedAt: baseTime}, - } - - _, cids := ops.GetBoundaryCIDs(operations) - - if len(cids) != 3 { - t.Errorf("expected all 3 CIDs, got %d", len(cids)) - } - }) - - t.Run("GetBoundaryCIDs_EmptyOperations", func(t *testing.T) { - operations := []plcclient.PLCOperation{} - boundaryTime, cids := ops.GetBoundaryCIDs(operations) - - if !boundaryTime.IsZero() { - t.Error("expected zero time for empty operations") - } - if len(cids) > 0 { - t.Error("expected nil or empty CID set") - } - }) - - t.Run("StripBoundaryDuplicates_ActualDuplication", func(t *testing.T) { - // CRITICAL: This prevents duplicate operations across bundle boundaries - baseTime := time.Now() - boundaryTimestamp := baseTime.Format(time.RFC3339Nano) - - prevBoundaryCIDs := map[string]bool{ - "cid3": true, - "cid4": true, - } - - operations := []plcclient.PLCOperation{ - {CID: "cid3", CreatedAt: baseTime}, // Duplicate - should be stripped - {CID: "cid4", CreatedAt: baseTime}, // Duplicate - should be stripped - {CID: "cid5", CreatedAt: baseTime}, // New - should be kept - {CID: "cid6", CreatedAt: baseTime.Add(1 * time.Second)}, // After boundary - kept - } - - result := ops.StripBoundaryDuplicates(operations, boundaryTimestamp, prevBoundaryCIDs) - - if len(result) != 2 { - t.Errorf("expected 2 operations after stripping, got %d", len(result)) - } - - if result[0].CID != "cid5" { - t.Errorf("expected cid5 first, got %s", result[0].CID) - } - if result[1].CID != "cid6" { - t.Errorf("expected cid6 second, got %s", result[1].CID) - } - }) - - t.Run("StripBoundaryDuplicates_NoDuplicates", func(t *testing.T) { - baseTime := time.Now() - boundaryTimestamp := baseTime.Format(time.RFC3339Nano) - - prevBoundaryCIDs := map[string]bool{ - "old_cid": true, - } - - operations := []plcclient.PLCOperation{ - {CID: "cid1", CreatedAt: baseTime.Add(1 * time.Second)}, - {CID: "cid2", CreatedAt: baseTime.Add(2 * time.Second)}, - } - - result := ops.StripBoundaryDuplicates(operations, boundaryTimestamp, prevBoundaryCIDs) - - if len(result) != 2 { - t.Errorf("expected 2 operations, got %d", len(result)) - } - }) - - t.Run("StripBoundaryDuplicates_EmptyPrevious", func(t *testing.T) { - baseTime := time.Now() - operations := makeTestOperations(10) - - result := ops.StripBoundaryDuplicates(operations, baseTime.Format(time.RFC3339Nano), nil) - - if len(result) != len(operations) { - t.Error("should not strip anything with no previous boundary CIDs") - } - }) -} - -// ==================================================================================== -// SERIALIZATION TESTS -// ==================================================================================== - -func TestStorageSerialization(t *testing.T) { - logger := &testLogger{t: t} - ops, err := storage.NewOperations(logger, false) - if err != nil { - t.Fatalf("NewOperations failed: %v", err) - } - defer ops.Close() - - t.Run("SerializeJSONL_PreservesRawJSON", func(t *testing.T) { - rawJSON := []byte(`{"did":"did:plc:test","cid":"bafytest","createdAt":"2024-01-01T00:00:00.000Z"}`) - op := plcclient.PLCOperation{ - DID: "did:plc:test", - CID: "bafytest", - CreatedAt: time.Now(), - RawJSON: rawJSON, - } - - result := ops.SerializeJSONL([]plcclient.PLCOperation{op}) - - // Should use RawJSON directly - if !containsBytes(result, rawJSON) { - t.Error("SerializeJSONL did not preserve RawJSON") - } - }) - - t.Run("SerializeJSONL_MarshalsFallback", func(t *testing.T) { - op := plcclient.PLCOperation{ - DID: "did:plc:test", - CID: "bafytest", - CreatedAt: time.Now(), - // No RawJSON - should marshal - } - - result := ops.SerializeJSONL([]plcclient.PLCOperation{op}) - - if len(result) == 0 { - t.Error("SerializeJSONL returned empty result") - } - - // Should contain the DID - if !containsBytes(result, []byte("did:plc:test")) { - t.Error("serialized data missing DID") - } - }) - - t.Run("ParseJSONL_RoundTrip", func(t *testing.T) { - original := makeTestOperations(100) - data := ops.SerializeJSONL(original) - - parsed, err := ops.ParseJSONL(data) - if err != nil { - t.Fatalf("ParseJSONL failed: %v", err) - } - - if len(parsed) != len(original) { - t.Errorf("count mismatch: got %d, want %d", len(parsed), len(original)) - } - - // Verify RawJSON is populated - for i, op := range parsed { - if len(op.RawJSON) == 0 { - t.Errorf("operation %d missing RawJSON", i) - } - } - }) -} - -// ==================================================================================== -// UTILITY FUNCTION TESTS -// ==================================================================================== - -func TestStorageUtilities(t *testing.T) { - tmpDir := t.TempDir() - logger := &testLogger{t: t} - ops, err := storage.NewOperations(logger, false) - if err != nil { - t.Fatalf("NewOperations failed: %v", err) - } - defer ops.Close() - - t.Run("ExtractUniqueDIDs", func(t *testing.T) { - operations := []plcclient.PLCOperation{ - {DID: "did:plc:aaa"}, - {DID: "did:plc:bbb"}, - {DID: "did:plc:aaa"}, // Duplicate - {DID: "did:plc:ccc"}, - {DID: "did:plc:bbb"}, // Duplicate - {DID: "did:plc:aaa"}, // Duplicate - } - - dids := ops.ExtractUniqueDIDs(operations) - - if len(dids) != 3 { - t.Errorf("expected 3 unique DIDs, got %d", len(dids)) - } - - // Verify all expected DIDs present - didSet := make(map[string]bool) - for _, did := range dids { - didSet[did] = true - } - - for _, expectedDID := range []string{"did:plc:aaa", "did:plc:bbb", "did:plc:ccc"} { - if !didSet[expectedDID] { - t.Errorf("missing expected DID: %s", expectedDID) - } - } - }) - - t.Run("ExtractUniqueDIDs_Empty", func(t *testing.T) { - dids := ops.ExtractUniqueDIDs([]plcclient.PLCOperation{}) - if len(dids) != 0 { - t.Error("expected empty result for empty input") - } - }) - - t.Run("FileExists", func(t *testing.T) { - existingFile := filepath.Join(tmpDir, "exists.txt") - os.WriteFile(existingFile, []byte("test"), 0644) - - if !ops.FileExists(existingFile) { - t.Error("FileExists returned false for existing file") - } - - if ops.FileExists(filepath.Join(tmpDir, "nonexistent.txt")) { - t.Error("FileExists returned true for nonexistent file") - } - }) - - t.Run("GetFileSize", func(t *testing.T) { - testFile := filepath.Join(tmpDir, "size_test.txt") - testData := []byte("exactly 12 b") - os.WriteFile(testFile, testData, 0644) - - size, err := ops.GetFileSize(testFile) - if err != nil { - t.Fatalf("GetFileSize failed: %v", err) - } - - if size != int64(len(testData)) { - t.Errorf("size mismatch: got %d, want %d", size, len(testData)) - } - }) -} - -// ==================================================================================== -// STREAMING TESTS -// ==================================================================================== - -func TestStorageStreaming(t *testing.T) { - tmpDir := t.TempDir() - logger := &testLogger{t: t} - ops, err := storage.NewOperations(logger, false) - if err != nil { - t.Fatalf("NewOperations failed: %v", err) - } - defer ops.Close() - - t.Run("StreamRaw", func(t *testing.T) { - operations := makeTestOperations(100) - path := filepath.Join(tmpDir, "stream_raw.jsonl.zst") - _, _, _, _, err := ops.SaveBundle(path, operations, bundleInfo) - if err != nil { - t.Fatalf("SaveBundle failed: %v", err) - } - - reader, err := ops.StreamRaw(path) - if err != nil { - t.Fatalf("StreamRaw failed: %v", err) - } - defer reader.Close() - - // Read all data - data := make([]byte, 1024*1024) - n, err := reader.Read(data) - if err != nil && err.Error() != "EOF" { - t.Fatalf("Read failed: %v", err) - } - - if n == 0 { - t.Error("StreamRaw returned no data") - } - }) - - t.Run("StreamDecompressed", func(t *testing.T) { - operations := makeTestOperations(100) - path := filepath.Join(tmpDir, "stream_decomp.jsonl.zst") - ops.SaveBundle(path, operations, bundleInfo) - - reader, err := ops.StreamDecompressed(path) - if err != nil { - t.Fatalf("StreamDecompressed failed: %v", err) - } - defer reader.Close() - - // Count JSONL lines - scanner := bufio.NewScanner(reader) - lineCount := 0 - for scanner.Scan() { - lineCount++ - } - - if lineCount != 100 { - t.Errorf("expected 100 lines, got %d", lineCount) - } - }) -} - -// ==================================================================================== -// PERFORMANCE / BENCHMARK TESTS -// ==================================================================================== - -func BenchmarkStorageOperations(b *testing.B) { - tmpDir := b.TempDir() - logger := &testLogger{t: &testing.T{}} - ops, _ := storage.NewOperations(logger, false) - defer ops.Close() - - operations := makeTestOperations(10000) - - b.Run("SaveBundle", func(b *testing.B) { - for i := 0; i < b.N; i++ { - path := filepath.Join(tmpDir, fmt.Sprintf("bench_%d.jsonl.zst", i)) - ops.SaveBundle(path, operations, bundleInfo) - } - }) - - // Create bundle for read benchmarks - testPath := filepath.Join(tmpDir, "bench_read.jsonl.zst") - ops.SaveBundle(testPath, operations, nil) - - b.Run("LoadBundle", func(b *testing.B) { - for i := 0; i < b.N; i++ { - ops.LoadBundle(testPath) - } - }) - - b.Run("LoadOperationAtPosition", func(b *testing.B) { - for i := 0; i < b.N; i++ { - ops.LoadOperationAtPosition(testPath, i%10000) - } - }) - - b.Run("Hash", func(b *testing.B) { - data := ops.SerializeJSONL(operations) - b.ResetTimer() - for i := 0; i < b.N; i++ { - ops.Hash(data) - } - }) - - b.Run("SerializeJSONL", func(b *testing.B) { - for i := 0; i < b.N; i++ { - ops.SerializeJSONL(operations) - } - }) -} - -// ==================================================================================== -// HELPER FUNCTIONS -// ==================================================================================== - -func makeTestOperations(count int) []plcclient.PLCOperation { - ops := make([]plcclient.PLCOperation, count) - baseTime := time.Now().Add(-time.Hour) - - for i := 0; i < count; i++ { - ops[i] = plcclient.PLCOperation{ - DID: fmt.Sprintf("did:plc:test%06d", i), - CID: fmt.Sprintf("bafy%06d", i), - CreatedAt: baseTime.Add(time.Duration(i) * time.Second), - } - } - - return ops -} - -func containsBytes(haystack, needle []byte) bool { - return bytes.Contains(haystack, needle) -} diff --git a/internal/storage/zstd.go b/internal/storage/zstd.go deleted file mode 100644 index 7273636..0000000 --- a/internal/storage/zstd.go +++ /dev/null @@ -1,295 +0,0 @@ -package storage - -import ( - "bytes" - "encoding/binary" - "encoding/json" - "fmt" - "io" - "os" - "path/filepath" - - "github.com/valyala/gozstd" -) - -// ============================================================================ -// ZSTD COMPRESSION ABSTRACTION LAYER -// ============================================================================ - -const ( - CompressionLevel = 1 - FrameSize = 100 - - SkippableMagicMetadata = 0x184D2A50 -) - -// ============================================================================ -// SKIPPABLE FRAME FUNCTIONS -// ============================================================================ - -// WriteSkippableFrame writes a skippable frame with the given data -func WriteSkippableFrame(w io.Writer, magicNumber uint32, data []byte) (int64, error) { - frameSize := uint32(len(data)) - - // Write magic number (little-endian) - if err := binary.Write(w, binary.LittleEndian, magicNumber); err != nil { - return 0, err - } - - // Write frame size (little-endian) - if err := binary.Write(w, binary.LittleEndian, frameSize); err != nil { - return 0, err - } - - // Write data - n, err := w.Write(data) - if err != nil { - return 0, err - } - - totalBytes := int64(4 + 4 + n) // magic + size + data - return totalBytes, nil -} - -// ReadSkippableFrame with debug -func (ops *Operations) ReadSkippableFrame(r io.Reader) (uint32, []byte, error) { - var magic uint32 - if err := binary.Read(r, binary.LittleEndian, &magic); err != nil { - return 0, nil, fmt.Errorf("failed to read magic: %w", err) - } - - if magic < 0x184D2A50 || magic > 0x184D2A5F { - return 0, nil, fmt.Errorf("not a skippable frame: magic=0x%08X (expected 0x184D2A50-0x184D2A5F)", magic) - } - - var frameSize uint32 - if err := binary.Read(r, binary.LittleEndian, &frameSize); err != nil { - return 0, nil, fmt.Errorf("failed to read frame size: %w", err) - } - - data := make([]byte, frameSize) - if _, err := io.ReadFull(r, data); err != nil { - return 0, nil, fmt.Errorf("failed to read frame data: %w", err) - } - - return magic, data, nil -} - -// WriteMetadataFrame writes bundle metadata as skippable frame (compact JSON) -func (op *Operations) WriteMetadataFrame(w io.Writer, meta *BundleMetadata) (int64, error) { - jsonData, err := json.Marshal(meta) - if err != nil { - return 0, fmt.Errorf("failed to marshal metadata: %w", err) - } - return WriteSkippableFrame(w, SkippableMagicMetadata, jsonData) -} - -// ReadMetadataFrame reads bundle metadata from skippable frame -func (ops *Operations) ReadMetadataFrame(r io.Reader) (*BundleMetadata, error) { - magic, data, err := ops.ReadSkippableFrame(r) - if err != nil { - return nil, err - } - - if magic != SkippableMagicMetadata { - return nil, fmt.Errorf("unexpected skippable frame magic: 0x%08X (expected 0x%08X)", - magic, SkippableMagicMetadata) - } - - var meta BundleMetadata - if err := json.Unmarshal(data, &meta); err != nil { - return nil, fmt.Errorf("failed to unmarshal metadata: %w", err) - } - - return &meta, nil -} - -// ExtractMetadataFromFile reads metadata without decompressing -func (ops *Operations) ExtractMetadataFromFile(path string) (*BundleMetadata, error) { - file, err := os.Open(path) - if err != nil { - return nil, err - } - defer file.Close() - - // Check first bytes - header := make([]byte, 8) - if _, err := file.Read(header); err != nil { - return nil, fmt.Errorf("failed to read header: %w", err) - } - - // Seek back to start - file.Seek(0, io.SeekStart) - - meta, err := ops.ReadMetadataFrame(file) - if err != nil { - return nil, fmt.Errorf("no metadata frame found: %w", err) - } - - return meta, nil -} - -// ExtractFrameIndexFromFile now just reads from metadata -func (ops *Operations) ExtractFrameIndexFromFile(path string) ([]int64, error) { - meta, err := ops.ExtractMetadataFromFile(path) - if err != nil { - return nil, err - } - - if len(meta.FrameOffsets) == 0 { - return nil, fmt.Errorf("metadata has no frame offsets") - } - - return meta.FrameOffsets, nil -} - -// DebugFrameOffsets extracts and displays frame offset information -func (ops *Operations) DebugFrameOffsets(path string) error { - meta, err := ops.ExtractMetadataFromFile(path) - if err != nil { - return fmt.Errorf("failed to extract metadata: %w", err) - } - - fmt.Printf("Frame Offset Debug for: %s\n\n", filepath.Base(path)) - fmt.Printf("Metadata:\n") - fmt.Printf(" Bundle: %d\n", meta.BundleNumber) - fmt.Printf(" Frames: %d\n", meta.FrameCount) - fmt.Printf(" Frame size: %d ops\n", meta.FrameSize) - fmt.Printf(" Total ops: %d\n", meta.OperationCount) - - fmt.Printf("\nFrame Offsets (%d total):\n", len(meta.FrameOffsets)) - for i, offset := range meta.FrameOffsets { - if i < len(meta.FrameOffsets)-1 { - nextOffset := meta.FrameOffsets[i+1] - frameSize := nextOffset - offset - fmt.Printf(" Frame %3d: offset %10d, size %10d bytes\n", i, offset, frameSize) - } else { - fmt.Printf(" End mark: offset %10d\n", offset) - } - } - - // Try to verify first frame - fmt.Printf("\nVerifying first frame...\n") - file, err := os.Open(path) - if err != nil { - return err - } - defer file.Close() - - if len(meta.FrameOffsets) < 2 { - return fmt.Errorf("not enough frame offsets") - } - - startOffset := meta.FrameOffsets[0] - endOffset := meta.FrameOffsets[1] - frameLength := endOffset - startOffset - - fmt.Printf(" Start: %d, End: %d, Length: %d\n", startOffset, endOffset, frameLength) - - compressedFrame := make([]byte, frameLength) - _, err = file.ReadAt(compressedFrame, startOffset) - if err != nil { - return fmt.Errorf("failed to read: %w", err) - } - - decompressed, err := DecompressFrame(compressedFrame) - if err != nil { - return fmt.Errorf("failed to decompress: %w", err) - } - - fmt.Printf(" ✓ Decompressed: %d bytes\n", len(decompressed)) - - // Count lines - lines := bytes.Count(decompressed, []byte("\n")) - fmt.Printf(" ✓ Lines: %d\n", lines) - - return nil -} - -// ============================================================================ -// COMPRESSION/DECOMPRESSION -// ============================================================================ - -func CompressFrame(data []byte) ([]byte, error) { - compressed := gozstd.CompressLevel(nil, data, CompressionLevel) - return compressed, nil -} - -func DecompressAll(compressed []byte) ([]byte, error) { - decompressed, err := gozstd.Decompress(nil, compressed) - if err != nil { - return nil, fmt.Errorf("decompression failed: %w", err) - } - return decompressed, nil -} - -func DecompressFrame(compressedFrame []byte) ([]byte, error) { - return gozstd.Decompress(nil, compressedFrame) -} - -func NewStreamingReader(r io.Reader) (StreamReader, error) { - reader := gozstd.NewReader(r) - return &gozstdReader{reader: reader}, nil -} - -func NewStreamingWriter(w io.Writer) (StreamWriter, error) { - writer := gozstd.NewWriterLevel(w, CompressionLevel) - return &gozstdWriter{writer: writer}, nil -} - -// ============================================================================ -// INTERFACES -// ============================================================================ - -type StreamReader interface { - io.Reader - io.WriterTo - Release() -} - -type StreamWriter interface { - io.Writer - io.Closer - Flush() error - Release() -} - -// ============================================================================ -// WRAPPER TYPES -// ============================================================================ - -type gozstdReader struct { - reader *gozstd.Reader -} - -func (r *gozstdReader) Read(p []byte) (int, error) { - return r.reader.Read(p) -} - -func (r *gozstdReader) WriteTo(w io.Writer) (int64, error) { - return r.reader.WriteTo(w) -} - -func (r *gozstdReader) Release() { - r.reader.Release() -} - -type gozstdWriter struct { - writer *gozstd.Writer -} - -func (w *gozstdWriter) Write(p []byte) (int, error) { - return w.writer.Write(p) -} - -func (w *gozstdWriter) Close() error { - return w.writer.Close() -} - -func (w *gozstdWriter) Flush() error { - return w.writer.Flush() -} - -func (w *gozstdWriter) Release() { - w.writer.Release() -} diff --git a/internal/sync/bundler.go b/internal/sync/bundler.go deleted file mode 100644 index c228343..0000000 --- a/internal/sync/bundler.go +++ /dev/null @@ -1,61 +0,0 @@ -package sync - -import ( - "time" - - "tangled.org/atscan.net/plcbundle/internal/plcclient" - "tangled.org/atscan.net/plcbundle/internal/storage" -) - -// CreateBundle creates a bundle structure from operations -// Note: This doesn't do hashing - that's done by Manager.SaveBundle -func CreateBundle( - bundleNumber int, - operations []plcclient.PLCOperation, - cursor string, - parent string, - ops *storage.Operations, -) *Bundle { - - dids := ops.ExtractUniqueDIDs(operations) - _, boundaryCIDs := ops.GetBoundaryCIDs(operations) - - cidSlice := make([]string, 0, len(boundaryCIDs)) - for cid := range boundaryCIDs { - cidSlice = append(cidSlice, cid) - } - - return &Bundle{ - BundleNumber: bundleNumber, - StartTime: operations[0].CreatedAt, - EndTime: operations[len(operations)-1].CreatedAt, - Operations: operations, - DIDCount: len(dids), - Cursor: cursor, - Parent: parent, - BoundaryCIDs: cidSlice, - Compressed: true, - CreatedAt: time.Now().UTC(), - } -} - -// Bundle is defined here temporarily - move to parent package later -type Bundle struct { - BundleNumber int - StartTime time.Time - EndTime time.Time - Operations []plcclient.PLCOperation - DIDCount int - - Hash string - ContentHash string - Parent string - - CompressedHash string - CompressedSize int64 - UncompressedSize int64 - Cursor string - BoundaryCIDs []string - Compressed bool - CreatedAt time.Time -} diff --git a/internal/sync/cloner.go b/internal/sync/cloner.go deleted file mode 100644 index 72d3fa7..0000000 --- a/internal/sync/cloner.go +++ /dev/null @@ -1,412 +0,0 @@ -package sync - -import ( - "context" - "fmt" - "io" - "net/http" - "os" - "path/filepath" - "strings" - "sync" - "time" - - "github.com/goccy/go-json" - "tangled.org/atscan.net/plcbundle/internal/bundleindex" - "tangled.org/atscan.net/plcbundle/internal/storage" - "tangled.org/atscan.net/plcbundle/internal/types" -) - -// Cloner handles cloning bundles from remote endpoints -type Cloner struct { - operations *storage.Operations - bundleDir string - logger types.Logger -} - -// NewCloner creates a new cloner -func NewCloner(operations *storage.Operations, bundleDir string, logger types.Logger) *Cloner { - return &Cloner{ - operations: operations, - bundleDir: bundleDir, - logger: logger, - } -} - -// CloneOptions configures cloning behavior -type CloneOptions struct { - RemoteURL string - Workers int - SkipExisting bool - ProgressFunc func(downloaded, total int, bytesDownloaded, bytesTotal int64) - SaveInterval time.Duration - Verbose bool - Logger types.Logger -} - -// CloneResult contains cloning results -type CloneResult struct { - RemoteBundles int - Downloaded int - Failed int - Skipped int - TotalBytes int64 - Duration time.Duration - Interrupted bool - FailedBundles []int -} - -// Clone performs the cloning operation -func (c *Cloner) Clone( - ctx context.Context, - opts CloneOptions, - localIndex *bundleindex.Index, - updateIndex func([]int, map[int]*bundleindex.BundleMetadata, bool) error, -) (*CloneResult, error) { - - if opts.Workers <= 0 { - opts.Workers = 4 - } - if opts.SaveInterval <= 0 { - opts.SaveInterval = 5 * time.Second - } - if opts.Logger == nil { - opts.Logger = c.logger - } - - result := &CloneResult{} - startTime := time.Now() - - // Step 1: Fetch remote index - opts.Logger.Printf("Fetching remote index from %s", opts.RemoteURL) - remoteIndex, err := c.loadRemoteIndex(opts.RemoteURL) - if err != nil { - return nil, fmt.Errorf("failed to load remote index: %w", err) - } - - remoteBundles := remoteIndex.GetBundles() - if len(remoteBundles) == 0 { - opts.Logger.Printf("Remote has no bundles") - return result, nil - } - - result.RemoteBundles = len(remoteBundles) - opts.Logger.Printf("Remote has %d bundles", len(remoteBundles)) - - // Step 2: Determine bundles to download - localBundleMap := make(map[int]*bundleindex.BundleMetadata) - for _, meta := range localIndex.GetBundles() { - localBundleMap[meta.BundleNumber] = meta - } - - remoteBundleMap := make(map[int]*bundleindex.BundleMetadata) - for _, meta := range remoteBundles { - remoteBundleMap[meta.BundleNumber] = meta - } - - var bundlesToDownload []int - var totalBytes int64 - - for _, meta := range remoteBundles { - if opts.SkipExisting && localBundleMap[meta.BundleNumber] != nil { - result.Skipped++ - if opts.Verbose { - opts.Logger.Printf("Skipping existing bundle %06d", meta.BundleNumber) - } - continue - } - bundlesToDownload = append(bundlesToDownload, meta.BundleNumber) - totalBytes += meta.CompressedSize - } - - if len(bundlesToDownload) == 0 { - opts.Logger.Printf("All bundles already exist locally") - return result, nil - } - - opts.Logger.Printf("Downloading %d bundles (%d bytes)", len(bundlesToDownload), totalBytes) - - // Step 3: Set up periodic index saving - saveCtx, saveCancel := context.WithCancel(ctx) - defer saveCancel() - - var downloadedBundles []int - var downloadedMu sync.Mutex - - saveDone := make(chan struct{}) - go func() { - defer close(saveDone) - ticker := time.NewTicker(opts.SaveInterval) - defer ticker.Stop() - - for { - select { - case <-saveCtx.Done(): - return - case <-ticker.C: - downloadedMu.Lock() - bundles := make([]int, len(downloadedBundles)) - copy(bundles, downloadedBundles) - downloadedMu.Unlock() - - if opts.Verbose { - opts.Logger.Printf("Periodic save: updating index with %d bundles", len(bundles)) - } - updateIndex(bundles, remoteBundleMap, false) - } - } - }() - - // Step 4: Download bundles - successList, failedList, bytes := c.downloadBundlesConcurrent( - ctx, - opts.RemoteURL, - bundlesToDownload, - remoteBundleMap, - totalBytes, - opts.Workers, - opts.ProgressFunc, - opts.Verbose, - &downloadedBundles, - &downloadedMu, - ) - - result.Downloaded = len(successList) - result.Failed = len(failedList) - result.TotalBytes = bytes - result.FailedBundles = failedList - result.Interrupted = ctx.Err() != nil - - // Stop periodic saves - saveCancel() - <-saveDone - - // Step 5: Final index update - opts.Logger.Printf("Updating local index...") - if err := updateIndex(successList, remoteBundleMap, opts.Verbose); err != nil { - return result, fmt.Errorf("failed to update index: %w", err) - } - - result.Duration = time.Since(startTime) - return result, nil -} - -// loadRemoteIndex loads index from remote URL -func (c *Cloner) loadRemoteIndex(baseURL string) (*bundleindex.Index, error) { - indexURL := strings.TrimSuffix(baseURL, "/") + "/index.json" - - client := &http.Client{Timeout: 30 * time.Second} - - resp, err := client.Get(indexURL) - if err != nil { - return nil, fmt.Errorf("failed to download: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode) - } - - data, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read response: %w", err) - } - - var idx bundleindex.Index - if err := json.Unmarshal(data, &idx); err != nil { - return nil, fmt.Errorf("failed to parse index: %w", err) - } - - return &idx, nil -} - -// downloadBundlesConcurrent downloads bundles using worker pool -func (c *Cloner) downloadBundlesConcurrent( - ctx context.Context, - baseURL string, - bundleNumbers []int, - remoteBundleMap map[int]*bundleindex.BundleMetadata, - totalBytes int64, - workers int, - progressFunc func(downloaded, total int, bytesDownloaded, bytesTotal int64), - verbose bool, - downloadedBundles *[]int, - downloadedMu *sync.Mutex, -) (successList []int, failedList []int, downloadedBytes int64) { - - type job struct { - bundleNum int - expectedHash string - } - - type result struct { - bundleNum int - success bool - bytes int64 - err error - } - - jobs := make(chan job, len(bundleNumbers)) - results := make(chan result, len(bundleNumbers)) - - // Shared state - var ( - mu sync.Mutex - processedCount int - processedBytes int64 - success []int - failed []int - ) - - // Start workers - var wg sync.WaitGroup - client := &http.Client{Timeout: 120 * time.Second} - - for w := 0; w < workers; w++ { - wg.Add(1) - go func() { - defer wg.Done() - for j := range jobs { - // Check cancellation - select { - case <-ctx.Done(): - results <- result{ - bundleNum: j.bundleNum, - success: false, - err: ctx.Err(), - } - continue - default: - } - - // Download bundle - bytes, err := c.downloadBundle(client, baseURL, j.bundleNum, j.expectedHash) - - // Update progress - mu.Lock() - processedCount++ - if err == nil { - processedBytes += bytes - success = append(success, j.bundleNum) - if downloadedMu != nil && downloadedBundles != nil { - downloadedMu.Lock() - *downloadedBundles = append(*downloadedBundles, j.bundleNum) - downloadedMu.Unlock() - } - } else { - failed = append(failed, j.bundleNum) - } - - if progressFunc != nil { - progressFunc(processedCount, len(bundleNumbers), processedBytes, totalBytes) - } - mu.Unlock() - - results <- result{ - bundleNum: j.bundleNum, - success: err == nil, - bytes: bytes, - err: err, - } - } - }() - } - - // Send jobs - for _, num := range bundleNumbers { - expectedHash := "" - if meta, exists := remoteBundleMap[num]; exists { - expectedHash = meta.CompressedHash - } - jobs <- job{ - bundleNum: num, - expectedHash: expectedHash, - } - } - close(jobs) - - // Wait for completion - go func() { - wg.Wait() - close(results) - }() - - // Collect results - for res := range results { - if res.err != nil && res.err != context.Canceled { - c.logger.Printf("Failed to download bundle %06d: %v", res.bundleNum, res.err) - } else if res.success && verbose { - c.logger.Printf("✓ Downloaded and verified bundle %06d (%d bytes)", res.bundleNum, res.bytes) - } - } - - mu.Lock() - successList = success - failedList = failed - downloadedBytes = processedBytes - mu.Unlock() - - return -} - -// downloadBundle downloads a single bundle and verifies hash -func (c *Cloner) downloadBundle(client *http.Client, baseURL string, bundleNum int, expectedHash string) (int64, error) { - url := fmt.Sprintf("%s/data/%d", strings.TrimSuffix(baseURL, "/"), bundleNum) - filename := fmt.Sprintf("%06d.jsonl.zst", bundleNum) - filepath := filepath.Join(c.bundleDir, filename) - - // Create request - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return 0, err - } - - // Download - resp, err := client.Do(req) - if err != nil { - return 0, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return 0, fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(body)) - } - - // Write to temp file - tempPath := filepath + ".tmp" - outFile, err := os.Create(tempPath) - if err != nil { - return 0, err - } - - written, err := io.Copy(outFile, resp.Body) - outFile.Close() - - if err != nil { - os.Remove(tempPath) - return 0, err - } - - // Verify hash - if expectedHash != "" { - valid, actualHash, err := c.operations.VerifyHash(tempPath, expectedHash) - if err != nil { - os.Remove(tempPath) - return 0, fmt.Errorf("hash verification failed: %w", err) - } - if !valid { - os.Remove(tempPath) - return 0, fmt.Errorf("hash mismatch: expected %s, got %s", - expectedHash[:16]+"...", actualHash[:16]+"...") - } - } - - // Rename to final location - if err := os.Rename(tempPath, filepath); err != nil { - os.Remove(tempPath) - return 0, err - } - - return written, nil -} diff --git a/internal/sync/fetcher.go b/internal/sync/fetcher.go deleted file mode 100644 index 3002fb6..0000000 --- a/internal/sync/fetcher.go +++ /dev/null @@ -1,237 +0,0 @@ -package sync - -import ( - "context" - "fmt" - "time" - - "tangled.org/atscan.net/plcbundle/internal/plcclient" - "tangled.org/atscan.net/plcbundle/internal/storage" - "tangled.org/atscan.net/plcbundle/internal/types" -) - -// Fetcher handles fetching operations from PLC directory -type Fetcher struct { - plcClient *plcclient.Client - operations *storage.Operations - logger types.Logger -} - -// MempoolInterface defines what we need from mempool -type MempoolInterface interface { - Add(ops []plcclient.PLCOperation) (int, error) - Save() error - SaveIfNeeded() error - Count() int - GetLastTime() string -} - -// NewFetcher creates a new fetcher -func NewFetcher(plcClient *plcclient.Client, operations *storage.Operations, logger types.Logger) *Fetcher { - return &Fetcher{ - plcClient: plcClient, - operations: operations, - logger: logger, - } -} - -// FetchToMempool fetches operations and adds them to mempool (with auto-save) -func (f *Fetcher) FetchToMempool( - ctx context.Context, - afterTime string, - prevBoundaryCIDs map[string]bool, - target int, - quiet bool, - mempool MempoolInterface, - totalFetchesSoFar int, -) ([]plcclient.PLCOperation, int, error) { - - seenCIDs := make(map[string]bool) - - // Initialize current boundaries from previous bundle (or empty if first fetch) - currentBoundaryCIDs := prevBoundaryCIDs - if currentBoundaryCIDs == nil { - currentBoundaryCIDs = make(map[string]bool) - } - - // Mark boundary CIDs as seen to prevent re-inclusion - for cid := range currentBoundaryCIDs { - seenCIDs[cid] = true - } - - if !quiet && len(currentBoundaryCIDs) > 0 { - f.logger.Printf(" Starting with %d boundary CIDs from previous bundle", len(currentBoundaryCIDs)) - } - - currentAfter := afterTime - maxFetches := 20 - var allNewOps []plcclient.PLCOperation - fetchesMade := 0 - totalReceived := 0 - totalDupes := 0 - - for fetchNum := 0; fetchNum < maxFetches; fetchNum++ { - fetchesMade++ - remaining := target - len(allNewOps) - if remaining <= 0 { - break - } - - // Smart batch sizing - var batchSize int - switch { - case remaining <= 50: - batchSize = 50 - case remaining <= 100: - batchSize = 100 - case remaining <= 500: - batchSize = 200 - default: - batchSize = 1000 - } - - fetchStart := time.Now() - - if !quiet { - f.logger.Printf(" Fetch #%d: requesting %d (need %d more, have %d/%d)", - totalFetchesSoFar+fetchesMade, batchSize, remaining, len(allNewOps), target) - } - - batch, err := f.plcClient.Export(ctx, plcclient.ExportOptions{ - Count: batchSize, - After: currentAfter, - }) - - fetchDuration := time.Since(fetchStart) - - if err != nil { - return allNewOps, fetchesMade, fmt.Errorf("export failed: %w", err) - } - - if len(batch) == 0 { - if !quiet { - f.logger.Printf(" No more operations available (in %s)", fetchDuration) - } - return allNewOps, fetchesMade, nil - } - - originalBatchSize := len(batch) - totalReceived += originalBatchSize - - // CRITICAL: Strip boundary duplicates using current boundaries - batch = f.operations.StripBoundaryDuplicates( - batch, - currentAfter, - currentBoundaryCIDs, - ) - - afterStripSize := len(batch) - strippedCount := originalBatchSize - afterStripSize - - if !quiet && strippedCount > 0 { - f.logger.Printf(" Stripped %d boundary duplicates from fetch", strippedCount) - } - - // Collect new ops (not in seenCIDs) - beforeDedup := len(allNewOps) - var batchNewOps []plcclient.PLCOperation - - for _, op := range batch { - if !seenCIDs[op.CID] { - batchNewOps = append(batchNewOps, op) - } - } - - uniqueInBatch := len(batchNewOps) - dupesFiltered := afterStripSize - uniqueInBatch - totalDupes += dupesFiltered + strippedCount - - // Try to add to mempool - if uniqueInBatch > 0 && mempool != nil { - _, addErr := mempool.Add(batchNewOps) - - if addErr != nil { - // Add failed - don't mark as seen - if !quiet { - f.logger.Printf(" ❌ Mempool add failed: %v", addErr) - } - mempool.Save() - return allNewOps, fetchesMade, fmt.Errorf("mempool add failed: %w", addErr) - } - - // Success - mark as seen - for _, op := range batchNewOps { - seenCIDs[op.CID] = true - } - allNewOps = append(allNewOps, batchNewOps...) - - uniqueAdded := len(allNewOps) - beforeDedup - - if !quiet { - opsPerSec := float64(originalBatchSize) / fetchDuration.Seconds() - if dupesFiltered+strippedCount > 0 { - f.logger.Printf(" → +%d unique (%d dupes, %d boundary) in %s • Running: %d/%d (%.0f ops/sec)", - uniqueAdded, dupesFiltered, strippedCount, fetchDuration, len(allNewOps), target, opsPerSec) - } else { - f.logger.Printf(" → +%d unique in %s • Running: %d/%d (%.0f ops/sec)", - uniqueAdded, fetchDuration, len(allNewOps), target, opsPerSec) - } - } - - // CRITICAL: Calculate NEW boundary CIDs from this fetch for next iteration - if len(batch) > 0 { - boundaryTime, newBoundaryCIDs := f.operations.GetBoundaryCIDs(batch) - currentBoundaryCIDs = newBoundaryCIDs - currentAfter = boundaryTime.Format(time.RFC3339Nano) - - if !quiet && len(newBoundaryCIDs) > 1 { - f.logger.Printf(" Updated boundaries: %d CIDs at %s", - len(newBoundaryCIDs), currentAfter[:19]) - } - } - - // Save if threshold met - if err := mempool.SaveIfNeeded(); err != nil { - f.logger.Printf(" Warning: failed to save mempool: %v", err) - } - - } else if uniqueInBatch > 0 { - // No mempool - just collect - for _, op := range batchNewOps { - seenCIDs[op.CID] = true - } - allNewOps = append(allNewOps, batchNewOps...) - - // Still update boundaries even without mempool - if len(batch) > 0 { - boundaryTime, newBoundaryCIDs := f.operations.GetBoundaryCIDs(batch) - currentBoundaryCIDs = newBoundaryCIDs - currentAfter = boundaryTime.Format(time.RFC3339Nano) - } - } - - // Check if incomplete batch (caught up) - if originalBatchSize < batchSize { - if !quiet { - f.logger.Printf(" Incomplete batch (%d/%d) → caught up", originalBatchSize, batchSize) - } - return allNewOps, fetchesMade, nil - } - - if len(allNewOps) >= target { - break - } - } - - // Summary - if !quiet && fetchesMade > 0 { - dedupRate := 0.0 - if totalReceived > 0 { - dedupRate = float64(totalDupes) / float64(totalReceived) * 100 - } - f.logger.Printf(" ✓ Collected %d unique ops from %d fetches (%.1f%% dedup)", - len(allNewOps), fetchesMade, dedupRate) - } - - return allNewOps, fetchesMade, nil -} diff --git a/internal/sync/sync_test.go b/internal/sync/sync_test.go deleted file mode 100644 index 3cb5340..0000000 --- a/internal/sync/sync_test.go +++ /dev/null @@ -1,758 +0,0 @@ -package sync_test - -import ( - "context" - "fmt" - "net/http" - "net/http/httptest" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/goccy/go-json" - "tangled.org/atscan.net/plcbundle/internal/plcclient" - "tangled.org/atscan.net/plcbundle/internal/storage" - internalsync "tangled.org/atscan.net/plcbundle/internal/sync" - "tangled.org/atscan.net/plcbundle/internal/types" -) - -type testLogger struct { - t *testing.T -} - -func (l *testLogger) Printf(format string, v ...interface{}) { - l.t.Logf(format, v...) -} - -func (l *testLogger) Println(v ...interface{}) { - l.t.Log(v...) -} - -// Mock mempool for testing -type mockMempool struct { - operations []plcclient.PLCOperation - mu sync.Mutex - saveCount int32 -} - -func newMockMempool() *mockMempool { - return &mockMempool{ - operations: make([]plcclient.PLCOperation, 0), - } -} - -func (m *mockMempool) Add(ops []plcclient.PLCOperation) (int, error) { - m.mu.Lock() - defer m.mu.Unlock() - - // Build existing CID set (like real mempool does) - existingCIDs := make(map[string]bool) - for _, op := range m.operations { - existingCIDs[op.CID] = true - } - - // Only add new operations (deduplicate by CID) - addedCount := 0 - for _, op := range ops { - if !existingCIDs[op.CID] { - m.operations = append(m.operations, op) - existingCIDs[op.CID] = true - addedCount++ - } - } - - return addedCount, nil -} - -func (m *mockMempool) Save() error { - atomic.AddInt32(&m.saveCount, 1) - return nil -} - -func (m *mockMempool) SaveIfNeeded() error { - return m.Save() -} - -func (m *mockMempool) Count() int { - m.mu.Lock() - defer m.mu.Unlock() - return len(m.operations) -} - -func (m *mockMempool) GetLastTime() string { - m.mu.Lock() - defer m.mu.Unlock() - if len(m.operations) == 0 { - return "" - } - return m.operations[len(m.operations)-1].CreatedAt.Format(time.RFC3339Nano) -} - -// ==================================================================================== -// FETCHER TESTS - DEDUPLICATION & RETRY LOGIC -// ==================================================================================== - -func TestFetcherDeduplication(t *testing.T) { - t.Run("BoundaryDuplicateHandling", func(t *testing.T) { - // Setup mock server - baseTime := time.Now() - boundaryTime := baseTime.Add(5 * time.Second) - - // Simulate operations at bundle boundary - mockOps := []plcclient.PLCOperation{ - {DID: "did:plc:001", CID: "cid1", CreatedAt: boundaryTime}, - {DID: "did:plc:002", CID: "cid2", CreatedAt: boundaryTime}, - {DID: "did:plc:003", CID: "cid3", CreatedAt: boundaryTime.Add(1 * time.Second)}, - } - - server := createMockPLCServer(t, mockOps) - defer server.Close() - - // Create fetcher - client := plcclient.NewClient(server.URL) - defer client.Close() - - logger := &testLogger{t: t} - ops, _ := storage.NewOperations(logger, false) - defer ops.Close() - - fetcher := internalsync.NewFetcher(client, ops, logger) - - // Previous bundle had cid1 and cid2 at boundary - prevBoundaryCIDs := map[string]bool{ - "cid1": true, - "cid2": true, - } - - mempool := newMockMempool() - - // Fetch - newOps, fetchCount, err := fetcher.FetchToMempool( - context.Background(), - boundaryTime.Add(-1*time.Second).Format(time.RFC3339Nano), - prevBoundaryCIDs, - 10, - true, // quiet - mempool, - 0, - ) - - if err != nil { - t.Fatalf("FetchToMempool failed: %v", err) - } - - // Should have filtered out cid1 and cid2 (duplicates) - // Only cid3 should be returned - if len(newOps) != 1 { - t.Errorf("expected 1 unique operation, got %d", len(newOps)) - } - - if len(newOps) > 0 && newOps[0].CID != "cid3" { - t.Errorf("expected cid3, got %s", newOps[0].CID) - } - - if fetchCount == 0 { - t.Error("expected at least one fetch") - } - }) - - t.Run("ConcurrentFetchDedup", func(t *testing.T) { - baseTime := time.Now() - mockOps := make([]plcclient.PLCOperation, 50) - for i := 0; i < 50; i++ { - mockOps[i] = plcclient.PLCOperation{ - DID: fmt.Sprintf("did:plc:%03d", i), - CID: fmt.Sprintf("cid%03d", i), - CreatedAt: baseTime.Add(time.Duration(i) * time.Second), - } - } - - server := createMockPLCServer(t, mockOps) - defer server.Close() - - client := plcclient.NewClient(server.URL) - defer client.Close() - - logger := &testLogger{t: t} - storageOps, _ := storage.NewOperations(logger, false) - defer storageOps.Close() - - fetcher := internalsync.NewFetcher(client, storageOps, logger) - mempool := newMockMempool() - - // First fetch - initialCount := mempool.Count() - _, _, err := fetcher.FetchToMempool( - context.Background(), - "", - nil, - 30, - true, - mempool, - 0, - ) - if err != nil { - t.Fatalf("First fetch failed: %v", err) - } - - countAfterFirst := mempool.Count() - addedFirst := countAfterFirst - initialCount - - if addedFirst == 0 { - t.Fatal("first fetch should add operations") - } - - // Second fetch with same cursor - mempool deduplicates - countBeforeSecond := mempool.Count() - _, _, err = fetcher.FetchToMempool( - context.Background(), - "", // Same cursor - fetches same data - nil, - 30, - true, - mempool, - 1, - ) - if err != nil { - t.Fatalf("Second fetch failed: %v", err) - } - - countAfterSecond := mempool.Count() - addedSecond := countAfterSecond - countBeforeSecond - - // Mempool's Add() method deduplicates by CID - // So second fetch should add 0 (all duplicates) - if addedSecond != 0 { - t.Errorf("expected 0 new ops in mempool after second fetch (duplicates), got %d", addedSecond) - } - - t.Logf("First fetch: +%d ops, Second fetch: +%d ops (deduped)", addedFirst, addedSecond) - }) - - t.Run("EmptyBoundaryCIDs", func(t *testing.T) { - baseTime := time.Now() - mockOps := []plcclient.PLCOperation{ - {DID: "did:plc:001", CID: "cid1", CreatedAt: baseTime}, - } - - server := createMockPLCServer(t, mockOps) - defer server.Close() - - client := plcclient.NewClient(server.URL) - defer client.Close() - - logger := &testLogger{t: t} - storageOps, _ := storage.NewOperations(logger, false) - defer storageOps.Close() - - fetcher := internalsync.NewFetcher(client, storageOps, logger) - mempool := newMockMempool() - - // Fetch with no boundary CIDs (genesis bundle) - newOps, _, err := fetcher.FetchToMempool( - context.Background(), - "", - nil, // No previous boundary - 10, - true, - mempool, - 0, - ) - - if err != nil { - t.Fatalf("FetchToMempool failed: %v", err) - } - - if len(newOps) != 1 { - t.Errorf("expected 1 operation, got %d", len(newOps)) - } - }) -} - -func TestFetcherRetry(t *testing.T) { - t.Run("TransientFailures", func(t *testing.T) { - attemptCount := 0 - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - attemptCount++ - - if attemptCount < 3 { - // Fail first 2 attempts - w.WriteHeader(500) - return - } - - // Succeed on 3rd attempt - w.Header().Set("Content-Type", "application/x-ndjson") - op := plcclient.PLCOperation{ - DID: "did:plc:test", - CID: "cid1", - CreatedAt: time.Now(), - } - json.NewEncoder(w).Encode(op) - })) - defer server.Close() - - client := plcclient.NewClient(server.URL) - defer client.Close() - - // Should retry and eventually succeed - _, err := client.Export(context.Background(), plcclient.ExportOptions{Count: 1}) - if err != nil { - t.Fatalf("expected retry to succeed, got error: %v", err) - } - - if attemptCount < 3 { - t.Errorf("expected at least 3 attempts, got %d", attemptCount) - } - }) - - t.Run("RateLimitHandling", func(t *testing.T) { - attemptCount := 0 - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - attemptCount++ - - if attemptCount == 1 { - // Return 429 with Retry-After - w.Header().Set("Retry-After", "1") - w.WriteHeader(429) - return - } - - // Success - w.Header().Set("Content-Type", "application/x-ndjson") - op := plcclient.PLCOperation{ - DID: "did:plc:test", - CID: "cid1", - CreatedAt: time.Now(), - } - json.NewEncoder(w).Encode(op) - })) - defer server.Close() - - client := plcclient.NewClient(server.URL) - defer client.Close() - - startTime := time.Now() - _, err := client.Export(context.Background(), plcclient.ExportOptions{Count: 1}) - elapsed := time.Since(startTime) - - if err != nil { - t.Fatalf("expected success after rate limit, got: %v", err) - } - - // Should have waited at least 1 second - if elapsed < 1*time.Second { - t.Errorf("expected wait for rate limit, elapsed: %v", elapsed) - } - - if attemptCount != 2 { - t.Errorf("expected 2 attempts, got %d", attemptCount) - } - }) - - t.Run("ContextCancellation", func(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Slow response - time.Sleep(5 * time.Second) - })) - defer server.Close() - - client := plcclient.NewClient(server.URL) - defer client.Close() - - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) - defer cancel() - - _, err := client.Export(ctx, plcclient.ExportOptions{Count: 1}) - if err == nil { - t.Error("expected timeout error, got nil") - } - }) -} - -func TestFetcherMempoolIntegration(t *testing.T) { - t.Run("AutoSaveAfterFetch", func(t *testing.T) { - baseTime := time.Now() - mockOps := []plcclient.PLCOperation{ - {DID: "did:plc:001", CID: "cid1", CreatedAt: baseTime}, - {DID: "did:plc:002", CID: "cid2", CreatedAt: baseTime.Add(1 * time.Second)}, - } - - server := createMockPLCServer(t, mockOps) - defer server.Close() - - client := plcclient.NewClient(server.URL) - defer client.Close() - - logger := &testLogger{t: t} - storageOps, _ := storage.NewOperations(logger, false) - defer storageOps.Close() - - fetcher := internalsync.NewFetcher(client, storageOps, logger) - mempool := newMockMempool() - - _, _, err := fetcher.FetchToMempool( - context.Background(), - "", - nil, - 10, - true, - mempool, - 0, - ) - - if err != nil { - t.Fatalf("FetchToMempool failed: %v", err) - } - - // Verify mempool.SaveIfNeeded was called - if mempool.saveCount == 0 { - t.Error("expected mempool to be saved after fetch") - } - }) -} - -// ==================================================================================== -// CLONER TESTS -// ==================================================================================== - -func TestClonerAtomicity(t *testing.T) { - // Note: Cloner tests would need more complex mocking - // Including mock HTTP server, file system operations, etc. - // This is a template showing what to test - - t.Run("InterruptedClone", func(t *testing.T) { - // TODO: Test context cancellation mid-download - // Verify: - // - .tmp files are cleaned up OR kept for resume - // - Index not updated for incomplete downloads - // - Partial progress can resume with --resume flag - }) - - t.Run("HashVerificationFailure", func(t *testing.T) { - // TODO: Mock server returns file with wrong hash - // Verify: - // - File is deleted (or .tmp is not renamed) - // - Bundle NOT added to index - // - Error returned to user - }) - - t.Run("IndexUpdateTiming", func(t *testing.T) { - // CRITICAL: Index must only update AFTER file write succeeds - // TODO: Implement test that verifies ordering - }) -} - -// ==================================================================================== -// SYNC LOOP TESTS -// ==================================================================================== - -func TestSyncLoopBehavior(t *testing.T) { - t.Run("CatchUpDetection", func(t *testing.T) { - // Mock manager - mockMgr := &mockSyncManager{ - lastBundle: 5, - mempoolCount: 500, - } - - logger := &testLogger{t: t} - config := &internalsync.SyncLoopConfig{ - MaxBundles: 0, - Verbose: false, - Logger: logger, - SkipDIDIndex: false, - } - - // First sync should detect "caught up" when no progress - synced, err := internalsync.SyncOnce(context.Background(), mockMgr, config) - - if err != nil { - t.Fatalf("SyncOnce failed: %v", err) - } - - // Should return 0 if already caught up - if synced != 0 { - t.Logf("Note: synced %d bundles (manager may not be caught up)", synced) - } - }) - - t.Run("MaxBundlesLimit", func(t *testing.T) { - mockMgr := &mockSyncManager{ - lastBundle: 0, - mempoolCount: 10000, // Always has enough for bundle - } - - logger := &testLogger{t: t} - config := &internalsync.SyncLoopConfig{ - MaxBundles: 3, - Verbose: false, - Logger: logger, - SkipDIDIndex: false, - } - - ctx := context.Background() - synced, err := internalsync.SyncOnce(ctx, mockMgr, config) - - if err != nil { - t.Fatalf("SyncOnce failed: %v", err) - } - - // Should respect max limit - if synced > 3 { - t.Errorf("synced %d bundles, but max was 3", synced) - } - }) - - t.Run("GracefulShutdown", func(t *testing.T) { - mockMgr := &mockSyncManager{ - lastBundle: 0, - mempoolCount: 10000, - fetchDelay: 50 * time.Millisecond, - } - - logger := &testLogger{t: t} - config := &internalsync.SyncLoopConfig{ - Interval: 100 * time.Millisecond, - MaxBundles: 0, - Verbose: false, - Logger: logger, - SkipDIDIndex: false, - } - - ctx, cancel := context.WithCancel(context.Background()) - - // Start sync loop in goroutine - done := make(chan error, 1) - go func() { - done <- internalsync.RunSyncLoop(ctx, mockMgr, config) - }() - - // Let it run briefly (should complete at least one cycle) - time.Sleep(250 * time.Millisecond) - - // Cancel context - cancel() - - // Should exit gracefully with context.Canceled error - select { - case err := <-done: - // Expected: context.Canceled or nil - if err != nil && err != context.Canceled { - t.Errorf("unexpected error on shutdown: %v", err) - } - t.Logf("Sync loop stopped cleanly: %v", err) - - case <-time.After(2 * time.Second): - t.Error("sync loop did not stop within timeout after context cancellation") - } - - // NOTE: Mempool saving on shutdown is handled by the caller (commands/server), - // not by the sync loop itself. The sync loop only respects context cancellation. - // - // For mempool save testing, see command-level tests. - }) -} - -// ==================================================================================== -// BUNDLER TESTS -// ==================================================================================== - -func TestBundlerCreateBundle(t *testing.T) { - logger := &testLogger{t: t} - storageOps, _ := storage.NewOperations(logger, false) - defer storageOps.Close() - - t.Run("BasicBundleCreation", func(t *testing.T) { - operations := makeTestOperations(10000) - cursor := operations[len(operations)-1].CreatedAt.Format(time.RFC3339Nano) - - bundle := internalsync.CreateBundle(1, operations, cursor, "", storageOps) - - if bundle.BundleNumber != 1 { - t.Errorf("wrong bundle number: got %d, want 1", bundle.BundleNumber) - } - - if len(bundle.Operations) != 10000 { - t.Errorf("wrong operation count: got %d, want 10000", len(bundle.Operations)) - } - - if bundle.DIDCount == 0 { - t.Error("DIDCount should not be zero") - } - - if len(bundle.BoundaryCIDs) == 0 { - t.Error("BoundaryCIDs should not be empty") - } - - if bundle.Cursor != cursor { - t.Error("cursor mismatch") - } - }) - - t.Run("GenesisBundle", func(t *testing.T) { - operations := makeTestOperations(10000) - cursor := operations[len(operations)-1].CreatedAt.Format(time.RFC3339Nano) - - bundle := internalsync.CreateBundle(1, operations, cursor, "", storageOps) - - // Genesis should have empty parent - if bundle.Parent != "" { - t.Errorf("genesis bundle should have empty parent, got %s", bundle.Parent) - } - }) - - t.Run("ChainedBundle", func(t *testing.T) { - operations := makeTestOperations(10000) - cursor := operations[len(operations)-1].CreatedAt.Format(time.RFC3339Nano) - parentHash := "parent_hash_from_bundle_1" - - bundle := internalsync.CreateBundle(2, operations, cursor, parentHash, storageOps) - - if bundle.Parent != parentHash { - t.Errorf("parent mismatch: got %s, want %s", bundle.Parent, parentHash) - } - - if bundle.BundleNumber != 2 { - t.Error("bundle number should be 2") - } - }) - - t.Run("BoundaryTimestamps", func(t *testing.T) { - baseTime := time.Now() - - // Create operations where last 5 share same timestamp - operations := makeTestOperations(10000) - for i := 9995; i < 10000; i++ { - operations[i].CreatedAt = baseTime - } - - cursor := baseTime.Format(time.RFC3339Nano) - bundle := internalsync.CreateBundle(1, operations, cursor, "", storageOps) - - // Should capture all 5 CIDs at boundary - if len(bundle.BoundaryCIDs) != 5 { - t.Errorf("expected 5 boundary CIDs, got %d", len(bundle.BoundaryCIDs)) - } - }) -} - -// ==================================================================================== -// MOCK SERVER & HELPERS -// ==================================================================================== - -func createMockPLCServer(_ *testing.T, operations []plcclient.PLCOperation) *httptest.Server { - return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/export" { - w.WriteHeader(404) - return - } - - w.Header().Set("Content-Type", "application/x-ndjson") - - // Return operations as JSONL - for _, op := range operations { - json.NewEncoder(w).Encode(op) - } - })) -} - -func makeTestOperations(count int) []plcclient.PLCOperation { - ops := make([]plcclient.PLCOperation, count) - baseTime := time.Now().Add(-time.Hour) - - for i := 0; i < count; i++ { - ops[i] = plcclient.PLCOperation{ - DID: fmt.Sprintf("did:plc:test%06d", i), - CID: fmt.Sprintf("bafy%06d", i), - CreatedAt: baseTime.Add(time.Duration(i) * time.Second), - } - } - - return ops -} - -// Mock sync manager for testing -type mockSyncManager struct { - lastBundle int - mempoolCount int - fetchDelay time.Duration - mempoolSaveCount int - mu sync.Mutex -} - -func (m *mockSyncManager) GetLastBundleNumber() int { - m.mu.Lock() - defer m.mu.Unlock() - return m.lastBundle -} - -func (m *mockSyncManager) UpdateDIDIndexSmart(ctx context.Context, progressCallback func(current, total int)) error { - m.mu.Lock() - defer m.mu.Unlock() - return nil -} - -func (m *mockSyncManager) BuildDIDIndex(ctx context.Context, progressCallback func(current, total int)) error { - m.mu.Lock() - defer m.mu.Unlock() - return nil -} - -func (m *mockSyncManager) GetMempoolCount() int { - m.mu.Lock() - defer m.mu.Unlock() - return m.mempoolCount -} - -func (m *mockSyncManager) FetchAndSaveNextBundle(ctx context.Context, verbose bool, quiet bool, skipDIDIndex bool) (int, *types.BundleProductionStats, error) { - m.mu.Lock() - defer m.mu.Unlock() - - if m.fetchDelay > 0 { - time.Sleep(m.fetchDelay) - } - - // Simulate creating bundle if we have enough ops - if m.mempoolCount >= 10000 { - m.lastBundle++ - m.mempoolCount -= 10000 - return m.lastBundle, nil, nil - } - - // Not enough ops - return 0, nil, fmt.Errorf("insufficient operations") -} - -func (m *mockSyncManager) SaveMempool() error { - m.mu.Lock() - defer m.mu.Unlock() - m.mempoolSaveCount++ - return nil -} - -func TestMockMempoolDeduplication(t *testing.T) { - m := newMockMempool() - - op1 := plcclient.PLCOperation{ - CID: "duplicate_cid", - DID: "did:plc:test", - CreatedAt: time.Now(), - } - - // Add first time - added, _ := m.Add([]plcclient.PLCOperation{op1}) - if added != 1 { - t.Fatalf("first add should return 1, got %d", added) - } - - // Add same CID again - added, _ = m.Add([]plcclient.PLCOperation{op1}) - if added != 0 { - t.Fatalf("duplicate add should return 0, got %d", added) - } - - if m.Count() != 1 { - t.Fatalf("count should be 1, got %d", m.Count()) - } -} diff --git a/internal/sync/syncer.go b/internal/sync/syncer.go deleted file mode 100644 index 7ad1e1f..0000000 --- a/internal/sync/syncer.go +++ /dev/null @@ -1,197 +0,0 @@ -// internal/sync/syncer.go -package sync - -import ( - "context" - "time" - - "tangled.org/atscan.net/plcbundle/internal/types" -) - -// SyncLoopConfig configures continuous syncing -type SyncLoopConfig struct { - Interval time.Duration - MaxBundles int // 0 = unlimited - Verbose bool - Logger types.Logger - OnBundleSynced func(bundleNum int, fetchedCount int, mempoolCount int, duration time.Duration, indexTime time.Duration) - SkipDIDIndex bool - Quiet bool -} - -// DefaultSyncLoopConfig returns default configuration -func DefaultSyncLoopConfig() *SyncLoopConfig { - return &SyncLoopConfig{ - Interval: 1 * time.Minute, - MaxBundles: 0, - Verbose: false, - SkipDIDIndex: false, - Quiet: false, - } -} - -// SyncManager is the minimal interface needed for syncing -type SyncManager interface { - GetLastBundleNumber() int - GetMempoolCount() int - // Returns: bundleNumber, indexUpdateTime, error - FetchAndSaveNextBundle(ctx context.Context, verbose bool, quiet bool, skipDIDIndex bool) (int, *types.BundleProductionStats, error) - SaveMempool() error - BuildDIDIndex(ctx context.Context, progressCallback func(current, total int)) error - UpdateDIDIndexSmart(ctx context.Context, progressCallback func(current, total int)) error -} - -// SyncOnce performs a single sync cycle - fetches until caught up -func SyncOnce(ctx context.Context, mgr SyncManager, config *SyncLoopConfig) (int, error) { - cycleStart := time.Now() - startMempool := mgr.GetMempoolCount() - - fetchedCount := 0 - var totalIndexTime time.Duration - - // Keep fetching until caught up (detect by checking if state changes) - for { - // Track state before fetch - bundleBefore := mgr.GetLastBundleNumber() - mempoolBefore := mgr.GetMempoolCount() - - // Attempt to fetch and save next bundle - bundleNum, stats, err := mgr.FetchAndSaveNextBundle(ctx, config.Verbose, config.Quiet, config.SkipDIDIndex) - - // Check if we made any progress - bundleAfter := mgr.GetLastBundleNumber() - mempoolAfter := mgr.GetMempoolCount() - - madeProgress := bundleAfter > bundleBefore || mempoolAfter > mempoolBefore - - if err != nil { - // If no progress and got error → caught up - if !madeProgress { - break - } - - // We added to mempool but couldn't complete bundle yet - // This is fine, just stop here - break - } - - // Success - fetchedCount++ - if stats != nil { - totalIndexTime += stats.IndexTime - } - - // Callback if provided - if config.OnBundleSynced != nil { - config.OnBundleSynced(bundleNum, fetchedCount, mempoolAfter, time.Since(cycleStart), totalIndexTime) - } - - // Small delay between bundles - time.Sleep(500 * time.Millisecond) - - // Check if we're still making progress - if !madeProgress { - break - } - } - - // Summary output - if config.Logger != nil { - mempoolAfter := mgr.GetMempoolCount() - addedOps := mempoolAfter - startMempool - duration := time.Since(cycleStart) - currentBundle := mgr.GetLastBundleNumber() - - if fetchedCount > 0 { - if totalIndexTime > 10*time.Millisecond { - config.Logger.Printf("[Sync] ✓ Bundle %06d | synced: %d | mempool: %d (+%d) | time: %s (index: %s)", - currentBundle, fetchedCount, mempoolAfter, addedOps, - duration.Round(time.Millisecond), totalIndexTime.Round(time.Millisecond)) - } else { - config.Logger.Printf("[Sync] ✓ Bundle %06d | synced: %d | mempool: %d (+%d) | time: %s", - currentBundle, fetchedCount, mempoolAfter, addedOps, duration.Round(time.Millisecond)) - } - } else if addedOps > 0 { - // No bundles but added to mempool - config.Logger.Printf("[Sync] ✓ Bundle %06d | mempool: %d (+%d) | time: %s", - currentBundle, mempoolAfter, addedOps, duration.Round(time.Millisecond)) - } else { - // Already up to date - config.Logger.Printf("[Sync] ✓ Bundle %06d | up to date | time: %s", - currentBundle, duration.Round(time.Millisecond)) - } - } - - return fetchedCount, nil -} - -// RunSyncLoop performs continuous syncing -func RunSyncLoop(ctx context.Context, mgr SyncManager, config *SyncLoopConfig) error { - if config == nil { - config = DefaultSyncLoopConfig() - } - - if config.Interval <= 0 { - config.Interval = 1 * time.Minute - } - - bundlesSynced := 0 - - // Initial sync - if config.Logger != nil && config.MaxBundles != 1 { - config.Logger.Printf("[Sync] Initial sync starting...") - } - - config.SkipDIDIndex = true - synced, err := SyncOnce(ctx, mgr, config) - if err != nil { - return err - } - bundlesSynced += synced - mgr.UpdateDIDIndexSmart(ctx, nil) - - // Check if reached limit - if config.MaxBundles > 0 && bundlesSynced >= config.MaxBundles { - if config.Logger != nil { - config.Logger.Printf("[Sync] Reached max bundles limit (%d)", config.MaxBundles) - } - return nil - } - - if config.Logger != nil { - config.Logger.Printf("[Sync] Loop started (interval: %s)", config.Interval) - } - - ticker := time.NewTicker(config.Interval) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - if config.Logger != nil { - config.Logger.Printf("[Sync] Stopped (total synced: %d bundles)", bundlesSynced) - } - return ctx.Err() - - case <-ticker.C: - // Each tick, do one sync cycle (which fetches until caught up) - synced, err := SyncOnce(ctx, mgr, config) - if err != nil { - if config.Logger != nil { - config.Logger.Printf("[Sync] Error: %v", err) - } - continue - } - - bundlesSynced += synced - - // Check if reached limit - if config.MaxBundles > 0 && bundlesSynced >= config.MaxBundles { - if config.Logger != nil { - config.Logger.Printf("[Sync] Reached max bundles limit (%d)", config.MaxBundles) - } - return nil - } - } - } -} diff --git a/internal/types/types.go b/internal/types/types.go deleted file mode 100644 index f745eb8..0000000 --- a/internal/types/types.go +++ /dev/null @@ -1,28 +0,0 @@ -package types - -import "time" - -// Logger is a simple logging interface used throughout plcbundle -type Logger interface { - Printf(format string, v ...interface{}) - Println(v ...interface{}) -} - -const ( - // BUNDLE_SIZE is the standard number of operations per bundle - BUNDLE_SIZE = 10000 - - // INDEX_FILE is the default index filename - INDEX_FILE = "plc_bundles.json" - - // INDEX_VERSION is the current index format version - INDEX_VERSION = "1.0" -) - -type BundleProductionStats struct { - TotalFetches int - TotalDuration time.Duration - AvgPerFetch float64 - Throughput float64 - IndexTime time.Duration -} diff --git a/internal/types/types_test.go b/internal/types/types_test.go deleted file mode 100644 index e99943b..0000000 --- a/internal/types/types_test.go +++ /dev/null @@ -1,218 +0,0 @@ -package types_test - -import ( - "bytes" - "fmt" - "testing" - - "tangled.org/atscan.net/plcbundle/internal/types" -) - -// ==================================================================================== -// CONSTANT VALIDATION TESTS -// ==================================================================================== - -func TestConstants(t *testing.T) { - t.Run("BundleSize", func(t *testing.T) { - if types.BUNDLE_SIZE != 10000 { - t.Errorf("BUNDLE_SIZE = %d, want 10000", types.BUNDLE_SIZE) - } - - // Ensure it's a reasonable size - if types.BUNDLE_SIZE < 1000 { - t.Error("BUNDLE_SIZE too small") - } - - if types.BUNDLE_SIZE > 100000 { - t.Error("BUNDLE_SIZE too large") - } - }) - - t.Run("IndexFile", func(t *testing.T) { - if types.INDEX_FILE != "plc_bundles.json" { - t.Errorf("INDEX_FILE = %s, want plc_bundles.json", types.INDEX_FILE) - } - - // Should be a valid filename - if types.INDEX_FILE == "" { - t.Error("INDEX_FILE should not be empty") - } - - // Should have .json extension - if len(types.INDEX_FILE) < 5 || types.INDEX_FILE[len(types.INDEX_FILE)-5:] != ".json" { - t.Error("INDEX_FILE should have .json extension") - } - }) - - t.Run("IndexVersion", func(t *testing.T) { - if types.INDEX_VERSION != "1.0" { - t.Errorf("INDEX_VERSION = %s, want 1.0", types.INDEX_VERSION) - } - - // Should follow semantic versioning format (at least major.minor) - if len(types.INDEX_VERSION) < 3 { - t.Error("INDEX_VERSION should follow semantic versioning") - } - }) -} - -// ==================================================================================== -// LOGGER INTERFACE COMPLIANCE TESTS -// ==================================================================================== - -func TestLoggerInterface(t *testing.T) { - t.Run("MockLoggerImplementsInterface", func(t *testing.T) { - var logger types.Logger = &mockLogger{} - - // Should compile and not panic - logger.Printf("test %s", "message") - logger.Println("test", "message") - }) - - t.Run("BufferedLoggerImplementation", func(t *testing.T) { - buf := &bytes.Buffer{} - logger := &bufferedLogger{buf: buf} - - // Cast to interface - var _ types.Logger = logger - - logger.Printf("formatted %s %d", "message", 42) - logger.Println("plain", "message") - - output := buf.String() - - if !containsString(output, "formatted message 42") { - t.Error("Printf output not captured") - } - - if !containsString(output, "plain message") { - t.Error("Println output not captured") - } - }) - - t.Run("NullLoggerImplementation", func(t *testing.T) { - // Logger that discards all output - logger := &nullLogger{} - - // Should not panic - var _ types.Logger = logger - logger.Printf("test %s", "ignored") - logger.Println("also", "ignored") - }) - - t.Run("MultiLoggerImplementation", func(t *testing.T) { - // Logger that writes to multiple destinations - buf1 := &bytes.Buffer{} - buf2 := &bytes.Buffer{} - - logger := &multiLogger{ - loggers: []types.Logger{ - &bufferedLogger{buf: buf1}, - &bufferedLogger{buf: buf2}, - }, - } - - var _ types.Logger = logger - - logger.Printf("test %s", "message") - - // Both buffers should have the message - if !containsString(buf1.String(), "test message") { - t.Error("first logger didn't receive message") - } - - if !containsString(buf2.String(), "test message") { - t.Error("second logger didn't receive message") - } - }) -} - -// ==================================================================================== -// CONSTANT USAGE IN CALCULATIONS -// ==================================================================================== - -func TestConstantUsage(t *testing.T) { - t.Run("GlobalPositionCalculation", func(t *testing.T) { - // Global position = bundleNumber * BUNDLE_SIZE + position - bundleNumber := 42 - position := 1337 - - globalPos := bundleNumber*types.BUNDLE_SIZE + position - expected := 420000 + 1337 - - if globalPos != expected { - t.Errorf("global position calculation incorrect: got %d, want %d", globalPos, expected) - } - }) - - t.Run("BundleFromGlobalPosition", func(t *testing.T) { - globalPos := 88410345 - - bundleNumber := globalPos / types.BUNDLE_SIZE - position := globalPos % types.BUNDLE_SIZE - - if bundleNumber != 8841 { - t.Errorf("bundle calculation wrong: got %d, want 8841", bundleNumber) - } - - if position != 345 { - t.Errorf("position calculation wrong: got %d, want 345", position) - } - }) - - t.Run("OperationCountPerBundle", func(t *testing.T) { - // Each bundle should have exactly BUNDLE_SIZE operations - bundleCount := 100 - totalOps := bundleCount * types.BUNDLE_SIZE - - if totalOps != 1000000 { - t.Errorf("total ops calculation: got %d, want 1000000", totalOps) - } - }) -} - -// ==================================================================================== -// HELPER IMPLEMENTATIONS -// ==================================================================================== - -type mockLogger struct{} - -func (l *mockLogger) Printf(format string, v ...interface{}) {} -func (l *mockLogger) Println(v ...interface{}) {} - -type bufferedLogger struct { - buf *bytes.Buffer -} - -func (l *bufferedLogger) Printf(format string, v ...interface{}) { - fmt.Fprintf(l.buf, format+"\n", v...) -} - -func (l *bufferedLogger) Println(v ...interface{}) { - fmt.Fprintln(l.buf, v...) -} - -type nullLogger struct{} - -func (l *nullLogger) Printf(format string, v ...interface{}) {} -func (l *nullLogger) Println(v ...interface{}) {} - -type multiLogger struct { - loggers []types.Logger -} - -func (l *multiLogger) Printf(format string, v ...interface{}) { - for _, logger := range l.loggers { - logger.Printf(format, v...) - } -} - -func (l *multiLogger) Println(v ...interface{}) { - for _, logger := range l.loggers { - logger.Println(v...) - } -} - -func containsString(haystack, needle string) bool { - return bytes.Contains([]byte(haystack), []byte(needle)) -} diff --git a/options.go b/options.go deleted file mode 100644 index 8c30036..0000000 --- a/options.go +++ /dev/null @@ -1,54 +0,0 @@ -package plcbundle - -import ( - "tangled.org/atscan.net/plcbundle/bundle" - "tangled.org/atscan.net/plcbundle/internal/plcclient" -) - -type config struct { - bundleConfig *bundle.Config - plcClient *plcclient.Client -} - -func defaultConfig() *config { - return &config{ - bundleConfig: bundle.DefaultConfig("./plc_bundles"), - } -} - -// Option configures the Manager -type Option func(*config) - -// WithDirectory sets the bundle storage directory -func WithDirectory(dir string) Option { - return func(c *config) { - c.bundleConfig.BundleDir = dir - } -} - -// WithPLCDirectory sets the PLC directory URL -func WithPLCDirectory(url string) Option { - return func(c *config) { - c.plcClient = plcclient.NewClient(url) - } -} - -// WithVerifyOnLoad enables/disables hash verification when loading bundles -func WithVerifyOnLoad(verify bool) Option { - return func(c *config) { - c.bundleConfig.VerifyOnLoad = verify - } -} - -// WithLogger sets a custom logger -func WithLogger(logger Logger) Option { - return func(c *config) { - c.bundleConfig.Logger = logger - } -} - -// Logger interface -type Logger interface { - Printf(format string, v ...interface{}) - Println(v ...interface{}) -} diff --git a/scripts/benchmark-detector.go b/scripts/benchmark-detector.go deleted file mode 100644 index 9e9060d..0000000 --- a/scripts/benchmark-detector.go +++ /dev/null @@ -1,165 +0,0 @@ -package main - -import ( - "bufio" - "bytes" - "fmt" - "os" - "path/filepath" - "strconv" - "strings" - "time" - - "github.com/goccy/go-json" - "github.com/valyala/gozstd" -) - -// Minimal operation struct -type Operation struct { - DID string `json:"did"` - Operation map[string]interface{} `json:"operation"` - CID string `json:"cid"` - Nullified interface{} `json:"nullified,omitempty"` - CreatedAt time.Time `json:"createdAt"` - - // RawJSON stores the original JSON bytes for exact reproduction - RawJSON []byte `json:"-"` -} - -// User's detect function -func detect(op *Operation) []string { - labels := []string{} - - if strings.HasPrefix(op.DID, "did:plc:aa") { - labels = append(labels, "test") - } - - // Log operation.sig (like console.log in JavaScript) - if sig, ok := op.Operation["sig"]; ok { - fmt.Fprintf(os.Stderr, "%v\n", sig) - } - - return labels -} - -func main() { - bundleDir := "./" - if len(os.Args) > 1 { - bundleDir = os.Args[1] - } - - startBundle := 1 - if len(os.Args) > 2 { - startBundle, _ = strconv.Atoi(os.Args[2]) - } - - endBundle := 100 - if len(os.Args) > 3 { - endBundle, _ = strconv.Atoi(os.Args[3]) - } - - fmt.Fprintf(os.Stderr, "Processing bundles %d-%d from %s\n", startBundle, endBundle, bundleDir) - fmt.Fprintf(os.Stderr, "\n") - - // Buffered stdout - writer := bufio.NewWriterSize(os.Stdout, 512*1024) - defer writer.Flush() - - // CSV header - writer.WriteString("bundle,position,cid,size,confidence,labels\n") - - totalOps := 0 - matchCount := 0 - totalBytes := int64(0) - matchedBytes := int64(0) - - startTime := time.Now() - - for bundleNum := startBundle; bundleNum <= endBundle; bundleNum++ { - bundleFile := filepath.Join(bundleDir, fmt.Sprintf("%06d.jsonl.zst", bundleNum)) - - // Read compressed file - compressed, err := os.ReadFile(bundleFile) - if err != nil { - fmt.Fprintf(os.Stderr, "\nError reading bundle %d: %v\n", bundleNum, err) - continue - } - - // Decompress with native gozstd (CGO, fast as native zstd) - decompressed, err := gozstd.Decompress(nil, compressed) - if err != nil { - fmt.Fprintf(os.Stderr, "\nError decompressing bundle %d: %v\n", bundleNum, err) - continue - } - - // Split into lines - lines := bytes.Split(decompressed, []byte("\n")) - - for position, line := range lines { - if len(line) == 0 { - continue - } - - totalOps++ - opSize := len(line) - totalBytes += int64(opSize) - - // Parse JSON with go-json - var op Operation - if err := json.Unmarshal(line, &op); err != nil { - fmt.Fprintf(os.Stderr, "Error parsing operation: %v\n", err) - continue - } - - // Run detect function - labels := detect(&op) - - if len(labels) > 0 { - matchCount++ - matchedBytes += int64(opSize) - - // Extract last 4 chars of CID - cidShort := op.CID - if len(cidShort) > 4 { - cidShort = cidShort[len(cidShort)-4:] - } - - // Write CSV line - writer.WriteString(strconv.Itoa(bundleNum)) - writer.WriteByte(',') - writer.WriteString(strconv.Itoa(position)) - writer.WriteByte(',') - writer.WriteString(cidShort) - writer.WriteByte(',') - writer.WriteString(strconv.Itoa(opSize)) - writer.WriteByte(',') - writer.WriteString("0.95") - writer.WriteByte(',') - writer.WriteString(strings.Join(labels, ";")) - writer.WriteByte('\n') - } - } - - // Progress - if bundleNum%10 == 0 { - elapsed := time.Since(startTime).Seconds() - opsPerSec := float64(totalOps) / elapsed - fmt.Fprintf(os.Stderr, "Processed %d/%d bundles | %d ops | %.0f ops/sec\r", - bundleNum, endBundle, totalOps, opsPerSec) - } - } - - elapsed := time.Since(startTime).Seconds() - - // Stats - fmt.Fprintf(os.Stderr, "\n\n") - fmt.Fprintf(os.Stderr, "✓ Detection complete\n") - fmt.Fprintf(os.Stderr, " Total operations: %d\n", totalOps) - fmt.Fprintf(os.Stderr, " Matches found: %d (%.2f%%)\n", matchCount, float64(matchCount)/float64(totalOps)*100) - fmt.Fprintf(os.Stderr, " Total size: %.1f MB\n", float64(totalBytes)/1e6) - fmt.Fprintf(os.Stderr, " Matched size: %.1f MB (%.2f%%)\n", float64(matchedBytes)/1e6, float64(matchedBytes)/float64(totalBytes)*100) - fmt.Fprintf(os.Stderr, "\n") - fmt.Fprintf(os.Stderr, " Time elapsed: %.2fs\n", elapsed) - fmt.Fprintf(os.Stderr, " Throughput: %.0f ops/sec\n", float64(totalOps)/elapsed) - fmt.Fprintf(os.Stderr, " Speed: %.1f MB/sec\n", float64(totalBytes)/elapsed/1e6) -} diff --git a/scripts/benchmark-detector.js b/scripts/benchmark-detector.js deleted file mode 100644 index c50196e..0000000 --- a/scripts/benchmark-detector.js +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env bun - -// User's detect function -function detect({ op }) { - const labels = []; - - if (op.did.startsWith('did:plc:aa')) { - labels.push('test') - } - - console.log(op.operation.sig) - - return labels; -} - -// ========================================== -// Pure Bun bundle processor with native zstd -// ========================================== - -const BUNDLE_DIR = process.argv[2] || './'; -const START_BUNDLE = parseInt(process.argv[3]) || 1; -const END_BUNDLE = parseInt(process.argv[4]) || 100; - -console.error(`Processing bundles ${START_BUNDLE}-${END_BUNDLE} from ${BUNDLE_DIR}`); -console.error(''); - -// CSV header -console.log('bundle,position,cid,size,confidence,labels'); - -let totalOps = 0; -let matchCount = 0; -let totalBytes = 0; -let matchedBytes = 0; - -const startTime = Date.now(); - -for (let bundleNum = START_BUNDLE; bundleNum <= END_BUNDLE; bundleNum++) { - const bundleFile = `${BUNDLE_DIR}/${bundleNum.toString().padStart(6, '0')}.jsonl.zst`; - - try { - // Read compressed bundle - const compressed = await Bun.file(bundleFile).arrayBuffer(); - - // Decompress using native Bun zstd (FAST!) - const decompressed = Bun.zstdDecompressSync(compressed); - - // Convert to text - const text = new TextDecoder().decode(decompressed); - - const lines = text.split('\n').filter(line => line.trim()); - - for (let position = 0; position < lines.length; position++) { - const line = lines[position]; - if (!line) continue; - - totalOps++; - const opSize = line.length; - totalBytes += opSize; - - try { - const op = JSON.parse(line); - const labels = detect({ op }); - - if (labels && labels.length > 0) { - matchCount++; - matchedBytes += opSize; - - // Extract last 4 chars of CID - const cidShort = op.cid.slice(-4); - - console.log( - `${bundleNum},${position},${cidShort},${opSize},0.95,${labels.join(';')}` - ); - } - } catch (err) { - console.error(`Error parsing operation: ${err.message}`); - } - } - - // Progress - if (bundleNum % 10 === 0) { - const elapsed = (Date.now() - startTime) / 1000; - const opsPerSec = (totalOps / elapsed).toFixed(0); - console.error(`Processed ${bundleNum}/${END_BUNDLE} bundles | ${totalOps} ops | ${opsPerSec} ops/sec\r`); - } - - } catch (err) { - console.error(`\nError loading bundle ${bundleNum}: ${err.message}`); - } -} - -const elapsed = (Date.now() - startTime) / 1000; - -// Stats -console.error('\n'); -console.error('✓ Detection complete'); -console.error(` Total operations: ${totalOps}`); -console.error(` Matches found: ${matchCount} (${(matchCount/totalOps*100).toFixed(2)}%)`); -console.error(` Total size: ${(totalBytes / 1e6).toFixed(1)} MB`); -console.error(` Matched size: ${(matchedBytes / 1e6).toFixed(1)} MB (${(matchedBytes/totalBytes*100).toFixed(2)}%)`); -console.error(''); -console.error(` Time elapsed: ${elapsed.toFixed(2)}s`); -console.error(` Throughput: ${(totalOps / elapsed).toFixed(0)} ops/sec`); -console.error(` Speed: ${(totalBytes / elapsed / 1e6).toFixed(1)} MB/sec`); diff --git a/scripts/benchmark_detector_rs/.gitignore b/scripts/benchmark_detector_rs/.gitignore deleted file mode 100644 index 1de5659..0000000 --- a/scripts/benchmark_detector_rs/.gitignore +++ /dev/null @@ -1 +0,0 @@ -target \ No newline at end of file diff --git a/scripts/benchmark_detector_rs/Cargo.lock b/scripts/benchmark_detector_rs/Cargo.lock deleted file mode 100644 index f102e7a..0000000 --- a/scripts/benchmark_detector_rs/Cargo.lock +++ /dev/null @@ -1,554 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "ahash" -version = "0.8.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" -dependencies = [ - "cfg-if", - "getrandom", - "once_cell", - "version_check", - "zerocopy", -] - -[[package]] -name = "benchmark_detector" -version = "0.1.0" -dependencies = [ - "serde", - "sonic-rs", - "zstd", -] - -[[package]] -name = "bumpalo" -version = "3.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" - -[[package]] -name = "bytes" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" - -[[package]] -name = "cc" -version = "1.2.43" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "739eb0f94557554b3ca9a86d2d37bebd49c5e6d0c1d2bda35ba5bdac830befc2" -dependencies = [ - "find-msvc-tools", - "jobserver", - "libc", - "shlex", -] - -[[package]] -name = "cfg-if" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" - -[[package]] -name = "equivalent" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" - -[[package]] -name = "faststr" -version = "0.2.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baec6a0289d7f1fe5665586ef7340af82e3037207bef60f5785e57569776f0c8" -dependencies = [ - "bytes", - "rkyv", - "serde", - "simdutf8", -] - -[[package]] -name = "find-msvc-tools" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127" - -[[package]] -name = "getrandom" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" -dependencies = [ - "cfg-if", - "libc", - "r-efi", - "wasip2", -] - -[[package]] -name = "hashbrown" -version = "0.15.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" - -[[package]] -name = "hashbrown" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" - -[[package]] -name = "indexmap" -version = "2.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6717a8d2a5a929a1a2eb43a12812498ed141a0bcfb7e8f7844fbdbe4303bba9f" -dependencies = [ - "equivalent", - "hashbrown 0.16.0", -] - -[[package]] -name = "itoa" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" - -[[package]] -name = "jobserver" -version = "0.1.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" -dependencies = [ - "getrandom", - "libc", -] - -[[package]] -name = "js-sys" -version = "0.3.82" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b011eec8cc36da2aab2d5cff675ec18454fad408585853910a202391cf9f8e65" -dependencies = [ - "once_cell", - "wasm-bindgen", -] - -[[package]] -name = "libc" -version = "0.2.177" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" - -[[package]] -name = "munge" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e17401f259eba956ca16491461b6e8f72913a0a114e39736ce404410f915a0c" -dependencies = [ - "munge_macro", -] - -[[package]] -name = "munge_macro" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "once_cell" -version = "1.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" - -[[package]] -name = "pkg-config" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" - -[[package]] -name = "proc-macro2" -version = "1.0.103" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "ptr_meta" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b9a0cf95a1196af61d4f1cbdab967179516d9a4a4312af1f31948f8f6224a79" -dependencies = [ - "ptr_meta_derive", -] - -[[package]] -name = "ptr_meta_derive" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "quote" -version = "1.0.41" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r-efi" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" - -[[package]] -name = "rancor" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a063ea72381527c2a0561da9c80000ef822bdd7c3241b1cc1b12100e3df081ee" -dependencies = [ - "ptr_meta", -] - -[[package]] -name = "ref-cast" -version = "1.0.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" -dependencies = [ - "ref-cast-impl", -] - -[[package]] -name = "ref-cast-impl" -version = "1.0.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "rend" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cadadef317c2f20755a64d7fdc48f9e7178ee6b0e1f7fce33fa60f1d68a276e6" - -[[package]] -name = "rkyv" -version = "0.8.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35a640b26f007713818e9a9b65d34da1cf58538207b052916a83d80e43f3ffa4" -dependencies = [ - "bytes", - "hashbrown 0.15.5", - "indexmap", - "munge", - "ptr_meta", - "rancor", - "rend", - "rkyv_derive", - "tinyvec", - "uuid", -] - -[[package]] -name = "rkyv_derive" -version = "0.8.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd83f5f173ff41e00337d97f6572e416d022ef8a19f371817259ae960324c482" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "rustversion" -version = "1.0.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" - -[[package]] -name = "ryu" -version = "1.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" - -[[package]] -name = "serde" -version = "1.0.228" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" -dependencies = [ - "serde_core", - "serde_derive", -] - -[[package]] -name = "serde_core" -version = "1.0.228" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.228" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - -[[package]] -name = "simdutf8" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" - -[[package]] -name = "sonic-number" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a74044c092f4f43ca7a6cfd62854cf9fb5ac8502b131347c990bf22bef1dfe" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "sonic-rs" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22540d56ba14521e4878ad436d498518c59698c39a89d5905c694932f0bf7134" -dependencies = [ - "ahash", - "bumpalo", - "bytes", - "cfg-if", - "faststr", - "itoa", - "ref-cast", - "ryu", - "serde", - "simdutf8", - "sonic-number", - "sonic-simd", - "thiserror", -] - -[[package]] -name = "sonic-simd" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b421f7b6aa4a5de8f685aaf398dfaa828346ee639d2b1c1061ab43d40baa6223" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "syn" -version = "2.0.108" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da58917d35242480a05c2897064da0a80589a2a0476c9a3f2fdc83b53502e917" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "thiserror" -version = "2.0.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "2.0.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tinyvec" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - -[[package]] -name = "unicode-ident" -version = "1.0.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" - -[[package]] -name = "uuid" -version = "1.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - -[[package]] -name = "wasip2" -version = "1.0.1+wasi-0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" -dependencies = [ - "wit-bindgen", -] - -[[package]] -name = "wasm-bindgen" -version = "0.2.105" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da95793dfc411fbbd93f5be7715b0578ec61fe87cb1a42b12eb625caa5c5ea60" -dependencies = [ - "cfg-if", - "once_cell", - "rustversion", - "wasm-bindgen-macro", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.105" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04264334509e04a7bf8690f2384ef5265f05143a4bff3889ab7a3269adab59c2" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.105" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "420bc339d9f322e562942d52e115d57e950d12d88983a14c79b86859ee6c7ebc" -dependencies = [ - "bumpalo", - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.105" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76f218a38c84bcb33c25ec7059b07847d465ce0e0a76b995e134a45adcb6af76" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "wit-bindgen" -version = "0.46.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" - -[[package]] -name = "zerocopy" -version = "0.8.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "zstd" -version = "0.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" -dependencies = [ - "zstd-safe", -] - -[[package]] -name = "zstd-safe" -version = "7.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" -dependencies = [ - "zstd-sys", -] - -[[package]] -name = "zstd-sys" -version = "2.0.16+zstd.1.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" -dependencies = [ - "cc", - "pkg-config", -] diff --git a/scripts/benchmark_detector_rs/Cargo.toml b/scripts/benchmark_detector_rs/Cargo.toml deleted file mode 100644 index 0de93b2..0000000 --- a/scripts/benchmark_detector_rs/Cargo.toml +++ /dev/null @@ -1,17 +0,0 @@ -[package] -name = "benchmark_detector" -version = "0.1.0" -edition = "2021" - -[dependencies] -sonic-rs = "0.5" -serde = { version = "1.0", features = ["derive"] } -zstd = "0.13" - -[profile.release] -lto = true -codegen-units = 1 -opt-level = 3 - -[build] -rustflags = ["-C", "target-cpu=native"] \ No newline at end of file diff --git a/scripts/benchmark_detector_rs/src/main.rs b/scripts/benchmark_detector_rs/src/main.rs deleted file mode 100644 index 4b6600f..0000000 --- a/scripts/benchmark_detector_rs/src/main.rs +++ /dev/null @@ -1,123 +0,0 @@ -use std::env; -use std::fs::File; -use std::io::{self, BufRead, BufReader, Write}; -use std::time::Instant; -use sonic_rs::{Deserialize, from_str, JsonValueTrait, Value}; - -#[derive(Deserialize)] -struct Operation { - did: String, - cid: String, - operation: Value, -} - -fn detect(op: &Operation) -> Vec { - let mut labels = Vec::new(); - - if op.did.starts_with("did:plc:aa") { - labels.push("test".to_string()); - } - - if let Some(sig) = op.operation.get("sig") { - eprintln!("{}", sig); - } - - labels -} - -fn main() -> io::Result<()> { - let args: Vec = env::args().collect(); - let bundle_dir = args.get(1).map(|s| s.as_str()).unwrap_or("./"); - let start_bundle: usize = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(1); - let end_bundle: usize = args.get(3).and_then(|s| s.parse().ok()).unwrap_or(100); - - eprintln!("Processing bundles {}-{} from {}", start_bundle, end_bundle, bundle_dir); - eprintln!(); - - let stdout = io::stdout(); - let mut writer = io::BufWriter::with_capacity(512 * 1024, stdout.lock()); - writeln!(writer, "bundle,position,cid,size,confidence,labels")?; - - let mut total_ops = 0; - let mut match_count = 0; - let mut total_bytes: u64 = 0; - let mut matched_bytes: u64 = 0; - let start_time = Instant::now(); - - for bundle_num in start_bundle..=end_bundle { - let bundle_file = format!("{}/{:06}.jsonl.zst", bundle_dir, bundle_num); - - let file = match File::open(&bundle_file) { - Ok(f) => f, - Err(_) => continue, - }; - - let decoder = match zstd::Decoder::new(file) { - Ok(d) => d, - Err(_) => continue, - }; - - let reader = BufReader::new(decoder); - - for (position, line) in reader.lines().enumerate() { - let line = match line { - Ok(l) => l, - Err(_) => continue, - }; - - if line.is_empty() { - continue; - } - - total_ops += 1; - let op_size = line.len(); - total_bytes += op_size as u64; - - let op: Operation = match from_str(&line) { - Ok(o) => o, - Err(_) => continue, - }; - - let labels = detect(&op); - - if !labels.is_empty() { - match_count += 1; - matched_bytes += op_size as u64; - - let cid_short = if op.cid.len() > 4 { - &op.cid[op.cid.len()-4..] - } else { - &op.cid - }; - - writeln!(writer, "{},{},{},{},0.95,{}", - bundle_num, position, cid_short, op_size, labels.join(";"))?; - } - } - - if bundle_num % 10 == 0 { - let elapsed = start_time.elapsed().as_secs_f64(); - let ops_per_sec = total_ops as f64 / elapsed; - eprint!("Processed {}/{} bundles | {} ops | {:.0} ops/sec\r", - bundle_num, end_bundle, total_ops, ops_per_sec); - } - } - - let elapsed = start_time.elapsed().as_secs_f64(); - writer.flush()?; - - eprintln!("\n\n✓ Detection complete"); - eprintln!(" Total operations: {}", total_ops); - eprintln!(" Matches found: {} ({:.2}%)", match_count, - match_count as f64 / total_ops as f64 * 100.0); - eprintln!(" Total size: {:.1} MB", total_bytes as f64 / 1e6); - eprintln!(" Matched size: {:.1} MB ({:.2}%)", - matched_bytes as f64 / 1e6, - matched_bytes as f64 / total_bytes as f64 * 100.0); - eprintln!(); - eprintln!(" Time elapsed: {:.2}s", elapsed); - eprintln!(" Throughput: {:.0} ops/sec", total_ops as f64 / elapsed); - eprintln!(" Speed: {:.1} MB/sec", total_bytes as f64 / elapsed / 1e6); - - Ok(()) -} diff --git a/scripts/bump-version.sh b/scripts/bump-version.sh deleted file mode 100755 index 83f5c37..0000000 --- a/scripts/bump-version.sh +++ /dev/null @@ -1,202 +0,0 @@ -#!/bin/bash - -set -e - -# Colors -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# Check if git is clean -if [[ -n $(git status -s) ]]; then - echo -e "${RED}Error: Git working directory is not clean${NC}" - echo "Please commit or stash your changes first" - exit 1 -fi - -# Get current version from git tags -CURRENT_VERSION=$(git describe --tags --abbrev=0 2>/dev/null || echo "v0.0.0") - -# Remove 'v' prefix if present -CURRENT_VERSION=${CURRENT_VERSION#v} - -# Parse version (supports semver with pre-release) -# Format: MAJOR.MINOR.PATCH[-PRERELEASE] -if [[ $CURRENT_VERSION =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)(-([a-zA-Z]+)\.?([0-9]+)?)?$ ]]; then - MAJOR="${BASH_REMATCH[1]}" - MINOR="${BASH_REMATCH[2]}" - PATCH="${BASH_REMATCH[3]}" - PRERELEASE="${BASH_REMATCH[5]}" # alpha, beta, rc, etc. - PRERELEASE_NUM="${BASH_REMATCH[6]}" # 1, 2, 3, etc. -else - echo -e "${RED}Error: Could not parse version: ${CURRENT_VERSION}${NC}" - exit 1 -fi - -# Function to show usage -show_usage() { - echo "Usage: $0 {major|minor|patch|alpha|beta|rc|pre|release} [--minor|--patch]" - echo "" - echo "Stable releases:" - echo " major Bump major version (x.0.0)" - echo " minor Bump minor version (0.x.0)" - echo " patch Bump patch version (0.0.x)" - echo "" - echo "Pre-releases:" - echo " alpha Create/bump alpha (0.x.0-alpha.1 by default)" - echo " beta Create/bump beta (0.x.0-beta.1 by default)" - echo " rc Create/bump RC (0.x.0-rc.1 by default)" - echo " pre Bump current pre-release number" - echo "" - echo "Combined (create pre-release with specific bump):" - echo " alpha --minor Bump minor and create alpha (0.x.0-alpha.1)" - echo " alpha --patch Bump patch and create alpha (0.0.x-alpha.1)" - echo " beta --minor Bump minor and create beta" - echo " rc --patch Bump patch and create rc" - echo "" - echo "Promotion:" - echo " release Remove pre-release suffix (promote to stable)" - echo "" - echo "Examples:" - echo " v0.4.26 → minor → v0.5.0" - echo " v0.4.26 → alpha --minor → v0.5.0-alpha.1" - echo " v0.4.26 → alpha --patch → v0.4.27-alpha.1" - echo " v0.5.0-alpha.1 → pre → v0.5.0-alpha.2" - echo " v0.5.0-alpha.2 → beta → v0.5.0-beta.1" - echo " v0.5.0-rc.1 → release → v0.5.0" - exit 1 -} - -# Parse optional flags (--minor, --patch) -BUMP_TYPE="" -if [[ "$2" == "--minor" ]]; then - BUMP_TYPE="minor" -elif [[ "$2" == "--patch" ]]; then - BUMP_TYPE="patch" -fi - -# Determine new version based on command -case "$1" in - major) - MAJOR=$((MAJOR + 1)) - MINOR=0 - PATCH=0 - PRERELEASE="" - PRERELEASE_NUM="" - ;; - minor) - MINOR=$((MINOR + 1)) - PATCH=0 - PRERELEASE="" - PRERELEASE_NUM="" - ;; - patch) - PATCH=$((PATCH + 1)) - PRERELEASE="" - PRERELEASE_NUM="" - ;; - alpha|beta|rc) - # If already on this pre-release type, bump the number - if [[ "$PRERELEASE" == "$1" ]]; then - PRERELEASE_NUM=$((PRERELEASE_NUM + 1)) - else - # Creating new pre-release type - if [[ -z "$PRERELEASE" ]]; then - # Coming from stable - determine what to bump - if [[ "$BUMP_TYPE" == "minor" ]]; then - MINOR=$((MINOR + 1)) - PATCH=0 - elif [[ "$BUMP_TYPE" == "patch" ]]; then - PATCH=$((PATCH + 1)) - else - # Default: bump minor for new pre-release - MINOR=$((MINOR + 1)) - PATCH=0 - fi - fi - # Set new pre-release - PRERELEASE="$1" - PRERELEASE_NUM=1 - fi - ;; - pre|prerelease) - if [[ -z "$PRERELEASE" ]]; then - echo -e "${RED}Error: Not currently on a pre-release version${NC}" - echo "Current version: ${CURRENT_VERSION}" - echo "Use 'alpha', 'beta', or 'rc' to create a pre-release" - exit 1 - fi - PRERELEASE_NUM=$((PRERELEASE_NUM + 1)) - ;; - release|stable) - if [[ -z "$PRERELEASE" ]]; then - echo -e "${YELLOW}Already on stable version: ${CURRENT_VERSION}${NC}" - exit 0 - fi - # Remove pre-release suffix - PRERELEASE="" - PRERELEASE_NUM="" - ;; - -h|--help|help) - show_usage - ;; - "") - echo -e "${RED}Error: No command specified${NC}" - echo "" - show_usage - ;; - *) - echo -e "${RED}Error: Invalid command: $1${NC}" - echo "" - show_usage - ;; -esac - -# Build new version string -if [[ -z "$PRERELEASE" ]]; then - NEW_VERSION="v${MAJOR}.${MINOR}.${PATCH}" -else - NEW_VERSION="v${MAJOR}.${MINOR}.${PATCH}-${PRERELEASE}.${PRERELEASE_NUM}" -fi - -# Show version change -echo "" -echo -e "${YELLOW}Current version: ${CURRENT_VERSION}${NC}" -echo -e "${GREEN}New version: ${NEW_VERSION#v}${NC}" - -# Show what changed -if [[ -n "$PRERELEASE" ]]; then - echo -e "${BLUE}Pre-release: ${PRERELEASE}.${PRERELEASE_NUM}${NC}" - echo -e "${YELLOW}⚠️ This is a pre-release (no stability guarantees)${NC}" -else - echo -e "${GREEN}✓ Stable release${NC}" -fi - -echo "" - -# Confirm -read -p "Create tag ${NEW_VERSION}? (y/N) " -n 1 -r -echo -if [[ ! $REPLY =~ ^[Yy]$ ]]; then - echo "Cancelled" - exit 0 -fi - -# Create git tag -if [[ -n "$PRERELEASE" ]]; then - # Pre-release tag - git tag -a "${NEW_VERSION}" -m "Pre-release ${NEW_VERSION}" - echo -e "${GREEN}✓ Created pre-release tag ${NEW_VERSION}${NC}" -else - # Stable release tag - git tag -a "${NEW_VERSION}" -m "Release ${NEW_VERSION}" - echo -e "${GREEN}✓ Created release tag ${NEW_VERSION}${NC}" -fi - -echo "" -echo "To push the tag, run:" -echo " git push origin ${NEW_VERSION}" -echo "" -echo "Or use 'make release' to push automatically" diff --git a/scripts/detector-template.js b/scripts/detector-template.js deleted file mode 100644 index cf73147..0000000 --- a/scripts/detector-template.js +++ /dev/null @@ -1,20 +0,0 @@ -function detect({ op }) { - // op.operation contains the PLC operation data - // op.did - DID identifier - // op.cid - Content ID - // op.createdAt - Timestamp - - const labels = []; - - // Add your detection logic here - // Return array of label strings - // Return empty array [] for no match - - //console.log(JSON.stringify(op)) - - if (op.did.match(/^did:plc:aaa/)) { - labels.push('test') - } - - return labels; -} diff --git a/scripts/did-resolve-benchmark-http-advanced.sh b/scripts/did-resolve-benchmark-http-advanced.sh deleted file mode 100755 index d820107..0000000 --- a/scripts/did-resolve-benchmark-http-advanced.sh +++ /dev/null @@ -1,392 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Colors -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -CYAN='\033[0;36m' -NC='\033[0m' - -# Default configuration -ENDPOINT="${ENDPOINT:-http://localhost:8080}" -NUM_REQUESTS="${NUM_REQUESTS:-100}" -CONCURRENCY="${CONCURRENCY:-4}" -BUNDLE_DIR="${BUNDLE_DIR:-.}" -ROUTE="${ROUTE:-document}" -WARMUP="${WARMUP:-0}" -OUTPUT_CSV="${OUTPUT_CSV:-}" -VERBOSE="${VERBOSE:-0}" - -# Parse arguments -while [[ $# -gt 0 ]]; do - case $1 in - -e|--endpoint) ENDPOINT="$2"; shift 2 ;; - -n|--requests) NUM_REQUESTS="$2"; shift 2 ;; - -c|--concurrency) CONCURRENCY="$2"; shift 2 ;; - -d|--dir) BUNDLE_DIR="$2"; shift 2 ;; - -r|--route) ROUTE="$2"; shift 2 ;; - -w|--warmup) WARMUP="$2"; shift 2 ;; - -o|--output) OUTPUT_CSV="$2"; shift 2 ;; - -v|--verbose) VERBOSE=1; shift ;; - -h|--help) - cat < /dev/null; then - echo -e "${RED}Error: '$cmd' not found. Install with: brew install $cmd${NC}" - exit 1 - fi -done - -echo -e "${BLUE}════════════════════════════════════════════════════════════════${NC}" -echo -e "${BLUE} DID Resolver HTTP Endpoint Benchmark${NC}" -echo -e "${BLUE}════════════════════════════════════════════════════════════════${NC}" -echo "" -echo "Configuration:" -echo " Endpoint: $ENDPOINT" -echo " Route: /$ROUTE" -echo " Requests: $NUM_REQUESTS" -echo " Concurrency: $CONCURRENCY" -if [[ $WARMUP -gt 0 ]]; then - echo " Warmup: $WARMUP" -fi -echo " Bundle Dir: $BUNDLE_DIR" -echo "" - -# Setup temp -TEMP_DIR=$(mktemp -d) -trap "rm -rf '$TEMP_DIR'" EXIT - -DID_LIST="$TEMP_DIR/dids.txt" -RESULTS_FILE="$TEMP_DIR/results.csv" - -# Phase 1: Extract DIDs -echo -e "${CYAN}━━━ Phase 1: Extracting DIDs ━━━${NC}" -echo "" - -# Find bundles -BUNDLES=() -while IFS= read -r line; do - BUNDLES+=("$line") -done < <(find "$BUNDLE_DIR" -name "*.jsonl.zst" -type f | sort) - -BUNDLE_COUNT=${#BUNDLES[@]} - -if [[ $BUNDLE_COUNT -eq 0 ]]; then - echo -e "${RED}Error: No bundles found in $BUNDLE_DIR${NC}" - exit 1 -fi - -echo " Found $BUNDLE_COUNT bundles" - -# Sample bundles -SAMPLE_SIZE=10 -if [[ $BUNDLE_COUNT -lt $SAMPLE_SIZE ]]; then - SAMPLE_SIZE=$BUNDLE_COUNT -fi - -echo " Extracting DIDs from $SAMPLE_SIZE bundles..." -echo "" - -# Extract DIDs properly (not in subshell) -> "$DID_LIST" # Create empty file - -for i in $(seq 0 $((SAMPLE_SIZE - 1))); do - BUNDLE="${BUNDLES[$i]}" - BUNDLE_NUM=$(basename "$BUNDLE" .jsonl.zst) - - echo -ne "\r Bundle $BUNDLE_NUM ($((i + 1))/$SAMPLE_SIZE)... " - - # Decompress and extract DIDs (show errors if verbose) - if [[ $VERBOSE -eq 1 ]]; then - zstd -dc "$BUNDLE" | jq -r '.did' >> "$DID_LIST" - else - zstd -dc "$BUNDLE" 2>/dev/null | jq -r '.did' 2>/dev/null >> "$DID_LIST" || { - echo -e "\n${YELLOW} Warning: Failed to process $BUNDLE_NUM${NC}" - continue - } - fi -done - -echo "" - -# Deduplicate -sort -u "$DID_LIST" -o "$DID_LIST" - -TOTAL_DIDS=$(wc -l < "$DID_LIST" | tr -d ' ') - -if [[ $TOTAL_DIDS -eq 0 ]]; then - echo -e "${RED}Error: No DIDs extracted. Check bundle files.${NC}" - if [[ $VERBOSE -eq 0 ]]; then - echo "Run with -v to see errors" - fi - exit 1 -fi - -echo -e "${GREEN} ✓ Extracted $TOTAL_DIDS unique DIDs${NC}" -echo "" - -# Adjust request count if needed -TOTAL_NEEDED=$((NUM_REQUESTS + WARMUP)) -if [[ $TOTAL_DIDS -lt $TOTAL_NEEDED ]]; then - echo -e "${YELLOW} Warning: Only $TOTAL_DIDS DIDs available${NC}" - if [[ $TOTAL_DIDS -le $WARMUP ]]; then - WARMUP=0 - NUM_REQUESTS=$TOTAL_DIDS - else - NUM_REQUESTS=$((TOTAL_DIDS - WARMUP)) - fi - TOTAL_NEEDED=$((NUM_REQUESTS + WARMUP)) - echo -e "${YELLOW} Adjusted: $WARMUP warmup + $NUM_REQUESTS benchmark${NC}" - echo "" -fi - -# Select random DIDs -RANDOM_DIDS="$TEMP_DIR/random_dids.txt" -shuf -n $TOTAL_NEEDED "$DID_LIST" > "$RANDOM_DIDS" - -# URL builder -get_url() { - local did=$1 - case $ROUTE in - data) echo "$ENDPOINT/$did/data" ;; - audit) echo "$ENDPOINT/$did/log/audit" ;; - *) echo "$ENDPOINT/$did" ;; - esac -} - -# Warmup -if [[ $WARMUP -gt 0 ]]; then - echo -e "${CYAN}━━━ Phase 2: Warmup ━━━${NC}" - echo "" - echo " Running $WARMUP warmup requests..." - - head -n $WARMUP "$RANDOM_DIDS" | while read did; do - curl -s -o /dev/null "$(get_url "$did")" 2>/dev/null || true - done - - echo -e "${GREEN} ✓ Complete${NC}" - echo "" - - tail -n +$((WARMUP + 1)) "$RANDOM_DIDS" > "$TEMP_DIR/bench.txt" - RANDOM_DIDS="$TEMP_DIR/bench.txt" -fi - -# Benchmark -PHASE_NUM=2 -if [[ $WARMUP -gt 0 ]]; then - PHASE_NUM=3 -fi - -echo -e "${CYAN}━━━ Phase $PHASE_NUM: Benchmark ━━━${NC}" -echo "" - -# CSV header -echo "did,status,time_ms,size" > "$RESULTS_FILE" - -# Benchmark function -benchmark_one() { - local did=$1 - local url=$(get_url "$did") - - local start=$(date +%s%N) - local output=$(curl -s -w "\n%{http_code}|%{size_download}" -o /dev/null "$url" 2>&1) - local end=$(date +%s%N) - - local status=$(echo "$output" | tail -1 | cut -d'|' -f1) - local size=$(echo "$output" | tail -1 | cut -d'|' -f2) - local ms=$(( (end - start) / 1000000 )) - - # Print URL if 500 error - if [[ "$status" == "500" ]]; then - echo -e "${RED}✗ 500 Error - URL: $url${NC}" >&2 - fi - - echo "$did,$status,$ms,$size" -} - -export -f benchmark_one -export -f get_url -export ENDPOINT ROUTE RED NC - -# Execute -echo " Running $NUM_REQUESTS requests (concurrency: $CONCURRENCY)..." - -BENCH_START=$(date +%s%N) - -if [[ $CONCURRENCY -eq 1 ]]; then - count=0 - while IFS= read -r did; do - benchmark_one "$did" >> "$RESULTS_FILE" - count=$((count + 1)) - if (( count % 10 == 0 )); then - printf "\r Progress: %d/%d (%d%%) " $count $NUM_REQUESTS $((count * 100 / NUM_REQUESTS)) - fi - done < "$RANDOM_DIDS" - echo "" -else - cat "$RANDOM_DIDS" | xargs -P $CONCURRENCY -I {} bash -c 'benchmark_one "$@"' _ {} >> "$RESULTS_FILE" 2>&1 - echo -e "${GREEN} ✓ Complete${NC}" -fi - -BENCH_END=$(date +%s%N) -BENCH_SEC=$(awk "BEGIN {printf \"%.3f\", ($BENCH_END - $BENCH_START) / 1000000000}") - -echo "" - -# Results -echo -e "${BLUE}════════════════════════════════════════════════════════════════${NC}" -echo -e "${BLUE} Results${NC}" -echo -e "${BLUE}════════════════════════════════════════════════════════════════${NC}" -echo "" - -# Status codes -echo "HTTP Status Codes:" -echo "──────────────────" - -tail -n +2 "$RESULTS_FILE" | cut -d',' -f2 | sort | uniq -c | sort -rn | while read count code; do - case $code in - 200) echo -e " ${GREEN}✓ 200${NC}: $count" ;; - 404) echo -e " ${YELLOW}⚠ 404${NC}: $count" ;; - 410) echo -e " ${YELLOW}⚠ 410${NC}: $count (deactivated)" ;; - 500) echo -e " ${RED}✗ 500${NC}: $count" ;; - *) echo " ? $code: $count" ;; - esac -done - -echo "" - -# Successful requests only -SUCCESS_FILE="$TEMP_DIR/success.csv" -tail -n +2 "$RESULTS_FILE" | awk -F',' '$2 == "200"' > "$SUCCESS_FILE" || true -SUCCESS_COUNT=$(wc -l < "$SUCCESS_FILE" | tr -d ' ') - -if [[ $SUCCESS_COUNT -gt 0 ]]; then - echo "Response Times:" - echo "───────────────" - - # Extract and sort times - TIMES="$TEMP_DIR/times.txt" - cut -d',' -f3 "$SUCCESS_FILE" | sort -n > "$TIMES" - - # Stats - MIN=$(head -1 "$TIMES") - MAX=$(tail -1 "$TIMES") - MEAN=$(awk '{sum+=$1} END {printf "%.1f", sum/NR}' "$TIMES") - - # Percentiles (safe calculation) - calc_line() { - local pct=$1 - local line=$(awk "BEGIN {print int($SUCCESS_COUNT * $pct)}") - [[ $line -lt 1 ]] && line=1 - echo $line - } - - P50=$(sed -n "$(calc_line 0.50)p" "$TIMES") - P90=$(sed -n "$(calc_line 0.90)p" "$TIMES") - P95=$(sed -n "$(calc_line 0.95)p" "$TIMES") - P99=$(sed -n "$(calc_line 0.99)p" "$TIMES") - - echo " Count: $SUCCESS_COUNT" - echo " Min: ${MIN} ms" - echo " Mean: ${MEAN} ms" - echo " p50: ${P50} ms" - echo " p90: ${P90} ms" - echo " p95: ${P95} ms" - echo " p99: ${P99} ms" - echo " Max: ${MAX} ms" - echo "" - - echo "Throughput:" - echo "───────────" - RPS=$(awk "BEGIN {printf \"%.2f\", $SUCCESS_COUNT / $BENCH_SEC}") - echo " Duration: ${BENCH_SEC}s" - echo " Requests/sec: $RPS" - echo "" - - # Distribution - echo "Distribution:" - echo "─────────────" - - ranges=( - "0:5:<5ms" - "5:10:5-10ms" - "10:20:10-20ms" - "20:50:20-50ms" - "50:100:50-100ms" - "100:500:100-500ms" - "500:999999:>500ms" - ) - - for range in "${ranges[@]}"; do - IFS=':' read -r min max label <<< "$range" - count=$(awk -F',' -v min=$min -v max=$max '$3 >= min && $3 < max {c++} END {print c+0}' "$SUCCESS_FILE") - - if [[ $count -gt 0 ]]; then - pct=$(awk "BEGIN {printf \"%.1f\", $count/$SUCCESS_COUNT*100}") - bar_len=$(awk "BEGIN {printf \"%.0f\", $count/$SUCCESS_COUNT*30}") - bar=$(printf '█%.0s' $(seq 1 $bar_len)) - printf " %-10s [%-30s] %4d (%5.1f%%)\n" "$label" "$bar" $count $pct - fi - done - -else - echo -e "${RED}No successful requests!${NC}" - echo "" - echo "Debugging info:" - echo " Test connection: curl -v $ENDPOINT/status" - echo " Check results: cat $RESULTS_FILE" -fi - -echo "" -echo -e "${BLUE}════════════════════════════════════════════════════════════════${NC}" - -# Save output -if [[ -n "$OUTPUT_CSV" ]]; then - cp "$RESULTS_FILE" "$OUTPUT_CSV" - echo -e "${GREEN}✓ Saved to: $OUTPUT_CSV${NC}" - echo "" -fi - -# Verbose extras -if [[ $VERBOSE -eq 1 ]] && [[ $SUCCESS_COUNT -gt 0 ]]; then - echo "" - echo "Top 5 slowest:" - tail -n +2 "$RESULTS_FILE" | awk -F',' '$2=="200"' | sort -t',' -k3 -rn | head -5 | \ - awk -F',' '{printf " %5d ms - %s\n", $3, $1}' - echo "" - echo "Top 5 fastest:" - tail -n +2 "$RESULTS_FILE" | awk -F',' '$2=="200"' | sort -t',' -k3 -n | head -5 | \ - awk -F',' '{printf " %5d ms - %s\n", $3, $1}' -fi - -echo -e "${BLUE}════════════════════════════════════════════════════════════════${NC}" - -# Exit code -if [[ ${SUCCESS_COUNT:-0} -eq 0 ]]; then - exit 1 -fi diff --git a/scripts/did-resolve-benchmark-http.sh b/scripts/did-resolve-benchmark-http.sh deleted file mode 100755 index 88b889e..0000000 --- a/scripts/did-resolve-benchmark-http.sh +++ /dev/null @@ -1,169 +0,0 @@ -#!/bin/bash -# did-resolve-benchmark-http.sh - Benchmark DID resolution over HTTP - -set -e - -SERVER_URL=${1:-"http://localhost:8080"} -BUNDLE=${2:-1} -SAMPLES=${3:-20} - -echo "═══════════════════════════════════════════════════════════" -echo " DID Resolution HTTP Performance Benchmark" -echo "═══════════════════════════════════════════════════════════" -echo "" -echo "Server: $SERVER_URL" -echo "Bundle: $BUNDLE" -echo "Samples per position range: $SAMPLES" -echo "" - -# Check if server is responding -if ! curl -s -f "$SERVER_URL/status" > /dev/null 2>&1; then - echo "❌ Error: Server not responding at $SERVER_URL" - echo "" - echo "Start server first:" - echo " plcbundle serve --port 8080" - exit 1 -fi - -echo "✓ Server is online" -echo "" - -# Extract DIDs at different positions from local bundle -echo "Extracting test DIDs from bundle $BUNDLE..." - -# Early positions (0-100) -EARLY_DIDS=$(plcbundle export --bundles $BUNDLE | head -100 | jq -r '.did' | head -$SAMPLES) - -# Middle positions (~5000) -MIDDLE_DIDS=$(plcbundle export --bundles $BUNDLE | head -5100 | tail -100 | jq -r '.did' | head -$SAMPLES) - -# Late positions (~9900) -LATE_DIDS=$(plcbundle export --bundles $BUNDLE | tail -100 | jq -r '.did' | head -$SAMPLES) - -echo "✓ Extracted test DIDs" -echo "" - -# Function to benchmark HTTP resolution -benchmark_http() { - local label="$1" - local dids="$2" - - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - echo "Testing: $label" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - - local total_time=0 - local total_dns=0 - local total_connect=0 - local total_starttransfer=0 - local count=0 - local min_time=999999 - local max_time=0 - local errors=0 - - # Arrays to store individual timings - local -a total_timings - local -a transfer_timings - - for did in $dids; do - # Use curl with timing - timing=$(curl -s -w "\n%{time_namelookup},%{time_connect},%{time_starttransfer},%{time_total},%{http_code}" \ - -o /dev/null \ - "$SERVER_URL/$did" 2>&1) - - # Parse timing (last line) - IFS=',' read -r dns connect starttransfer total http_code <<< "$(echo "$timing" | tail -1)" - - if [ "$http_code" == "200" ]; then - # Convert to milliseconds - total_ms=$(echo "scale=2; $total * 1000" | bc) - transfer_ms=$(echo "scale=2; $starttransfer * 1000" | bc) - dns_ms=$(echo "scale=2; $dns * 1000" | bc) - connect_ms=$(echo "scale=2; $connect * 1000" | bc) - - total_timings+=($total_ms) - transfer_timings+=($transfer_ms) - - total_time=$(echo "$total_time + $total_ms" | bc) - total_dns=$(echo "$total_dns + $dns_ms" | bc) - total_connect=$(echo "$total_connect + $connect_ms" | bc) - total_starttransfer=$(echo "$total_starttransfer + $transfer_ms" | bc) - count=$((count + 1)) - - # Update min/max (total time) - if (( $(echo "$total_ms < $min_time" | bc -l) )); then - min_time=$total_ms - fi - if (( $(echo "$total_ms > $max_time" | bc -l) )); then - max_time=$total_ms - fi - - printf "." - else - printf "E" - errors=$((errors + 1)) - fi - done - - echo "" - - if [ $count -gt 0 ]; then - avg_total=$(echo "scale=2; $total_time / $count" | bc) - avg_dns=$(echo "scale=2; $total_dns / $count" | bc) - avg_connect=$(echo "scale=2; $total_connect / $count" | bc) - avg_transfer=$(echo "scale=2; $total_starttransfer / $count" | bc) - - # Calculate median total time - IFS=$'\n' sorted=($(sort -n <<<"${total_timings[*]}")) - median_idx=$((count / 2)) - median_total=${sorted[$median_idx]} - - # Calculate median transfer time - IFS=$'\n' sorted_transfer=($(sort -n <<<"${transfer_timings[*]}")) - median_transfer=${sorted_transfer[$median_idx]} - - echo "" - echo "Results ($count successful, $errors errors):" - echo "" - echo " Total Response Time:" - echo " Average: ${avg_total}ms" - echo " Median: ${median_total}ms" - echo " Min: ${min_time}ms" - echo " Max: ${max_time}ms" - echo "" - echo " Breakdown (average):" - echo " DNS lookup: ${avg_dns}ms" - echo " TCP connect: ${avg_connect}ms" - echo " Time to first byte: ${avg_transfer}ms" - echo " Transfer: $(echo "scale=2; $avg_total - $avg_transfer" | bc)ms" - else - echo " ❌ All requests failed" - fi - - echo "" -} - -# Run HTTP benchmarks -benchmark_http "Early Positions (0-100)" "$EARLY_DIDS" -benchmark_http "Middle Positions (~5000)" "$MIDDLE_DIDS" -benchmark_http "Late Positions (~9900)" "$LATE_DIDS" - -echo "═══════════════════════════════════════════════════════════" -echo " HTTP Benchmark Complete" -echo "═══════════════════════════════════════════════════════════" -echo "" -echo "Note: HTTP adds overhead (~2-5ms) compared to local resolution" -echo "The 'Time to first byte' shows server processing time" -echo "" - -# Bonus: Test server status endpoint -echo "Server Status:" -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -curl -s "$SERVER_URL/status" | jq -r ' - " Bundles: \(.bundles.count)", - " Operations: \(.bundles.total_operations // 0)", - " Uptime: \(.server.uptime_seconds)s", - " WebSocket: \(.server.websocket_enabled)", - if .mempool then " Mempool: \(.mempool.count) ops" else " Mempool: disabled" end -' -echo "" diff --git a/scripts/did-resolve-benchmark.sh b/scripts/did-resolve-benchmark.sh deleted file mode 100755 index ed6b23e..0000000 --- a/scripts/did-resolve-benchmark.sh +++ /dev/null @@ -1,123 +0,0 @@ -#!/bin/bash -# did-resolve-benchmar.sh - Benchmark DID resolution performance - -set -e - -BUNDLE=${1:-1} -SAMPLES=${2:-20} - -echo "═══════════════════════════════════════════════════════════" -echo " DID Resolution Performance Benchmark" -echo "═══════════════════════════════════════════════════════════" -echo "" -echo "Bundle: $BUNDLE" -echo "Samples per position range: $SAMPLES" -echo "" - -# Extract DIDs at different positions from a bundle -echo "Extracting test DIDs from bundle $BUNDLE..." - -# Early positions (0-100) -EARLY_DIDS=$(plcbundle export --bundles $BUNDLE | head -100 | jq -r '.did' | head -$SAMPLES) - -# Middle positions (~5000) -MIDDLE_DIDS=$(plcbundle export --bundles $BUNDLE | head -5100 | tail -100 | jq -r '.did' | head -$SAMPLES) - -# Late positions (~9900) -LATE_DIDS=$(plcbundle export --bundles $BUNDLE | tail -100 | jq -r '.did' | head -$SAMPLES) - -echo "✓ Extracted test DIDs" -echo "" - -# Function to benchmark a set of DIDs -benchmark_dids() { - local label="$1" - local dids="$2" - - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - echo "Testing: $label" - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - - local total_time=0 - local count=0 - local min_time=999999 - local max_time=0 - - # Arrays to store individual timings - local -a timings - - for did in $dids; do - # Run resolve and extract timing - output=$(plcbundle index resolve "$did" 2>&1) - - # Extract "Total: XXms" from stderr - if [[ $output =~ Total:\ ([0-9.]+)([µm]?s) ]]; then - time_value="${BASH_REMATCH[1]}" - time_unit="${BASH_REMATCH[2]}" - - # Convert to milliseconds - if [[ $time_unit == "µs" ]]; then - time_ms=$(echo "scale=3; $time_value / 1000" | bc) - elif [[ $time_unit == "ms" ]]; then - time_ms=$time_value - else - # Assume seconds - time_ms=$(echo "scale=3; $time_value * 1000" | bc) - fi - - timings+=($time_ms) - total_time=$(echo "$total_time + $time_ms" | bc) - count=$((count + 1)) - - # Update min/max - if (( $(echo "$time_ms < $min_time" | bc -l) )); then - min_time=$time_ms - fi - if (( $(echo "$time_ms > $max_time" | bc -l) )); then - max_time=$time_ms - fi - - printf "." - fi - done - - echo "" - - if [ $count -gt 0 ]; then - avg_time=$(echo "scale=2; $total_time / $count" | bc) - - # Calculate median (sort and take middle) - IFS=$'\n' sorted=($(sort -n <<<"${timings[*]}")) - median_idx=$((count / 2)) - median_time=${sorted[$median_idx]} - - echo "" - echo "Results ($count samples):" - echo " Average: ${avg_time}ms" - echo " Median: ${median_time}ms" - echo " Min: ${min_time}ms" - echo " Max: ${max_time}ms" - else - echo " No successful timings" - fi - - echo "" -} - -# Run benchmarks -benchmark_dids "Early Positions (0-100)" "$EARLY_DIDS" -benchmark_dids "Middle Positions (~5000)" "$MIDDLE_DIDS" -benchmark_dids "Late Positions (~9900)" "$LATE_DIDS" - -echo "═══════════════════════════════════════════════════════════" -echo " Benchmark Complete" -echo "═══════════════════════════════════════════════════════════" -echo "" -echo "Expected results with LoadOperation optimization:" -echo " Early: ~2-5ms (only decompress first 1%)" -echo " Middle: ~10-15ms (decompress ~50%)" -echo " Late: ~20-30ms (decompress ~99%)" -echo "" -echo "If all positions show similar timing (~18ms), the optimization" -echo "isn't working and LoadBundle is still being called." -echo "" diff --git a/scripts/get-random-dids.sh b/scripts/get-random-dids.sh deleted file mode 100755 index a9b01b2..0000000 --- a/scripts/get-random-dids.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -# get-random-dids.sh - Extract N DIDs from random positions - -N=${1:-10} - -# Get bundle range -read FIRST LAST < <(plcbundle info 2>/dev/null | awk '/Range:/ {print $2, $4}' | tr -d '→') - -echo "Sampling $N DIDs from bundles $FIRST-$LAST" >&2 -echo "" >&2 - -for i in $(seq 1 $N); do - BUNDLE=$(jot -r 1 $FIRST $LAST) - POS=$(jot -r 1 0 9999) - - echo "[$i/$N] Bundle $BUNDLE, position $POS" >&2 - - plcbundle get-op $BUNDLE $POS 2>/dev/null | jq -r '.did' -done | jq -R . | jq -s . - -echo "" >&2 -echo "✓ Done" >&2 diff --git a/scripts/release.sh b/scripts/release.sh deleted file mode 100755 index e042cf9..0000000 --- a/scripts/release.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -set -e - -# Colors -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -NC='\033[0m' - -# Get latest tag -LATEST_TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "") - -if [[ -z "$LATEST_TAG" ]]; then - echo -e "${RED}Error: No tags found${NC}" - echo "Create a tag first using: make bump-patch|bump-minor|bump-major" - exit 1 -fi - -echo -e "${GREEN}Releasing version: ${LATEST_TAG}${NC}" -echo "" - -# Check if tag is already pushed -if git ls-remote --tags origin | grep -q "${LATEST_TAG}"; then - echo -e "${YELLOW}Warning: Tag ${LATEST_TAG} already exists on remote${NC}" - read -p "Continue anyway? (y/N) " -n 1 -r - echo - if [[ ! $REPLY =~ ^[Yy]$ ]]; then - exit 0 - fi -fi - -# Push tag -echo "Pushing tag ${LATEST_TAG}..." -git push origin "${LATEST_TAG}" - -# Also push main/master branch -BRANCH=$(git rev-parse --abbrev-ref HEAD) -echo "Pushing branch ${BRANCH}..." -git push origin "${BRANCH}" - -echo "" -echo -e "${GREEN}✓ Release ${LATEST_TAG} published${NC}" -echo "" -echo "View releases at:" -echo " https://tangled.org/@atscan.net/plcbundle/tags" diff --git a/scripts/resolve-benchmark-simple.sh b/scripts/resolve-benchmark-simple.sh deleted file mode 100755 index 2f202bc..0000000 --- a/scripts/resolve-benchmark-simple.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env bash - -ENDPOINT="${1:-http://localhost:8080}" -NUM="${2:-100}" -DIR="${3:-.}" - -echo "=== DID Resolver Benchmark ===" -echo "Endpoint: $ENDPOINT" -echo "Requests: $NUM" -echo "" - -# Extract DIDs -echo "Extracting DIDs..." -TEMP=$(mktemp) -trap "rm -f $TEMP ${TEMP}.*" EXIT - -find "$DIR" -name "*.jsonl.zst" | head -5 | while read f; do - zstd -dc "$f" | jq -r '.did' -done | sort -u | shuf -n $NUM > "$TEMP" - -TOTAL=$(wc -l < "$TEMP" | tr -d ' ') -echo "Testing $TOTAL DIDs" -echo "" - -# Run benchmark -echo "Benchmarking..." -SUCCESS=0 -FAILED=0 - -while read did; do - START=$(date +%s%N) - CODE=$(curl -s -o /dev/null -w "%{http_code}" "$ENDPOINT/$did" 2>/dev/null) - END=$(date +%s%N) - - MS=$(( (END - START) / 1000000 )) - - if [[ $CODE == "200" ]]; then - SUCCESS=$((SUCCESS + 1)) - echo "$MS" >> "${TEMP}.times" - else - FAILED=$((FAILED + 1)) - fi - - DONE=$((SUCCESS + FAILED)) - if (( DONE % 10 == 0 )); then - printf "\r %d/%d " $DONE $TOTAL - fi -done < "$TEMP" - -echo "" -echo "" -echo "Results:" -echo " Success: $SUCCESS" -echo " Failed: $FAILED" - -if [[ $SUCCESS -gt 0 ]]; then - sort -n "${TEMP}.times" -o "${TEMP}.times" - - MIN=$(head -1 "${TEMP}.times") - MAX=$(tail -1 "${TEMP}.times") - AVG=$(awk '{s+=$1} END {printf "%.0f", s/NR}' "${TEMP}.times") - - # Percentiles - P50=$(sed -n "$(awk "BEGIN {print int($SUCCESS * 0.50)}")p" "${TEMP}.times") - P90=$(sed -n "$(awk "BEGIN {print int($SUCCESS * 0.90)}")p" "${TEMP}.times") - P95=$(sed -n "$(awk "BEGIN {print int($SUCCESS * 0.95)}")p" "${TEMP}.times") - - echo "" - echo " Min: ${MIN}ms" - echo " Avg: ${AVG}ms" - echo " p50: ${P50}ms" - echo " p90: ${P90}ms" - echo " p95: ${P95}ms" - echo " Max: ${MAX}ms" -fi \ No newline at end of file diff --git a/scripts/test-did-resolution.sh b/scripts/test-did-resolution.sh deleted file mode 100755 index 76ebc84..0000000 --- a/scripts/test-did-resolution.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/bin/bash -# test-did-resolution.sh -# Compares local DID resolution with plc.directory - -set -e - -BUNDLE_RANGE="${1:-1}" -PLC_URL="https://plc.wtf" -TEMP_DIR=$(mktemp -d) -trap "rm -rf $TEMP_DIR" EXIT - -# Colors -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -NC='\033[0m' # No Color - -echo "Testing DID resolution for bundle(s): $BUNDLE_RANGE" -echo "Temp directory: $TEMP_DIR" -echo "" - -# Extract unique DIDs from bundle(s) -echo "Extracting DIDs from bundles..." -DIDS=$(plcbundle export --bundles "$BUNDLE_RANGE" | jq -r '.did' | sort -u) -DID_COUNT=$(echo "$DIDS" | wc -l | tr -d ' ') - -echo "Found $DID_COUNT unique DIDs" -echo "" - -# Test counters -TOTAL=0 -PASSED=0 -FAILED=0 -ERRORS=0 - -# Test each DID -while IFS= read -r DID; do - TOTAL=$((TOTAL + 1)) - - # Progress - echo -ne "\r[$TOTAL/$DID_COUNT] Testing $DID..." - - # Resolve locally - LOCAL_FILE="$TEMP_DIR/local.json" - if ! plcbundle index resolve "$DID" 2>/dev/null | jq --sort-keys . > "$LOCAL_FILE" 2>/dev/null; then - echo -e "\r[$TOTAL/$DID_COUNT] ${RED}ERROR${NC} $DID (local resolution failed)" - ERRORS=$((ERRORS + 1)) - continue - fi - - # Resolve remotely - REMOTE_FILE="$TEMP_DIR/remote.json" - if ! curl -s "$PLC_URL/$DID" | jq --sort-keys . > "$REMOTE_FILE" 2>/dev/null; then - echo -e "\r[$TOTAL/$DID_COUNT] ${YELLOW}SKIP${NC} $DID (remote fetch failed)" - ERRORS=$((ERRORS + 1)) - continue - fi - - # Compare - if diff -q "$LOCAL_FILE" "$REMOTE_FILE" > /dev/null 2>&1; then - echo -e "\r[$TOTAL/$DID_COUNT] ${GREEN}✓${NC} $DID" - PASSED=$((PASSED + 1)) - else - echo -e "\r[$TOTAL/$DID_COUNT] ${RED}✗${NC} $DID" - echo "" - echo "Differences found:" - diff -u "$REMOTE_FILE" "$LOCAL_FILE" | head -20 - echo "" - FAILED=$((FAILED + 1)) - - # Ask to continue - if [ $FAILED -ge 3 ]; then - echo "" - read -p "Continue testing? (y/N) " -n 1 -r - echo - if [[ ! $REPLY =~ ^[Yy]$ ]]; then - break - fi - fi - fi - - # Rate limit - sleep 0.1 - -done <<< "$DIDS" - -# Summary -echo "" -echo "=========================================" -echo "Test Results" -echo "=========================================" -echo -e "Total: $TOTAL" -echo -e "${GREEN}Passed: $PASSED${NC}" -echo -e "${RED}Failed: $FAILED${NC}" -echo -e "${YELLOW}Errors: $ERRORS${NC}" -echo "" - -if [ $FAILED -eq 0 ] && [ $ERRORS -eq 0 ]; then - echo -e "${GREEN}✓ All tests passed!${NC}" - exit 0 -else - echo -e "${RED}✗ Some tests failed${NC}" - exit 1 -fi diff --git a/server/handlers.go b/server/handlers.go deleted file mode 100644 index 10489b3..0000000 --- a/server/handlers.go +++ /dev/null @@ -1,1093 +0,0 @@ -package server - -import ( - "context" - "fmt" - "io" - "net/http" - "runtime" - "strconv" - "strings" - "time" - - "github.com/goccy/go-json" - "tangled.org/atscan.net/plcbundle/internal/plcclient" - "tangled.org/atscan.net/plcbundle/internal/types" -) - -func (s *Server) handleRoot() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "text/plain; charset=utf-8") - - index := s.manager.GetIndex() - stats := index.GetStats() - bundleCount := stats["bundle_count"].(int) - - baseURL := getBaseURL(r) - wsURL := getWSURL(r) - - var sb strings.Builder - - sb.WriteString(` - - ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠄⠀⡀⠀⠀⠀⠀⠀⠀⢀⠀⠀⡀⠀⢀⠀⢀⡀⣤⡢⣤⡤⡀⡄⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ -⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⡄⡄⠐⡀⠈⣀⠀⡠⡠⠀⣢⣆⢌⡾⢙⠺⢽⠾⡋⣻⡷⡫⢵⣭⢦⣴⠦⠀⢠⠀⠀⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ -⠀⠀⠀⠀⠀⠀⠀⠀⢠⣤⣽⣥⡈⠧⣂⢧⢾⠕⠞⠡⠊⠁⣐⠉⠀⠉⢍⠀⠉⠌⡉⠀⠂⠁⠱⠉⠁⢝⠻⠎⣬⢌⡌⣬⣡⣀⣢⣄⡄⠀⡀⠀⠀⠀⠀⠀⠀⠀⠀ -⠀⠀⠀⠀⠀⠀⠀⢀⢸⣿⣿⢿⣾⣯⣑⢄⡂⠀⠄⠂⠀⠀⢀⠀⠀⠐⠀⠀⠀⠀⠀⠀⠀⠀⠄⠐⠀⠀⠀⠀⣄⠭⠂⠈⠜⣩⣿⢝⠃⠀⠁⠀⠀⠀⠀⠀⠀⠀⠀ -⠀⠀⠀⠀⠀⠀⠀⢀⣻⡟⠏⠀⠚⠈⠚⡉⡝⢶⣱⢤⣅⠈⠀⠄⠀⠀⠀⠀⠀⠠⠀⠀⡂⠐⣤⢕⡪⢼⣈⡹⡇⠏⠏⠋⠅⢃⣪⡏⡇⡍⠀⠀⠀⠀⠀⠀⠀⠀⠀ -⠀⠀⠀⠀⠀⠀⠀⠀⠺⣻⡄⠀⠀⠀⢠⠌⠃⠐⠉⢡⠱⠧⠝⡯⣮⢶⣴⣤⡆⢐⣣⢅⣮⡟⠦⠍⠉⠀⠁⠐⠀⠀⠀⠄⠐⠡⣽⡸⣎⢁⠀⠀⠀⠀⠀⠀⠀⠀⠀ -⠀⠀⠀⠀⠀⠀⠀⢈⡻⣧⠀⠁⠐⠀⠀⠀⠀⠀⠀⠊⠀⠕⢀⡉⠈⡫⠽⡿⡟⠿⠟⠁⠀⠀⠄⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⠬⠥⣋⡯⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ -⠀⠀⠀⠀⠀⠀⠀⡀⣾⡍⠕⡀⠀⠀⠀⠄⠠⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠥⣤⢌⠀⠀⠀⠀⠀⠀⠀⠀⠀⠁⠀⠀⠄⢀⠀⢝⢞⣫⡆⡄⠀⠀⠀⠀⠀⠀⠀⠀⠀ -⠀⠀⠀⠀⠀⠀⠀⠀⣽⡶⡄⠐⡀⠀⠀⠀⠀⠀⠀⢀⠀⠄⠀⠀⠀⠄⠁⠇⣷⡆⠀⠀⠀⢀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⡸⢝⣮⠍⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ -⠀⠀⠀⠀⠀⠀⢀⠀⢾⣷⠀⠠⡀⠀⠀⠀⠀⢀⠀⠀⠀⠀⠀⠁⡁⠀⠀⣾⡥⠖⠀⠀⠀⠂⠀⠀⠀⠀⠀⠁⠀⡀⠁⠀⠀⠻⢳⣻⢄⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ -⠀⠀⠀⠀⠀⠀⠀⠀⣞⡙⠨⣀⠠⠄⠀⠂⠀⠀⠀⠈⢀⠀⠀⠀⠀⠀⠤⢚⢢⣟⠀⠀⠀⠀⡐⠀⠀⡀⠀⠀⠀⠀⠁⠈⠌⠊⣯⣮⡏⠡⠂⠀⠀⠀⠀⠀⠀⠀⠀ -⠀⠀⠀⠀⠀⠀⠀⠀⣻⡟⡄⡡⣄⠀⠠⠀⠀⡅⠀⠐⠀⡀⠀⡀⠀⠄⠈⠃⠳⠪⠤⠀⠀⠀⠀⡀⠀⠂⠀⠀⠀⠁⠈⢠⣠⠒⠻⣻⡧⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ -⠀⠀⠀⠀⠀⠀⠀⠀⠪⡎⠠⢌⠑⡀⠂⠀⠄⠠⠀⠠⠀⠁⡀⠠⠠⡀⣀⠜⢏⡅⠀⠀⡀⠁⠀⠀⠁⠁⠐⠄⡀⢀⠂⠀⠄⢑⣿⣿⣿⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀ -⠀⠀⠀⠀⠀⠀⠀⠀⠼⣻⠧⣣⣀⠐⠨⠁⠕⢈⢀⢀⡁⠀⠈⠠⢀⠀⠐⠜⣽⡗⡤⠀⠂⠀⠠⠀⢂⠠⠀⠁⠄⠀⠔⠀⠑⣨⣿⢯⠋⡅⠀⠀⠀⠀⠀⠀⠀⠀⠀ -⠀⠀⠀⠀⠀⠀⠀⠀⡚⣷⣭⠎⢃⡗⠄⡄⢀⠁⠀⠅⢀⢅⡀⠠⠀⢠⡀⡩⠷⢇⠀⡀⠄⡠⠤⠆⣀⡀⠄⠉⣠⠃⠴⠀⠈⢁⣿⡛⡯⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ -⠀⠀⠀⠀⠀⠀⠀⠘⡬⡿⣿⡏⡻⡯⠌⢁⢛⠠⠓⠐⠐⠐⠌⠃⠋⠂⡢⢰⣈⢏⣰⠂⠈⠀⠠⠒⠡⠌⠫⠭⠩⠢⡬⠆⠿⢷⢿⡽⡧⠉⠊⠀⠀⠀⠀⠀⠀⠀⠀ -⠀⠀⠀⠀⠀⠀⠀⠀⠺⣷⣺⣗⣿⡶⡎⡅⣣⢎⠠⡅⣢⡖⠴⠬⡈⠂⡨⢡⠾⣣⣢⠀⠀⡹⠄⡄⠄⡇⣰⡖⡊⠔⢹⣄⣿⣭⣵⣿⢷⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀ -⠀⠀⠀⠀⠀⠀⠀⠀⠩⣿⣿⣲⣿⣷⣟⣼⠟⣬⢉⡠⣪⢜⣂⣁⠥⠓⠚⡁⢶⣷⣠⠂⡄⡢⣀⡐⠧⢆⣒⡲⡳⡫⢟⡃⢪⡧⣟⡟⣯⠐⠀⠀⠀⠀⠀⠀⠀⠀⠀ -⠀⠀⠀⠀⠀⠀⠀⠀⢺⠟⢿⢟⢻⡗⡮⡿⣲⢷⣆⣏⣇⡧⣄⢖⠾⡷⣿⣤⢳⢷⣣⣦⡜⠗⣭⢂⠩⣹⢿⡲⢎⡧⣕⣖⣓⣽⡿⡖⡿⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ -⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⠉⠂⠂⠏⠿⢻⣥⡪⢽⣳⣳⣥⡶⣫⣍⢐⣥⣻⣾⡻⣅⢭⡴⢭⣿⠕⣧⡭⣞⣻⣣⣻⢿⠟⠛⠙⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ -⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠄⠋⠫⠯⣍⢻⣿⣿⣷⣕⣵⣹⣽⣿⣷⣇⡏⣿⡿⣍⡝⠵⠯⠁⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ -⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠐⠠⠁⠋⢣⠓⡍⣫⠹⣿⣿⣷⡿⠯⠺⠁⠁⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ -⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠋⢀⠋⢈⡿⠿⠁⠉⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ - - plcbundle server - -*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~* -| ⚠️ Preview Version – Do Not Use In Production! | -*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~* -| This project and plcbundle specification is currently | -| unstable and under heavy development. Things can break at | -| any time. Do not use this for production systems. | -| Please wait for the 1.0 release. | -|________________________________________________________________| - -`) - - sb.WriteString("\nplcbundle server\n\n") - sb.WriteString("What is PLC Bundle?\n") - sb.WriteString("━━━━━━━━━━━━━━━━━━━━\n") - sb.WriteString("plcbundle archives AT Protocol's DID PLC Directory operations into\n") - sb.WriteString("immutable, cryptographically-chained bundles of 10,000 operations.\n\n") - sb.WriteString("More info: https://tangled.org/@atscan.net/plcbundle\n\n") - - origin := s.manager.GetPLCOrigin() - - if bundleCount > 0 { - sb.WriteString("Bundles\n") - sb.WriteString("━━━━━━━\n") - sb.WriteString(fmt.Sprintf(" Origin: %s\n", origin)) - sb.WriteString(fmt.Sprintf(" Bundle count: %d\n", bundleCount)) - - firstBundle := stats["first_bundle"].(int) - lastBundle := stats["last_bundle"].(int) - totalSize := stats["total_size"].(int64) - totalUncompressed := stats["total_uncompressed_size"].(int64) - - sb.WriteString(fmt.Sprintf(" Last bundle: %d (%s)\n", lastBundle, - stats["updated_at"].(time.Time).Format("2006-01-02 15:04:05"))) - sb.WriteString(fmt.Sprintf(" Range: %06d - %06d\n", firstBundle, lastBundle)) - sb.WriteString(fmt.Sprintf(" Total size: %.2f MB\n", float64(totalSize)/(1000*1000))) - sb.WriteString(fmt.Sprintf(" Uncompressed: %.2f MB (%.2fx)\n", - float64(totalUncompressed)/(1000*1000), - float64(totalUncompressed)/float64(totalSize))) - - if gaps, ok := stats["gaps"].(int); ok && gaps > 0 { - sb.WriteString(fmt.Sprintf(" ⚠ Gaps: %d missing bundles\n", gaps)) - } - - firstMeta, err := index.GetBundle(firstBundle) - if err == nil { - sb.WriteString(fmt.Sprintf("\n Root: %s\n", firstMeta.Hash)) - } - - lastMeta, err := index.GetBundle(lastBundle) - if err == nil { - sb.WriteString(fmt.Sprintf(" Head: %s\n", lastMeta.Hash)) - } - } - - if s.config.SyncMode { - mempoolStats := s.manager.GetMempoolStats() - count := mempoolStats["count"].(int) - targetBundle := mempoolStats["target_bundle"].(int) - - sb.WriteString("\nMempool\n") - sb.WriteString("━━━━━━━\n") - sb.WriteString(fmt.Sprintf(" Target bundle: %d\n", targetBundle)) - sb.WriteString(fmt.Sprintf(" Operations: %d / %d\n", count, types.BUNDLE_SIZE)) - - if count > 0 { - progress := float64(count) / float64(types.BUNDLE_SIZE) * 100 - sb.WriteString(fmt.Sprintf(" Progress: %.1f%%\n", progress)) - - barWidth := 50 - filled := int(float64(barWidth) * float64(count) / float64(types.BUNDLE_SIZE)) - if filled > barWidth { - filled = barWidth - } - bar := strings.Repeat("█", filled) + strings.Repeat("░", barWidth-filled) - sb.WriteString(fmt.Sprintf(" [%s]\n", bar)) - - if firstTime, ok := mempoolStats["first_time"].(time.Time); ok { - sb.WriteString(fmt.Sprintf(" First op: %s\n", firstTime.Format("2006-01-02 15:04:05"))) - } - if lastTime, ok := mempoolStats["last_time"].(time.Time); ok { - sb.WriteString(fmt.Sprintf(" Last op: %s\n", lastTime.Format("2006-01-02 15:04:05"))) - } - } else { - sb.WriteString(" (empty)\n") - } - } - - if s.config.EnableResolver { - - sb.WriteString("\nResolver\n") - sb.WriteString("━━━━━━━━\n") - sb.WriteString(" Status: enabled\n") - - if didStats := s.manager.GetDIDIndexStats(); didStats["exists"].(bool) { - indexedDIDs := didStats["indexed_dids"].(int64) - mempoolDIDs := didStats["mempool_dids"].(int64) - totalDIDs := didStats["total_dids"].(int64) - - if mempoolDIDs > 0 { - sb.WriteString(fmt.Sprintf(" Total DIDs: %s (%s indexed + %s mempool)\n", - formatNumber(int(totalDIDs)), - formatNumber(int(indexedDIDs)), - formatNumber(int(mempoolDIDs)))) - } else { - sb.WriteString(fmt.Sprintf(" Total DIDs: %s\n", formatNumber(int(totalDIDs)))) - } - } - sb.WriteString("\n") - } - - sb.WriteString("Server Stats\n") - sb.WriteString("━━━━━━━━━━━━\n") - sb.WriteString(fmt.Sprintf(" Version: %s\n", s.config.Version)) - sb.WriteString(fmt.Sprintf(" Sync mode: %v\n", s.config.SyncMode)) - sb.WriteString(fmt.Sprintf(" WebSocket: %v\n", s.config.EnableWebSocket)) - sb.WriteString(fmt.Sprintf(" Handle Resolver: %v\n", s.manager.GetHandleResolver().GetBaseURL())) - sb.WriteString(fmt.Sprintf(" Uptime: %s\n", time.Since(s.startTime).Round(time.Second))) - - sb.WriteString("\n\nAPI Endpoints\n") - sb.WriteString("━━━━━━━━━━━━━\n") - sb.WriteString(" GET / This info page\n") - sb.WriteString(" GET /index.json Full bundle index\n") - sb.WriteString(" GET /bundle/:number Bundle metadata (JSON)\n") - sb.WriteString(" GET /data/:number Raw bundle (zstd compressed)\n") - sb.WriteString(" GET /jsonl/:number Decompressed JSONL stream\n") - sb.WriteString(" GET /op/:pointer Get single operation\n") - sb.WriteString(" GET /status Server status\n") - sb.WriteString(" GET /mempool Mempool operations (JSONL)\n") - - if s.config.EnableResolver { - sb.WriteString("\nDID Resolution\n") - sb.WriteString("━━━━━━━━━━━━━━\n") - sb.WriteString(" GET /:did DID Document (W3C format)\n") - sb.WriteString(" GET /:did/data PLC State (raw format)\n") - sb.WriteString(" GET /:did/log/audit Operation history\n") - } - - if s.config.EnableWebSocket { - sb.WriteString("\nWebSocket Endpoints\n") - sb.WriteString("━━━━━━━━━━━━━━━━━━━\n") - sb.WriteString(" WS /ws Live stream (new operations only)\n") - sb.WriteString(" WS /ws?cursor=0 Stream all from beginning\n") - sb.WriteString(" WS /ws?cursor=N Stream from cursor N\n\n") - sb.WriteString("Cursor Format:\n") - sb.WriteString(" Global record number: (bundleNumber × 10,000) + position\n") - sb.WriteString(" Example: 88410345 = bundle 8841, position 345\n") - sb.WriteString(" Default: starts from latest (skips all historical data)\n") - - latestCursor := s.manager.GetCurrentCursor() - bundledOps := len(index.GetBundles()) * types.BUNDLE_SIZE - mempoolOps := latestCursor - bundledOps - - if s.config.SyncMode && mempoolOps > 0 { - sb.WriteString(fmt.Sprintf(" Current latest: %d (%d bundled + %d mempool)\n", - latestCursor, bundledOps, mempoolOps)) - } else { - sb.WriteString(fmt.Sprintf(" Current latest: %d (%d bundles)\n", - latestCursor, len(index.GetBundles()))) - } - } - - sb.WriteString("\nExamples\n") - sb.WriteString("━━━━━━━━\n") - sb.WriteString(fmt.Sprintf(" curl %s/bundle/1\n", baseURL)) - sb.WriteString(fmt.Sprintf(" curl %s/data/42 -o 000042.jsonl.zst\n", baseURL)) - sb.WriteString(fmt.Sprintf(" curl %s/jsonl/1\n", baseURL)) - sb.WriteString(fmt.Sprintf(" curl %s/op/0\n", baseURL)) - - if s.config.EnableWebSocket { - sb.WriteString(fmt.Sprintf(" websocat %s/ws\n", wsURL)) - sb.WriteString(fmt.Sprintf(" websocat '%s/ws?cursor=0'\n", wsURL)) - } - - if s.config.SyncMode { - sb.WriteString(fmt.Sprintf(" curl %s/status\n", baseURL)) - sb.WriteString(fmt.Sprintf(" curl %s/mempool\n", baseURL)) - } - - sb.WriteString("\n────────────────────────────────────────────────────────────────\n") - sb.WriteString("https://tangled.org/@atscan.net/plcbundle\n") - - w.Write([]byte(sb.String())) - } -} - -func (s *Server) handleIndexJSON() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - index := s.manager.GetIndex() - sendJSON(w, 200, index) - } -} - -func (s *Server) handleBundle() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - bundleNum, err := strconv.Atoi(r.PathValue("number")) - if err != nil { - sendJSON(w, 400, map[string]string{"error": "Invalid bundle number"}) - return - } - - meta, err := s.manager.GetIndex().GetBundle(bundleNum) - if err != nil { - sendJSON(w, 404, map[string]string{"error": "Bundle not found"}) - return - } - - sendJSON(w, 200, meta) - } -} - -func (s *Server) handleBundleData() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - bundleNum, err := strconv.Atoi(r.PathValue("number")) - if err != nil { - sendJSON(w, 400, map[string]string{"error": "Invalid bundle number"}) - return - } - - reader, err := s.manager.StreamBundleRaw(context.Background(), bundleNum) - if err != nil { - if strings.Contains(err.Error(), "not in index") || strings.Contains(err.Error(), "not found") { - sendJSON(w, 404, map[string]string{"error": "Bundle not found"}) - } else { - sendJSON(w, 500, map[string]string{"error": err.Error()}) - } - return - } - defer reader.Close() - - w.Header().Set("Content-Type", "application/zstd") - w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%06d.jsonl.zst", bundleNum)) - - io.Copy(w, reader) - } -} - -func (s *Server) handleBundleJSONL() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - bundleNum, err := strconv.Atoi(r.PathValue("number")) - if err != nil { - sendJSON(w, 400, map[string]string{"error": "Invalid bundle number"}) - return - } - - reader, err := s.manager.StreamBundleDecompressed(context.Background(), bundleNum) - if err != nil { - if strings.Contains(err.Error(), "not in index") || strings.Contains(err.Error(), "not found") { - sendJSON(w, 404, map[string]string{"error": "Bundle not found"}) - } else { - sendJSON(w, 500, map[string]string{"error": err.Error()}) - } - return - } - defer reader.Close() - - w.Header().Set("Content-Type", "application/x-ndjson") - w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%06d.jsonl", bundleNum)) - - io.Copy(w, reader) - } -} - -func (s *Server) handleStatus() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - index := s.manager.GetIndex() - indexStats := index.GetStats() - - response := StatusResponse{ - Server: ServerStatus{ - Version: s.config.Version, - UptimeSeconds: int(time.Since(s.startTime).Seconds()), - SyncMode: s.config.SyncMode, - WebSocketEnabled: s.config.EnableWebSocket, - ResolverEnabled: s.config.EnableResolver, - Origin: s.manager.GetPLCOrigin(), - }, - Bundles: BundleStatus{ - Count: indexStats["bundle_count"].(int), - TotalSize: indexStats["total_size"].(int64), - UncompressedSize: indexStats["total_uncompressed_size"].(int64), - UpdatedAt: indexStats["updated_at"].(time.Time), - }, - } - - if resolver := s.manager.GetHandleResolver(); resolver != nil { - response.Server.HandleResolver = resolver.GetBaseURL() - } - - if s.config.SyncMode && s.config.SyncInterval > 0 { - response.Server.SyncIntervalSeconds = int(s.config.SyncInterval.Seconds()) - } - - if bundleCount := response.Bundles.Count; bundleCount > 0 { - firstBundle := indexStats["first_bundle"].(int) - lastBundle := indexStats["last_bundle"].(int) - - response.Bundles.FirstBundle = firstBundle - response.Bundles.LastBundle = lastBundle - response.Bundles.StartTime = indexStats["start_time"].(time.Time) - response.Bundles.EndTime = indexStats["end_time"].(time.Time) - - if firstMeta, err := index.GetBundle(firstBundle); err == nil { - response.Bundles.RootHash = firstMeta.Hash - } - - if lastMeta, err := index.GetBundle(lastBundle); err == nil { - response.Bundles.HeadHash = lastMeta.Hash - response.Bundles.HeadAgeSeconds = int(time.Since(lastMeta.EndTime).Seconds()) - } - - if gaps, ok := indexStats["gaps"].(int); ok { - response.Bundles.Gaps = gaps - response.Bundles.HasGaps = gaps > 0 - if gaps > 0 { - response.Bundles.GapNumbers = index.FindGaps() - } - } - - totalOps := bundleCount * types.BUNDLE_SIZE - response.Bundles.TotalOperations = totalOps - - duration := response.Bundles.EndTime.Sub(response.Bundles.StartTime) - if duration.Hours() > 0 { - response.Bundles.AvgOpsPerHour = int(float64(totalOps) / duration.Hours()) - } - } - - if s.config.SyncMode { - mempoolStats := s.manager.GetMempoolStats() - - if count, ok := mempoolStats["count"].(int); ok { - mempool := &MempoolStatus{ - Count: count, - TargetBundle: mempoolStats["target_bundle"].(int), - CanCreateBundle: mempoolStats["can_create_bundle"].(bool), - MinTimestamp: mempoolStats["min_timestamp"].(time.Time), - Validated: mempoolStats["validated"].(bool), - ProgressPercent: float64(count) / float64(types.BUNDLE_SIZE) * 100, - BundleSize: types.BUNDLE_SIZE, - OperationsNeeded: types.BUNDLE_SIZE - count, - } - - if firstTime, ok := mempoolStats["first_time"].(time.Time); ok { - mempool.FirstTime = firstTime - mempool.TimespanSeconds = int(time.Since(firstTime).Seconds()) - } - if lastTime, ok := mempoolStats["last_time"].(time.Time); ok { - mempool.LastTime = lastTime - mempool.LastOpAgeSeconds = int(time.Since(lastTime).Seconds()) - } - - if count > 100 && count < types.BUNDLE_SIZE { - if !mempool.FirstTime.IsZero() && !mempool.LastTime.IsZero() { - timespan := mempool.LastTime.Sub(mempool.FirstTime) - if timespan.Seconds() > 0 { - opsPerSec := float64(count) / timespan.Seconds() - remaining := types.BUNDLE_SIZE - count - mempool.EtaNextBundleSeconds = int(float64(remaining) / opsPerSec) - } - } - } - - response.Mempool = mempool - } - } - - // DID Index stats - didStats := s.manager.GetDIDIndexStats() - if didStats["enabled"].(bool) { - didIndex := &DIDIndexStatus{ - Enabled: true, - Exists: didStats["exists"].(bool), - TotalDIDs: didStats["total_dids"].(int64), - IndexedDIDs: didStats["indexed_dids"].(int64), - LastBundle: didStats["last_bundle"].(int), - ShardCount: didStats["shard_count"].(int), - CachedShards: didStats["cached_shards"].(int), - CacheLimit: didStats["cache_limit"].(int), - UpdatedAt: didStats["updated_at"].(time.Time), - } - - // Mempool DIDs - if mempoolDIDs, ok := didStats["mempool_dids"].(int64); ok && mempoolDIDs > 0 { - didIndex.MempoolDIDs = mempoolDIDs - } - - // Version and format - if s.manager.GetDIDIndex() != nil { - config := s.manager.GetDIDIndex().GetConfig() - didIndex.Version = config.Version - didIndex.Format = config.Format - } - - // Hot shards - if cacheOrder, ok := didStats["cache_order"].([]int); ok && len(cacheOrder) > 0 { - maxShards := 10 - if len(cacheOrder) < maxShards { - maxShards = len(cacheOrder) - } - didIndex.HotShards = cacheOrder[:maxShards] - } - - // Cache performance - if cacheHitRate, ok := didStats["cache_hit_rate"].(float64); ok { - didIndex.CacheHitRate = cacheHitRate - } - if cacheHits, ok := didStats["cache_hits"].(int64); ok { - didIndex.CacheHits = cacheHits - } - if cacheMisses, ok := didStats["cache_misses"].(int64); ok { - didIndex.CacheMisses = cacheMisses - } - if totalLookups, ok := didStats["total_lookups"].(int64); ok { - didIndex.TotalLookups = totalLookups - } - - // Lookup performance metrics - if avgTime, ok := didStats["avg_lookup_time_ms"].(float64); ok { - didIndex.AvgLookupTimeMs = avgTime - } - if recentAvg, ok := didStats["recent_avg_lookup_time_ms"].(float64); ok { - didIndex.RecentAvgLookupTimeMs = recentAvg - } - if minTime, ok := didStats["min_lookup_time_ms"].(float64); ok { - didIndex.MinLookupTimeMs = minTime - } - if maxTime, ok := didStats["max_lookup_time_ms"].(float64); ok { - didIndex.MaxLookupTimeMs = maxTime - } - if p50, ok := didStats["p50_lookup_time_ms"].(float64); ok { - didIndex.P50LookupTimeMs = p50 - } - if p95, ok := didStats["p95_lookup_time_ms"].(float64); ok { - didIndex.P95LookupTimeMs = p95 - } - if p99, ok := didStats["p99_lookup_time_ms"].(float64); ok { - didIndex.P99LookupTimeMs = p99 - } - if sampleSize, ok := didStats["recent_sample_size"].(int); ok { - didIndex.RecentSampleSize = sampleSize - } - - response.DIDIndex = didIndex - } - - // Resolver performance stats - if s.config.EnableResolver { - resolverStats := s.manager.GetResolverStats() - - if totalRes, ok := resolverStats["total_resolutions"].(int64); ok && totalRes > 0 { - resolver := &ResolverStatus{ - Enabled: true, - TotalResolutions: totalRes, - } - - // Handle resolver URL - if hr := s.manager.GetHandleResolver(); hr != nil { - resolver.HandleResolver = hr.GetBaseURL() - } - - // Counts - if v, ok := resolverStats["mempool_hits"].(int64); ok { - resolver.MempoolHits = v - } - if v, ok := resolverStats["bundle_hits"].(int64); ok { - resolver.BundleHits = v - } - if v, ok := resolverStats["errors"].(int64); ok { - resolver.Errors = v - } - if v, ok := resolverStats["success_rate"].(float64); ok { - resolver.SuccessRate = v - } - if v, ok := resolverStats["mempool_hit_rate"].(float64); ok { - resolver.MempoolHitRate = v - } - - // Overall averages - if v, ok := resolverStats["avg_total_time_ms"].(float64); ok { - resolver.AvgTotalTimeMs = v - } - if v, ok := resolverStats["avg_mempool_time_ms"].(float64); ok { - resolver.AvgMempoolTimeMs = v - } - if v, ok := resolverStats["avg_index_time_ms"].(float64); ok { - resolver.AvgIndexTimeMs = v - } - if v, ok := resolverStats["avg_load_op_time_ms"].(float64); ok { - resolver.AvgLoadOpTimeMs = v - } - - // Recent averages - if v, ok := resolverStats["recent_avg_total_time_ms"].(float64); ok { - resolver.RecentAvgTotalTimeMs = v - } - if v, ok := resolverStats["recent_avg_mempool_time_ms"].(float64); ok { - resolver.RecentAvgMempoolTimeMs = v - } - if v, ok := resolverStats["recent_avg_index_time_ms"].(float64); ok { - resolver.RecentAvgIndexTimeMs = v - } - if v, ok := resolverStats["recent_avg_load_time_ms"].(float64); ok { - resolver.RecentAvgLoadTimeMs = v - } - if v, ok := resolverStats["recent_sample_size"].(int); ok { - resolver.RecentSampleSize = v - } - - // Percentiles - if v, ok := resolverStats["min_total_time_ms"].(float64); ok { - resolver.MinTotalTimeMs = v - } - if v, ok := resolverStats["max_total_time_ms"].(float64); ok { - resolver.MaxTotalTimeMs = v - } - if v, ok := resolverStats["p50_total_time_ms"].(float64); ok { - resolver.P50TotalTimeMs = v - } - if v, ok := resolverStats["p95_total_time_ms"].(float64); ok { - resolver.P95TotalTimeMs = v - } - if v, ok := resolverStats["p99_total_time_ms"].(float64); ok { - resolver.P99TotalTimeMs = v - } - if v, ok := resolverStats["p95_index_time_ms"].(float64); ok { - resolver.P95IndexTimeMs = v - } - if v, ok := resolverStats["p95_load_op_time_ms"].(float64); ok { - resolver.P95LoadOpTimeMs = v - } - - response.Resolver = resolver - } else { - // No resolutions yet, but resolver is enabled - response.Resolver = &ResolverStatus{ - Enabled: true, - TotalResolutions: 0, - } - - if hr := s.manager.GetHandleResolver(); hr != nil { - response.Resolver.HandleResolver = hr.GetBaseURL() - } - } - } - - sendJSON(w, 200, response) - } -} - -func (s *Server) handleMempool() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - ops, err := s.manager.GetMempoolOperations() - if err != nil { - sendJSON(w, 500, map[string]string{"error": err.Error()}) - return - } - - w.Header().Set("Content-Type", "application/x-ndjson") - - if len(ops) == 0 { - return - } - - for _, op := range ops { - if len(op.RawJSON) > 0 { - w.Write(op.RawJSON) - } else { - data, _ := json.Marshal(op) - w.Write(data) - } - w.Write([]byte("\n")) - } - } -} - -func (s *Server) handleDebugMemory() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - var m runtime.MemStats - runtime.ReadMemStats(&m) - - didStats := s.manager.GetDIDIndexStats() - - beforeAlloc := m.Alloc / 1024 / 1024 - - runtime.GC() - runtime.ReadMemStats(&m) - afterAlloc := m.Alloc / 1024 / 1024 - - response := fmt.Sprintf(`Memory Stats: - Alloc: %d MB - TotalAlloc: %d MB - Sys: %d MB - NumGC: %d - -DID Index: - Cached shards: %d/%d - -After GC: - Alloc: %d MB -`, - beforeAlloc, - m.TotalAlloc/1024/1024, - m.Sys/1024/1024, - m.NumGC, - didStats["cached_shards"], - didStats["cache_limit"], - afterAlloc) - - w.Header().Set("Content-Type", "text/plain") - w.Write([]byte(response)) - } -} - -func (s *Server) handleDIDRouting(w http.ResponseWriter, r *http.Request) { - path := strings.TrimPrefix(r.URL.Path, "/") - - parts := strings.SplitN(path, "/", 2) - input := parts[0] - - // Ignore common browser files before any validation - if isCommonBrowserFile(input) { - w.WriteHeader(http.StatusNotFound) - return - } - - // Quick validation: must be either a DID or a valid handle format - if !isValidDIDOrHandle(input) { - sendJSON(w, 404, map[string]string{"error": "not found"}) - return - } - - // Route to appropriate handler - if len(parts) == 1 { - s.handleDIDDocument(input)(w, r) - } else if parts[1] == "data" { - s.handleDIDData(input)(w, r) - } else if parts[1] == "log/audit" { - s.handleDIDAuditLog(input)(w, r) - } else { - sendJSON(w, 404, map[string]string{"error": "not found"}) - } -} - -func isCommonBrowserFile(path string) bool { - // Common files browsers request automatically - commonFiles := []string{ - "favicon.ico", - "robots.txt", - "sitemap.xml", - "apple-touch-icon.png", - "apple-touch-icon-precomposed.png", - ".well-known", - } - - for _, file := range commonFiles { - if path == file || strings.HasPrefix(path, file) { - return true - } - } - - // Common file extensions that are NOT DIDs/handles - commonExtensions := []string{ - ".ico", ".png", ".jpg", ".jpeg", ".gif", ".svg", - ".css", ".js", ".woff", ".woff2", ".ttf", ".eot", - ".xml", ".txt", ".html", ".webmanifest", - } - - for _, ext := range commonExtensions { - if strings.HasSuffix(path, ext) { - return true - } - } - - return false -} - -// isValidDIDOrHandle does quick format check before expensive resolution -func isValidDIDOrHandle(input string) bool { - // Empty input - if input == "" { - return false - } - - // If it's a DID - if strings.HasPrefix(input, "did:") { - // Only accept did:plc: method (reject other methods at routing level) - if !strings.HasPrefix(input, "did:plc:") { - return false // Returns 404 for did:web:, did:key:, did:invalid:, etc - } - - // Accept any did:plc:* - let handler validate exact format - // This allows invalid formats to reach handler and get proper 400 errors - return true - } - - // Not a DID - validate as handle - // Must have at least one dot (domain.tld) - if !strings.Contains(input, ".") { - return false - } - - // Must not have invalid characters for a domain - // Simple check: alphanumeric, dots, hyphens only - for _, c := range input { - if !((c >= 'a' && c <= 'z') || - (c >= 'A' && c <= 'Z') || - (c >= '0' && c <= '9') || - c == '.' || c == '-') { - return false - } - } - - // Basic length check (DNS max is 253) - if len(input) > 253 { - return false - } - - // Must not start or end with dot or hyphen - if strings.HasPrefix(input, ".") || strings.HasSuffix(input, ".") || - strings.HasPrefix(input, "-") || strings.HasSuffix(input, "-") { - return false - } - - return true -} - -func (s *Server) handleDIDDocument(input string) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - if r.Method == "OPTIONS" { - return - } - - // Resolve handle to DID - did, handleResolveTime, err := s.manager.ResolveHandleOrDID(r.Context(), input) - if err != nil { - if strings.Contains(err.Error(), "appears to be a handle") { - sendJSON(w, 400, map[string]string{ - "error": "Handle resolver not configured", - "hint": "Start server with --handle-resolver flag", - }) - } else { - sendJSON(w, 400, map[string]string{"error": err.Error()}) - } - return - } - - resolvedHandle := "" - if handleResolveTime > 0 { - resolvedHandle = input - } - - // Single call gets both document AND operation metadata - result, err := s.manager.ResolveDID(r.Context(), did) - if err != nil { - if strings.Contains(err.Error(), "deactivated") { - sendJSON(w, 410, map[string]string{"error": "DID has been deactivated"}) - } else if strings.Contains(err.Error(), "not found") { - sendJSON(w, 404, map[string]string{"error": "DID not found"}) - } else { - sendJSON(w, 500, map[string]string{"error": err.Error()}) - } - return - } - - // Early ETag check - operation is already in result.LatestOperation - if result.LatestOperation != nil { - etag := fmt.Sprintf(`"%s"`, result.LatestOperation.CID) - - if match := r.Header.Get("If-None-Match"); match != "" { - // Strip quotes if present - matchClean := strings.Trim(match, `"`) - if matchClean == result.LatestOperation.CID { - // Set minimal headers for 304 response - w.Header().Set("ETag", etag) - w.Header().Set("Cache-Control", "public, max-age=300") - w.WriteHeader(http.StatusNotModified) - return - } - } - } - - // Set all headers (now with result.LatestOperation available) - setDIDDocumentHeaders(w, r, did, resolvedHandle, result, handleResolveTime) - - w.Header().Set("Content-Type", "application/did+ld+json") - sendJSON(w, 200, result.Document) - } -} - -func (s *Server) handleDIDData(input string) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - // Resolve handle to DID - did, _, err := s.manager.ResolveHandleOrDID(r.Context(), input) - if err != nil { - sendJSON(w, 400, map[string]string{"error": err.Error()}) - return - } - - operations, _, err := s.manager.GetDIDOperations(context.Background(), did, false) - if err != nil { - sendJSON(w, 500, map[string]string{"error": err.Error()}) - return - } - - if len(operations) == 0 { - sendJSON(w, 404, map[string]string{"error": "DID not found"}) - return - } - - state, err := plcclient.BuildDIDState(did, operations) - if err != nil { - if strings.Contains(err.Error(), "deactivated") { - sendJSON(w, 410, map[string]string{"error": "DID has been deactivated"}) - } else { - sendJSON(w, 500, map[string]string{"error": err.Error()}) - } - return - } - - sendJSON(w, 200, state) - } -} - -func (s *Server) handleDIDAuditLog(input string) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - did, _, err := s.manager.ResolveHandleOrDID(r.Context(), input) - if err != nil { - sendJSON(w, 400, map[string]string{"error": err.Error()}) - return - } - - operations, _, err := s.manager.GetDIDOperations(context.Background(), did, false) - if err != nil { - sendJSON(w, 500, map[string]string{"error": err.Error()}) - return - } - - if len(operations) == 0 { - sendJSON(w, 404, map[string]string{"error": "DID not found"}) - return - } - - auditLog := plcclient.FormatAuditLog(operations) - sendJSON(w, 200, auditLog) - } -} - -// handleOperation gets a single operation with detailed timing headers -func (s *Server) handleOperation() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - pointer := r.PathValue("pointer") - - // Parse pointer format: "bundle:position" or global position - bundleNum, position, err := parseOperationPointer(pointer) - if err != nil { - sendJSON(w, 400, map[string]string{"error": err.Error()}) - return - } - - // Validate position range - if position < 0 || position >= types.BUNDLE_SIZE { - sendJSON(w, 400, map[string]string{ - "error": fmt.Sprintf("Position must be 0-%d", types.BUNDLE_SIZE-1), - }) - return - } - - // Time the entire request - totalStart := time.Now() - - // Time the operation load - loadStart := time.Now() - op, err := s.manager.LoadOperation(r.Context(), bundleNum, position) - loadDuration := time.Since(loadStart) - - if err != nil { - if strings.Contains(err.Error(), "not in index") || - strings.Contains(err.Error(), "not found") { - sendJSON(w, 404, map[string]string{"error": "Operation not found"}) - } else { - sendJSON(w, 500, map[string]string{"error": err.Error()}) - } - return - } - - totalDuration := time.Since(totalStart) - - // Calculate global position - globalPos := (bundleNum * types.BUNDLE_SIZE) + position - - // Calculate operation age - opAge := time.Since(op.CreatedAt) - - // Set response headers with useful metadata - setOperationHeaders(w, op, bundleNum, position, globalPos, loadDuration, totalDuration, opAge) - - // Send raw JSON if available (faster, preserves exact format) - if len(op.RawJSON) > 0 { - w.Header().Set("Content-Type", "application/json") - w.Write(op.RawJSON) - } else { - sendJSON(w, 200, op) - } - } -} - -// parseOperationPointer parses pointer in format "bundle:position" or global position -func parseOperationPointer(pointer string) (bundleNum, position int, err error) { - // Check if it's the "bundle:position" format - if strings.Contains(pointer, ":") { - parts := strings.Split(pointer, ":") - if len(parts) != 2 { - return 0, 0, fmt.Errorf("invalid pointer format: use 'bundle:position' or global position") - } - - bundleNum, err = strconv.Atoi(parts[0]) - if err != nil { - return 0, 0, fmt.Errorf("invalid bundle number: %w", err) - } - - position, err = strconv.Atoi(parts[1]) - if err != nil { - return 0, 0, fmt.Errorf("invalid position: %w", err) - } - - if bundleNum < 1 { - return 0, 0, fmt.Errorf("bundle number must be >= 1") - } - - return bundleNum, position, nil - } - - // Parse as global position - globalPos, err := strconv.Atoi(pointer) - if err != nil { - return 0, 0, fmt.Errorf("invalid position: must be number or 'bundle:position' format") - } - - if globalPos < 0 { - return 0, 0, fmt.Errorf("global position must be >= 0") - } - - // Handle small numbers as shorthand for bundle 1 - if globalPos < types.BUNDLE_SIZE { - return 1, globalPos, nil - } - - // Convert global position to bundle + position - bundleNum = globalPos / types.BUNDLE_SIZE - position = globalPos % types.BUNDLE_SIZE - - // Minimum bundle number is 1 - if bundleNum < 1 { - bundleNum = 1 - } - - return bundleNum, position, nil -} - -// setOperationHeaders sets useful response headers -func setOperationHeaders( - w http.ResponseWriter, - op *plcclient.PLCOperation, - bundleNum, position, globalPos int, - loadDuration, totalDuration, opAge time.Duration, -) { - // === Location Information === - w.Header().Set("X-Bundle-Number", fmt.Sprintf("%d", bundleNum)) - w.Header().Set("X-Position", fmt.Sprintf("%d", position)) - w.Header().Set("X-Global-Position", fmt.Sprintf("%d", globalPos)) - w.Header().Set("X-Pointer", fmt.Sprintf("%d:%d", bundleNum, position)) - - // === Operation Metadata === - w.Header().Set("X-Operation-DID", op.DID) - w.Header().Set("X-Operation-CID", op.CID) - w.Header().Set("X-Operation-Created", op.CreatedAt.Format(time.RFC3339)) - w.Header().Set("X-Operation-Age-Seconds", fmt.Sprintf("%d", int(opAge.Seconds()))) - - // Nullification status - if op.IsNullified() { - w.Header().Set("X-Operation-Nullified", "true") - if nullCID := op.GetNullifyingCID(); nullCID != "" { - w.Header().Set("X-Operation-Nullified-By", nullCID) - } - } else { - w.Header().Set("X-Operation-Nullified", "false") - } - - // === Size Information === - if len(op.RawJSON) > 0 { - w.Header().Set("X-Operation-Size", fmt.Sprintf("%d", len(op.RawJSON))) - } - - // === Performance Metrics (in milliseconds with precision) === - w.Header().Set("X-Load-Time-Ms", fmt.Sprintf("%.3f", float64(loadDuration.Microseconds())/1000.0)) - w.Header().Set("X-Total-Time-Ms", fmt.Sprintf("%.3f", float64(totalDuration.Microseconds())/1000.0)) - - // === Caching Hints === - // Set cache control (operations are immutable once bundled) - w.Header().Set("Cache-Control", "public, max-age=31536000, immutable") - w.Header().Set("ETag", op.CID) // CID is perfect for ETag -} - -// handleDIDIndexStats returns detailed DID index performance metrics -func (s *Server) handleDebugDIDIndex() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - didStats := s.manager.GetDIDIndexStats() - - if !didStats["enabled"].(bool) || !didStats["exists"].(bool) { - sendJSON(w, 404, map[string]string{ - "error": "DID index not available", - }) - return - } - - // Return all stats (more detailed than /status) - sendJSON(w, 200, didStats) - } -} - -func (s *Server) handleDebugResolver() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - resolverStats := s.manager.GetResolverStats() - - if resolverStats == nil { - sendJSON(w, 404, map[string]string{ - "error": "Resolver not enabled", - }) - return - } - - // Return all stats (more detailed than /status) - sendJSON(w, 200, resolverStats) - } -} diff --git a/server/helpers.go b/server/helpers.go deleted file mode 100644 index aeee781..0000000 --- a/server/helpers.go +++ /dev/null @@ -1,151 +0,0 @@ -package server - -import ( - "fmt" - "net/http" - "time" - - "tangled.org/atscan.net/plcbundle/bundle" -) - -// getScheme determines the HTTP scheme -func getScheme(r *http.Request) string { - if r.TLS != nil { - return "https" - } - - if proto := r.Header.Get("X-Forwarded-Proto"); proto != "" { - return proto - } - - if r.Header.Get("X-Forwarded-Ssl") == "on" { - return "https" - } - - return "http" -} - -// getWSScheme determines the WebSocket scheme -func getWSScheme(r *http.Request) string { - if getScheme(r) == "https" { - return "wss" - } - return "ws" -} - -// getBaseURL returns the base URL for HTTP -func getBaseURL(r *http.Request) string { - scheme := getScheme(r) - host := r.Host - return fmt.Sprintf("%s://%s", scheme, host) -} - -// getWSURL returns the base URL for WebSocket -func getWSURL(r *http.Request) string { - scheme := getWSScheme(r) - host := r.Host - return fmt.Sprintf("%s://%s", scheme, host) -} - -// formatNumber formats numbers with thousand separators -func formatNumber(n int) string { - s := fmt.Sprintf("%d", n) - var result []byte - for i, c := range s { - if i > 0 && (len(s)-i)%3 == 0 { - result = append(result, ',') - } - result = append(result, byte(c)) - } - return string(result) -} - -func setDIDDocumentHeaders( - w http.ResponseWriter, - _ *http.Request, - did string, - resolvedHandle string, - result *bundle.ResolveDIDResult, - handleResolveTime time.Duration, -) { - // === Identity === - w.Header().Set("X-DID", did) - - if resolvedHandle != "" { - w.Header().Set("X-Handle-Resolved", resolvedHandle) - w.Header().Set("X-Handle-Resolution-Time-Ms", - fmt.Sprintf("%.3f", float64(handleResolveTime.Microseconds())/1000.0)) - w.Header().Set("X-Request-Type", "handle") - } else { - w.Header().Set("X-Request-Type", "did") - } - - // === Resolution Source & Location === - w.Header().Set("X-Resolution-Source", result.Source) - - if result.Source == "bundle" { - w.Header().Set("X-Bundle-Number", fmt.Sprintf("%d", result.BundleNumber)) - w.Header().Set("X-Bundle-Position", fmt.Sprintf("%d", result.Position)) - globalPos := (result.BundleNumber * 10000) + result.Position - w.Header().Set("X-Global-Position", fmt.Sprintf("%d", globalPos)) - w.Header().Set("X-Pointer", fmt.Sprintf("%d:%d", result.BundleNumber, result.Position)) - } else { - w.Header().Set("X-Mempool", "true") - } - - // === Operation Metadata (from result.LatestOperation) === - if result.LatestOperation != nil { - op := result.LatestOperation - - w.Header().Set("X-Operation-CID", op.CID) - w.Header().Set("X-Operation-Created", op.CreatedAt.Format(time.RFC3339)) - - opAge := time.Since(op.CreatedAt) - w.Header().Set("X-Operation-Age-Seconds", fmt.Sprintf("%d", int(opAge.Seconds()))) - - if len(op.RawJSON) > 0 { - w.Header().Set("X-Operation-Size", fmt.Sprintf("%d", len(op.RawJSON))) - } - - // Nullification status - if op.IsNullified() { - w.Header().Set("X-Operation-Nullified", "true") - if nullCID := op.GetNullifyingCID(); nullCID != "" { - w.Header().Set("X-Operation-Nullified-By", nullCID) - } - } else { - w.Header().Set("X-Operation-Nullified", "false") - } - - // Standard HTTP headers - w.Header().Set("Last-Modified", op.CreatedAt.UTC().Format(http.TimeFormat)) - w.Header().Set("ETag", fmt.Sprintf(`"%s"`, op.CID)) - } - - // === Performance Metrics === - totalTime := handleResolveTime + result.TotalTime - w.Header().Set("X-Resolution-Time-Ms", - fmt.Sprintf("%.3f", float64(totalTime.Microseconds())/1000.0)) - w.Header().Set("X-Mempool-Time-Ms", - fmt.Sprintf("%.3f", float64(result.MempoolTime.Microseconds())/1000.0)) - - if result.Source == "bundle" { - w.Header().Set("X-Index-Time-Ms", - fmt.Sprintf("%.3f", float64(result.IndexTime.Microseconds())/1000.0)) - w.Header().Set("X-Load-Time-Ms", - fmt.Sprintf("%.3f", float64(result.LoadOpTime.Microseconds())/1000.0)) - } - - // === Caching Strategy === - if result.Source == "bundle" { - // Bundled data: cache 5min, stale-while-revalidate 10min - w.Header().Set("Cache-Control", - "public, max-age=300, stale-while-revalidate=600, stale-if-error=3600") - } else { - // Mempool data: cache 1min, stale-while-revalidate 5min - w.Header().Set("Cache-Control", - "public, max-age=60, stale-while-revalidate=300, stale-if-error=600") - } - - w.Header().Set("Vary", "Accept, If-None-Match") -} diff --git a/server/helpers_test.go b/server/helpers_test.go deleted file mode 100644 index af40561..0000000 --- a/server/helpers_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package server_test - -import ( - "io" - "net/http" - "net/http/httptest" - "testing" -) - -func TestServerHelperFunctions(t *testing.T) { - // Note: Many helper functions are unexported, so we test them indirectly - - t.Run("FormatNumber_ViaOutput", func(t *testing.T) { - // This tests the formatNumber function indirectly - srv, _, cleanup := setupTestServer(t, false) - defer cleanup() - - ts := httptest.NewServer(srv) - defer ts.Close() - - resp, _ := http.Get(ts.URL + "/") - body, _ := io.ReadAll(resp.Body) - resp.Body.Close() - - // Large numbers should be formatted with commas - // Check if output looks reasonable - if len(body) == 0 { - t.Error("root page is empty") - } - }) -} diff --git a/server/middleware.go b/server/middleware.go deleted file mode 100644 index 21d8196..0000000 --- a/server/middleware.go +++ /dev/null @@ -1,52 +0,0 @@ -package server - -import ( - "net/http" - - "github.com/goccy/go-json" -) - -// corsMiddleware adds CORS headers -func corsMiddleware(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // Skip CORS for WebSocket upgrade requests - if r.Header.Get("Upgrade") == "websocket" { - next.ServeHTTP(w, r) - return - } - - // Set CORS headers for all requests - w.Header().Set("Access-Control-Allow-Origin", "*") - w.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS") - w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization") - w.Header().Set("Access-Control-Max-Age", "86400") - w.Header().Set("Access-Control-Expose-Headers", - "X-Bundle-Number, X-Position, X-Global-Position, X-Pointer, "+ - "X-Operation-DID, X-Operation-CID, X-Load-Time-Ms, X-Total-Time-Ms, "+ - "X-Resolution-Time-Ms, X-Resolution-Source, X-Index-Time-Ms") - - // Handle OPTIONS preflight - return immediately WITHOUT calling handler - if r.Method == "OPTIONS" { - w.WriteHeader(http.StatusNoContent) // 204 - return // STOP HERE - don't call next - } - - // Only call handler for non-OPTIONS requests - next.ServeHTTP(w, r) - }) -} - -// sendJSON sends a JSON response -func sendJSON(w http.ResponseWriter, statusCode int, data interface{}) { - w.Header().Set("Content-Type", "application/json") - - jsonData, err := json.Marshal(data) - if err != nil { - w.WriteHeader(500) - w.Write([]byte(`{"error":"failed to marshal JSON"}`)) - return - } - - w.WriteHeader(statusCode) - w.Write(jsonData) -} diff --git a/server/server.go b/server/server.go deleted file mode 100644 index 1d13d0d..0000000 --- a/server/server.go +++ /dev/null @@ -1,100 +0,0 @@ -package server - -import ( - "context" - "net/http" - "time" - - "tangled.org/atscan.net/plcbundle/bundle" -) - -// New creates a new HTTP server -func New(manager *bundle.Manager, config *Config) *Server { - if config.Version == "" { - config.Version = "dev" - } - - s := &Server{ - manager: manager, - addr: config.Addr, - config: config, - startTime: time.Now(), - } - - handler := s.createHandler() - - s.httpServer = &http.Server{ - Addr: config.Addr, - Handler: handler, - } - - return s -} - -// createHandler creates the HTTP handler with all routes -func (s *Server) createHandler() http.Handler { - mux := http.NewServeMux() - - // Specific routes first - mux.HandleFunc("GET /index.json", s.handleIndexJSON()) - mux.HandleFunc("GET /bundle/{number}", s.handleBundle()) - mux.HandleFunc("GET /data/{number}", s.handleBundleData()) - mux.HandleFunc("GET /jsonl/{number}", s.handleBundleJSONL()) - mux.HandleFunc("GET /op/{pointer}", s.handleOperation()) - mux.HandleFunc("GET /status", s.handleStatus()) - mux.HandleFunc("GET /debug/memory", s.handleDebugMemory()) - mux.HandleFunc("GET /debug/didindex", s.handleDebugDIDIndex()) - mux.HandleFunc("GET /debug/resolver", s.handleDebugResolver()) - - // WebSocket - if s.config.EnableWebSocket { - mux.HandleFunc("GET /ws", s.handleWebSocket()) - } - - // Sync mode endpoints - if s.config.SyncMode { - mux.HandleFunc("GET /mempool", s.handleMempool()) - } - - // Root and DID resolver - mux.HandleFunc("GET /", func(w http.ResponseWriter, r *http.Request) { - path := r.URL.Path - - if path == "/" { - s.handleRoot()(w, r) - return - } - - if s.config.EnableResolver { - s.handleDIDRouting(w, r) - return - } - - sendJSON(w, 404, map[string]string{"error": "not found"}) - }) - - // Apply middleware in correct order: - handler := corsMiddleware(mux) - - return handler -} - -// ListenAndServe starts the HTTP server -func (s *Server) ListenAndServe() error { - return s.httpServer.ListenAndServe() -} - -// Shutdown gracefully shuts down the server -func (s *Server) Shutdown(ctx context.Context) error { - return s.httpServer.Shutdown(ctx) -} - -// GetStartTime returns when the server started -func (s *Server) GetStartTime() time.Time { - return s.startTime -} - -// Handler returns the configured HTTP handler -func (s *Server) Handler() http.Handler { - return s.createHandler() -} diff --git a/server/server_test.go b/server/server_test.go deleted file mode 100644 index 728c43e..0000000 --- a/server/server_test.go +++ /dev/null @@ -1,1243 +0,0 @@ -package server_test - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "net/http/httptest" - "path/filepath" - "strings" - "sync" - "testing" - "time" - - "github.com/gorilla/websocket" - "tangled.org/atscan.net/plcbundle/bundle" - "tangled.org/atscan.net/plcbundle/internal/bundleindex" - "tangled.org/atscan.net/plcbundle/internal/plcclient" - "tangled.org/atscan.net/plcbundle/internal/storage" - "tangled.org/atscan.net/plcbundle/server" -) - -type testLogger struct { - t *testing.T -} - -func (l *testLogger) Printf(format string, v ...interface{}) { - l.t.Logf(format, v...) -} - -func (l *testLogger) Println(v ...interface{}) { - l.t.Log(v...) -} - -var ( - bundleInfo = &storage.BundleInfo{ - BundleNumber: 1, - Origin: "test-origin", - ParentHash: "", - Cursor: "", - CreatedBy: "test", - Hostname: "test-host", - } -) - -// ==================================================================================== -// HTTP ENDPOINT TESTS -// ==================================================================================== - -func TestServerHTTPEndpoints(t *testing.T) { - handler, _, cleanup := setupTestServer(t, false) - defer cleanup() - - ts := httptest.NewServer(handler) - defer ts.Close() - - t.Run("RootEndpoint", func(t *testing.T) { - resp, err := http.Get(ts.URL + "/") - if err != nil { - t.Fatalf("GET / failed: %v", err) - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - t.Errorf("expected 200, got %d", resp.StatusCode) - } - - body, _ := io.ReadAll(resp.Body) - bodyStr := string(body) - - // Should contain welcome message - if !strings.Contains(bodyStr, "plcbundle server") { - t.Error("root page missing title") - } - - // Should show API endpoints - if !strings.Contains(bodyStr, "API Endpoints") { - t.Error("root page missing API documentation") - } - }) - - t.Run("IndexJSON", func(t *testing.T) { - resp, err := http.Get(ts.URL + "/index.json") - if err != nil { - t.Fatalf("GET /index.json failed: %v", err) - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - t.Errorf("expected 200, got %d", resp.StatusCode) - } - - // Should be JSON - contentType := resp.Header.Get("Content-Type") - if !strings.Contains(contentType, "application/json") { - t.Errorf("wrong content type: %s", contentType) - } - - // Parse JSON - var idx bundleindex.Index - if err := json.NewDecoder(resp.Body).Decode(&idx); err != nil { - t.Fatalf("failed to parse index JSON: %v", err) - } - - if idx.Version != "1.0" { - t.Errorf("index version mismatch: got %s", idx.Version) - } - }) - - t.Run("BundleMetadata", func(t *testing.T) { - resp, err := http.Get(ts.URL + "/bundle/1") - if err != nil { - t.Fatalf("GET /bundle/1 failed: %v", err) - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - t.Errorf("expected 200, got %d", resp.StatusCode) - } - - var meta bundleindex.BundleMetadata - if err := json.NewDecoder(resp.Body).Decode(&meta); err != nil { - t.Fatalf("failed to parse bundle metadata: %v", err) - } - - if meta.BundleNumber != 1 { - t.Error("wrong bundle returned") - } - - // Verify it has the fields we set - if meta.ContentHash == "" { - t.Error("metadata missing content hash") - } - }) - - t.Run("BundleMetadata_NotFound", func(t *testing.T) { - resp, err := http.Get(ts.URL + "/bundle/9999") - if err != nil { - t.Fatalf("GET /bundle/9999 failed: %v", err) - } - defer resp.Body.Close() - - if resp.StatusCode != 404 { - t.Errorf("expected 404 for nonexistent bundle, got %d", resp.StatusCode) - } - }) - - t.Run("BundleMetadata_InvalidNumber", func(t *testing.T) { - resp, err := http.Get(ts.URL + "/bundle/invalid") - if err != nil { - t.Fatalf("GET /bundle/invalid failed: %v", err) - } - defer resp.Body.Close() - - if resp.StatusCode != 400 { - t.Errorf("expected 400 for invalid bundle number, got %d", resp.StatusCode) - } - }) - - t.Run("BundleData_Raw", func(t *testing.T) { - resp, err := http.Get(ts.URL + "/data/1") - if err != nil { - t.Fatalf("GET /data/1 failed: %v", err) - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - // If 500, read error body - if resp.StatusCode == 500 { - body, _ := io.ReadAll(resp.Body) - t.Fatalf("expected 200, got 500. Error: %s", string(body)) - } - t.Errorf("expected 200, got %d", resp.StatusCode) - } - - // Should be zstd compressed - contentType := resp.Header.Get("Content-Type") - if !strings.Contains(contentType, "application/zstd") { - t.Errorf("wrong content type for raw data: %s", contentType) - } - - // Should have content-disposition header - disposition := resp.Header.Get("Content-Disposition") - if !strings.Contains(disposition, "000001.jsonl.zst") { - t.Errorf("wrong disposition header: %s", disposition) - } - - // Should have data - data, _ := io.ReadAll(resp.Body) - if len(data) == 0 { - t.Error("bundle data is empty") - } - - t.Logf("Bundle data size: %d bytes", len(data)) - }) - - t.Run("BundleJSONL_Decompressed", func(t *testing.T) { - resp, err := http.Get(ts.URL + "/jsonl/1") - if err != nil { - t.Fatalf("GET /jsonl/1 failed: %v", err) - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - t.Errorf("expected 200, got %d", resp.StatusCode) - } - - // Should be JSONL - contentType := resp.Header.Get("Content-Type") - if !strings.Contains(contentType, "application/x-ndjson") { - t.Errorf("wrong content type for JSONL: %s", contentType) - } - - // Count lines - data, _ := io.ReadAll(resp.Body) - lines := bytes.Count(data, []byte("\n")) - - if lines == 0 { - t.Error("JSONL should have lines") - } - }) - - t.Run("StatusEndpoint", func(t *testing.T) { - resp, err := http.Get(ts.URL + "/status") - if err != nil { - t.Fatalf("GET /status failed: %v", err) - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - t.Errorf("expected 200, got %d", resp.StatusCode) - } - - var status server.StatusResponse - if err := json.NewDecoder(resp.Body).Decode(&status); err != nil { - t.Fatalf("failed to parse status JSON: %v", err) - } - - // Verify structure - if status.Server.Version == "" { - t.Error("status missing server version") - } - - if status.Bundles.Count < 0 { - t.Error("invalid bundle count") - } - - if status.Server.UptimeSeconds < 0 { - t.Error("invalid uptime") - } - }) -} - -// ==================================================================================== -// DID RESOLUTION ENDPOINT TESTS -// ==================================================================================== - -func TestServerDIDResolution(t *testing.T) { - handler, _, cleanup := setupTestServerWithResolver(t) - defer cleanup() - - ts := httptest.NewServer(handler) - defer ts.Close() - - // Use valid did:plc format: "did:plc:" + 24 chars base32 (a-z, 2-7 only) - testDID := "did:plc:abc234def567ghi234jkl456" // Valid format - - t.Run("DIDDocument", func(t *testing.T) { - resp, err := http.Get(ts.URL + "/" + testDID) - if err != nil { - t.Fatalf("GET /%s failed: %v", testDID, err) - } - defer resp.Body.Close() - - // Should be 404 (not in test data) or 500 (no DID index) - // NOT 400 (that means invalid format) - if resp.StatusCode == 400 { - body, _ := io.ReadAll(resp.Body) - t.Fatalf("got 400 (invalid DID format): %s", string(body)) - } - - if resp.StatusCode == 500 { - t.Log("Expected 500 (no DID index)") - return - } - - if resp.StatusCode == 404 { - t.Log("Expected 404 (DID not found)") - return - } - - if resp.StatusCode == 200 { - var doc plcclient.DIDDocument - if err := json.NewDecoder(resp.Body).Decode(&doc); err != nil { - t.Fatalf("failed to parse DID document: %v", err) - } - } - }) - - t.Run("DIDData_RawState", func(t *testing.T) { - resp, err := http.Get(ts.URL + "/" + testDID + "/data") - if err != nil { - t.Fatalf("GET /%s/data failed: %v", testDID, err) - } - defer resp.Body.Close() - - // /data endpoint validates format, so 400 is NOT acceptable for valid DID - if resp.StatusCode == 400 { - body, _ := io.ReadAll(resp.Body) - t.Fatalf("got 400 for valid DID format: %s", string(body)) - } - - // 404 or 500 acceptable (no data / no index) - if resp.StatusCode == 500 || resp.StatusCode == 404 { - t.Logf("Expected error (no DID index): status %d", resp.StatusCode) - return - } - - if resp.StatusCode == 200 { - var state plcclient.DIDState - json.NewDecoder(resp.Body).Decode(&state) - } - }) - - t.Run("DIDAuditLog", func(t *testing.T) { - resp, err := http.Get(ts.URL + "/" + testDID + "/log/audit") - if err != nil { - t.Fatalf("request failed: %v", err) - } - defer resp.Body.Close() - - // Should NOT be 400 for valid DID - if resp.StatusCode == 400 { - body, _ := io.ReadAll(resp.Body) - t.Fatalf("got 400 for valid DID format: %s", string(body)) - } - - // 404, 500 acceptable - if resp.StatusCode == 500 || resp.StatusCode == 404 { - t.Logf("Expected error (no DID index): status %d", resp.StatusCode) - return - } - }) - - // Test invalid formats on /data endpoint (which validates properly) - t.Run("InvalidDIDFormat_OnDataEndpoint", func(t *testing.T) { - // Test DIDs that START with "did:plc:" but are still invalid - // (routing checks prefix first, so "did:invalid:" returns 404 before validation) - invalidDIDs := []string{ - "did:plc:short", // Too short (< 24 chars) - "did:plc:tooshort2345", // Still too short - "did:plc:contains0189invalidchars456", // Has 0,1,8,9 (invalid in base32) - "did:plc:UPPERCASENOTALLOWED1234", // Has uppercase - "did:plc:has-dashes-not-allowed12", // Has dashes - "did:plc:waytoolonggggggggggggggggg", // Too long (> 24 chars) - } - - for _, invalidDID := range invalidDIDs { - resp, err := http.Get(ts.URL + "/" + invalidDID + "/data") - if err != nil { - t.Fatalf("request to %s failed: %v", invalidDID, err) - } - - body, _ := io.ReadAll(resp.Body) - resp.Body.Close() - - // /data endpoint validates format and should return 400 - if resp.StatusCode != 400 { - t.Logf("DID %s: got %d (body: %s)", invalidDID, resp.StatusCode, string(body)) - // Some might also return 500 if they pass initial checks - // but fail deeper validation - that's also acceptable - if resp.StatusCode != 500 { - t.Errorf("DID %s: expected 400 or 500, got %d", invalidDID, resp.StatusCode) - } - } - } - }) - - t.Run("InvalidDIDMethod_Returns400", func(t *testing.T) { - // These now return 400 (validation error) instead of 404 (routing rejection) - wrongMethodDIDs := []string{ - "did:invalid:format", - "did:web:example.com", - "did:key:z6MkhaXgBZDvotDkL5257faiztiGiC2QtKLGpbnnEGta2doK", - } - - for _, did := range wrongMethodDIDs { - resp, err := http.Get(ts.URL + "/" + did + "/data") - if err != nil { - t.Fatalf("request failed: %v", err) - } - resp.Body.Close() - - // Now expect 400 (invalid DID format) or 404 (routing rejection) - if resp.StatusCode != 400 && resp.StatusCode != 404 { - t.Errorf("DID %s: expected 400 or 404, got %d", did, resp.StatusCode) - } - } - }) - - t.Run("HandleLikePathWithoutResolver", func(t *testing.T) { - // Need to create a fresh manager without resolver for this test - tmpDir := t.TempDir() - config := bundle.DefaultConfig(tmpDir) - config.AutoInit = true - config.HandleResolverURL = "" // ← DISABLE resolver - - mgr, err := bundle.NewManager(config, nil) - if err != nil { - t.Fatalf("failed to create manager: %v", err) - } - defer mgr.Close() - - serverConfig := &server.Config{ - Addr: ":8080", - EnableResolver: true, - Version: "test", - } - - srv := server.New(mgr, serverConfig) - ts := httptest.NewServer(srv.Handler()) - defer ts.Close() - - // Now test handle resolution without resolver configured - resp, err := http.Get(ts.URL + "/tree.fail") - if err != nil { - t.Fatalf("request failed: %v", err) - } - body, _ := io.ReadAll(resp.Body) - resp.Body.Close() - - // Should get 400 (resolver not configured) - if resp.StatusCode != 400 { - t.Errorf("expected 400 (resolver not configured), got %d: %s", - resp.StatusCode, string(body)) - return - } - - // Verify error message - var errResp map[string]string - json.Unmarshal(body, &errResp) - - if !strings.Contains(errResp["error"], "resolver") && - !strings.Contains(errResp["hint"], "resolver") { - t.Errorf("expected resolver error, got: %v", errResp) - } - }) - - t.Run("HandleResolutionWithIndex", func(t *testing.T) { - // The default setupTestServerWithResolver has resolver configured - // So this tests the normal flow: handle → DID → document - - resp, err := http.Get(ts.URL + "/tree.fail") - if err != nil { - t.Fatalf("request failed: %v", err) - } - body, _ := io.ReadAll(resp.Body) - resp.Body.Close() - - // Could be: - // - 500: No DID index (expected in test) - // - 404: DID not found in index - // - 200: Success (if test data includes this DID) - - switch resp.StatusCode { - case 500: - // No DID index - expected in test environment - var errResp map[string]string - json.Unmarshal(body, &errResp) - if !strings.Contains(errResp["error"], "DID index") { - t.Errorf("expected DID index error, got: %s", errResp["error"]) - } - t.Log("Expected: no DID index configured") - - case 404: - // DID not found - also acceptable - t.Log("Expected: DID not found in index") - - case 200: - // Success - would need DID index + test data - var doc plcclient.DIDDocument - json.Unmarshal(body, &doc) - t.Logf("Success: resolved to %s", doc.ID) - - default: - t.Errorf("unexpected status: %d, body: %s", resp.StatusCode, string(body)) - } - - // Verify we got handle resolution header - if resolvedHandle := resp.Header.Get("X-Handle-Resolved"); resolvedHandle != "" { - if resolvedHandle != "tree.fail" { - t.Errorf("wrong handle in header: %s", resolvedHandle) - } - t.Log("✓ Handle resolution header present") - } - }) - - t.Run("InvalidDIDMethod_Returns404", func(t *testing.T) { - // These should be rejected by routing (404) not validation (400) - wrongMethodDIDs := []string{ - "did:invalid:format", - "did:web:example.com", - "did:key:z6MkhaXgBZDvotDkL5257faiztiGiC2QtKLGpbnnEGta2doK", - } - - for _, did := range wrongMethodDIDs { - resp, err := http.Get(ts.URL + "/" + did + "/data") - if err != nil { - t.Fatalf("request failed: %v", err) - } - resp.Body.Close() - - // With smart routing, these get 404 (not supported) - if resp.StatusCode != 404 { - t.Errorf("DID %s: expected 404 from routing, got %d", did, resp.StatusCode) - } - } - }) - - t.Run("NotADIDPath", func(t *testing.T) { - resp, err := http.Get(ts.URL + "/notadid") - if err != nil { - t.Fatalf("request failed: %v", err) - } - defer resp.Body.Close() - - // "notadid" has no dot, rejected by isValidDIDOrHandle - if resp.StatusCode != 404 { - t.Errorf("expected 404 for non-DID path, got %d", resp.StatusCode) - } - }) - - t.Run("ValidHandleFormat", func(t *testing.T) { - // These should pass routing validation (have dots, valid chars) - validHandles := []string{ - "user.bsky.social", - "tree.fail", - "example.com", - } - - for _, handle := range validHandles { - resp, err := http.Get(ts.URL + "/" + handle) - if err != nil { - t.Fatalf("request failed: %v", err) - } - resp.Body.Close() - - // Should NOT be 404 (routing accepts it) - // Will be 400 (no resolver), 500 (no index), or 404 (not found) - if resp.StatusCode == 404 { - body, _ := io.ReadAll(resp.Body) - // 404 is OK if it's "DID not found", not "route not found" - var errResp map[string]string - resp.Body = io.NopCloser(bytes.NewReader(body)) - json.NewDecoder(resp.Body).Decode(&errResp) - - if errResp["error"] == "not found" && !strings.Contains(errResp["error"], "DID") { - t.Errorf("Handle %s: got routing 404, should be accepted", handle) - } - } - - t.Logf("Handle %s: status %d (400/500/404 all acceptable)", handle, resp.StatusCode) - } - }) - - t.Run("NotADIDPath", func(t *testing.T) { - resp, err := http.Get(ts.URL + "/notadid") - if err != nil { - t.Fatalf("request failed: %v", err) - } - defer resp.Body.Close() - - if resp.StatusCode != 404 { - t.Errorf("expected 404 for non-DID path, got %d", resp.StatusCode) - } - }) -} - -// ==================================================================================== -// CORS MIDDLEWARE TESTS -// ==================================================================================== - -func TestServerCORS(t *testing.T) { - srv, _, cleanup := setupTestServer(t, false) - defer cleanup() - - ts := httptest.NewServer(srv) - defer ts.Close() - - t.Run("CORS_Headers_GET", func(t *testing.T) { - resp, err := http.Get(ts.URL + "/index.json") - if err != nil { - t.Fatalf("request failed: %v", err) - } - defer resp.Body.Close() - - // Check CORS headers - if resp.Header.Get("Access-Control-Allow-Origin") != "*" { - t.Error("missing or wrong Access-Control-Allow-Origin header") - } - - methods := resp.Header.Get("Access-Control-Allow-Methods") - if !strings.Contains(methods, "GET") { - t.Errorf("Access-Control-Allow-Methods missing GET: %s", methods) - } - }) - - t.Run("CORS_Preflight_OPTIONS", func(t *testing.T) { - req, _ := http.NewRequest("OPTIONS", ts.URL+"/index.json", nil) - req.Header.Set("Access-Control-Request-Method", "GET") - req.Header.Set("Access-Control-Request-Headers", "Content-Type") - - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("OPTIONS request failed: %v", err) - } - defer resp.Body.Close() - - if resp.StatusCode != 204 { - t.Errorf("expected 204 for OPTIONS, got %d", resp.StatusCode) - } - - if resp.Header.Get("Access-Control-Allow-Origin") != "*" { - t.Error("CORS headers missing on OPTIONS") - } - - maxAge := resp.Header.Get("Access-Control-Max-Age") - if maxAge != "86400" { - t.Errorf("wrong max-age: %s", maxAge) - } - }) -} - -// ==================================================================================== -// WEBSOCKET TESTS -// ==================================================================================== - -func TestServerWebSocket(t *testing.T) { - srv, _, cleanup := setupTestServer(t, true) // Enable WebSocket - defer cleanup() - - ts := httptest.NewServer(srv) - defer ts.Close() - - wsURL := "ws" + strings.TrimPrefix(ts.URL, "http") + "/ws" - - t.Run("WebSocket_Connect", func(t *testing.T) { - ws, _, err := websocket.DefaultDialer.Dial(wsURL, nil) - if err != nil { - t.Fatalf("WebSocket dial failed: %v", err) - } - defer ws.Close() - - // Should connect successfully - t.Log("WebSocket connected successfully") - }) - - t.Run("WebSocket_ReceiveOperations", func(t *testing.T) { - ws, _, err := websocket.DefaultDialer.Dial(wsURL+"?cursor=0", nil) - if err != nil { - t.Fatalf("WebSocket dial failed: %v", err) - } - defer ws.Close() - - // Set read deadline - ws.SetReadDeadline(time.Now().Add(5 * time.Second)) - - // Read a message (should get operations or timeout) - _, message, err := ws.ReadMessage() - if err != nil { - // Timeout is OK (no operations available) - if !strings.Contains(err.Error(), "timeout") { - t.Logf("Read error (may be OK if no ops): %v", err) - } - return - } - - // If we got a message, verify it's valid JSON - var op plcclient.PLCOperation - if err := json.Unmarshal(message, &op); err != nil { - t.Errorf("received invalid operation JSON: %v", err) - } - - t.Logf("Received operation: %s", op.CID) - }) - - t.Run("WebSocket_InvalidCursor", func(t *testing.T) { - resp, err := http.Get(ts.URL + "/ws?cursor=invalid") - if err != nil { - t.Fatalf("request failed: %v", err) - } - defer resp.Body.Close() - - if resp.StatusCode != 400 { - t.Errorf("expected 400 for invalid cursor, got %d", resp.StatusCode) - } - }) - - t.Run("WebSocket_CloseGracefully", func(t *testing.T) { - ws, _, err := websocket.DefaultDialer.Dial(wsURL, nil) - if err != nil { - t.Fatalf("WebSocket dial failed: %v", err) - } - - // Close immediately - err = ws.WriteMessage(websocket.CloseMessage, - websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) - if err != nil { - t.Logf("close message error (may be OK): %v", err) - } - - ws.Close() - t.Log("WebSocket closed gracefully") - }) -} - -// ==================================================================================== -// SYNC MODE TESTS -// ==================================================================================== - -func TestServerSyncMode(t *testing.T) { - srv, _, cleanup := setupTestServer(t, true) - defer cleanup() - - ts := httptest.NewServer(srv) - defer ts.Close() - - t.Run("MempoolEndpoint", func(t *testing.T) { - resp, err := http.Get(ts.URL + "/mempool") - if err != nil { - t.Fatalf("GET /mempool failed: %v", err) - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - t.Errorf("expected 200, got %d", resp.StatusCode) - } - - // Should be JSONL - contentType := resp.Header.Get("Content-Type") - if !strings.Contains(contentType, "application/x-ndjson") { - t.Errorf("wrong content type: %s", contentType) - } - }) - - t.Run("StatusWithMempool", func(t *testing.T) { - resp, err := http.Get(ts.URL + "/status") - if err != nil { - t.Fatalf("GET /status failed: %v", err) - } - defer resp.Body.Close() - - var status server.StatusResponse - if err := json.NewDecoder(resp.Body).Decode(&status); err != nil { - t.Fatalf("failed to parse status: %v", err) - } - - // Sync mode should include mempool stats - if status.Server.SyncMode { - if status.Mempool == nil { - t.Error("sync mode status missing mempool") - } - } - }) -} - -// ==================================================================================== -// CONCURRENT REQUEST TESTS -// ==================================================================================== - -func TestServerConcurrency(t *testing.T) { - srv, _, cleanup := setupTestServer(t, false) - defer cleanup() - - ts := httptest.NewServer(srv) - defer ts.Close() - - t.Run("ConcurrentIndexRequests", func(t *testing.T) { - var wg sync.WaitGroup - errors := make(chan error, 100) - - for i := 0; i < 100; i++ { - wg.Add(1) - go func() { - defer wg.Done() - - resp, err := http.Get(ts.URL + "/index.json") - if err != nil { - errors <- err - return - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - errors <- fmt.Errorf("status %d", resp.StatusCode) - } - }() - } - - wg.Wait() - close(errors) - - for err := range errors { - t.Errorf("concurrent request error: %v", err) - } - }) - - t.Run("ConcurrentBundleRequests", func(t *testing.T) { - var wg sync.WaitGroup - errors := make(chan error, 50) - - for i := 0; i < 50; i++ { - wg.Add(1) - go func(bundleNum int) { - defer wg.Done() - - resp, err := http.Get(fmt.Sprintf("%s/bundle/%d", ts.URL, bundleNum%3+1)) - if err != nil { - errors <- err - return - } - defer resp.Body.Close() - - if resp.StatusCode != 200 && resp.StatusCode != 404 { - errors <- fmt.Errorf("unexpected status %d", resp.StatusCode) - } - }(i) - } - - wg.Wait() - close(errors) - - for err := range errors { - t.Errorf("concurrent request error: %v", err) - } - }) - - t.Run("MixedEndpointConcurrency", func(t *testing.T) { - var wg sync.WaitGroup - - endpoints := []string{ - "/", - "/index.json", - "/bundle/1", - "/data/1", - "/jsonl/1", - "/status", - } - - for i := 0; i < 30; i++ { - wg.Add(1) - go func(id int) { - defer wg.Done() - - endpoint := endpoints[id%len(endpoints)] - resp, err := http.Get(ts.URL + endpoint) - if err != nil { - t.Errorf("request to %s failed: %v", endpoint, err) - return - } - defer resp.Body.Close() - - // Read body to completion - io.ReadAll(resp.Body) - }(i) - } - - wg.Wait() - }) -} - -// ==================================================================================== -// ERROR HANDLING TESTS -// ==================================================================================== - -func TestServerErrorHandling(t *testing.T) { - srv, _, cleanup := setupTestServer(t, false) - defer cleanup() - - ts := httptest.NewServer(srv) - defer ts.Close() - - t.Run("404_NotFound", func(t *testing.T) { - resp, err := http.Get(ts.URL + "/nonexistent") - if err != nil { - t.Fatalf("request failed: %v", err) - } - defer resp.Body.Close() - - if resp.StatusCode != 404 { - t.Errorf("expected 404, got %d", resp.StatusCode) - } - }) - - t.Run("405_MethodNotAllowed", func(t *testing.T) { - // POST to GET-only endpoint - resp, err := http.Post(ts.URL+"/index.json", "application/json", bytes.NewReader([]byte("{}"))) - if err != nil { - t.Fatalf("request failed: %v", err) - } - defer resp.Body.Close() - - if resp.StatusCode != 404 && resp.StatusCode != 405 { - t.Logf("Note: Got status %d (404/405 both acceptable)", resp.StatusCode) - } - }) - - t.Run("LargeRequestHandling", func(t *testing.T) { - // Request very large bundle number - resp, err := http.Get(ts.URL + "/bundle/999999") - if err != nil { - t.Fatalf("request failed: %v", err) - } - defer resp.Body.Close() - - if resp.StatusCode != 404 { - t.Errorf("expected 404 for large bundle number, got %d", resp.StatusCode) - } - }) -} - -// ==================================================================================== -// MIDDLEWARE TESTS -// ==================================================================================== - -func TestServerMiddleware(t *testing.T) { - srv, _, cleanup := setupTestServer(t, false) - defer cleanup() - - ts := httptest.NewServer(srv) - defer ts.Close() - - t.Run("JSON_ContentType", func(t *testing.T) { - resp, err := http.Get(ts.URL + "/index.json") - if err != nil { - t.Fatalf("request failed: %v", err) - } - defer resp.Body.Close() - - contentType := resp.Header.Get("Content-Type") - if !strings.Contains(contentType, "application/json") { - t.Errorf("wrong content type: %s", contentType) - } - }) - - t.Run("CORS_AllowsAllOrigins", func(t *testing.T) { - req, _ := http.NewRequest("GET", ts.URL+"/index.json", nil) - req.Header.Set("Origin", "https://example.com") - - resp, err := http.DefaultClient.Do(req) - if err != nil { - t.Fatalf("request failed: %v", err) - } - defer resp.Body.Close() - - allowOrigin := resp.Header.Get("Access-Control-Allow-Origin") - if allowOrigin != "*" { - t.Errorf("CORS not allowing all origins: %s", allowOrigin) - } - }) -} - -// ==================================================================================== -// HELPER FUNCTIONS & FORMATTERS -// ==================================================================================== - -func TestServerHelpers(t *testing.T) { - t.Run("FormatNumber", func(t *testing.T) { - // Note: formatNumber is not exported, so we test indirectly - // through endpoints that use it (like root page) - - srv, _, cleanup := setupTestServer(t, false) - defer cleanup() - - ts := httptest.NewServer(srv) - defer ts.Close() - - resp, _ := http.Get(ts.URL + "/") - body, _ := io.ReadAll(resp.Body) - resp.Body.Close() - - // Should have formatted numbers with commas - // (if there are any large numbers in output) - t.Logf("Root page length: %d bytes", len(body)) - }) -} - -// ==================================================================================== -// MEMORY & PERFORMANCE TESTS -// ==================================================================================== - -func TestServerPerformance(t *testing.T) { - if testing.Short() { - t.Skip("skipping performance test in short mode") - } - - srv, _, cleanup := setupTestServer(t, false) - defer cleanup() - - ts := httptest.NewServer(srv) - defer ts.Close() - - t.Run("MemoryDebugEndpoint", func(t *testing.T) { - resp, err := http.Get(ts.URL + "/debug/memory") - if err != nil { - t.Fatalf("GET /debug/memory failed: %v", err) - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - t.Errorf("expected 200, got %d", resp.StatusCode) - } - - body, _ := io.ReadAll(resp.Body) - bodyStr := string(body) - - if !strings.Contains(bodyStr, "Memory Stats") { - t.Error("memory debug output missing stats") - } - - if !strings.Contains(bodyStr, "Alloc:") { - t.Error("memory debug missing allocation info") - } - }) - - t.Run("ResponseTime", func(t *testing.T) { - // Measure response time for index - start := time.Now() - resp, err := http.Get(ts.URL + "/index.json") - elapsed := time.Since(start) - - if err != nil { - t.Fatalf("request failed: %v", err) - } - resp.Body.Close() - - // Should be fast (< 100ms for index) - if elapsed > 100*time.Millisecond { - t.Logf("Warning: slow response time: %v", elapsed) - } - - t.Logf("Index response time: %v", elapsed) - }) -} - -// ==================================================================================== -// SERVER LIFECYCLE TESTS -// ==================================================================================== - -func TestServerLifecycle(t *testing.T) { - t.Run("StartAndStop", func(t *testing.T) { - mgr, mgrCleanup := setupTestManager(t) - defer mgrCleanup() - - config := &server.Config{ - Addr: "127.0.0.1:0", // Random port - SyncMode: false, - EnableWebSocket: false, - EnableResolver: false, - Version: "test", - } - - srv := server.New(mgr, config) - - // Start in goroutine - errChan := make(chan error, 1) - go func() { - // This will block - errChan <- srv.ListenAndServe() - }() - - // Give it time to start - time.Sleep(100 * time.Millisecond) - - // Shutdown - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - if err := srv.Shutdown(ctx); err != nil { - t.Errorf("shutdown failed: %v", err) - } - - // Should exit - select { - case err := <-errChan: - if err != nil && err != http.ErrServerClosed { - t.Errorf("unexpected error: %v", err) - } - case <-time.After(2 * time.Second): - t.Error("server did not stop after shutdown") - } - }) - - t.Run("GetStartTime", func(t *testing.T) { - mgr, cleanup := setupTestManager(t) - defer cleanup() - - config := &server.Config{ - Addr: ":0", - Version: "test", - } - - before := time.Now() - srv := server.New(mgr, config) - after := time.Now() - - startTime := srv.GetStartTime() - - if startTime.Before(before) || startTime.After(after) { - t.Error("start time not in expected range") - } - }) -} - -// ==================================================================================== -// SETUP HELPERS -// ==================================================================================== - -func setupTestServer(t *testing.T, enableWebSocket bool) (http.Handler, *server.Server, func()) { - mgr, cleanup := setupTestManager(t) - - config := &server.Config{ - Addr: ":8080", - SyncMode: true, - SyncInterval: 1 * time.Minute, - EnableWebSocket: enableWebSocket, - EnableResolver: false, - Version: "test", - } - - srv := server.New(mgr, config) - - // Get handler from server - handler := srv.Handler() // Use new method - - return handler, srv, cleanup -} - -func setupTestServerWithResolver(t *testing.T) (http.Handler, *server.Server, func()) { - mgr, cleanup := setupTestManager(t) - - config := &server.Config{ - Addr: ":8080", - SyncMode: false, - EnableWebSocket: false, - EnableResolver: true, - Version: "test", - } - - srv := server.New(mgr, config) - handler := srv.Handler() - - return handler, srv, cleanup -} - -func setupTestManager(t *testing.T) (*bundle.Manager, func()) { - tmpDir := t.TempDir() - - config := bundle.DefaultConfig(tmpDir) - config.AutoInit = true - config.VerifyOnLoad = false // Disable verification in tests - - // Create storage operations ONCE and reuse - logger := &testLogger{t: t} - storageOps, err := storage.NewOperations(logger, false) - if err != nil { - t.Fatalf("failed to create storage operations: %v", err) - } - - mgr, err := bundle.NewManager(config, nil) - if err != nil { - storageOps.Close() - t.Fatalf("failed to create manager: %v", err) - } - - // Add test bundles with actual files - for i := 1; i <= 3; i++ { - // Create actual bundle file FIRST - path := filepath.Join(tmpDir, fmt.Sprintf("%06d.jsonl.zst", i)) - ops := makeMinimalTestOperations(10000, i*10000) // Unique ops per bundle - - contentHash, compHash, uncompSize, compSize, err := storageOps.SaveBundle(path, ops, bundleInfo) - if err != nil { - t.Fatalf("failed to save test bundle %d: %v", i, err) - } - - // Create metadata that matches the actual file - meta := &bundleindex.BundleMetadata{ - BundleNumber: i, - StartTime: ops[0].CreatedAt, - EndTime: ops[len(ops)-1].CreatedAt, - OperationCount: len(ops), - DIDCount: len(ops), // All unique in test data - Hash: fmt.Sprintf("hash%d", i), - ContentHash: contentHash, // Use actual hash - CompressedHash: compHash, // Use actual hash - CompressedSize: compSize, // Use actual size - UncompressedSize: uncompSize, // Use actual size - CreatedAt: time.Now(), - } - - mgr.GetIndex().AddBundle(meta) - } - - if err := mgr.SaveIndex(); err != nil { - t.Fatalf("failed to save index: %v", err) - } - - cleanup := func() { - storageOps.Close() - mgr.Close() - } - - return mgr, cleanup -} - -func makeMinimalTestOperations(count int, offset int) []plcclient.PLCOperation { - ops := make([]plcclient.PLCOperation, count) - baseTime := time.Now().Add(-time.Hour) - - for i := 0; i < count; i++ { - idx := offset + i - - // Create valid base32 DID identifier (24 chars, only a-z and 2-7) - // Convert index to base32-like string - identifier := fmt.Sprintf("%024d", idx) - // Replace invalid chars (0,1,8,9) with valid ones - identifier = strings.ReplaceAll(identifier, "0", "a") - identifier = strings.ReplaceAll(identifier, "1", "b") - identifier = strings.ReplaceAll(identifier, "8", "c") - identifier = strings.ReplaceAll(identifier, "9", "d") - - ops[i] = plcclient.PLCOperation{ - DID: "did:plc:" + identifier, - CID: fmt.Sprintf("bafytest%012d", idx), - CreatedAt: baseTime.Add(time.Duration(idx) * time.Second), - } - } - - return ops -} diff --git a/server/types.go b/server/types.go deleted file mode 100644 index 8afacb3..0000000 --- a/server/types.go +++ /dev/null @@ -1,164 +0,0 @@ -package server - -import ( - "net/http" - "time" - - "tangled.org/atscan.net/plcbundle/bundle" -) - -// Server serves bundle data over HTTP -type Server struct { - manager *bundle.Manager - addr string - config *Config - startTime time.Time - httpServer *http.Server -} - -// Config configures the server -type Config struct { - Addr string - SyncMode bool - SyncInterval time.Duration - EnableWebSocket bool - EnableResolver bool - Version string -} - -// StatusResponse is the /status endpoint response -type StatusResponse struct { - Server ServerStatus `json:"server"` - Bundles BundleStatus `json:"bundles"` - Mempool *MempoolStatus `json:"mempool,omitempty"` - DIDIndex *DIDIndexStatus `json:"didindex,omitempty"` - Resolver *ResolverStatus `json:"resolver,omitempty"` -} - -// ServerStatus contains server information -type ServerStatus struct { - Version string `json:"version"` - Origin string `json:"origin,omitempty"` - SyncMode bool `json:"sync_mode"` - SyncIntervalSeconds int `json:"sync_interval_seconds,omitempty"` - WebSocketEnabled bool `json:"websocket_enabled"` - ResolverEnabled bool `json:"resolver_enabled"` - HandleResolver string `json:"handle_resolver,omitempty"` - UptimeSeconds int `json:"uptime_seconds"` -} - -// DIDIndexStatus contains DID index statistics -type DIDIndexStatus struct { - Enabled bool `json:"enabled"` - Exists bool `json:"exists"` - TotalDIDs int64 `json:"total_dids"` - IndexedDIDs int64 `json:"indexed_dids"` - MempoolDIDs int64 `json:"mempool_dids,omitempty"` - LastBundle int `json:"last_bundle"` - ShardCount int `json:"shard_count"` - CachedShards int `json:"cached_shards"` - CacheLimit int `json:"cache_limit"` - CacheHitRate float64 `json:"cache_hit_rate"` - CacheHits int64 `json:"cache_hits"` - CacheMisses int64 `json:"cache_misses"` - TotalLookups int64 `json:"total_lookups"` - UpdatedAt time.Time `json:"updated_at"` - Version int `json:"version,omitempty"` - Format string `json:"format,omitempty"` - HotShards []int `json:"hot_shards,omitempty"` - - // Lookup performance metrics - AvgLookupTimeMs float64 `json:"avg_lookup_time_ms"` // All-time average - RecentAvgLookupTimeMs float64 `json:"recent_avg_lookup_time_ms"` // Recent average - MinLookupTimeMs float64 `json:"min_lookup_time_ms,omitempty"` // Fastest - MaxLookupTimeMs float64 `json:"max_lookup_time_ms,omitempty"` // Slowest - P50LookupTimeMs float64 `json:"p50_lookup_time_ms,omitempty"` // Median - P95LookupTimeMs float64 `json:"p95_lookup_time_ms,omitempty"` // 95th percentile - P99LookupTimeMs float64 `json:"p99_lookup_time_ms,omitempty"` // 99th percentile - RecentSampleSize int `json:"recent_sample_size,omitempty"` // How many samples -} - -// ResolverStatus contains DID document resolver performance metrics -type ResolverStatus struct { - Enabled bool `json:"enabled"` - HandleResolver string `json:"handle_resolver,omitempty"` - - // Resolution counts - TotalResolutions int64 `json:"total_resolutions"` - MempoolHits int64 `json:"mempool_hits"` - BundleHits int64 `json:"bundle_hits"` - Errors int64 `json:"errors"` - SuccessRate float64 `json:"success_rate"` - MempoolHitRate float64 `json:"mempool_hit_rate"` - - // Overall timing (all-time averages) - AvgTotalTimeMs float64 `json:"avg_total_time_ms"` - AvgMempoolTimeMs float64 `json:"avg_mempool_time_ms"` - AvgIndexTimeMs float64 `json:"avg_index_time_ms,omitempty"` - AvgLoadOpTimeMs float64 `json:"avg_load_op_time_ms,omitempty"` - - // Recent performance (last N resolutions) - RecentAvgTotalTimeMs float64 `json:"recent_avg_total_time_ms"` - RecentAvgMempoolTimeMs float64 `json:"recent_avg_mempool_time_ms"` - RecentAvgIndexTimeMs float64 `json:"recent_avg_index_time_ms,omitempty"` - RecentAvgLoadTimeMs float64 `json:"recent_avg_load_time_ms,omitempty"` - RecentSampleSize int `json:"recent_sample_size"` - - // Percentiles (total response time) - MinTotalTimeMs float64 `json:"min_total_time_ms,omitempty"` - MaxTotalTimeMs float64 `json:"max_total_time_ms,omitempty"` - P50TotalTimeMs float64 `json:"p50_total_time_ms,omitempty"` - P95TotalTimeMs float64 `json:"p95_total_time_ms,omitempty"` - P99TotalTimeMs float64 `json:"p99_total_time_ms,omitempty"` - - // Breakdown percentiles (for bundle resolutions only) - P95IndexTimeMs float64 `json:"p95_index_time_ms,omitempty"` - P95LoadOpTimeMs float64 `json:"p95_load_op_time_ms,omitempty"` -} - -// BundleStatus contains bundle statistics -type BundleStatus struct { - Count int `json:"count"` - FirstBundle int `json:"first_bundle,omitempty"` - LastBundle int `json:"last_bundle,omitempty"` - TotalSize int64 `json:"total_size"` - UncompressedSize int64 `json:"uncompressed_size,omitempty"` - CompressionRatio float64 `json:"compression_ratio,omitempty"` - TotalOperations int `json:"total_operations,omitempty"` - AvgOpsPerHour int `json:"avg_ops_per_hour,omitempty"` - StartTime time.Time `json:"start_time,omitempty"` - EndTime time.Time `json:"end_time,omitempty"` - UpdatedAt time.Time `json:"updated_at"` - HeadAgeSeconds int `json:"head_age_seconds,omitempty"` - RootHash string `json:"root_hash,omitempty"` - HeadHash string `json:"head_hash,omitempty"` - Gaps int `json:"gaps,omitempty"` - HasGaps bool `json:"has_gaps"` - GapNumbers []int `json:"gap_numbers,omitempty"` -} - -// MempoolStatus contains mempool statistics -type MempoolStatus struct { - Count int `json:"count"` - TargetBundle int `json:"target_bundle"` - CanCreateBundle bool `json:"can_create_bundle"` - MinTimestamp time.Time `json:"min_timestamp"` - Validated bool `json:"validated"` - ProgressPercent float64 `json:"progress_percent"` - BundleSize int `json:"bundle_size"` - OperationsNeeded int `json:"operations_needed"` - FirstTime time.Time `json:"first_time,omitempty"` - LastTime time.Time `json:"last_time,omitempty"` - TimespanSeconds int `json:"timespan_seconds,omitempty"` - LastOpAgeSeconds int `json:"last_op_age_seconds,omitempty"` - EtaNextBundleSeconds int `json:"eta_next_bundle_seconds,omitempty"` -} - -// RequestLog represents a logged HTTP request -type RequestLog struct { - Timestamp time.Time `json:"timestamp"` - Method string `json:"method"` - Path string `json:"path"` - UserAgent string `json:"user_agent"` - RemoteAddr string `json:"remote_addr"` -} diff --git a/server/types_test.go b/server/types_test.go deleted file mode 100644 index 0fa89ec..0000000 --- a/server/types_test.go +++ /dev/null @@ -1,122 +0,0 @@ -package server_test - -import ( - "encoding/json" - "testing" - "time" - - "tangled.org/atscan.net/plcbundle/server" -) - -func TestServerResponseTypes(t *testing.T) { - t.Run("StatusResponse_JSON", func(t *testing.T) { - response := server.StatusResponse{ - Server: server.ServerStatus{ - Version: "1.0.0", - UptimeSeconds: 3600, - SyncMode: true, - SyncIntervalSeconds: 60, - WebSocketEnabled: true, - Origin: "https://plc.directory", - }, - Bundles: server.BundleStatus{ - Count: 100, - FirstBundle: 1, - LastBundle: 100, - TotalSize: 1024000, - UncompressedSize: 5120000, - CompressionRatio: 5.0, - TotalOperations: 1000000, - AvgOpsPerHour: 10000, - UpdatedAt: time.Now(), - HeadAgeSeconds: 30, - RootHash: "root_hash", - HeadHash: "head_hash", - Gaps: 0, - HasGaps: false, - }, - } - - // Should marshal to JSON - data, err := json.Marshal(response) - if err != nil { - t.Fatalf("failed to marshal StatusResponse: %v", err) - } - - // Should unmarshal back - var decoded server.StatusResponse - if err := json.Unmarshal(data, &decoded); err != nil { - t.Fatalf("failed to unmarshal StatusResponse: %v", err) - } - - // Verify round-trip - if decoded.Server.Version != "1.0.0" { - t.Error("version not preserved") - } - - if decoded.Bundles.Count != 100 { - t.Error("bundle count not preserved") - } - }) - - t.Run("MempoolStatus_JSON", func(t *testing.T) { - status := server.MempoolStatus{ - Count: 500, - TargetBundle: 42, - CanCreateBundle: false, - MinTimestamp: time.Now(), - Validated: true, - ProgressPercent: 5.0, - BundleSize: 10000, - OperationsNeeded: 9500, - FirstTime: time.Now().Add(-time.Hour), - LastTime: time.Now(), - TimespanSeconds: 3600, - LastOpAgeSeconds: 10, - EtaNextBundleSeconds: 1800, - } - - data, err := json.Marshal(status) - if err != nil { - t.Fatalf("failed to marshal MempoolStatus: %v", err) - } - - var decoded server.MempoolStatus - if err := json.Unmarshal(data, &decoded); err != nil { - t.Fatalf("failed to unmarshal MempoolStatus: %v", err) - } - - if decoded.Count != 500 { - t.Error("count not preserved") - } - - if decoded.ProgressPercent != 5.0 { - t.Error("progress not preserved") - } - }) - - t.Run("BundleStatus_WithGaps", func(t *testing.T) { - status := server.BundleStatus{ - Count: 100, - Gaps: 3, - HasGaps: true, - GapNumbers: []int{5, 23, 67}, - } - - data, err := json.Marshal(status) - if err != nil { - t.Fatalf("marshal failed: %v", err) - } - - var decoded server.BundleStatus - json.Unmarshal(data, &decoded) - - if !decoded.HasGaps { - t.Error("HasGaps flag not preserved") - } - - if len(decoded.GapNumbers) != 3 { - t.Error("gap numbers not preserved") - } - }) -} diff --git a/server/websocket.go b/server/websocket.go deleted file mode 100644 index 80247d2..0000000 --- a/server/websocket.go +++ /dev/null @@ -1,243 +0,0 @@ -package server - -import ( - "bufio" - "context" - "fmt" - "net/http" - "os" - "strconv" - "time" - - "github.com/goccy/go-json" - "github.com/gorilla/websocket" - "tangled.org/atscan.net/plcbundle/internal/plcclient" - "tangled.org/atscan.net/plcbundle/internal/types" -) - -var upgrader = websocket.Upgrader{ - ReadBufferSize: 1024, - WriteBufferSize: 1024, - CheckOrigin: func(r *http.Request) bool { - return true - }, -} - -func (s *Server) handleWebSocket() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - cursorStr := r.URL.Query().Get("cursor") - var cursor int - - if cursorStr == "" { - cursor = s.manager.GetCurrentCursor() - } else { - var err error - cursor, err = strconv.Atoi(cursorStr) - if err != nil || cursor < 0 { - http.Error(w, "Invalid cursor: must be non-negative integer", 400) - return - } - } - - conn, err := upgrader.Upgrade(w, r, nil) - if err != nil { - fmt.Fprintf(os.Stderr, "WebSocket upgrade failed: %v\n", err) - return - } - defer conn.Close() - - conn.SetPongHandler(func(string) error { - conn.SetReadDeadline(time.Now().Add(60 * time.Second)) - return nil - }) - - done := make(chan struct{}) - - go func() { - defer close(done) - for { - _, _, err := conn.ReadMessage() - if err != nil { - if websocket.IsCloseError(err, websocket.CloseNormalClosure, websocket.CloseGoingAway) { - fmt.Fprintf(os.Stderr, "WebSocket: client closed connection\n") - } - return - } - } - }() - - bgCtx := context.Background() - - if err := s.streamLive(bgCtx, conn, cursor, done); err != nil { - fmt.Fprintf(os.Stderr, "WebSocket stream error: %v\n", err) - } - } -} - -func (s *Server) streamLive(ctx context.Context, conn *websocket.Conn, startCursor int, done chan struct{}) error { - index := s.manager.GetIndex() - bundles := index.GetBundles() - currentRecord := startCursor - - // Stream existing bundles - if len(bundles) > 0 { - startBundleIdx := startCursor / types.BUNDLE_SIZE - startPosition := startCursor % types.BUNDLE_SIZE - - if startBundleIdx < len(bundles) { - for i := startBundleIdx; i < len(bundles); i++ { - skipUntil := 0 - if i == startBundleIdx { - skipUntil = startPosition - } - - newRecordCount, err := s.streamBundle(ctx, conn, bundles[i].BundleNumber, skipUntil, done) - if err != nil { - return err - } - currentRecord += newRecordCount - } - } - } - - lastSeenMempoolCount := 0 - if err := s.streamMempool(conn, startCursor, len(bundles)*types.BUNDLE_SIZE, ¤tRecord, &lastSeenMempoolCount, done); err != nil { - return err - } - - ticker := time.NewTicker(500 * time.Millisecond) - defer ticker.Stop() - - lastBundleCount := len(bundles) - - for { - select { - case <-done: - return nil - - case <-ticker.C: - index = s.manager.GetIndex() - bundles = index.GetBundles() - - if len(bundles) > lastBundleCount { - newBundleCount := len(bundles) - lastBundleCount - currentRecord += newBundleCount * types.BUNDLE_SIZE - lastBundleCount = len(bundles) - lastSeenMempoolCount = 0 - } - - if err := s.streamMempool(conn, startCursor, len(bundles)*types.BUNDLE_SIZE, ¤tRecord, &lastSeenMempoolCount, done); err != nil { - return err - } - - if err := conn.WriteMessage(websocket.PingMessage, nil); err != nil { - return err - } - } - } -} - -func (s *Server) streamBundle(ctx context.Context, conn *websocket.Conn, bundleNumber int, skipUntil int, done chan struct{}) (int, error) { - reader, err := s.manager.StreamBundleDecompressed(ctx, bundleNumber) - if err != nil { - return 0, nil - } - defer reader.Close() - - scanner := bufio.NewScanner(reader) - buf := make([]byte, 0, 64*1024) - scanner.Buffer(buf, 1024*1024) - - position := 0 - streamed := 0 - - for scanner.Scan() { - line := scanner.Bytes() - if len(line) == 0 { - continue - } - - if position < skipUntil { - position++ - continue - } - - select { - case <-done: - return streamed, nil - default: - } - - if err := conn.WriteMessage(websocket.TextMessage, line); err != nil { - return streamed, err - } - - position++ - streamed++ - - if streamed%1000 == 0 { - conn.WriteMessage(websocket.PingMessage, nil) - } - } - - if err := scanner.Err(); err != nil { - return streamed, fmt.Errorf("scanner error on bundle %d: %w", bundleNumber, err) - } - - return streamed, nil -} - -func (s *Server) streamMempool(conn *websocket.Conn, startCursor int, bundleRecordBase int, currentRecord *int, lastSeenCount *int, done chan struct{}) error { - mempoolOps, err := s.manager.GetMempoolOperations() - if err != nil { - return nil - } - - if len(mempoolOps) <= *lastSeenCount { - return nil - } - - for i := *lastSeenCount; i < len(mempoolOps); i++ { - recordNum := bundleRecordBase + i - if recordNum < startCursor { - continue - } - - select { - case <-done: - return nil - default: - } - - if err := sendOperation(conn, mempoolOps[i]); err != nil { - return err - } - *currentRecord++ - } - - *lastSeenCount = len(mempoolOps) - return nil -} - -func sendOperation(conn *websocket.Conn, op plcclient.PLCOperation) error { - var data []byte - var err error - - if len(op.RawJSON) > 0 { - data = op.RawJSON - } else { - data, err = json.Marshal(op) - if err != nil { - return nil - } - } - - if err := conn.WriteMessage(websocket.TextMessage, data); err != nil { - if !websocket.IsCloseError(err, websocket.CloseNormalClosure, websocket.CloseGoingAway) { - fmt.Fprintf(os.Stderr, "WebSocket write error: %v\n", err) - } - return err - } - - return nil -} diff --git a/docs/specification.md b/spec/format.md similarity index 99% rename from docs/specification.md rename to spec/format.md index c229cb0..3bfe90d 100644 --- a/docs/specification.md +++ b/spec/format.md @@ -1,4 +1,4 @@ -# plcbundle V1 (draft) Specification +# plcbundle V1 (draft) Format Specification > ⚠️ **Preview Version - Request for Comments!** diff --git a/spec/http-api.md b/spec/http-api.md new file mode 100644 index 0000000..2e1cfea --- /dev/null +++ b/spec/http-api.md @@ -0,0 +1,3 @@ +# plcbundle V1 HTTP API + +TODO \ No newline at end of file diff --git a/types.go b/types.go deleted file mode 100644 index a8c90fa..0000000 --- a/types.go +++ /dev/null @@ -1,46 +0,0 @@ -package plcbundle - -import ( - "time" - - "tangled.org/atscan.net/plcbundle/internal/plcclient" -) - -// Bundle represents a PLC bundle (public version) -type Bundle struct { - BundleNumber int - StartTime time.Time - EndTime time.Time - Operations []plcclient.PLCOperation - DIDCount int - Hash string - CompressedSize int64 - UncompressedSize int64 -} - -// BundleInfo provides metadata about a bundle -type BundleInfo struct { - BundleNumber int `json:"bundle_number"` - StartTime time.Time `json:"start_time"` - EndTime time.Time `json:"end_time"` - OperationCount int `json:"operation_count"` - DIDCount int `json:"did_count"` - Hash string `json:"hash"` - CompressedSize int64 `json:"compressed_size"` - UncompressedSize int64 `json:"uncompressed_size"` -} - -// IndexStats provides statistics about the bundle index -type IndexStats struct { - BundleCount int - FirstBundle int - LastBundle int - TotalSize int64 - MissingBundles []int -} - -// Helper to convert internal bundle to public -func toBundlePublic(_ interface{}) *Bundle { - // Implement conversion from internal bundle to public Bundle - return &Bundle{} // placeholder -}