diff --git a/README.md b/README.md index 55669d9..919ebc9 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,14 @@ Start the local database: just db-up ``` +Bootstrap the schema once: + +```bash +just api-build +DATABASE_URL="postgresql://localhost/${USER}_dev?sslmode=disable" \ + ./packages/api/twister migrate +``` + Run the mobile app: ```bash @@ -79,9 +87,11 @@ If `ADMIN_AUTH_TOKEN` is present, the smoke script also checks admin status. ## Deployment -Production deployment now uses Coolify plus a separate Coolify-managed -PostgreSQL instance. The backend services are defined in -`docker-compose.prod.yaml`. +`docker-compose.prod.yaml` is now the source-of-truth VPS stack. It runs +`postgres`, `migrate`, `api`, `indexer`, `tap`, and `llama-embeddings`. + +The llama.cpp service is only deployment groundwork for a later embedding +adapter. Search remains keyword-only for now. See [`docs/reference/deployment-walkthrough.md`](docs/reference/deployment-walkthrough.md) for the full setup, bootstrap, backup, and cutover flow. diff --git a/docker-compose.prod.yaml b/docker-compose.prod.yaml index b00f48b..d46f8f1 100644 --- a/docker-compose.prod.yaml +++ b/docker-compose.prod.yaml @@ -1,4 +1,39 @@ services: + postgres: + image: pgvector/pgvector:pg17 + restart: unless-stopped + environment: + POSTGRES_USER: ${POSTGRES_USER:-twisted} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-twisted} + POSTGRES_DB: ${POSTGRES_DB:-twisted} + volumes: + - twisted-postgres:/var/lib/postgresql/data + healthcheck: + test: + [ + "CMD-SHELL", + "pg_isready -h localhost -p 5432 -U ${POSTGRES_USER:-twisted} -d ${POSTGRES_DB:-twisted}", + ] + interval: 10s + timeout: 5s + retries: 5 + expose: + - "5432" + + migrate: + build: + context: ./packages/api + dockerfile: Dockerfile + command: ["twister", "migrate"] + restart: "no" + environment: + DATABASE_URL: ${DATABASE_URL:-postgresql://${POSTGRES_USER:-twisted}:${POSTGRES_PASSWORD:-twisted}@postgres:5432/${POSTGRES_DB:-twisted}?sslmode=disable} + LOG_LEVEL: ${LOG_LEVEL:-info} + LOG_FORMAT: ${LOG_FORMAT:-json} + depends_on: + postgres: + condition: service_healthy + api: build: context: ./packages/api @@ -6,7 +41,7 @@ services: command: ["twister", "api"] restart: unless-stopped environment: - DATABASE_URL: ${DATABASE_URL:?DATABASE_URL is required} + DATABASE_URL: ${DATABASE_URL:-postgresql://${POSTGRES_USER:-twisted}:${POSTGRES_PASSWORD:-twisted}@postgres:5432/${POSTGRES_DB:-twisted}?sslmode=disable} HTTP_BIND_ADDR: ${HTTP_BIND_ADDR:-:8080} LOG_LEVEL: ${LOG_LEVEL:-info} LOG_FORMAT: ${LOG_FORMAT:-json} @@ -21,6 +56,9 @@ services: ADMIN_AUTH_TOKEN: ${ADMIN_AUTH_TOKEN:-} OAUTH_CLIENT_ID: ${OAUTH_CLIENT_ID:-} OAUTH_REDIRECT_URIS: ${OAUTH_REDIRECT_URIS:-} + depends_on: + migrate: + condition: service_completed_successfully expose: - "8080" @@ -31,7 +69,7 @@ services: command: ["twister", "indexer"] restart: unless-stopped environment: - DATABASE_URL: ${DATABASE_URL:?DATABASE_URL is required} + DATABASE_URL: ${DATABASE_URL:-postgresql://${POSTGRES_USER:-twisted}:${POSTGRES_PASSWORD:-twisted}@postgres:5432/${POSTGRES_DB:-twisted}?sslmode=disable} INDEXER_HEALTH_ADDR: ${INDEXER_HEALTH_ADDR:-:9090} LOG_LEVEL: ${LOG_LEVEL:-info} LOG_FORMAT: ${LOG_FORMAT:-json} @@ -39,6 +77,11 @@ services: TAP_AUTH_PASSWORD: ${TAP_AUTH_PASSWORD:?TAP_AUTH_PASSWORD is required} INDEXED_COLLECTIONS: ${INDEXED_COLLECTIONS:?INDEXED_COLLECTIONS is required} ENABLE_INGEST_ENRICHMENT: ${ENABLE_INGEST_ENRICHMENT:-true} + depends_on: + migrate: + condition: service_completed_successfully + tap: + condition: service_started expose: - "9090" @@ -56,5 +99,32 @@ services: expose: - "2480" + llama-embeddings: + image: ghcr.io/ggml-org/llama.cpp:server + restart: unless-stopped + environment: + HF_HOME: /models/hf-cache + LLAMA_CACHE: /models + command: + - --hf-repo + - ${LLAMA_MODEL_REPO:-nomic-ai/nomic-embed-text-v1.5-GGUF} + - --hf-file + - ${LLAMA_MODEL_FILE:-nomic-embed-text-v1.5.Q8_0.gguf} + - --embeddings + - --pooling + - mean + - --ctx-size + - ${LLAMA_CONTEXT_SIZE:-2048} + - --host + - 0.0.0.0 + - --port + - "8080" + volumes: + - twisted-llama:/models + expose: + - "8080" + volumes: + twisted-postgres: tap-data: + twisted-llama: diff --git a/docs/adr/storage.md b/docs/adr/storage.md index 4772b72..d1bf03e 100644 --- a/docs/adr/storage.md +++ b/docs/adr/storage.md @@ -7,15 +7,16 @@ status: accepted ## Decision Twisted will use PostgreSQL as the primary database backend for search, -indexing, queue state, and activity cache. The production deploy target is a -Coolify application for `api`, `indexer`, and `tap` plus a separate -Coolify-managed PostgreSQL instance. +indexing, queue state, and activity cache. The production deploy target is one +Compose stack for `postgres`, `migrate`, `api`, `indexer`, `tap`, and +`llama-embeddings`. Coolify can host that stack, but the Compose file remains +the source of truth. ## Why - PostgreSQL is the better fit for long-running multi-service deployment. -- Coolify gives the project a straightforward Git-to-deploy path with built-in - Traefik and a managed database resource. +- Coolify still gives the project a straightforward Git-to-deploy path with + built-in Traefik when we want it. - The current service shape already wants two long-lived processes writing to one shared database. - A local PostgreSQL workflow keeps development closer to production than the @@ -28,6 +29,7 @@ Coolify-managed PostgreSQL instance. - one mainstream database for local and remote environments - simpler production backups and restore story - easier operational model for `api` and `indexer` +- explicit migration ownership through a one-shot `migrate` command - no dependency on Turso-specific SQLite extension behavior ### Negative @@ -53,7 +55,7 @@ expected to match the previous FTS5 implementation. 1. add PostgreSQL connection/config support and local defaults 2. add a primary PostgreSQL migration set 3. move search and store implementations to PostgreSQL -4. deploy `api`, `indexer`, and `tap` from `docker-compose.prod.yaml` +4. deploy the Compose stack from `docker-compose.prod.yaml` 5. rebuild data through `backfill`, `enrich`, and `reindex` 6. cut traffic over only after smoke checks pass diff --git a/docs/reference/api.md b/docs/reference/api.md index 48aa3e6..4881d3c 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -13,6 +13,7 @@ temporary local SQLite fallback behind `--local`. | --- | --- | | `api` | HTTP API server | | `indexer` | Tap consumer and index writer | +| `migrate` | apply embedded SQL migrations | | `backfill` | register repos with Tap | | `enrich` | fill missing repo names, handles, and web URLs | | `reindex` | re-upsert documents and finalize the search index | @@ -57,6 +58,9 @@ Main tables: `documents` stores a generated weighted `tsvector` column plus a GIN index for keyword search. +No embedding tables are active yet. `llama-embeddings` is deployed only as +infra groundwork for a later semantic-search milestone. + ## Configuration Primary env vars: @@ -88,11 +92,15 @@ Start local Postgres with the repo compose file: ```sh just db-up +just api-build +DATABASE_URL="postgresql://localhost/${USER}_dev?sslmode=disable" \ + ./packages/api/twister migrate just api-dev just api-run-indexer ``` That dev compose file also runs Tap locally at `ws://localhost:2480/channel`. +`api` and `indexer` no longer auto-apply schema changes on startup. Use `just api-dev sqlite` only when you need the temporary SQLite rollback path. @@ -100,9 +108,9 @@ Use `just api-dev sqlite` only when you need the temporary SQLite rollback path. Production uses: -- Coolify Application with `docker-compose.prod.yaml` -- separate Coolify-managed PostgreSQL resource -- private Tap service from the pinned Indigo image -- built-in Coolify Traefik for the public `api` domain +- `docker-compose.prod.yaml` as the VPS stack source of truth +- in-stack PostgreSQL on `pgvector/pgvector:pg17` +- a one-shot `migrate` service before long-lived services start +- private Tap and llama.cpp embedding services on the internal Compose network See `docs/reference/deployment-walkthrough.md` for the full production flow. diff --git a/docs/reference/deployment-walkthrough.md b/docs/reference/deployment-walkthrough.md index b22212b..0304878 100644 --- a/docs/reference/deployment-walkthrough.md +++ b/docs/reference/deployment-walkthrough.md @@ -1,114 +1,107 @@ # Deployment Walkthrough -Twisted deploys to Coolify as one Compose application with three services: +Twisted now deploys as one VPS Compose stack defined by +`docker-compose.prod.yaml`. Coolify can still host that stack, but the Compose +file is the source of truth. +## Services + +- `postgres`: primary database on `pgvector/pgvector:pg17` +- `migrate`: one-shot schema bootstrap via `twister migrate` - `api`: public HTTP service - `indexer`: private Tap consumer - `tap`: private Indigo Tap service - -PostgreSQL is a separate Coolify-managed resource. - -## Files - -- production compose: `docker-compose.prod.yaml` -- local dev compose: `docker-compose.dev.yaml` -- app image build: `packages/api/Dockerfile` -- Tap image: `ghcr.io/bluesky-social/indigo/tap:sha-4f47add43060c27e8a37d9d76482ecddf001fcd8` +- `llama-embeddings`: private llama.cpp server for future embedding work ## Prerequisites -- Coolify access -- one Coolify PostgreSQL resource -- this repo connected to Coolify +- one VPS or Coolify host with Docker Compose +- this repo available to the host - explicit `INDEXED_COLLECTIONS` and `READ_THROUGH_COLLECTIONS` -- one shared Tap admin password +- one shared `TAP_AUTH_PASSWORD` +- a production `POSTGRES_PASSWORD` -## Provision PostgreSQL +Use explicit search collections. Do not use `sh.tangled.*` in production. -Create the PostgreSQL resource first. +## Environment -- keep the generated connection string in Coolify secrets as `DATABASE_URL` -- use PostgreSQL backups from the database resource -- point both `api` and `indexer` at the same database +Required: -## Create The Coolify App +- `POSTGRES_PASSWORD` +- `TAP_AUTH_PASSWORD` +- `INDEXED_COLLECTIONS` -In Coolify: +Common overrides: -1. create a new Application -2. choose the Docker Compose build pack -3. point it at this repo -4. set base directory to `/` -5. set compose file location to `/docker-compose.prod.yaml` +- `POSTGRES_USER` default `twisted` +- `POSTGRES_DB` default `twisted` +- `LOG_LEVEL=info` +- `LOG_FORMAT=json` +- `READ_THROUGH_MODE=missing` +- `READ_THROUGH_COLLECTIONS=` +- `READ_THROUGH_MAX_ATTEMPTS=5` +- `HTTP_BIND_ADDR=:8080` +- `INDEXER_HEALTH_ADDR=:9090` +- `LLAMA_MODEL_REPO=nomic-ai/nomic-embed-text-v1.5-GGUF` +- `LLAMA_MODEL_FILE=nomic-embed-text-v1.5.Q8_0.gguf` -Do not add your own Traefik container. Coolify already provides the proxy. +`llama-embeddings` is private and not part of search yet. It only keeps the +model warm and cached for later adapter work. -## Set Environment Variables +## Bootstrap -Shared: +From the repo root: -- `DATABASE_URL` -- `INDEXED_COLLECTIONS` -- `LOG_LEVEL=info` -- `LOG_FORMAT=json` -- `TAP_AUTH_PASSWORD=` +```sh +just vps-up +``` -`api`: +That sequence: -- `HTTP_BIND_ADDR=:8080` -- `SEARCH_DEFAULT_LIMIT=20` -- `SEARCH_MAX_LIMIT=100` -- `READ_THROUGH_MODE=missing` -- `READ_THROUGH_COLLECTIONS=` -- `READ_THROUGH_MAX_ATTEMPTS=5` -- `ENABLE_ADMIN_ENDPOINTS=false` -- `ADMIN_AUTH_TOKEN=` -- `OAUTH_CLIENT_ID=` -- `OAUTH_REDIRECT_URIS=` +1. starts PostgreSQL and Tap +2. runs `migrate` +3. starts `api`, `indexer`, and `llama-embeddings` -`indexer`: +## Clean Reset -- `INDEXER_HEALTH_ADDR=:9090` -- `TAP_URL=ws://tap:2480/channel` -- `ENABLE_INGEST_ENRICHMENT=true` +For a fresh VM or full reset: -`tap`: +```sh +just vps-reset +``` -- `TAP_COLLECTION_FILTERS=` -- optional persistent volume override if you do not want the default `/data` +This removes named volumes, recreates PostgreSQL, reapplies migrations, and +starts the full stack from zero. -Use explicit search collections. Do not use `sh.tangled.*` in production. +## Coolify Notes + +If you run this through Coolify: -## Domains And Health Checks +1. create one Docker Compose application +2. point it at `/docker-compose.prod.yaml` +3. set the env vars above in Coolify +4. expose only the `api` service through Traefik -Expose only `api` publicly. +Do not add a separate PostgreSQL resource for this stack. -- assign the domain in Coolify to the `api` service -- if `api` stays on `:8080`, include that internal port in the Coolify mapping -- configure readiness checks against `GET /readyz` -- keep `indexer` and `tap` private -- monitor `indexer` with `GET /health` +## Checks -## First Bootstrap +- `api`: `GET /readyz` returns `200` +- `indexer`: `GET /health` returns `200` +- `indexer` can reach `ws://tap:2480/channel` +- `llama-embeddings` serves embeddings on its internal port -1. deploy `tap` -2. deploy `api` -3. deploy `indexer` -4. confirm `api` returns `200` from `/readyz` -5. confirm `indexer` returns `200` from `/health` -6. confirm `indexer` can reach `ws://tap:2480/channel` -7. open a Coolify terminal in the `indexer` service and run: +Then rebuild the serving dataset: ```sh -twister backfill -twister enrich -twister reindex +docker compose -f docker-compose.prod.yaml exec indexer twister backfill +docker compose -f docker-compose.prod.yaml exec indexer twister enrich +docker compose -f docker-compose.prod.yaml exec indexer twister reindex ``` -This rebuilds the serving dataset from authoritative sources. Do not import the -old Turso data as the default migration path. +Do not import old Turso data as the default migration path. -## Point The App At Coolify +## App Target For local app builds: @@ -116,10 +109,7 @@ For local app builds: VITE_TWISTER_API_BASE_URL=https:// ``` -Then run the app normally with `pnpm --dir apps/twisted dev` or `build`. - -## Rollback Notes +## Rollback -- keep the SQLite `--local` path only as a temporary development fallback -- rollback production by restoring PostgreSQL and redeploying the prior app -- treat PostgreSQL restore as the database rollback primitive +- PostgreSQL restore is the rollback primitive +- keep `--local` only as a temporary development fallback diff --git a/docs/reference/resync.md b/docs/reference/resync.md index 1e5b051..42f1fd5 100644 --- a/docs/reference/resync.md +++ b/docs/reference/resync.md @@ -3,17 +3,21 @@ title: Backfill & Resync Playbook updated: 2026-03-26 --- -Twisted has three recovery tools. Choose based on what broke. +Twisted has four recovery tools. Choose based on what broke. | Situation | Recovery path | | --- | --- | | Search results wrong but documents exist | `twister reindex` | | Documents missing because Tap never delivered them | `twister backfill` | | Documents exist but derived metadata is empty or stale | `twister enrich` | -| Full database loss or migration to a fresh PostgreSQL instance | backfill, enrich, reindex | +| Full database loss or migration to a fresh PostgreSQL instance | migrate, backfill, enrich, reindex | ## Commands +### `twister migrate` + +Applies the embedded schema migrations for the configured database. + ### `twister indexer` Runs the Tap consumer continuously. Persists its cursor in `sync_state`. @@ -89,7 +93,7 @@ twister reindex Use this after restoring to a fresh database or moving to a new PostgreSQL instance. -1. start `api` once so migrations run +1. run `twister migrate` 2. start `indexer` 3. run `twister backfill` 4. run `twister enrich` diff --git a/docs/roadmap.md b/docs/roadmap.md index 99619ea..b4bfdae 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -56,6 +56,18 @@ Improve keyword search quality without external dependencies. - [ ] Structured metrics: ingestion rate, search latency, embedding throughput - [ ] Dashboard or log-based monitoring +## API: Embedding Adapter + +**Depends on:** API: Search Stabilization + +- [ ] Add an embedding provider interface in the API +- [ ] Add a `llama.cpp` HTTP adapter targeting `llama-embeddings` +- [ ] Add PostgreSQL embedding schema and jobs using `pgvector` +- [ ] Define source-code chunking and repo re-embed triggers +- [ ] Add `embed` and `reembed` operational commands +- [ ] Add queue-depth, latency, and throughput metrics for embeddings +- [ ] Keep semantic or hybrid retrieval behind a non-default search mode + ## App: Search & Discovery Wire the Explore tab to the search API and add activity feed. diff --git a/docs/specs/search.md b/docs/specs/search.md index eb2c3a5..11fe353 100644 --- a/docs/specs/search.md +++ b/docs/specs/search.md @@ -10,8 +10,9 @@ small set of rebuild tools. - primary storage: PostgreSQL - local default URL: `postgresql://localhost/${USER}_dev?sslmode=disable` -- production deploy target: Coolify application plus managed PostgreSQL +- production deploy target: `docker-compose.prod.yaml` on a VPS or Coolify host - legacy fallback: local SQLite behind `--local` +- `llama-embeddings` is deployed only as future embedding groundwork ## Goals diff --git a/justfile b/justfile index 9200f3f..55bbbde 100644 --- a/justfile +++ b/justfile @@ -49,6 +49,24 @@ api-dev mode="local": api-run-indexer mode="local": just --justfile packages/api/justfile run-indexer {{mode}} +vps-up: + docker compose -f docker-compose.prod.yaml up -d postgres tap + docker compose -f docker-compose.prod.yaml up --build migrate --exit-code-from migrate + docker compose -f docker-compose.prod.yaml up -d --build api indexer llama-embeddings + +vps-down: + docker compose -f docker-compose.prod.yaml down + +vps-migrate: + docker compose -f docker-compose.prod.yaml up -d postgres + docker compose -f docker-compose.prod.yaml up --build migrate --exit-code-from migrate + +vps-reset: + docker compose -f docker-compose.prod.yaml down -v + docker compose -f docker-compose.prod.yaml up -d postgres + docker compose -f docker-compose.prod.yaml up --build migrate --exit-code-from migrate + docker compose -f docker-compose.prod.yaml up -d --build tap api indexer llama-embeddings + db-up: docker compose -f docker-compose.dev.yaml up -d postgres tap diff --git a/packages/api/doc.go b/packages/api/doc.go index 1d8eb6d..eabca3a 100644 --- a/packages/api/doc.go +++ b/packages/api/doc.go @@ -9,6 +9,8 @@ // // cd /Users/owais/Projects/Twisted // just db-up +// just api-build +// DATABASE_URL=postgresql://localhost/${USER}_dev?sslmode=disable ./packages/api/twister migrate // just api-dev // just api-run-indexer // @@ -54,6 +56,7 @@ // // twister api // twister indexer +// twister migrate // twister backfill // twister reindex // twister enrich @@ -61,6 +64,9 @@ // // # Deployment // -// Production uses Coolify for the `api`, `indexer`, and `tap` services plus a -// separate Coolify-managed PostgreSQL resource. See docs/reference/deployment-walkthrough.md. +// Production uses docker-compose.prod.yaml as the source-of-truth VPS stack: +// PostgreSQL, migrate, api, indexer, tap, and llama-embeddings. +// +// Semantic search is still deferred. The llama.cpp service is present only as +// operational groundwork for a later embedding adapter. package main diff --git a/packages/api/internal/embed/embed.go b/packages/api/internal/embed/embed.go deleted file mode 100644 index e92b434..0000000 --- a/packages/api/internal/embed/embed.go +++ /dev/null @@ -1 +0,0 @@ -package embed diff --git a/packages/api/main.go b/packages/api/main.go index 0e94fdf..0cc168a 100644 --- a/packages/api/main.go +++ b/packages/api/main.go @@ -47,6 +47,7 @@ func main() { root.AddCommand( newAPICmd(&local), newIndexerCmd(&local), + newMigrateCmd(&local), newBackfillCmd(&local), newReindexCmd(&local), newEnrichCmd(&local), @@ -89,10 +90,6 @@ func newAPICmd(local *bool) *cobra.Command { } defer db.Close() - if err := store.Migrate(db, cfg.DatabaseURL); err != nil { - return fmt.Errorf("migrate database: %w", err) - } - st := store.New(cfg.DatabaseURL, db) searchRepo := search.NewRepository(cfg.DatabaseURL, db) @@ -147,10 +144,6 @@ func newIndexerCmd(local *bool) *cobra.Command { } defer db.Close() - if err := store.Migrate(db, cfg.DatabaseURL); err != nil { - return fmt.Errorf("migrate database: %w", err) - } - st := store.New(cfg.DatabaseURL, db) registry := normalize.NewRegistry() tap := tapclient.New(cfg.TapURL, cfg.TapAuthPassword, log) @@ -207,6 +200,34 @@ func newIndexerCmd(local *bool) *cobra.Command { } } +func newMigrateCmd(local *bool) *cobra.Command { + return &cobra.Command{ + Use: "migrate", + Short: "Apply database schema migrations", + RunE: func(cmd *cobra.Command, args []string) error { + cfg, err := config.Load(config.LoadOptions{Local: *local}) + if err != nil { + return fmt.Errorf("config: %w", err) + } + log := observability.NewLogger(cfg) + log.Info("starting migrate", slog.String("service", "migrate"), slog.String("version", version)) + + db, err := store.Open(cfg.DatabaseURL) + if err != nil { + return fmt.Errorf("open database: %w", err) + } + defer db.Close() + + if err := store.Migrate(db, cfg.DatabaseURL); err != nil { + return fmt.Errorf("migrate database: %w", err) + } + + log.Info("migrate finished") + return nil + }, + } +} + func newBackfillCmd(local *bool) *cobra.Command { var opts backfill.Options @@ -231,10 +252,6 @@ func newBackfillCmd(local *bool) *cobra.Command { } defer db.Close() - if err := store.Migrate(db, cfg.DatabaseURL); err != nil { - return fmt.Errorf("migrate database: %w", err) - } - tapAdmin, err := backfill.NewHTTPTapAdmin(cfg.TapURL, cfg.TapAuthPassword) if err != nil { return fmt.Errorf("tap admin client: %w", err) @@ -299,10 +316,6 @@ func newReindexCmd(local *bool) *cobra.Command { } defer db.Close() - if err := store.Migrate(db, cfg.DatabaseURL); err != nil { - return fmt.Errorf("migrate database: %w", err) - } - ctx, cancel := baseContext() defer cancel() @@ -347,10 +360,6 @@ func newEnrichCmd(local *bool) *cobra.Command { } defer db.Close() - if err := store.Migrate(db, cfg.DatabaseURL); err != nil { - return fmt.Errorf("migrate database: %w", err) - } - xrpcClient := xrpc.NewClient( xrpc.WithPLCDirectory(cfg.PLCDirectoryURL), xrpc.WithIdentityService(cfg.IdentityServiceURL),