diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 2d962fb..7cbd202 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -38,11 +38,14 @@ jobs: - name: Setup Blacksmith Builder uses: useblacksmith/setup-docker-builder@v1 + with: + platforms: linux/amd64,linux/arm64 - name: Build and push Docker image uses: useblacksmith/build-push-action@v2 with: context: . push: true + platforms: linux/amd64,linux/arm64 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} diff --git a/Dockerfile b/Dockerfile index d569d70..fdb5bbe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM oven/bun:1 AS frontend-builder +FROM --platform=$BUILDPLATFORM oven/bun:1 AS frontend-builder WORKDIR /app/web COPY web/package.json web/bun.lock ./ @@ -6,16 +6,31 @@ RUN bun install COPY web/ ./ RUN bun run build -FROM golang:1.25-alpine AS backend-builder +FROM --platform=$BUILDPLATFORM golang:1.25-alpine AS backend-builder -RUN apk add --no-cache gcc musl-dev +ARG BUILDPLATFORM +ARG TARGETPLATFORM +ARG TARGETOS +ARG TARGETARCH + +RUN apk add --no-cache gcc musl-dev \ + && if [ "$TARGETARCH" = "arm64" ] && [ "$BUILDPLATFORM" != "$TARGETPLATFORM" ]; then \ + apk add --no-cache aarch64-linux-musl-cross; \ + fi WORKDIR /app COPY backend/go.mod backend/go.sum ./ RUN go mod download COPY backend/ ./ -RUN CGO_ENABLED=1 GOOS=linux go build -a -ldflags '-linkmode external -extldflags "-static"' -o margin-server ./cmd/server +RUN if [ "$TARGETARCH" = "arm64" ] && [ "$BUILDPLATFORM" != "$TARGETPLATFORM" ]; then \ + CC=aarch64-linux-musl-gcc \ + CGO_ENABLED=1 GOOS=$TARGETOS GOARCH=$TARGETARCH \ + go build -a -ldflags '-linkmode external -extldflags "-static"' -o margin-server ./cmd/server; \ + else \ + CGO_ENABLED=1 GOOS=$TARGETOS GOARCH=$TARGETARCH \ + go build -a -ldflags '-linkmode external -extldflags "-static"' -o margin-server ./cmd/server; \ + fi FROM node:20-alpine diff --git a/backend/cmd/server/main.go b/backend/cmd/server/main.go index 83ef349..68e4910 100644 --- a/backend/cmd/server/main.go +++ b/backend/cmd/server/main.go @@ -48,14 +48,18 @@ func main() { ticker := time.NewTicker(1 * time.Hour) defer ticker.Stop() - if err := database.DeleteExpiredSessions(); err != nil { - logger.Error("Failed to run initial cleanup of expired sessions: %v", err) + runCleanup := func() { + _, err := database.TryAdvisoryLock(context.Background(), db.LockSessionCleanup, func() error { + return database.DeleteExpiredSessions() + }) + if err != nil { + logger.Error("Failed to run cleanup of expired sessions: %v", err) + } } + runCleanup() for range ticker.C { - if err := database.DeleteExpiredSessions(); err != nil { - logger.Error("Failed to delete expired sessions: %v", err) - } + runCleanup() } }() @@ -92,32 +96,35 @@ func main() { return default: } - logger.Info("Starting recommendation backfill...") - if err := recService.BackfillDocumentEmbeddings(200); err != nil { - logger.Error("Document embedding backfill error: %v", err) - } - if backfillCtx.Err() != nil { - return - } - annCount, err := recService.BackfillAnnotationEmbeddings(200) - if err != nil { - logger.Error("Annotation embedding backfill error: %v", err) - } - if backfillCtx.Err() != nil { - return - } - hlCount, err := recService.BackfillHighlightEmbeddings(200) - if err != nil { - logger.Error("Highlight embedding backfill error: %v", err) - } - if backfillCtx.Err() != nil { - return - } - profileCount, err := recService.RebuildAllProfiles() - if err != nil { - logger.Error("Profile rebuild error: %v", err) - } - logger.Info("Recommendation backfill complete (annotations: %d, highlights: %d, profiles: %d)", annCount, hlCount, profileCount) + database.TryAdvisoryLock(backfillCtx, db.LockBackfill, func() error { //nolint:errcheck + logger.Info("Starting recommendation backfill...") + if err := recService.BackfillDocumentEmbeddings(200); err != nil { + logger.Error("Document embedding backfill error: %v", err) + } + if backfillCtx.Err() != nil { + return nil + } + annCount, err := recService.BackfillAnnotationEmbeddings(200) + if err != nil { + logger.Error("Annotation embedding backfill error: %v", err) + } + if backfillCtx.Err() != nil { + return nil + } + hlCount, err := recService.BackfillHighlightEmbeddings(200) + if err != nil { + logger.Error("Highlight embedding backfill error: %v", err) + } + if backfillCtx.Err() != nil { + return nil + } + profileCount, err := recService.RebuildAllProfiles() + if err != nil { + logger.Error("Profile rebuild error: %v", err) + } + logger.Info("Recommendation backfill complete (annotations: %d, highlights: %d, profiles: %d)", annCount, hlCount, profileCount) + return nil + }) }() } else { logger.Info("Recommendation backfill disabled (DISABLE_BACKFILL is set)") @@ -125,7 +132,10 @@ func main() { } go func() { - if err := ingester.Start(context.Background()); err != nil { + _, err := database.TryAdvisoryLock(context.Background(), db.LockFirehose, func() error { + return ingester.Start(context.Background()) + }) + if err != nil { logger.Error("Firehose ingester error: %v", err) } }() diff --git a/backend/internal/db/db.go b/backend/internal/db/db.go index b4d8552..7d4fdd8 100644 --- a/backend/internal/db/db.go +++ b/backend/internal/db/db.go @@ -1,6 +1,7 @@ package db import ( + "context" "database/sql" "encoding/json" "fmt" @@ -60,6 +61,40 @@ func New(dsn string) (*DB, error) { func (db *DB) Close() error { return db.DB.Close() } +func (db *DB) AdvisoryLock(ctx context.Context, key int64, fn func() error) (bool, error) { + conn, err := db.Conn(ctx) + if err != nil { + return false, fmt.Errorf("advisory lock: acquire conn: %w", err) + } + defer conn.Close() + + if _, err := conn.ExecContext(ctx, "SELECT pg_advisory_lock($1)", key); err != nil { + return false, fmt.Errorf("advisory lock: acquire: %w", err) + } + defer conn.ExecContext(ctx, "SELECT pg_advisory_unlock($1)", key) //nolint:errcheck + + return true, fn() +} + +func (db *DB) TryAdvisoryLock(ctx context.Context, key int64, fn func() error) (bool, error) { + conn, err := db.Conn(ctx) + if err != nil { + return false, fmt.Errorf("try advisory lock: acquire conn: %w", err) + } + defer conn.Close() + + var acquired bool + if err := conn.QueryRowContext(ctx, "SELECT pg_try_advisory_lock($1)", key).Scan(&acquired); err != nil { + return false, fmt.Errorf("try advisory lock: %w", err) + } + if !acquired { + return false, nil + } + defer conn.ExecContext(ctx, "SELECT pg_advisory_unlock($1)", key) //nolint:errcheck + + return true, fn() +} + func ParseSelector(selectorJSON *string) (*Selector, error) { if selectorJSON == nil || *selectorJSON == "" { return nil, nil diff --git a/backend/internal/db/migrate.go b/backend/internal/db/migrate.go index 3542dab..9b9c90c 100644 --- a/backend/internal/db/migrate.go +++ b/backend/internal/db/migrate.go @@ -1,6 +1,7 @@ package db import ( + "context" "database/sql" "embed" "time" @@ -11,14 +12,31 @@ import ( //go:embed migrations var migrationsFS embed.FS +const ( + LockMigrate int64 = 0x6D617267696E + LockFirehose int64 = 0x6669726568736F65 + LockSessionCleanup int64 = 0x73657373696F6E + LockBackfill int64 = 0x6261636B66696C6C +) + func (db *DB) Migrate() error { - goose.SetBaseFS(migrationsFS) + ctx := context.Background() - if err := goose.SetDialect("postgres"); err != nil { + acquired, err := db.TryAdvisoryLock(ctx, LockMigrate, func() error { + goose.SetBaseFS(migrationsFS) + if err := goose.SetDialect("postgres"); err != nil { + return err + } + return goose.Up(db.DB, "migrations") + }) + if err != nil { return err } - - return goose.Up(db.DB, "migrations") + if !acquired { + _, err = db.AdvisoryLock(ctx, LockMigrate, func() error { return nil }) + return err + } + return nil } func (db *DB) GetCursor(id string) (int64, error) {