From 1c1bf4ab301b9ded7a5b20e7a97449a3fbc84564 Mon Sep 17 00:00:00 2001
From: dawn <dawn@tangled.org>
Date: Sun, 4 Oct 2026 21:21:47 +0300
Subject: [PATCH] tapc: batch AddRepos so tap stays under sqlite's variable cap

Signed-off-by: dawn <dawn@tangled.org>
---
 tapc/tap.go      | 37 +++++++++++++--------------
 tapc/tap_test.go | 65 ++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 82 insertions(+), 20 deletions(-)
 create mode 100644 tapc/tap_test.go

diff --git a/tapc/tap.go b/tapc/tap.go
index c44ca19b5..c0e155cbf 100644
--- a/tapc/tap.go
+++ b/tapc/tap.go
@@ -40,35 +40,32 @@ func NewClient(url, adminPassword string) Client {
 	}
 }
 
-func (c *Client) AddRepos(ctx context.Context, dids []syntax.DID) error {
-	body, err := json.Marshal(map[string][]syntax.DID{"dids": dids})
-	if err != nil {
-		return err
-	}
-	req, err := http.NewRequestWithContext(ctx, "POST", c.Url+"/repos/add", bytes.NewReader(body))
-	if err != nil {
-		return err
-	}
-	req.SetBasicAuth("admin", c.AdminPassword)
-	req.Header.Set("Content-Type", "application/json")
+// tap inserts one /repos/add body as a single sqlite statement at about 9
+// variables per did, so a spindle's whole member list blows past sqlite's
+// 32766 cap and tap 500s
+const addReposBatchSize = 1000
 
-	resp, err := c.HTTPClient.Do(req)
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("tap: /repos/add failed with status %d", resp.StatusCode)
+func (c *Client) AddRepos(ctx context.Context, dids []syntax.DID) error {
+	for len(dids) > 0 {
+		n := min(len(dids), addReposBatchSize)
+		if err := c.postDids(ctx, "/repos/add", dids[:n]); err != nil {
+			return err
+		}
+		dids = dids[n:]
 	}
 	return nil
 }
 
 func (c *Client) RemoveRepos(ctx context.Context, dids []syntax.DID) error {
+	return c.postDids(ctx, "/repos/remove", dids)
+}
+
+func (c *Client) postDids(ctx context.Context, path string, dids []syntax.DID) error {
 	body, err := json.Marshal(map[string][]syntax.DID{"dids": dids})
 	if err != nil {
 		return err
 	}
-	req, err := http.NewRequestWithContext(ctx, "POST", c.Url+"/repos/remove", bytes.NewReader(body))
+	req, err := http.NewRequestWithContext(ctx, "POST", c.Url+path, bytes.NewReader(body))
 	if err != nil {
 		return err
 	}
@@ -81,7 +78,7 @@ func (c *Client) RemoveRepos(ctx context.Context, dids []syntax.DID) error {
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("tap: /repos/remove failed with status %d", resp.StatusCode)
+		return fmt.Errorf("tap: %s failed with status %d", path, resp.StatusCode)
 	}
 	return nil
 }
diff --git a/tapc/tap_test.go b/tapc/tap_test.go
new file mode 100644
index 000000000..7f236d233
--- /dev/null
+++ b/tapc/tap_test.go
@@ -0,0 +1,65 @@
+package tapc
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"slices"
+	"testing"
+
+	"github.com/bluesky-social/indigo/atproto/syntax"
+)
+
+func TestAddReposBatchesLargeLists(t *testing.T) {
+	var batches [][]syntax.DID
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var body struct {
+			DIDs []syntax.DID `json:"dids"`
+		}
+		if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+			t.Errorf("decode body: %v", err)
+		}
+		// same failure real tap gives once sqlite runs out of variables
+		if len(body.DIDs) > addReposBatchSize {
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+		batches = append(batches, body.DIDs)
+	}))
+	defer srv.Close()
+
+	dids := make([]syntax.DID, 2*addReposBatchSize+500)
+	for i := range dids {
+		dids[i] = syntax.DID(fmt.Sprintf("did:plc:%024d", i))
+	}
+
+	c := NewClient(srv.URL, "pw")
+	if err := c.AddRepos(context.Background(), dids); err != nil {
+		t.Fatalf("AddRepos: %v", err)
+	}
+
+	if len(batches) != 3 {
+		t.Fatalf("got %d batches, want 3", len(batches))
+	}
+	var sent []syntax.DID
+	for _, b := range batches {
+		sent = append(sent, b...)
+	}
+	if !slices.Equal(sent, dids) {
+		t.Fatalf("sent %d dids, want all %d in order", len(sent), len(dids))
+	}
+}
+
+func TestAddReposEmptySendsNothing(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		t.Errorf("unexpected request to %s", r.URL.Path)
+	}))
+	defer srv.Close()
+
+	c := NewClient(srv.URL, "pw")
+	if err := c.AddRepos(context.Background(), nil); err != nil {
+		t.Fatalf("AddRepos: %v", err)
+	}
+}
-- 
knotmirror 0.0.1