zoekt-webserver-1  | {"SeverityText":"ERROR","Timestamp":1782128343011021640,"InstrumentationScope":"searchShards","Caller":"search/shards.go:949","Function":"github.com/sourcegraph/zoekt/search.logShardCrash","Body":"crashed shard","Resource":{"service.name":"zoekt-webserver","service.instance.id":"9539c8c3cc17"},"Attributes":{"operation":"search","shard":"shard(/data/index/did%3Aplc%3Ayfvb2abqvemh2qawtidbcv7t_v16.00000.zoekt)","query":"substr:\"parse\"","stacktrace":"goroutine 168 [running]:\nruntime/debug.Stack()\n\truntime/debug/stack.go:26 +0x5e\ngithub.com/sourcegraph/zoekt/search.searchOneShard.func1()\n\tgithub.com/sourcegraph/zoekt/search/shards.go:957 +0x85\npanic({0x1327ba0?, 0x220439d068?})\n\truntime/panic.go:860 +0x13a\ngithub.com/sourcegraph/zoekt/index.(*candidateMatch).matchContent(0x22043c8000, {0x4047912249?, 0x414845?, 0x27e2d661bebfa3fc?})\n\tgithub.com/sourcegraph/zoekt/index/matchiter.go:64 +0xcd\ngithub.com/sourcegraph/zoekt/index.(*substrMatchTree).matches(0x22034b1580, 0x2204452160, 0x22043637b8?, 0x30?)\n\tgithub.com/sourcegraph/zoekt/index/matchtree.go:980 +0x14c\ngithub.com/sourcegraph/zoekt/index.evalMatchTree(0x2204452160, 0x2, 0x220439fb60, {0x15b4e70, 0x22034b1580})\n\tgithub.com/sourcegraph/zoekt/index/matchtree.go:929 +0x7a\ngithub.com/sourcegraph/zoekt/index.(*orMatchTree).matches(0x1229760?, 0x2204452160, 0x2, 0x220439fb60)\n\tgithub.com/sourcegraph/zoekt/index/matchtree.go:792 +0x7f\ngithub.com/sourcegraph/zoekt/index.evalMatchTree(0x2204452160, 0x2, 0x220439fb60, {0x15b4ed0, 0x2204222fa8})\n\tgithub.com/sourcegraph/zoekt/index/matchtree.go:929 +0x7a\ngithub.com/sourcegraph/zoekt/index.(*indexData).Search(0x220417c408, {0x15b8118, 0x220349a8c0}, {0x15a9e40, 0x2204222d20}, 0x220418f740)\n\tgithub.com/sourcegraph/zoekt/index/eval.go:277 +0x87a\ngithub.com/sourcegraph/zoekt/search.searchOneShard({0x15b8118, 0x220349a8c0}, {0x15b8838, 0x22042103f0}, {0x15a9e40, 0x2204222d20}, 0x220418f740)\n\tgithub.com/sourcegraph/zoekt/search/shards.go:966 +0x102\ngithub.com/sourcegraph/zoekt/search.streamSearch.func3()\n\tgithub.com/sourcegraph/zoekt/search/shards.go:740 +0x11e\ncreated by github.com/sourcegraph/zoekt/search.streamSearch in goroutine 164\n\tgithub.com/sourcegraph/zoekt/search/shards.go:737 +0x546\n","error":"runtime error: slice bounds out of range [14514462:19654]"}}
zoekt-webserver-1  | unexpected fault address 0xe87be500e87bca
zoekt-webserver-1  | fatal error: fault
zoekt-webserver-1  | [signal SIGSEGV: segmentation violation code=0x1 addr=0xe87be500e87bca pc=0x1039819]
zoekt-webserver-1  |
zoekt-webserver-1  | goroutine 164 gp=0x2203281680 m=3 mp=0x220328a008 [running]:
zoekt-webserver-1  | runtime.throw({0x13d9ad8?, 0x22043dcdd0?})
zoekt-webserver-1  | 	runtime/panic.go:1229 +0x48 fp=0x22043dcd08 sp=0x22043dccd8 pc=0x48d528
zoekt-webserver-1  | runtime.sigpanic()
zoekt-webserver-1  | 	runtime/signal_unix.go:945 +0x285 fp=0x22043dcd68 sp=0x22043dcd08 pc=0x48f885
zoekt-webserver-1  | github.com/sourcegraph/zoekt/search.sendByRepository(0x22043cc000?, 0x22043dcfb8?, {0x15aa160?, 0x2203fea6e0?})
zoekt-webserver-1  | 	github.com/sourcegraph/zoekt/search/shards.go:832 +0x39 fp=0x22043dcf70 sp=0x22043dcd68 pc=0x1039819
zoekt-webserver-1  | github.com/sourcegraph/zoekt/search.streamSearch({0x15b80a8, 0x2204256320}, 0x2204222ed0, {0x15a9e40, 0x2204222d20}, 0x220418f740, {0x220328d810, 0x1, 0x1}, {0x15aa160, ...})
zoekt-webserver-1  | 	github.com/sourcegraph/zoekt/search/shards.go:818 +0xa6a fp=0x22043dd130 sp=0x22043dcf70 pc=0x103926a
zoekt-webserver-1  | github.com/sourcegraph/zoekt/search.(*shardedSearcher).Search(0x22041f3d00, {0x15b8070?, 0x22046049c0?}, {0x15a9e40, 0x2204222d20}, 0x220418f740)
zoekt-webserver-1  | 	github.com/sourcegraph/zoekt/search/shards.go:568 +0x348 fp=0x22043dd258 sp=0x22043dd130 pc=0x10378c8
zoekt-webserver-1  | github.com/sourcegraph/zoekt/search.(*directorySearcher).Search(0x15b8070?, {0x15b8070?, 0x22046049c0?}, {0x15a9e40?, 0x2204222d20?}, 0x2204222d20?)
zoekt-webserver-1  | 	<autogenerated>:1 +0x2f fp=0x22043dd298 sp=0x22043dd258 pc=0x104014f
zoekt-webserver-1  | github.com/sourcegraph/zoekt/search.(*typeRepoSearcher).Search(0x22041fcad0, {0x15b8070?, 0x2204604900?}, {0x15a9e40, 0x2204222d20}, 0x220418f740)
zoekt-webserver-1  | 	github.com/sourcegraph/zoekt/search/eval.go:41 +0x22b fp=0x22043dd338 sp=0x22043dd298 pc=0x1032bcb
zoekt-webserver-1  | main.(*loggedSearcher).Search(0x2203ff83a0, {0x15b8070, 0x2204604900}, {0x15a9e40, 0x2204222d20}, 0x220418f740)
zoekt-webserver-1  | 	github.com/sourcegraph/zoekt/cmd/zoekt-webserver/main.go:503 +0x115 fp=0x22043dd3e8 sp=0x22043dd338 pc=0x104de95
zoekt-webserver-1  | github.com/sourcegraph/zoekt/web.traceAwareSearcher.Search({{0x15bd660?, 0x2203ff83a0?}}, {0x15b80a8?, 0x2204256230?}, {0x15a9e40, 0x2204222d20}, 0x1f?)
zoekt-webserver-1  | 	github.com/sourcegraph/zoekt/web/trace.go:40 +0x297 fp=0x22043dd4d8 sp=0x22043dd3e8 pc=0x10488d7
zoekt-webserver-1  | github.com/sourcegraph/zoekt/web.(*traceAwareSearcher).Search(0x15b80a8?, {0x15b80a8?, 0x2204256230?}, {0x15a9e40?, 0x2204222d20?}, 0x22041fcb80?)
zoekt-webserver-1  | 	<autogenerated>:1 +0x48 fp=0x22043dd520 sp=0x22043dd4d8 pc=0x10490a8
zoekt-webserver-1  | github.com/sourcegraph/zoekt/internal/json.(*jsonSearcher).jsonSearch(0x22041fcb90, {0x15b4900, 0x22042541e0}, 0x22040ed7c0)
zoekt-webserver-1  | 	github.com/sourcegraph/zoekt/internal/json/json.go:93 +0x573 fp=0x22043dd678 sp=0x22043dd520 pc=0x1040bd3
zoekt-webserver-1  | github.com/sourcegraph/zoekt/internal/json.(*jsonSearcher).jsonSearch-fm({0x15b4900?, 0x22042541e0?}, 0x22043dd6e8?)
zoekt-webserver-1  | 	<autogenerated>:1 +0x2d fp=0x22043dd6a8 sp=0x22043dd678 pc=0x104164d
zoekt-webserver-1  | net/http.HandlerFunc.ServeHTTP(0x22032b15c0?, {0x15b4900?, 0x22042541e0?}, 0x22043dd710?)
zoekt-webserver-1  | 	net/http/server.go:2286 +0x29 fp=0x22043dd6d0 sp=0x22043dd6a8 pc=0x782389
zoekt-webserver-1  | net/http.(*ServeMux).ServeHTTP(0x0?, {0x15b4900, 0x22042541e0}, 0x22040ed7c0)
zoekt-webserver-1  | 	net/http/server.go:2828 +0x1c7 fp=0x22043dd720 sp=0x22043dd6d0 pc=0x784187
zoekt-webserver-1  | github.com/sourcegraph/zoekt/web.NewMux.StripPrefix.func1({0x15b4900, 0x22042541e0}, 0x22040ed540)
zoekt-webserver-1  | 	net/http/server.go:2348 +0x2be fp=0x22043dd7a8 sp=0x22043dd720 pc=0x10432de
zoekt-webserver-1  | net/http.HandlerFunc.ServeHTTP(0x22032b1080?, {0x15b4900?, 0x22042541e0?}, 0x22034b0680?)
zoekt-webserver-1  | 	net/http/server.go:2286 +0x29 fp=0x22043dd7d0 sp=0x22043dd7a8 pc=0x782389
zoekt-webserver-1  | net/http.(*ServeMux).ServeHTTP(0x22043dd848?, {0x15b4900, 0x22042541e0}, 0x22040ed540)
zoekt-webserver-1  | 	net/http/server.go:2828 +0x1c7 fp=0x22043dd820 sp=0x22043dd7d0 pc=0x784187
zoekt-webserver-1  | main.main.Middleware.func2({0x15b4900, 0x22042541e0}, 0x22040ed540)
zoekt-webserver-1  | 	github.com/sourcegraph/zoekt/internal/trace/middleware.go:19 +0xc2 fp=0x22043dd878 sp=0x22043dd820 pc=0x104c102
zoekt-webserver-1  | net/http.HandlerFunc.ServeHTTP(0x13f5224?, {0x15b4900?, 0x22042541e0?}, 0x22043dd880?)
zoekt-webserver-1  | 	net/http/server.go:2286 +0x29 fp=0x22043dd8a0 sp=0x22043dd878 pc=0x782389
zoekt-webserver-1  | main.main.MultiplexGRPC.func6({0x15b4900?, 0x22042541e0?}, 0x0?)
zoekt-webserver-1  | 	github.com/sourcegraph/zoekt/grpc/grpcutil/util.go:31 +0xa5 fp=0x22043dd8e8 sp=0x22043dd8a0 pc=0x104be85
zoekt-webserver-1  | net/http.HandlerFunc.ServeHTTP(0x2204604330?, {0x15b4900?, 0x22042541e0?}, 0x22043dd988?)
zoekt-webserver-1  | 	net/http/server.go:2286 +0x29 fp=0x22043dd910 sp=0x22043dd8e8 pc=0x782389
zoekt-webserver-1  | golang.org/x/net/http2/h2c.h2cHandler.ServeHTTP({{0x15a7ac0?, 0x2203ff8780?}, 0x220418ed20?}, {0x15b4900, 0x22042541e0}, 0x22040ed540)
zoekt-webserver-1  | 	golang.org/x/net@v0.53.0/http2/h2c/h2c.go:125 +0x602 fp=0x22043ddb20 sp=0x22043dd910 pc=0xd00922
zoekt-webserver-1  | golang.org/x/net/http2/h2c.(*h2cHandler).ServeHTTP(0x4259e5?, {0x15b4900?, 0x22042541e0?}, 0x2204254101?)
zoekt-webserver-1  | 	<autogenerated>:1 +0x3f fp=0x22043ddb60 sp=0x22043ddb20 pc=0xd0161f
zoekt-webserver-1  | net/http.serverHandler.ServeHTTP({0x15b0950?}, {0x15b4900?, 0x22042541e0?}, 0x6?)
zoekt-webserver-1  | 	net/http/server.go:3311 +0x8e fp=0x22043ddb90 sp=0x22043ddb60 pc=0x7a2a4e
zoekt-webserver-1  | net/http.(*conn).serve(0x2204518480, {0x15b8070, 0x22042102d0})
zoekt-webserver-1  | 	net/http/server.go:2073 +0x650 fp=0x22043ddfb8 sp=0x22043ddb90 pc=0x780470
zoekt-webserver-1  | net/http.(*Server).Serve.gowrap3()
zoekt-webserver-1  | 	net/http/server.go:3464 +0x1f fp=0x22043ddfe0 sp=0x22043ddfb8 pc=0x78605f
zoekt-webserver-1  | runtime.goexit({})
zoekt-webserver-1  | 	runtime/asm_amd64.s:1771 +0x1 fp=0x22043ddfe8 sp=0x22043ddfe0 pc=0x495261
zoekt-webserver-1  | created by net/http.(*Server).Serve in goroutine 79
zoekt-webserver-1  | 	net/http/server.go:3464 +0x485
zoekt-webserver-1  |
zoekt-webserver-1  | goroutine 1 gp=0x22031ec1e0 m=nil [chan receive, 6 minutes]:
zoekt-webserver-1  | runtime.gopark(0x424f01?, 0x1?, 0xc0?, 0xba?, 0x22042e9a58?)
zoekt-webserver-1  | 	runtime/proc.go:462 +0xce fp=0x22040919d8 sp=0x22040919b8 pc=0x48d64e
zoekt-webserver-1  | runtime.chanrecv(0x22034900e0, 0x0, 0x1)
zoekt-webserver-1  | 	runtime/chan.go:667 +0x4ae fp=0x2204091a50 sp=0x22040919d8 pc=0x41c66e
zoekt-webserver-1  | runtime.chanrecv1(0x22031ec1e0?, 0x104b6b9?)
zoekt-webserver-1  | 	runtime/chan.go:509 +0x12 fp=0x2204091a78 sp=0x2204091a50 pc=0x41c192
zoekt-webserver-1  | main.closeOnSignal(0x220413fb00)
zoekt-webserver-1  | 	github.com/sourcegraph/zoekt/cmd/zoekt-webserver/main.go:363 +0x25 fp=0x2204091a98 sp=0x2204091a78 pc=0x104c8e5
zoekt-webserver-1  | main.main()
zoekt-webserver-1  | 	github.com/sourcegraph/zoekt/cmd/zoekt-webserver/main.go:321 +0x1434 fp=0x2204091f48 sp=0x2204091a98 pc=0x104b6d4
zoekt-webserver-1  | runtime.main()
zoekt-webserver-1  | 	runtime/proc.go:290 +0x2d5 fp=0x2204091fe0 sp=0x2204091f48 pc=0x456615
zoekt-webserver-1  | runtime.goexit({})
zoekt-webserver-1  | 	runtime/asm_amd64.s:1771 +0x1 fp=0x2204091fe8 sp=0x2204091fe0 pc=0x495261
zoekt-webserver-1  |
zoekt-webserver-1  | goroutine 17 gp=0x2203280000 m=nil [force gc (idle), 2 minutes]:
zoekt-webserver-1  | runtime.gopark(0xc28056f94fae?, 0x0?, 0x0?, 0x0?, 0x0?)
zoekt-webserver-1  | 	runtime/proc.go:462 +0xce fp=0x22032407a8 sp=0x2203240788 pc=0x48d64e
zoekt-webserver-1  | runtime.goparkunlock(...)
zoekt-webserver-1  | 	runtime/proc.go:468
zoekt-webserver-1  | runtime.forcegchelper()
zoekt-webserver-1  | 	runtime/proc.go:375 +0xb3 fp=0x22032407e0 sp=0x22032407a8 pc=0x456933
zoekt-webserver-1  | runtime.goexit({})
zoekt-webserver-1  | 	runtime/asm_amd64.s:1771 +0x1 fp=0x22032407e8 sp=0x22032407e0 pc=0x495261