Something went wrong. Try again.
an attempt to make a lightweight, easily self-hostable, scoped bluesky appview
Something went wrong. Try again.
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167import { indexHandlerContext } from "./index/types.ts";
import { assertRecord, validateRecord } from "./utils/records.ts";import {buildBlobUrl, getSlingshotRecord, resolveIdentity, searchParamsToJson, withCors,} from "./utils/server.ts";import * as IndexServerTypes from "./utils/indexservertypes.ts";import { Database } from "jsr:@db/sqlite@0.11";import { setupUserDb } from "./utils/dbuser.ts";// import { systemDB } from "./env.ts";import { JetstreamManager, SpacedustManager } from "./utils/sharders.ts";import { handleSpacedust, SpacedustLinkMessage } from "./index/spacedust.ts";import { handleJetstream } from "./index/jetstream.ts";import * as ATPAPI from "npm:@atproto/api";import { AtUri } from "npm:@atproto/api";import * as IndexServerAPI from "./indexclient/index.ts";import * as IndexServerUtils from "./indexclient/util.ts";import { isPostView } from "./indexclient/types/app/bsky/feed/defs.ts";
export interface IndexServerConfig { baseDbPath: string; systemDbPath: string;}
interface BaseRow { uri: string; did: string; cid: string | null; rev: string | null; createdat: number | null; indexedat: number; json: string | null;}interface GeneratorRow extends BaseRow { displayname: string | null; description: string | null; avatarcid: string | null;}interface LikeRow extends BaseRow { subject: string;}interface RepostRow extends BaseRow { subject: string;}interface BacklinkRow { srcuri: string; srcdid: string;}
const FEED_LIMIT = 50;
export class IndexServer { private config: IndexServerConfig; public userManager: IndexServerUserManager; public systemDB: Database;
constructor(config: IndexServerConfig) { this.config = config;
// We will initialize the system DB and user manager here this.systemDB = new Database(this.config.systemDbPath); // TODO: We need to setup the system DB schema if it's new
this.userManager = new IndexServerUserManager(this); // Pass the server instance }
public start() { // This is where we'll kick things off, like the cold start this.userManager.coldStart(this.systemDB); console.log("IndexServer started."); }
public async handleRequest(req: Request): Promise<Response> { const url = new URL(req.url); // We will add routing logic here later to call our handlers if (url.pathname.startsWith("/xrpc/")) { return this.indexServerHandler(req); } if (url.pathname.startsWith("/links")) { return this.constellationAPIHandler(req); } return new Response("Not Found", { status: 404 }); }
public handlesDid(did: string): boolean { return this.userManager.handlesDid(did); } async unspeccedGetRegisteredUsers(): Promise<{ did: string; role: string; registrationdate: string; onboardingstatus: string; pfp?: string; displayname: string; handle: string; }[]|undefined> { const stmt = this.systemDB.prepare(` SELECT * FROM users; `); const result = stmt.all() as { did: string; role: string; registrationdate: string; onboardingstatus: string; }[]; const hydrated = await Promise.all( result.map(async (user)=>{ const identity = await resolveIdentity(user.did); const profile = (await getSlingshotRecord(identity.did,"app.bsky.actor.profile","self")).value as ATPAPI.AppBskyActorProfile.Record; const avatarcid = uncid(profile.avatar?.ref); const avatar = avatarcid ? buildBlobUrl(identity.pds, identity.did, avatarcid) : undefined; return {...user,handle: identity.handle,pfp: avatar, displayname:profile.displayName ?? identity.handle } })) //const exists = result !== undefined; return hydrated; }
// We will move all the global functions into this class as methods... async indexServerHandler(req: Request): Promise<Response> { const url = new URL(req.url); const pathname = url.pathname; //const bskyUrl = `https://api.bsky.app${pathname}${url.search}`; //const hasAuth = req.headers.has("authorization"); const xrpcMethod = pathname.startsWith("/xrpc/") ? pathname.slice("/xrpc/".length) : null; const searchParams = searchParamsToJson(url.searchParams); console.log(JSON.stringify(searchParams, null, 2)); const jsonUntyped = searchParams;
switch (xrpcMethod) { case "app.bsky.actor.getProfile": { const jsonTyped = jsonUntyped as IndexServerTypes.AppBskyActorGetProfile.QueryParams;
const res = await this.queryProfileView(jsonTyped.actor, "Detailed"); if (!res) return new Response( JSON.stringify({ error: "User not found", }), { status: 404, headers: withCors({ "Content-Type": "application/json" }), } ); const response: IndexServerTypes.AppBskyActorGetProfile.OutputSchema = res;
return new Response(JSON.stringify(response), { headers: withCors({ "Content-Type": "application/json" }), }); } case "app.bsky.actor.getProfiles": { const jsonTyped = jsonUntyped as IndexServerTypes.AppBskyActorGetProfiles.QueryParams;
if (typeof jsonUntyped?.actors === "string") { const res = await this.queryProfileView( jsonUntyped.actors as string, "Detailed" ); if (!res) return new Response( JSON.stringify({ error: "User not found", }), { status: 404, headers: withCors({ "Content-Type": "application/json" }), } ); const response: IndexServerTypes.AppBskyActorGetProfiles.OutputSchema = { profiles: [res], };
return new Response(JSON.stringify(response), { headers: withCors({ "Content-Type": "application/json" }), }); }
const res: ATPAPI.AppBskyActorDefs.ProfileViewDetailed[] = await Promise.all( jsonTyped.actors .map(async (actor) => { return await this.queryProfileView(actor, "Detailed"); }) .filter( ( x ): x is Promise<ATPAPI.AppBskyActorDefs.ProfileViewDetailed> => x !== undefined ) );
if (!res) return new Response( JSON.stringify({ error: "User not found", }), { status: 404, headers: withCors({ "Content-Type": "application/json" }), } );
const response: IndexServerTypes.AppBskyActorGetProfiles.OutputSchema = { profiles: res, };
return new Response(JSON.stringify(response), { headers: withCors({ "Content-Type": "application/json" }), }); } case "app.bsky.feed.getActorFeeds": { const jsonTyped = jsonUntyped as IndexServerTypes.AppBskyFeedGetActorFeeds.QueryParams;
const qresult = await this.queryActorFeeds(jsonTyped.actor);
const response: IndexServerTypes.AppBskyFeedGetActorFeeds.OutputSchema = { feeds: qresult, };
return new Response(JSON.stringify(response), { headers: withCors({ "Content-Type": "application/json" }), }); } case "app.bsky.feed.getFeedGenerator": { const jsonTyped = jsonUntyped as IndexServerTypes.AppBskyFeedGetFeedGenerator.QueryParams;
const qresult = await this.queryFeedGenerator(jsonTyped.feed); if (!qresult) { return new Response( JSON.stringify({ error: "Feed not found", }), { status: 404, headers: withCors({ "Content-Type": "application/json" }), } ); }
const response: IndexServerTypes.AppBskyFeedGetFeedGenerator.OutputSchema = { view: qresult, isOnline: true, // lmao isValid: true, // lmao };
return new Response(JSON.stringify(response), { headers: withCors({ "Content-Type": "application/json" }), }); } case "app.bsky.feed.getFeedGenerators": { const jsonTyped = jsonUntyped as IndexServerTypes.AppBskyFeedGetFeedGenerators.QueryParams;
const qresult = await this.queryFeedGenerators(jsonTyped.feeds); if (!qresult) { return new Response( JSON.stringify({ error: "Feed not found", }), { status: 404, headers: withCors({ "Content-Type": "application/json" }), } ); }
const response: IndexServerTypes.AppBskyFeedGetFeedGenerators.OutputSchema = { feeds: qresult, };
return new Response(JSON.stringify(response), { headers: withCors({ "Content-Type": "application/json" }), }); } case "app.bsky.feed.getPosts": { const jsonTyped = jsonUntyped as IndexServerTypes.AppBskyFeedGetPosts.QueryParams;
const posts: IndexServerTypes.AppBskyFeedGetPosts.OutputSchema["posts"] = ( await Promise.all( jsonTyped.uris.map((uri) => this.queryPostView(uri)) ) ).filter((p): p is ATPAPI.AppBskyFeedDefs.PostView => Boolean(p));
const response: IndexServerTypes.AppBskyFeedGetPosts.OutputSchema = { posts, };
return new Response(JSON.stringify(response), { headers: withCors({ "Content-Type": "application/json" }), }); } case "party.whey.app.bsky.feed.getActorLikesPartial": { const jsonTyped = jsonUntyped as IndexServerTypes.PartyWheyAppBskyFeedGetActorLikesPartial.QueryParams;
// TODO: not partial yet, currently skips refs
const qresult = await this.queryActorLikesPartial( jsonTyped.actor, jsonTyped.cursor ); if (!qresult) { return new Response( JSON.stringify({ error: "Feed not found", }), { status: 404, headers: withCors({ "Content-Type": "application/json" }), } ); }
const response: IndexServerTypes.PartyWheyAppBskyFeedGetActorLikesPartial.OutputSchema = { feed: qresult.items as ATPAPI.$Typed<ATPAPI.AppBskyFeedDefs.FeedViewPost>[], cursor: qresult.cursor, };
return new Response(JSON.stringify(response), { headers: withCors({ "Content-Type": "application/json" }), }); } case "party.whey.app.bsky.feed.getAuthorFeedPartial": { const jsonTyped = jsonUntyped as IndexServerTypes.PartyWheyAppBskyFeedGetAuthorFeedPartial.QueryParams;
// TODO: not partial yet, currently skips refs
const qresult = await this.queryAuthorFeedPartial( jsonTyped.actor, jsonTyped.cursor ); if (!qresult) { return new Response( JSON.stringify({ error: "Feed not found", }), { status: 404, headers: withCors({ "Content-Type": "application/json" }), } ); }
const response: IndexServerTypes.PartyWheyAppBskyFeedGetAuthorFeedPartial.OutputSchema = { feed: qresult.items as ATPAPI.$Typed<ATPAPI.AppBskyFeedDefs.FeedViewPost>[], cursor: qresult.cursor, };
return new Response(JSON.stringify(response), { headers: withCors({ "Content-Type": "application/json" }), }); } case "party.whey.app.bsky.feed.getLikesPartial": { const jsonTyped = jsonUntyped as IndexServerTypes.PartyWheyAppBskyFeedGetLikesPartial.QueryParams;
// TODO: not partial yet, currently skips refs
const qresult = this.queryLikes(jsonTyped.uri); if (!qresult) { return new Response( JSON.stringify({ error: "Feed not found", }), { status: 404, headers: withCors({ "Content-Type": "application/json" }), } ); } const response: IndexServerTypes.PartyWheyAppBskyFeedGetLikesPartial.OutputSchema = { // @ts-ignore whatever i dont care TODO: fix ts ignores likes: qresult, };
return new Response(JSON.stringify(response), { headers: withCors({ "Content-Type": "application/json" }), }); } case "party.whey.app.bsky.feed.getPostThreadPartial": { const jsonTyped = jsonUntyped as IndexServerTypes.PartyWheyAppBskyFeedGetPostThreadPartial.QueryParams;
// TODO: not partial yet, currently skips refs
const qresult = await this.queryPostThreadPartial(jsonTyped.uri); if (!qresult) { return new Response( JSON.stringify({ error: "Feed not found", }), { status: 404, headers: withCors({ "Content-Type": "application/json" }), } ); } const response: IndexServerTypes.PartyWheyAppBskyFeedGetPostThreadPartial.OutputSchema = qresult;
return new Response(JSON.stringify(response), { headers: withCors({ "Content-Type": "application/json" }), }); } case "party.whey.app.bsky.feed.getQuotesPartial": { const jsonTyped = jsonUntyped as IndexServerTypes.PartyWheyAppBskyFeedGetQuotesPartial.QueryParams;
// TODO: not partial yet, currently skips refs
const qresult = await this.queryQuotes(jsonTyped.uri); if (!qresult) { return new Response( JSON.stringify({ error: "Feed not found", }), { status: 404, headers: withCors({ "Content-Type": "application/json" }), } ); } const response: IndexServerTypes.PartyWheyAppBskyFeedGetQuotesPartial.OutputSchema = { uri: jsonTyped.uri, posts: qresult.map((feedviewpost) => { return feedviewpost.post as ATPAPI.$Typed<ATPAPI.AppBskyFeedDefs.PostView>; }), };
return new Response(JSON.stringify(response), { headers: withCors({ "Content-Type": "application/json" }), }); } case "party.whey.app.bsky.feed.getRepostedByPartial": { const jsonTyped = jsonUntyped as IndexServerTypes.PartyWheyAppBskyFeedGetRepostedByPartial.QueryParams;
// TODO: not partial yet, currently skips refs
const qresult = await this.queryReposts(jsonTyped.uri); if (!qresult) { return new Response( JSON.stringify({ error: "Feed not found", }), { status: 404, headers: withCors({ "Content-Type": "application/json" }), } ); } const response: IndexServerTypes.PartyWheyAppBskyFeedGetRepostedByPartial.OutputSchema = { uri: jsonTyped.uri, repostedBy: qresult as ATPAPI.$Typed<ATPAPI.AppBskyActorDefs.ProfileView>[], };
return new Response(JSON.stringify(response), { headers: withCors({ "Content-Type": "application/json" }), }); } // TODO: too hard for now // case "party.whey.app.bsky.feed.getListFeedPartial": { // const jsonTyped = // jsonUntyped as IndexServerTypes.PartyWheyAppBskyFeedGetListFeedPartial.QueryParams;
// const response: IndexServerTypes.PartyWheyAppBskyFeedGetListFeedPartial.OutputSchema = // {};
// return new Response(JSON.stringify(response), { // headers: withCors({ "Content-Type": "application/json" }), // }); // } /* three more coming soon app.bsky.graph.getLists app.bsky.graph.getList app.bsky.graph.getActorStarterPacks */ default: { return new Response( JSON.stringify({ error: "XRPCNotSupported", message: "HEY hello there my name is whey dot party and you have used my custom appview that is very cool but have you considered that XRPC Not Supported", }), { status: 404, headers: withCors({ "Content-Type": "application/json" }), } ); } }
// return new Response("Not Found", { status: 404 }); }
constellationAPIHandler(req: Request): Response { const url = new URL(req.url); const pathname = url.pathname; const searchParams = searchParamsToJson(url.searchParams) as linksQuery; const jsonUntyped = searchParams;
if (!jsonUntyped.target) { return new Response( JSON.stringify({ error: "Missing required parameter: target" }), { status: 400, headers: withCors({ "Content-Type": "application/json" }), } ); }
const did = isDid(searchParams.target) ? searchParams.target : new AtUri(searchParams.target).host; const db = this.userManager.getDbForDid(did); if (!db) { return new Response( JSON.stringify({ error: "User not found", }), { status: 404, headers: withCors({ "Content-Type": "application/json" }), } ); }
const limit = 16; //Math.min(parseInt(searchParams.limit || "50", 10), 100); const offset = parseInt(searchParams.cursor || "0", 10);
switch (pathname) { case "/links": { const jsonTyped = jsonUntyped as linksQuery; if (!jsonTyped.collection || !jsonTyped.path) { return new Response( JSON.stringify({ error: "Missing required parameters: collection, path", }), { status: 400, headers: withCors({ "Content-Type": "application/json" }), } ); }
const field = `${jsonTyped.collection}:${jsonTyped.path.replace( /^\./, "" )}`;
const paginatedSql = `${SQL.links} LIMIT ? OFFSET ?`; const rows = db .prepare(paginatedSql) .all(jsonTyped.target, jsonTyped.collection, field, limit, offset);
const countResult = db .prepare(SQL.count) .get(jsonTyped.target, jsonTyped.collection, field); const total = countResult ? Number(countResult.total) : 0;
const linking_records: linksRecord[] = rows.map((row: any) => { const rkey = row.srcuri.split("/").pop()!; return { did: row.srcdid, collection: row.srccol, rkey, }; });
const response: linksRecordsResponse = { total: total.toString(), linking_records, };
const nextCursor = offset + linking_records.length; if (nextCursor < total) { response.cursor = nextCursor.toString(); }
return new Response(JSON.stringify(response), { headers: withCors({ "Content-Type": "application/json" }), }); } case "/links/distinct-dids": { const jsonTyped = jsonUntyped as linksQuery; if (!jsonTyped.collection || !jsonTyped.path) { return new Response( JSON.stringify({ error: "Missing required parameters: collection, path", }), { status: 400, headers: withCors({ "Content-Type": "application/json" }), } ); }
const field = `${jsonTyped.collection}:${jsonTyped.path.replace( /^\./, "" )}`;
const paginatedSql = `${SQL.distinctDids} LIMIT ? OFFSET ?`; const rows = db .prepare(paginatedSql) .all(jsonTyped.target, jsonTyped.collection, field, limit, offset);
const countResult = db .prepare(SQL.countDistinctDids) .get(jsonTyped.target, jsonTyped.collection, field); const total = countResult ? Number(countResult.total) : 0;
const linking_dids: string[] = rows.map((row: any) => row.srcdid);
const response: linksDidsResponse = { total: total.toString(), linking_dids, };
const nextCursor = offset + linking_dids.length; if (nextCursor < total) { response.cursor = nextCursor.toString(); }
return new Response(JSON.stringify(response), { headers: withCors({ "Content-Type": "application/json" }), }); } case "/links/count": { const jsonTyped = jsonUntyped as linksQuery; if (!jsonTyped.collection || !jsonTyped.path) { return new Response( JSON.stringify({ error: "Missing required parameters: collection, path", }), { status: 400, headers: withCors({ "Content-Type": "application/json" }), } ); }
const field = `${jsonTyped.collection}:${jsonTyped.path.replace( /^\./, "" )}`;
const result = db .prepare(SQL.count) .get(jsonTyped.target, jsonTyped.collection, field);
const response: linksCountResponse = { total: result && result.total ? result.total.toString() : "0", };
return new Response(JSON.stringify(response), { headers: withCors({ "Content-Type": "application/json" }), }); } case "/links/count/distinct-dids": { const jsonTyped = jsonUntyped as linksQuery; if (!jsonTyped.collection || !jsonTyped.path) { return new Response( JSON.stringify({ error: "Missing required parameters: collection, path", }), { status: 400, headers: withCors({ "Content-Type": "application/json" }), } ); }
const field = `${jsonTyped.collection}:${jsonTyped.path.replace( /^\./, "" )}`;
const result = db .prepare(SQL.countDistinctDids) .get(jsonTyped.target, jsonTyped.collection, field);
const response: linksCountResponse = { total: result && result.total ? result.total.toString() : "0", };
return new Response(JSON.stringify(response), { headers: withCors({ "Content-Type": "application/json" }), }); } case "/links/all": { const jsonTyped = jsonUntyped as linksAllQuery;
const rows = db.prepare(SQL.all).all(jsonTyped.target) as any[];
const links: linksAllResponse["links"] = {};
for (const row of rows) { if (!links[row.suburi]) { links[row.suburi] = {}; } links[row.suburi][row.srccol] = { records: row.records, distinct_dids: row.distinct_dids, }; }
const response: linksAllResponse = { links, };
return new Response(JSON.stringify(response), { headers: withCors({ "Content-Type": "application/json" }), }); } default: { return new Response( JSON.stringify({ error: "NotSupported", message: "The requested endpoint is not supported by this Constellation implementation.", }), { status: 404, headers: withCors({ "Content-Type": "application/json" }), } ); } } }
indexServerIndexer(ctx: indexHandlerContext) { const record = assertRecord(ctx.value); //const record = validateRecord(ctx.value); const db = this.userManager.getDbForDid(ctx.doer); if (!db) return; console.log("indexering"); switch (record?.$type) { case "app.bsky.feed.like": { return; } case "app.bsky.actor.profile": { console.log("bsky profuile");
try { const stmt = db.prepare(` INSERT OR IGNORE INTO app_bsky_actor_profile ( uri, did, cid, rev, createdat, indexedat, json, displayname, description, avatarcid, avatarmime, bannercid, bannermime ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `); console.log({ uri: ctx.aturi, did: ctx.doer, cid: ctx.cid, rev: ctx.rev, createdat: record.createdAt, indexedat: Date.now(), json: JSON.stringify(record), displayname: record.displayName, description: record.description, avatarcid: uncid(record.avatar?.ref), avatarmime: record.avatar?.mimeType, bannercid: uncid(record.banner?.ref), bannermime: record.banner?.mimeType, }); stmt.run( ctx.aturi ?? null, ctx.doer ?? null, ctx.cid ?? null, ctx.rev ?? null, record.createdAt ?? null, Date.now(), JSON.stringify(record),
record.displayName ?? null, record.description ?? null, uncid(record.avatar?.ref) ?? null, record.avatar?.mimeType ?? null, uncid(record.banner?.ref) ?? null, record.banner?.mimeType ?? null // TODO please add pinned posts ); } catch (err) { console.error("stmt.run failed:", err); } return; } case "app.bsky.feed.post": { console.log("bsky post"); const stmt = db.prepare(` INSERT OR IGNORE INTO app_bsky_feed_post ( uri, did, cid, rev, createdat, indexedat, json, text, replyroot, replyparent, quote, imagecount, image1cid, image1mime, image1aspect, image2cid, image2mime, image2aspect, image3cid, image3mime, image3aspect, image4cid, image4mime, image4aspect, videocount, videocid, videomime, videoaspect ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `);
const embed = record.embed;
const images = extractImages(embed); const video = extractVideo(embed); const quoteUri = extractQuoteUri(embed); try { stmt.run( ctx.aturi ?? null, ctx.doer ?? null, ctx.cid ?? null, ctx.rev ?? null, record.createdAt, Date.now(), JSON.stringify(record),
record.text ?? null, record.reply?.root?.uri ?? null, record.reply?.parent?.uri ?? null,
quoteUri,
images.length, uncid(images[0]?.image?.ref) ?? null, images[0]?.image?.mimeType ?? null, images[0]?.aspectRatio && images[0].aspectRatio.width && images[0].aspectRatio.height ? `${images[0].aspectRatio.width}:${images[0].aspectRatio.height}` : null,
uncid(images[1]?.image?.ref) ?? null, images[1]?.image?.mimeType ?? null, images[1]?.aspectRatio && images[1].aspectRatio.width && images[1].aspectRatio.height ? `${images[1].aspectRatio.width}:${images[1].aspectRatio.height}` : null,
uncid(images[2]?.image?.ref) ?? null, images[2]?.image?.mimeType ?? null, images[2]?.aspectRatio && images[2].aspectRatio.width && images[2].aspectRatio.height ? `${images[2].aspectRatio.width}:${images[2].aspectRatio.height}` : null,
uncid(images[3]?.image?.ref) ?? null, images[3]?.image?.mimeType ?? null, images[3]?.aspectRatio && images[3].aspectRatio.width && images[3].aspectRatio.height ? `${images[3].aspectRatio.width}:${images[3].aspectRatio.height}` : null,
uncid(video?.video) ? 1 : 0, uncid(video?.video) ?? null, uncid(video?.video) ? "video/mp4" : null, video?.aspectRatio ? `${video.aspectRatio.width}:${video.aspectRatio.height}` : null ); } catch (err) { console.error("stmt.run failed:", err); } return; } default: { // what the hell return; } } }
// user data async queryProfileView( did: string, type: "" ): Promise<ATPAPI.AppBskyActorDefs.ProfileView | undefined>; async queryProfileView( did: string, type: "Basic" ): Promise<ATPAPI.AppBskyActorDefs.ProfileViewBasic | undefined>; async queryProfileView( did: string, type: "Detailed" ): Promise<ATPAPI.AppBskyActorDefs.ProfileViewDetailed | undefined>; async queryProfileView( did: string, type: "" | "Basic" | "Detailed" ): Promise< | ATPAPI.AppBskyActorDefs.ProfileView | ATPAPI.AppBskyActorDefs.ProfileViewBasic | ATPAPI.AppBskyActorDefs.ProfileViewDetailed | undefined > { if (!this.isRegisteredIndexUser(did)) return; const db = this.userManager.getDbForDid(did); if (!db) return;
const stmt = db.prepare(` SELECT * FROM app_bsky_actor_profile WHERE did = ? LIMIT 1; `);
const row = stmt.get(did) as ProfileRow;
const identity = await resolveIdentity(did); const avatar = row.avatarcid ? buildBlobUrl( identity.pds, identity.did, row.avatarcid ) : undefined const banner = row.bannercid ? buildBlobUrl( identity.pds, identity.did, row.bannercid ) : undefined // simulate different types returned switch (type) { case "": { const result: ATPAPI.AppBskyActorDefs.ProfileView = { $type: "app.bsky.actor.defs#profileView", did: did, handle: identity.handle, // TODO: Resolve user identity here for the handle displayName: row.displayname ?? undefined, description: row.description ?? undefined, avatar: avatar, // create profile URL from resolved identity //associated?: ProfileAssociated, indexedAt: row.createdat ? new Date(row.createdat).toISOString() : undefined, createdAt: row.createdat ? new Date(row.createdat).toISOString() : undefined, //viewer?: ViewerState, //labels?: ComAtprotoLabelDefs.Label[], //verification?: VerificationState, //status?: StatusView, }; return result; } case "Basic": { const result: ATPAPI.AppBskyActorDefs.ProfileViewBasic = { $type: "app.bsky.actor.defs#profileViewBasic", did: did, handle: identity.handle, // TODO: Resolve user identity here for the handle displayName: row.displayname ?? undefined, avatar: avatar, // create profile URL from resolved identity //associated?: ProfileAssociated, createdAt: row.createdat ? new Date(row.createdat).toISOString() : undefined, //viewer?: ViewerState, //labels?: ComAtprotoLabelDefs.Label[], //verification?: VerificationState, //status?: StatusView, }; return result; } case "Detailed": { // Query for follower count from the backlink_skeleton table const followersStmt = db.prepare(` SELECT COUNT(*) as count FROM backlink_skeleton WHERE subdid = ? AND srccol = 'app.bsky.graph.follow' `); const followersResult = followersStmt.get(did) as { count: number }; const followersCount = followersResult?.count ?? 0;
// Query for following count from the app_bsky_graph_follow table const followingStmt = db.prepare(` SELECT COUNT(*) as count FROM app_bsky_graph_follow WHERE did = ? `); const followingResult = followingStmt.get(did) as { count: number }; const followsCount = followingResult?.count ?? 0;
// Query for post count from the app_bsky_feed_post table const postsStmt = db.prepare(` SELECT COUNT(*) as count FROM app_bsky_feed_post WHERE did = ? `); const postsResult = postsStmt.get(did) as { count: number }; const postsCount = postsResult?.count ?? 0;
const result: ATPAPI.AppBskyActorDefs.ProfileViewDetailed = { $type: "app.bsky.actor.defs#profileViewDetailed", did: did, handle: identity.handle, // TODO: Resolve user identity here for the handle displayName: row.displayname ?? undefined, description: row.description ?? undefined, avatar: avatar, // TODO: create profile URL from resolved identity banner: banner, // same here followersCount: followersCount, followsCount: followsCount, postsCount: postsCount, //associated?: ProfileAssociated, //joinedViaStarterPack?: // AppBskyGraphDefs.StarterPackViewBasic; indexedAt: row.createdat ? new Date(row.createdat).toISOString() : undefined, createdAt: row.createdat ? new Date(row.createdat).toISOString() : undefined, //viewer?: ViewerState, //labels?: ComAtprotoLabelDefs.Label[], pinnedPost: undefined, //row.; // TODO: i forgot to put pinnedp posts in db schema oops //verification?: VerificationState, //status?: StatusView, }; return result; } default: throw new Error("Invalid type"); } }
// post hydration async queryPostView( uri: string ): Promise<ATPAPI.AppBskyFeedDefs.PostView | undefined> { const URI = new AtUri(uri); const did = URI.host; if (!this.isRegisteredIndexUser(did)) return; const db = this.userManager.getDbForDid(did); if (!db) return;
const stmt = db.prepare(` SELECT * FROM app_bsky_feed_post WHERE uri = ? LIMIT 1; `);
const row = stmt.get(uri) as PostRow; const profileView = await this.queryProfileView(did, "Basic"); if (!row || !row.cid || !profileView || !row.json) return; const value = JSON.parse(row.json) as ATPAPI.AppBskyFeedPost.Record;
const post: ATPAPI.AppBskyFeedDefs.PostView = { uri: row.uri, cid: row.cid, author: profileView, record: value, indexedAt: new Date(row.indexedat).toISOString(), embed: value.embed, };
return post; }
constructPostViewRef( uri: string ): IndexServerAPI.PartyWheyAppBskyFeedDefs.PostViewRef { const post: IndexServerAPI.PartyWheyAppBskyFeedDefs.PostViewRef = { uri: uri, cid: "cid.invalid", // oh shit we dont know the cid TODO: major design flaw };
return post; }
async queryFeedViewPost( uri: string ): Promise<ATPAPI.AppBskyFeedDefs.FeedViewPost | undefined> { const post = await this.queryPostView(uri); if (!post) return;
const feedviewpost: ATPAPI.AppBskyFeedDefs.FeedViewPost = { $type: "app.bsky.feed.defs#feedViewPost", post: post, //reply: ReplyRef, //reason: , };
return feedviewpost; }
constructFeedViewPostRef( uri: string ): IndexServerAPI.PartyWheyAppBskyFeedDefs.FeedViewPostRef { const post = this.constructPostViewRef(uri);
const feedviewpostref: IndexServerAPI.PartyWheyAppBskyFeedDefs.FeedViewPostRef = { $type: "party.whey.app.bsky.feed.defs#feedViewPostRef", post: post as IndexServerUtils.$Typed<IndexServerAPI.PartyWheyAppBskyFeedDefs.PostViewRef>, };
return feedviewpostref; }
// user feedgens
async queryActorFeeds( did: string ): Promise<ATPAPI.AppBskyFeedDefs.GeneratorView[]> { if (!this.isRegisteredIndexUser(did)) return []; const db = this.userManager.getDbForDid(did); if (!db) return [];
const stmt = db.prepare(` SELECT uri, cid, did, json, indexedat FROM app_bsky_feed_generator WHERE did = ? ORDER BY createdat DESC; `);
const rows = stmt.all(did) as unknown as GeneratorRow[]; const creatorView = await this.queryProfileView(did, "Basic"); if (!creatorView) return [];
return rows .map((row) => { try { if (!row.json) return; const record = JSON.parse( row.json ) as ATPAPI.AppBskyFeedGenerator.Record; return { $type: "app.bsky.feed.defs#generatorView", uri: row.uri, cid: row.cid, did: row.did, creator: creatorView, displayName: record.displayName, description: record.description, descriptionFacets: record.descriptionFacets, avatar: record.avatar, likeCount: 0, // TODO: this should be easy indexedAt: new Date(row.indexedat).toISOString(), } as ATPAPI.AppBskyFeedDefs.GeneratorView; } catch { return undefined; } }) .filter((v): v is ATPAPI.AppBskyFeedDefs.GeneratorView => !!v); }
async queryFeedGenerator( uri: string ): Promise<ATPAPI.AppBskyFeedDefs.GeneratorView | undefined> { const gens = await this.queryFeedGenerators([uri]); // gens: GeneratorView[] return gens[0]; }
async queryFeedGenerators( uris: string[] ): Promise<ATPAPI.AppBskyFeedDefs.GeneratorView[]> { const generators: ATPAPI.AppBskyFeedDefs.GeneratorView[] = []; const urisByDid = new Map<string, string[]>();
for (const uri of uris) { try { const { host: did } = new AtUri(uri); if (!urisByDid.has(did)) { urisByDid.set(did, []); } urisByDid.get(did)!.push(uri); } catch {} }
for (const [did, didUris] of urisByDid.entries()) { if (!this.isRegisteredIndexUser(did)) continue; const db = this.userManager.getDbForDid(did); if (!db) continue;
const placeholders = didUris.map(() => "?").join(","); const stmt = db.prepare(` SELECT uri, cid, did, json, indexedat FROM app_bsky_feed_generator WHERE uri IN (${placeholders}); `);
const rows = stmt.all(...didUris) as unknown as GeneratorRow[]; if (rows.length === 0) continue;
const creatorView = await this.queryProfileView(did, ""); if (!creatorView) continue;
for (const row of rows) { try { if (!row.json || !row.cid) continue; const record = JSON.parse( row.json ) as ATPAPI.AppBskyFeedGenerator.Record; generators.push({ $type: "app.bsky.feed.defs#generatorView", uri: row.uri, cid: row.cid, did: row.did, creator: creatorView, displayName: record.displayName, description: record.description, descriptionFacets: record.descriptionFacets, avatar: record.avatar as string | undefined, likeCount: 0, indexedAt: new Date(row.indexedat).toISOString(), }); } catch {} } } return generators; }
// user feeds
async queryAuthorFeedPartial( did: string, cursor?: string ): Promise< | { items: ( | ATPAPI.AppBskyFeedDefs.FeedViewPost | IndexServerAPI.PartyWheyAppBskyFeedDefs.FeedViewPostRef )[]; cursor: string | undefined; } | undefined > { if (!this.isRegisteredIndexUser(did)) return; const db = this.userManager.getDbForDid(did); if (!db) return;
const subquery = ` SELECT uri, cid, indexedat, 'post' as type, null as subject FROM app_bsky_feed_post WHERE did = ? UNION ALL SELECT uri, cid, indexedat, 'repost' as type, subject FROM app_bsky_feed_repost WHERE did = ? `;
let query = `SELECT * FROM (${subquery}) as feed_items`; const params: (string | number)[] = [did, did];
if (cursor) { const [indexedat, cid] = cursor.split("::"); query += ` WHERE (indexedat < ? OR (indexedat = ? AND cid < ?))`; params.push(parseInt(indexedat, 10), parseInt(indexedat, 10), cid); }
query += ` ORDER BY indexedat DESC, cid DESC LIMIT ${FEED_LIMIT}`;
const stmt = db.prepare(query); const rows = stmt.all(...params) as { uri: string; indexedat: number; cid: string; type: "post" | "repost"; subject: string | null; }[];
const authorProfile = await this.queryProfileView(did, "Basic");
const items = await Promise.all( rows .map((row) => { if (row.type === "repost" && row.subject) { const subjectDid = new AtUri(row.subject).host;
const originalPost = this.handlesDid(subjectDid) ? this.queryFeedViewPost(row.subject) : this.constructFeedViewPostRef(row.subject);
if (!originalPost || !authorProfile) return null;
return { post: originalPost, reason: { $type: "app.bsky.feed.defs#reasonRepost", by: authorProfile, indexedAt: new Date(row.indexedat).toISOString(), }, }; } else { return this.queryFeedViewPost(row.uri); } }) .filter((p): p is Promise<ATPAPI.AppBskyFeedDefs.FeedViewPost> => !!p) );
const lastItem = rows[rows.length - 1]; const nextCursor = lastItem ? `${lastItem.indexedat}::${lastItem.cid}` : undefined;
return { items, cursor: nextCursor }; }
queryListFeed( uri: string, cursor?: string ): | { items: ATPAPI.AppBskyFeedDefs.FeedViewPost[]; cursor: string | undefined; } | undefined { return { items: [], cursor: undefined }; }
async queryActorLikesPartial( did: string, cursor?: string ): Promise< | { items: ( | ATPAPI.AppBskyFeedDefs.FeedViewPost | IndexServerAPI.PartyWheyAppBskyFeedDefs.FeedViewPostRef )[]; cursor: string | undefined; } | undefined > { // early return only if the actor did is not registered if (!this.isRegisteredIndexUser(did)) return; const db = this.userManager.getDbForDid(did); if (!db) return;
let query = ` SELECT subject, indexedat, cid FROM app_bsky_feed_like WHERE did = ? `; const params: (string | number)[] = [did];
if (cursor) { const [indexedat, cid] = cursor.split("::"); query += ` AND (indexedat < ? OR (indexedat = ? AND cid < ?))`; params.push(parseInt(indexedat, 10), parseInt(indexedat, 10), cid); }
query += ` ORDER BY indexedat DESC, cid DESC LIMIT ${FEED_LIMIT}`;
const stmt = db.prepare(query); const rows = stmt.all(...params) as { subject: string; indexedat: number; cid: string; }[];
const items = await Promise.all( rows .map(async (row) => { const subjectDid = new AtUri(row.subject).host;
if (this.handlesDid(subjectDid)) { return await this.queryFeedViewPost(row.subject); } else { return this.constructFeedViewPostRef(row.subject); } }) .filter( ( p ): p is Promise< | ATPAPI.AppBskyFeedDefs.FeedViewPost | IndexServerAPI.PartyWheyAppBskyFeedDefs.FeedViewPostRef > => !!p ) );
const lastItem = rows[rows.length - 1]; const nextCursor = lastItem ? `${lastItem.indexedat}::${lastItem.cid}` : undefined;
return { items, cursor: nextCursor }; }
// post metadata
async queryLikes( uri: string ): Promise<ATPAPI.AppBskyFeedGetLikes.Like[] | undefined> { const postUri = new AtUri(uri); const postAuthorDid = postUri.hostname; if (!this.isRegisteredIndexUser(postAuthorDid)) return; const db = this.userManager.getDbForDid(postAuthorDid); if (!db) return;
const stmt = db.prepare(` SELECT b.srcdid, b.srcuri FROM backlink_skeleton AS b WHERE b.suburi = ? AND b.srccol = 'app_bsky_feed_like' ORDER BY b.id DESC; `);
const rows = stmt.all(uri) as unknown as BacklinkRow[];
return await Promise.all( rows .map(async (row) => { const actor = await this.queryProfileView(row.srcdid, ""); if (!actor) return;
return { // TODO write indexedAt for spacedust indexes createdAt: new Date(Date.now()).toISOString(), indexedAt: new Date(Date.now()).toISOString(), actor: actor, }; }) .filter( (like): like is Promise<ATPAPI.AppBskyFeedGetLikes.Like> => !!like ) ); }
async queryReposts( uri: string ): Promise<ATPAPI.AppBskyActorDefs.ProfileView[]> { const postUri = new AtUri(uri); const postAuthorDid = postUri.hostname; if (!this.isRegisteredIndexUser(postAuthorDid)) return []; const db = this.userManager.getDbForDid(postAuthorDid); if (!db) return [];
const stmt = db.prepare(` SELECT srcdid FROM backlink_skeleton WHERE suburi = ? AND srccol = 'app_bsky_feed_repost' ORDER BY id DESC; `);
const rows = stmt.all(uri) as { srcdid: string }[];
return await Promise.all( rows .map(async (row) => await this.queryProfileView(row.srcdid, "")) .filter((p): p is Promise<ATPAPI.AppBskyActorDefs.ProfileView> => !!p) ); }
async queryQuotes( uri: string ): Promise<ATPAPI.AppBskyFeedDefs.FeedViewPost[]> { const postUri = new AtUri(uri); const postAuthorDid = postUri.hostname; if (!this.isRegisteredIndexUser(postAuthorDid)) return []; const db = this.userManager.getDbForDid(postAuthorDid); if (!db) return [];
const stmt = db.prepare(` SELECT srcuri FROM backlink_skeleton WHERE suburi = ? AND srccol = 'app_bsky_feed_post' AND srcfield = 'quote' ORDER BY id DESC; `);
const rows = stmt.all(uri) as { srcuri: string }[];
return await Promise.all( rows .map(async (row) => await this.queryFeedViewPost(row.srcuri)) .filter((p): p is Promise<ATPAPI.AppBskyFeedDefs.FeedViewPost> => !!p) ); } async _getPostViewUnion( uri: string ): Promise< | ATPAPI.AppBskyFeedDefs.PostView | IndexServerAPI.PartyWheyAppBskyFeedDefs.PostViewRef | undefined > { try { const postDid = new AtUri(uri).hostname; if (this.handlesDid(postDid)) { return await this.queryPostView(uri); } else { return this.constructPostViewRef(uri); } } catch (_e) { return undefined; } } async queryPostThreadPartial( uri: string ): Promise< | IndexServerTypes.PartyWheyAppBskyFeedGetPostThreadPartial.OutputSchema | undefined > { const post = await this._getPostViewUnion(uri);
if (!post) { return { thread: { $type: "app.bsky.feed.defs#notFoundPost", uri: uri, notFound: true, } as ATPAPI.$Typed<ATPAPI.AppBskyFeedDefs.NotFoundPost>, }; }
const thread: IndexServerAPI.PartyWheyAppBskyFeedDefs.ThreadViewPostRef = { $type: "party.whey.app.bsky.feed.defs#threadViewPostRef", post: post as | ATPAPI.$Typed<ATPAPI.AppBskyFeedDefs.PostView> | IndexServerUtils.$Typed<IndexServerAPI.PartyWheyAppBskyFeedDefs.PostViewRef>, replies: [], };
let current = thread; // we can only climb the parent tree if we have the full post record. // which is not implemented yet (sad i know) if ( isPostView(current.post) && isFeedPostRecord(current.post.record) && current.post.record?.reply?.parent?.uri ) { let parentUri: string | undefined = current.post.record.reply.parent.uri;
// keep climbing as long as we find a valid parent post. while (parentUri) { const parentPost = await this._getPostViewUnion(parentUri); if (!parentPost) break; // stop if a parent in the chain is not found.
const parentThread: IndexServerAPI.PartyWheyAppBskyFeedDefs.ThreadViewPostRef = { $type: "party.whey.app.bsky.feed.defs#threadViewPostRef", post: parentPost as ATPAPI.$Typed<ATPAPI.AppBskyFeedDefs.PostView>, replies: [ current as IndexServerUtils.$Typed<IndexServerAPI.PartyWheyAppBskyFeedDefs.ThreadViewPostRef>, ], }; current.parent = parentThread as IndexServerUtils.$Typed<IndexServerAPI.PartyWheyAppBskyFeedDefs.ThreadViewPostRef>; current = parentThread;
// check if the new current post has a parent to continue the loop parentUri = isPostView(current.post) && isFeedPostRecord(current.post.record) ? current.post.record?.reply?.parent?.uri : undefined; } }
const seenUris = new Set<string>(); const fetchReplies = async ( parentThread: IndexServerAPI.PartyWheyAppBskyFeedDefs.ThreadViewPostRef ) => { if (!parentThread.post || !("uri" in parentThread.post)) { return; } if (seenUris.has(parentThread.post.uri)) return; seenUris.add(parentThread.post.uri);
const parentUri = new AtUri(parentThread.post.uri); const parentAuthorDid = parentUri.hostname;
// replies can only be discovered for local posts where we have the backlink data if (!this.handlesDid(parentAuthorDid)) return;
const db = this.userManager.getDbForDid(parentAuthorDid); if (!db) return;
const stmt = db.prepare(` SELECT srcuri FROM backlink_skeleton WHERE suburi = ? AND srccol = 'app_bsky_feed_post' AND srcfield = 'replyparent' `); const replyRows = stmt.all(parentThread.post.uri) as { srcuri: string }[];
const replies = await Promise.all( replyRows .map(async (row) => await this._getPostViewUnion(row.srcuri)) .filter( ( p ): p is Promise< | ATPAPI.AppBskyFeedDefs.PostView | IndexServerAPI.PartyWheyAppBskyFeedDefs.PostViewRef > => !!p ) );
for (const replyPost of replies) { const replyThread: IndexServerAPI.PartyWheyAppBskyFeedDefs.ThreadViewPostRef = { $type: "party.whey.app.bsky.feed.defs#threadViewPostRef", post: replyPost as | ATPAPI.$Typed<ATPAPI.AppBskyFeedDefs.PostView> | IndexServerUtils.$Typed<IndexServerAPI.PartyWheyAppBskyFeedDefs.PostViewRef>, parent: parentThread as IndexServerUtils.$Typed<IndexServerAPI.PartyWheyAppBskyFeedDefs.ThreadViewPostRef>, replies: [], }; parentThread.replies?.push( replyThread as IndexServerUtils.$Typed<IndexServerAPI.PartyWheyAppBskyFeedDefs.ThreadViewPostRef> ); fetchReplies(replyThread); // recurse } };
fetchReplies(thread);
const returned = current as unknown as IndexServerAPI.PartyWheyAppBskyFeedDefs.ThreadViewPostRef;
return { thread: returned as IndexServerUtils.$Typed<IndexServerAPI.PartyWheyAppBskyFeedDefs.ThreadViewPostRef>, }; }
/** * please do not use this, use openDbForDid() instead * @param did * @returns */ internalCreateDbForDid(did: string): Database { const path = `${this.config.baseDbPath}/${did}.sqlite`; const db = new Database(path); setupUserDb(db); //await db.exec(/* CREATE IF NOT EXISTS statements */); return db; } /** * @deprecated use handlesDid() instead * @param did * @returns */ isRegisteredIndexUser(did: string): boolean { const stmt = this.systemDB.prepare(` SELECT 1 FROM users WHERE did = ? AND onboardingstatus != 'onboarding-backfill' LIMIT 1; `); const result = stmt.value<[number]>(did); const exists = result !== undefined; return exists; }}
export class IndexServerUserManager { public indexServer: IndexServer;
constructor(indexServer: IndexServer) { this.indexServer = indexServer; }
public users = new Map<string, UserIndexServer>(); public handlesDid(did: string): boolean { return this.users.has(did); }
/*async*/ addUser(did: string) { if (this.users.has(did)) return; const instance = new UserIndexServer(this, did); //await instance.initialize(); this.users.set(did, instance); }
// async handleRequest({ // did, // route, // req, // }: { // did: string; // route: string; // req: Request; // }) { // if (!this.users.has(did)) await this.addUser(did); // const user = this.users.get(did)!; // return await user.handleHttpRequest(route, req); // }
removeUser(did: string) { const instance = this.users.get(did); if (!instance) return; /*await*/ instance.shutdown(); this.users.delete(did); }
getDbForDid(did: string): Database | null { if (!this.users.has(did)) { return null; } return this.users.get(did)?.db ?? null; }
coldStart(db: Database) { const rows = db.prepare("SELECT did FROM users").all(); for (const row of rows) { this.addUser(row.did); } }}
class UserIndexServer { public indexServerUserManager: IndexServerUserManager; did: string; db: Database; // | undefined; jetstream: JetstreamManager; // | undefined; spacedust: SpacedustManager; // | undefined;
constructor(indexServerUserManager: IndexServerUserManager, did: string) { this.did = did; this.indexServerUserManager = indexServerUserManager; this.db = this.indexServerUserManager.indexServer.internalCreateDbForDid( this.did ); // should probably put the params of exactly what were listening to here this.jetstream = new JetstreamManager((msg) => { console.log("Received Jetstream message: ", msg);
const op = msg.commit.operation; const doer = msg.did; const rev = msg.commit.rev; const aturi = `${msg.did}/${msg.commit.collection}/${msg.commit.rkey}`; const value = msg.commit.record;
if (!doer || !value) return; this.indexServerUserManager.indexServer.indexServerIndexer({ op, doer, cid: msg.commit.cid, rev, aturi, value, indexsrc: `jetstream-${op}`, db: this.db, }); }); this.jetstream.start({ // for realsies pls get from db or something instead of this shit wantedDids: [ this.did, // "did:plc:mn45tewwnse5btfftvd3powc", // "did:plc:yy6kbriyxtimkjqonqatv2rb", // "did:plc:zzhzjga3ab5fcs2vnsv2ist3", // "did:plc:jz4ibztn56hygfld6j6zjszg", ], wantedCollections: [ "app.bsky.actor.profile", "app.bsky.feed.generator", "app.bsky.feed.like", "app.bsky.feed.post", "app.bsky.feed.repost", "app.bsky.feed.threadgate", "app.bsky.graph.block", "app.bsky.graph.follow", "app.bsky.graph.list", "app.bsky.graph.listblock", "app.bsky.graph.listitem", "app.bsky.notification.declaration", ], }); //await connectToJetstream(this.did, this.db); this.spacedust = new SpacedustManager((msg: SpacedustLinkMessage) => { console.log("Received Spacedust message: ", msg); const operation = msg.link.operation;
const sourceURI = new AtUri(msg.link.source_record); const srcUri = msg.link.source_record; const srcDid = sourceURI.host; const srcField = msg.link.source; const srcCol = sourceURI.collection; const subjectURI = new AtUri(msg.link.subject); const subUri = msg.link.subject; const subDid = subjectURI.host; const subCol = subjectURI.collection;
if (operation === "delete") { this.db.run( `DELETE FROM backlink_skeleton WHERE srcuri = ? AND srcfield = ? AND suburi = ?`, [srcUri, srcField, subUri] ); } else if (operation === "create") { this.db.run( `INSERT OR REPLACE INTO backlink_skeleton ( srcuri, srcdid, srcfield, srccol, suburi, subdid, subcol, indexedAt ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, [ srcUri, // full AT URI of the source record srcDid, // did: of the source srcField, // e.g., "reply.parent.uri" or "facets.features.did" srcCol, // e.g., "app.bsky.feed.post" subUri, // full AT URI of the subject (linked record) subDid, // did: of the subject subCol, // subject collection (can be inferred or passed) Date.now() ] ); } }); this.spacedust.start({ wantedSources: [ "app.bsky.feed.like:subject.uri", // like "app.bsky.feed.like:via.uri", // liked repost "app.bsky.feed.repost:subject.uri", // repost "app.bsky.feed.repost:via.uri", // reposted repost "app.bsky.feed.post:reply.root.uri", // thread OP "app.bsky.feed.post:reply.parent.uri", // direct parent "app.bsky.feed.post:embed.media.record.record.uri", // quote with media "app.bsky.feed.post:embed.record.uri", // quote without media "app.bsky.feed.threadgate:post", // threadgate subject "app.bsky.feed.threadgate:hiddenReplies", // threadgate items (array) "app.bsky.feed.post:facets.features.did", // facet item (array): mention "app.bsky.graph.block:subject", // blocks "app.bsky.graph.follow:subject", // follow "app.bsky.graph.listblock:subject", // list item (blocks) "app.bsky.graph.listblock:list", // blocklist mention (might not exist) "app.bsky.graph.listitem:subject", // list item (blocks) "app.bsky.graph.listitem:list", // list mention ], // should be getting from DB but whatever right wantedSubjects: [ // as noted i dont need to write down each post, just the user to listen to ! // hell yeah // "at://did:plc:mn45tewwnse5btfftvd3powc/app.bsky.feed.post/3lvybv7b6ic2h", // "at://did:plc:mn45tewwnse5btfftvd3powc/app.bsky.feed.post/3lvybws4avc2h", // "at://did:plc:mn45tewwnse5btfftvd3powc/app.bsky.feed.post/3lvvkcxcscs2h", // "at://did:plc:yy6kbriyxtimkjqonqatv2rb/app.bsky.feed.post/3l63ogxocq42f", // "at://did:plc:yy6kbriyxtimkjqonqatv2rb/app.bsky.feed.post/3lw3wamvflu23", ], wantedSubjectDids: [ this.did, //"did:plc:mn45tewwnse5btfftvd3powc", //"did:plc:yy6kbriyxtimkjqonqatv2rb", //"did:plc:zzhzjga3ab5fcs2vnsv2ist3", //"did:plc:jz4ibztn56hygfld6j6zjszg", ], instant: ["true"] }); //await connectToConstellation(this.did, this.db); }
// initialize() {
// }
// async handleHttpRequest(route: string, req: Request): Promise<Response> { // if (route === "posts") { // const posts = await this.queryPosts(); // return new Response(JSON.stringify(posts), { // headers: { "content-type": "application/json" }, // }); // }
// return new Response("Unknown route", { status: 404 }); // }
// private async queryPosts() { // return this.db.run( // "SELECT * FROM posts ORDER BY created_at DESC LIMIT 100" // ); // }
shutdown() { this.jetstream.stop(); this.spacedust.stop(); this.db.close?.(); }}
// /**// * please do not use this, use openDbForDid() instead// * @param did// * @returns// */// function internalCreateDbForDid(did: string): Database {// const path = `./dbs/${did}.sqlite`;// const db = new Database(path);// setupUserDb(db);// //await db.exec(/* CREATE IF NOT EXISTS statements */);// return db;// }
// function getDbForDid(did: string): Database | undefined {// const db = indexerUserManager.getDbForDid(did);// if (!db) return;// return db;// }
// async function connectToJetstream(did: string, db: Database): Promise<WebSocket> {// const url = `${jetstreamurl}/xrpc/com.atproto.sync.subscribeRepos?did=${did}`;// const ws = new WebSocket(url);// ws.onmessage = (msg) => {// //handleJetstreamMessage(evt.data, db)
// const op = msg.commit.operation;// const doer = msg.did;// const rev = msg.commit.rev;// const aturi = `${msg.did}/${msg.commit.collection}/${msg.commit.rkey}`;// const value = msg.commit.record;
// if (!doer || !value) return;// indexServerIndexer({// op,// doer,// rev,// aturi,// value,// indexsrc: "onboarding_backfill",// userdbname: did,// })// };
// return ws;// }
// async function connectToConstellation(did: string, db: D1Database): Promise<WebSocket> {// const url = `wss://bsky.social/xrpc/com.atproto.sync.subscribeLabels?did=${did}`;// const ws = new WebSocket(url);// ws.onmessage = (evt) => handleConstellationMessage(evt.data, db);// return ws;// }
type PostRow = { uri: string; did: string; cid: string | null; rev: string | null; createdat: number | null; indexedat: number; json: string | null;
text: string | null; replyroot: string | null; replyparent: string | null; quote: string | null;
imagecount: number | null; image1cid: string | null; image1mime: string | null; image1aspect: string | null; image2cid: string | null; image2mime: string | null; image2aspect: string | null; image3cid: string | null; image3mime: string | null; image3aspect: string | null; image4cid: string | null; image4mime: string | null; image4aspect: string | null;
videocount: number | null; videocid: string | null; videomime: string | null; videoaspect: string | null;};
type ProfileRow = { uri: string; cid: string | null; rev: string | null; createdat: number | null; indexedat: number; json: string | null; displayname: string | null; description: string | null; avatarcid: string | null; avatarmime: string | null; bannercid: string | null; bannermime: string | null;};
type linksQuery = { target: string; collection: string; path: string; cursor?: string;};type linksRecord = { did: string; collection: string; rkey: string;};type linksRecordsResponse = { total: string; linking_records: linksRecord[]; cursor?: string;};type linksDidsResponse = { total: string; linking_dids: string[]; cursor?: string;};type linksCountResponse = { total: string;};type linksAllResponse = { links: Record< string, Record< string, { records: number; distinct_dids: number; } > >;};
type linksAllQuery = { target: string;};
const SQL = { links: ` SELECT srcuri, srcdid, srccol FROM backlink_skeleton WHERE suburi = ? AND srccol = ? AND srcfield = ? `, distinctDids: ` SELECT DISTINCT srcdid FROM backlink_skeleton WHERE suburi = ? AND srccol = ? AND srcfield = ? `, count: ` SELECT COUNT(*) as total FROM backlink_skeleton WHERE suburi = ? AND srccol = ? AND srcfield = ? `, countDistinctDids: ` SELECT COUNT(DISTINCT srcdid) as total FROM backlink_skeleton WHERE suburi = ? AND srccol = ? AND srcfield = ? `, all: ` SELECT suburi, srccol, COUNT(*) as records, COUNT(DISTINCT srcdid) as distinct_dids FROM backlink_skeleton WHERE suburi = ? GROUP BY suburi, srccol `,};
export function isDid(str: string): boolean { return typeof str === "string" && str.startsWith("did:");}
function isFeedPostRecord( post: unknown): post is ATPAPI.AppBskyFeedPost.Record { return ( typeof post === "object" && post !== null && "$type" in post && (post as any).$type === "app.bsky.feed.post" );}
function isImageEmbed(embed: unknown): embed is ATPAPI.AppBskyEmbedImages.Main { return ( typeof embed === "object" && embed !== null && "$type" in embed && (embed as any).$type === "app.bsky.embed.images" );}
function isVideoEmbed(embed: unknown): embed is ATPAPI.AppBskyEmbedVideo.Main { return ( typeof embed === "object" && embed !== null && "$type" in embed && (embed as any).$type === "app.bsky.embed.video" );}
function isRecordEmbed( embed: unknown): embed is ATPAPI.AppBskyEmbedRecord.Main { return ( typeof embed === "object" && embed !== null && "$type" in embed && (embed as any).$type === "app.bsky.embed.record" );}
function isRecordWithMediaEmbed( embed: unknown): embed is ATPAPI.AppBskyEmbedRecordWithMedia.Main { return ( typeof embed === "object" && embed !== null && "$type" in embed && (embed as any).$type === "app.bsky.embed.recordWithMedia" );}
export function uncid(anything: any): string | null { return ( ((anything as Record<string, unknown>)?.["$link"] as string | null) || null );}
function extractImages(embed: unknown) { if (isImageEmbed(embed)) return embed.images; if (isRecordWithMediaEmbed(embed) && isImageEmbed(embed.media)) return embed.media.images; return [];}
function extractVideo(embed: unknown) { if (isVideoEmbed(embed)) return embed; if (isRecordWithMediaEmbed(embed) && isVideoEmbed(embed.media)) return embed.media; return null;}
function extractQuoteUri(embed: unknown): string | null { if (isRecordEmbed(embed)) return embed.record.uri; if (isRecordWithMediaEmbed(embed)) return embed.record.record.uri; return null;}