Something went wrong. Try again.
Read-only ActivityPub → atproto bridge for the threadiverse using Coves lexicons
Something went wrong. Try again.
18 kB · 432 lines
Go
at main
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433package materialize
import ( "context" "fmt" "strings"
"tidepool/internal/ap" "tidepool/internal/apobject" "tidepool/internal/errors" "tidepool/internal/store")
// MaterializePost translates a Lemmy Page (or Article) into a// social.coves.community.postv2 record, written into the AUTHOR's repo// (PLAN.md decision 20: authorship IS the repo, so the record carries no// `author` field, and `community` names the community it was submitted to).// Community and author profiles are ensured — and therefore committed —// before the post itself, preserving the emission-ordering guarantee.//// An object the bridge has already materialized is re-committed EXACTLY// where its mapping says it lives — same repo, collection and rkey. That is// what keeps a post from being relocated by an upstream edit, and it is also// how the two eras coexist: a post first materialized under the deprecated// collection keeps being updated there (no migration is planned; Coves// indexes both), while everything new is a postv2.func (m *Materializer) MaterializePost(ctx context.Context, page *ap.Object, communityIRI string) (*Result, error) { if err := requireBoundCommunityIRI(communityIRI); err != nil { return nil, err } if page == nil || page.ID == "" { return nil, errors.NewValidationError("page", "must carry an AP object id") } if page.Type != ap.TypePage && page.Type != ap.TypeArticle { return nil, errors.NewValidationError("page", "object "+page.ID+" has type "+page.Type+", want Page or Article") } rkey, err := recordRKey(page) if err != nil { return nil, err }
authorRef := page.AttributedTo.First() if authorRef == nil || authorRef.ID == "" { return nil, skip(page.ID, "post has no attributedTo author") } // Before anything is minted: a forged attribution must not cost the actor // it names a DID, nor the community it claims a repo. if err := requireSameAuthorityAuthor(page, authorRef); err != nil { return nil, err }
groupRef := communityRef(page) if groupRef == nil { return nil, skip(page.ID, "post names no community (no audience/to group IRI)") } // The delivering community can only speak for itself. Checked before the // Group is bridged, so a Page naming some other Group cannot get it a repo. if groupRef.ID != communityIRI { return nil, skip(page.ID, "post names community "+groupRef.ID+", not the delivering community "+communityIRI) }
// The coordinates below are taken from an existing mapping wholesale, so a // Page whose id the bridge holds as something else (a bridged actor's // same-host id, say) would be written over that record: a post committed at // the actor's profile collection and rkey "self". Refused before anything // is ensured, so the refusal costs no community or author a DID. existing, err := m.objects.GetByAPID(ctx, page.ID) switch { case err == nil: if existing.Collection != CollectionPost && existing.Collection != CollectionPostV2 { return nil, skip(page.ID, fmt.Sprintf( "id already maps a %s record; refusing to materialize it as a post", existing.Collection)) } // A post's community is fixed at first materialization, so the // audience above proves nothing about a post already stored: an edit // retargeted at C is still D's post, and C may not re-commit it. stored, storedErr := CommunityDIDOf(ctx, m.repos, existing) if storedErr != nil { return nil, storedErr } if err := m.requireBoundCommunity(ctx, stored, communityIRI, page.ID); err != nil { return nil, err } case errors.IsNotFound(err): existing = nil default: return nil, fmt.Errorf("materialize: check mapping for %s: %w", page.ID, err) }
community, err := m.EnsureCommunity(ctx, groupRef) if err != nil { return nil, err } author, err := m.EnsureActor(ctx, authorRef) if err != nil { return nil, err }
did, collection, authorDID := author.DID, CollectionPostV2, author.DID if existing != nil { did, collection, rkey = existing.DID, existing.Collection, existing.RKey // The repo a postv2 lives in IS its authorship claim, so authorship is // fixed at first materialization. attributedTo on an updated Page is // proposed by whoever delivered the update: honouring a changed value // would write the record into an unrelated bridged user's repo — the // strongest authorship statement atproto has — and strand the real // author's copy. The stored mapping is the authority on who authored a // bridged object, exactly as the stored record is on its community. if existing.AuthorDID != "" { authorDID = existing.AuthorDID } }
// The blob DID is the repo the record lands in, not the community: a blob // ref resolves against the repo its record lives in, so embed media // fetched anywhere else is unresolvable for every consumer. var record map[string]any if collection == CollectionPost { record = m.buildPostRecord(ctx, page, community.DID, author.DID, did) } else { // Provenance is built from the FETCHED actor document, never from the // page's own attributedTo: on a content path that object is authored by // whoever sent the content. actorDoc(allowEmbedded=false) is the same // authority-bound fetch the actor path trusts, and it is re-read here // rather than borrowed from EnsureActor because EnsureActor skips the // fetch for a profile still inside its TTL — provenance that appeared // or vanished with profile freshness would churn the record (and the // firehose) on every re-materialization. authorDoc, err := m.actorDoc(ctx, authorRef, false) if err != nil { return nil, err } record = m.buildPostV2Record(ctx, page, community.DID, author, authorDoc, did) } res, err := m.commitRecord(ctx, did, collection, rkey, record, page, authorDID, community.DID) if err != nil { return nil, err } if collection == CollectionPostV2 { // The community is read back off the COMMITTED record, not from the // EnsureCommunity above: on an update whose audience was retargeted, // carryForwardFields has restored the immutable stored community by // now, and accepting into the audience's community instead would plant // an attestation in a community that was never given the post. // // A failure here fails the whole materialization on purpose. The // alternative — logging and returning the post as materialized — would // leave a post that exists and is invisible, with nothing left to // retry it; this way redelivery heals it. acceptingDID, _ := record["community"].(string) if err := m.acceptPost(ctx, acceptingDID, res.ATURI, res.CID, page.Published.Time); err != nil { return nil, err } } return res, nil}
// buildPostV2Record translates the Page per the community.postv2 lexicon:// the shared post fields, plus the provenance a bridged record owes its// readers. There is deliberately no `author` — the record's repo is the// authorship claim, and a field restating it would invite a consumer to// trust the field over the repo.func (m *Materializer) buildPostV2Record(ctx context.Context, page *ap.Object, communityDID string, author *store.BridgedActor, authorDoc *ap.Object, repoDID string) map[string]any { record := map[string]any{ "$type": CollectionPostV2, "community": communityDID, "createdAt": recordDatetime(page.Published.Time), } m.fillPostContent(ctx, record, page, repoDID, apobject.MaximumPostContentGraphemes, apobject.MaximumPostContentBytes) record["originalAuthor"] = originalAuthorOf(author, authorDoc) record["federatedFrom"] = federatedFromOf(page) return record}
// originalAuthorOf names the account on the ORIGIN platform. It is provenance,// never an authorship claim about the repo (the lexicon says so explicitly),// and empty fields are omitted rather than written blank — a reader cannot// tell an empty string from an assertion that the value is empty.func originalAuthorOf(author *store.BridgedActor, authorDoc *ap.Object) map[string]any { out := map[string]any{"apId": author.APActorID} if instance := (&ap.Object{ID: author.APActorID}).Host(); instance != "" { out["instance"] = instance } if handle := originAccountName(author); handle != "" { out["handle"] = handle } // ONLY the fetched document may name the account. The inline attributedTo // object a page carries is written by the sending instance, so honouring it // would let any remote host hang an arbitrary display name on a real // account in a field consumers render. A document that asserts no name // leaves the field absent: an absence is not licence to substitute one. if authorDoc != nil && authorDoc.Name != "" { out["displayName"] = authorDoc.Name } return out}
// originAccountName is the account's name ON THE ORIGIN PLATFORM — the// username a reader would type to find them there — NOT the bridged// tidepool handle, which names a repo on this side of the bridge and would// mislead anyone trying to trace the post back to its source.//// It is read off the actor's CANONICAL AP id, whose last path segment is the// origin username (Lemmy `/u/Name`, PieFed and Mastodon alike). That id is// authority-bound at mint time, so unlike an inlined `attributedTo` — which// is attacker-influenced on every content path — it cannot be spelled by// whoever authored the post. The bridged handle's local part is the fallback// for an id with no usable segment; it is the same derivation// preferredUsernameFor already uses, and it loses the original casing, which// is why it is second rather than first.func originAccountName(author *store.BridgedActor) string { if idx := strings.LastIndexByte(author.APActorID, '/'); idx >= 0 { if name := author.APActorID[idx+1:]; name != "" { return name } } if idx := strings.IndexByte(author.Handle, '.'); idx > 0 { return author.Handle[:idx] } return author.Handle}
// federatedFromOf names where the post came from. Lemmy is the only origin// platform the bridge speaks (PLAN.md decision 9 defers PieFed/Mbin to their// own verification), so the platform is a constant rather than a guess off// the instance's software.func federatedFromOf(page *ap.Object) map[string]any { out := map[string]any{"platform": "lemmy", "apId": page.ID} if instance := page.Host(); instance != "" { out["instance"] = instance } return out}
// communityRef finds the community Group IRI a post belongs to: Lemmy sets// `audience` (FEP-1b12); older objects carry the group in `to`/`cc` next to// the public collection. Public-collection IRIs are skipped in EVERY list,// audience included — the ingest layer's communityIRIFrom does the same, and// a divergence there is not cosmetic: an `audience: ["as:Public"]` post that// ingest reads as "names no community" would arrive here as// EnsureCommunity("as:Public"), whose failure is retryable and would back the// whole ordering key off into poison.func communityRef(page *ap.Object) *ap.Object { for _, iri := range page.Audience { if iri != "" && !isPublicCollection(iri) { return &ap.Object{ID: iri} } } for _, list := range []ap.Audience{page.To, page.Cc} { for _, iri := range list { if iri == "" || isPublicCollection(iri) { continue } // Heuristic for addressing lists that mix users and groups: // Lemmy community IRIs live under /c/. if strings.Contains(iri, "/c/") { return &ap.Object{ID: iri} } } } return nil}
// isPublicCollection reports whether an addressing IRI is the AS2 public// collection. All three spellings appear in the wild (Lemmy always writes the// full IRI); ap.Object.IsPublic and ingest.isPublicIRI accept the same set.func isPublicCollection(iri string) bool { return iri == ap.PublicAudience || iri == "as:Public" || iri == "Public"}
// buildPostRecord translates the Page fields per the DEPRECATED// community.post lexicon. Reached only for a post already materialized under// that collection, which keeps being updated where it lies.func (m *Materializer) buildPostRecord(ctx context.Context, page *ap.Object, communityDID, authorDID, repoDID string) map[string]any { record := map[string]any{ "$type": CollectionPost, "community": communityDID, "author": authorDID, "createdAt": recordDatetime(page.Published.Time), } // The deprecated community.post lexicon keeps the 10000-grapheme / // 100000-byte body limit postv2 has outgrown. m.fillPostContent(ctx, record, page, repoDID, 10000, 100000) return record}
// fillPostContent adds the fields both post eras share — title, body and// facets, langs, self-labels, embed. The body is truncated to the era's// lexicon limits. Embed media is fetched into repoDID's blob store (the repo// holding the record); media failures degrade to a post without the affected// embed.func (m *Materializer) fillPostContent(ctx context.Context, record map[string]any, page *ap.Object, repoDID string, maximumContentGraphemes, maximumContentBytes int) { if page.Name != "" { record["title"] = truncateText(page.Name, 300, 3000) } if content := markdownFromObject(page); content != "" { // bridgedRichText can reduce an HTML-only body to nothing once tags // are stripped; a post is still valid without content (title/embed // carry it), so the field is simply omitted. if body, facets := bridgedRichText(content, maximumContentGraphemes, maximumContentBytes); body != "" { record["content"] = body if len(facets) > 0 { record["facets"] = facets } } } if langs := recordLangs(page.Language); len(langs) > 0 { record["langs"] = langs } if page.Sensitive != nil && *page.Sensitive { record["labels"] = selfLabels("nsfw") } if embed := m.buildPostEmbed(ctx, page, repoDID); embed != nil { record["embed"] = embed }}
// buildPostEmbed maps the Page's attachments onto the post embed union:// image attachments → social.coves.embed.images (blobs in repoDID, the repo// that will hold the record); a link attachment (or `url`) →// social.coves.embed.external with the Lemmy-provided thumbnail as thumb (no// og-image fetching in v1 — the AP object already carries the thumbnail when// there is one).func (m *Materializer) buildPostEmbed(ctx context.Context, page *ap.Object, repoDID string) map[string]any { var images []any var externalLink *ap.Link for i := range page.Attach { link := &page.Attach[i] href := link.Href if href == "" { href = link.URL // PieFed image attachments carry url } if href == "" { continue } if isImageAttachment(link) { if len(images) >= 8 { continue // lexicon maxLength } blob := m.fetchBlob(ctx, repoDID, href, slotEmbedImage) if blob == nil { continue } image := map[string]any{"image": *blob} if link.Name != "" { image["alt"] = truncateText(link.Name, 1000, 10000) } images = append(images, image) } else if externalLink == nil && isSafeLinkScheme(href) { // Remember the resolved href (Href or the PieFed `url` fallback), // not link.Href — which may be empty when only `url` was set. externalLink = &ap.Link{Href: href, Name: link.Name} } } if len(images) > 0 { return map[string]any{ "$type": "social.coves.embed.images", "images": images, } }
uri := "" title := "" if externalLink != nil { uri = externalLink.Href title = externalLink.Name } else if u := page.URLString(); u != "" && u != page.ID && isSafeLinkScheme(u) { // Older Lemmy put the shared link in `url` instead of `attachment`. uri = u } if uri == "" { // No link, no images. A Lemmy thumbnail alone (image posts sometimes // deliver the pictrs file in `image`) still makes an images embed. if thumbURL := imageURL(page.Image); thumbURL != "" { if blob := m.fetchBlob(ctx, repoDID, thumbURL, slotEmbedImage); blob != nil { return map[string]any{ "$type": "social.coves.embed.images", "images": []any{map[string]any{"image": *blob}}, } } } return nil }
external := map[string]any{"uri": uri} if title != "" { external["title"] = truncateGraphemes(title, 500) } if thumbURL := imageURL(page.Image); thumbURL != "" { if blob := m.fetchBlob(ctx, repoDID, thumbURL, slotExternalThumb); blob != nil { external["thumb"] = *blob } } return map[string]any{ "$type": "social.coves.embed.external", "external": external, }}
// isSafeLinkScheme restricts bridge-authored clickable URIs (external// embeds, link facets) to http/https. The lexicon's format:"uri" accepts// javascript:/data:/vbscript:, which a downstream client rendering the// remote-actor-controlled link as clickable would treat as a scripting URI.// Fail closed: an unsafe scheme drops the embed or facet rather than// erroring the whole record.func isSafeLinkScheme(uri string) bool { lower := strings.ToLower(strings.TrimSpace(uri)) return strings.HasPrefix(lower, "http://") || strings.HasPrefix(lower, "https://")}
// isImageAttachment reports whether an attachment is an image: an explicit// image/* mediaType, an AS2 Image type, or a href with a well-known image// extension (Lemmy usually sets mediaType; pictrs URLs carry extensions).func isImageAttachment(link *ap.Link) bool { if strings.HasPrefix(strings.ToLower(link.MediaType), "image/") { return true } if link.Type == ap.TypeImage { return true } href := strings.ToLower(link.Href) if href == "" { href = strings.ToLower(link.URL) } for _, ext := range []string{".png", ".jpg", ".jpeg", ".webp", ".gif", ".avif"} { if strings.HasSuffix(href, ext) { return true } } return false}