From 2ee88865b27b62011ff66a4db347ce2e2cb5fdb0 Mon Sep 17 00:00:00 2001 From: Michael Chernigin Date: Thu, 4 Jun 2026 16:23:37 +0400 Subject: [PATCH] fix: decode html entities in titles --- app/components/article-reader.tsx | 3 ++- app/components/feed-reader.tsx | 5 ++-- app/components/link-list.tsx | 3 ++- app/lib/html-entities.ts | 39 ++++++++++++++++++++++++++++++ app/lib/rss.ts | 1 + app/routes/articles.$linkId.tsx | 3 ++- app/routes/feeds.items.$itemId.tsx | 3 ++- app/services/extraction.server.ts | 7 +++--- tests/html-entities.test.ts | 10 ++++++++ tests/rss.test.ts | 24 ++++++++++++++++++ 10 files changed, 89 insertions(+), 9 deletions(-) create mode 100644 app/lib/html-entities.ts create mode 100644 tests/html-entities.test.ts diff --git a/app/components/article-reader.tsx b/app/components/article-reader.tsx index 6633c1c..8dd8ac1 100644 --- a/app/components/article-reader.tsx +++ b/app/components/article-reader.tsx @@ -6,11 +6,12 @@ import { Card, CardContent } from "~/components/ui/card"; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "~/components/ui/tooltip"; import { MarkdownContent } from "~/components/markdown-content"; import { WebsiteIcon } from "~/components/website-icon"; +import { decodeHtmlEntities } from "~/lib/html-entities"; import { formatDate, readingTime } from "~/lib/utils"; import type { ArticleSnapshot, SavedLink } from "~/db/schema"; export function ArticleReader({ link, snapshot }: { link: SavedLink; snapshot: ArticleSnapshot | null }) { - const title = snapshot?.title || link.title || link.url; + const title = decodeHtmlEntities(snapshot?.title || link.title || link.url); const readTime = readingTime(snapshot?.wordCount); const archiveFetcher = useFetcher(); const updateFetcher = useFetcher(); diff --git a/app/components/feed-reader.tsx b/app/components/feed-reader.tsx index 90065cb..b47c994 100644 --- a/app/components/feed-reader.tsx +++ b/app/components/feed-reader.tsx @@ -9,6 +9,7 @@ import { Popover, PopoverContent, PopoverTrigger } from "~/components/ui/popover import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "~/components/ui/tooltip"; import { WebsiteIcon } from "~/components/website-icon"; import type { FeedItem, FeedSubscription } from "~/db/schema"; +import { decodeHtmlEntities } from "~/lib/html-entities"; import { formatDate } from "~/lib/utils"; export type FeedItemWithSubscription = FeedItem & { subscription: FeedSubscription }; @@ -274,7 +275,7 @@ export function FeedItemList({ function FeedItemRow({ item }: { item: FeedItemWithSubscription }) { const fetcher = useFetcher(); - const title = item.title || item.url || "Untitled feed item"; + const title = decodeHtmlEntities(item.title || item.url || "Untitled feed item"); const source = feedDomain(item.subscription) || item.subscription.title || item.subscription.url; return ( @@ -400,7 +401,7 @@ export function FeedItemDetail({ item, error }: { item: FeedItemWithSubscription
-

{item.title || item.url || "Untitled feed item"}

+

{decodeHtmlEntities(item.title || item.url || "Untitled feed item")}

diff --git a/app/components/link-list.tsx b/app/components/link-list.tsx index 19cec64..bb7eec2 100644 --- a/app/components/link-list.tsx +++ b/app/components/link-list.tsx @@ -5,6 +5,7 @@ import { Button } from "~/components/ui/button"; import { Card, CardContent } from "~/components/ui/card"; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "~/components/ui/tooltip"; import { WebsiteIcon } from "~/components/website-icon"; +import { decodeHtmlEntities } from "~/lib/html-entities"; import { formatDate, readingTime } from "~/lib/utils"; import type { ArticleSnapshot, SavedLink } from "~/db/schema"; @@ -12,7 +13,7 @@ type LinkWithSnapshot = SavedLink & { snapshot: ArticleSnapshot | null }; function LinkListItem({ item }: { item: LinkWithSnapshot }) { const fetcher = useFetcher(); - const title = item.snapshot?.title || item.title || item.url; + const title = decodeHtmlEntities(item.snapshot?.title || item.title || item.url); const readTime = readingTime(item.snapshot?.wordCount); return ( diff --git a/app/lib/html-entities.ts b/app/lib/html-entities.ts new file mode 100644 index 0000000..411d8ba --- /dev/null +++ b/app/lib/html-entities.ts @@ -0,0 +1,39 @@ +const namedEntities: Record = { + amp: "&", + apos: "'", + gt: ">", + hellip: "…", + laquo: "«", + ldquo: "“", + lsquo: "‘", + lt: "<", + mdash: "—", + nbsp: "\u00a0", + ndash: "–", + quot: '"', + raquo: "»", + rdquo: "”", + rsquo: "’", +}; + +export function decodeHtmlEntities(value: string): string; +export function decodeHtmlEntities(value: string | null | undefined): string | null; +export function decodeHtmlEntities(value: string | null | undefined) { + if (value == null) return null; + + return value.replace(/&(?:#(\d+)|#x([\da-f]+)|([a-z][\w:-]*));/gi, (entity, decimal: string | undefined, hex: string | undefined, named: string | undefined) => { + if (decimal || hex) { + const codePoint = Number.parseInt(decimal ?? hex ?? "", decimal ? 10 : 16); + if (Number.isFinite(codePoint)) { + try { + return String.fromCodePoint(codePoint); + } catch { + return entity; + } + } + } + + if (named) return namedEntities[named.toLowerCase()] ?? entity; + return entity; + }); +} diff --git a/app/lib/rss.ts b/app/lib/rss.ts index 6655dbb..a193eca 100644 --- a/app/lib/rss.ts +++ b/app/lib/rss.ts @@ -36,6 +36,7 @@ const parser = new XMLParser({ parseTagValue: false, parseAttributeValue: false, trimValues: true, + htmlEntities: true, }); export function normalizeFeedUrl(input: string) { diff --git a/app/routes/articles.$linkId.tsx b/app/routes/articles.$linkId.tsx index a5140f4..c818c5b 100644 --- a/app/routes/articles.$linkId.tsx +++ b/app/routes/articles.$linkId.tsx @@ -2,11 +2,12 @@ import { useEffect } from "react"; import type { ActionFunctionArgs, LoaderFunctionArgs, MetaFunction } from "react-router"; import { redirect, useLoaderData, useRevalidator } from "react-router"; import { ArticleReader } from "~/components/article-reader"; +import { decodeHtmlEntities } from "~/lib/html-entities"; import { canRetry, kickOffExtraction } from "~/services/extraction.server"; import { deleteSavedLink, getSavedLinkForUser, updateSavedLinkState } from "~/services/links.server"; import { requireUser } from "~/services/session.server"; -export const meta: MetaFunction = ({ data }) => [{ title: `${data?.link.title || data?.snapshot?.title || "Article"} · Later` }]; +export const meta: MetaFunction = ({ data }) => [{ title: `${decodeHtmlEntities(data?.link.title || data?.snapshot?.title || "Article")} · Later` }]; export async function loader({ request, params }: LoaderFunctionArgs) { const user = await requireUser(request); diff --git a/app/routes/feeds.items.$itemId.tsx b/app/routes/feeds.items.$itemId.tsx index 4cb5579..f0e3668 100644 --- a/app/routes/feeds.items.$itemId.tsx +++ b/app/routes/feeds.items.$itemId.tsx @@ -1,11 +1,12 @@ import type { ActionFunctionArgs, LoaderFunctionArgs, MetaFunction } from "react-router"; import { data, useActionData, useLoaderData } from "react-router"; import { FeedItemDetail } from "~/components/feed-reader"; +import { decodeHtmlEntities } from "~/lib/html-entities"; import { handleFeedAction } from "~/services/feed-actions.server"; import { getFeedItemForUser } from "~/services/feeds.server"; import { requireUser } from "~/services/session.server"; -export const meta: MetaFunction = ({ data }) => [{ title: `${data?.item.title || "Feed item"} · Later` }]; +export const meta: MetaFunction = ({ data }) => [{ title: `${decodeHtmlEntities(data?.item.title || "Feed item")} · Later` }]; export async function loader({ request, params }: LoaderFunctionArgs) { const user = await requireUser(request); diff --git a/app/services/extraction.server.ts b/app/services/extraction.server.ts index a6a5602..2e4b395 100644 --- a/app/services/extraction.server.ts +++ b/app/services/extraction.server.ts @@ -3,6 +3,7 @@ import { and, eq, sql } from "drizzle-orm"; import { db } from "~/db/db.server"; import { articleSnapshots, savedLinks } from "~/db/schema"; import { sanitizeArticleHtml } from "~/lib/article-sanitize"; +import { decodeHtmlEntities } from "~/lib/html-entities"; import { cacheFaviconForPage } from "~/services/favicon.server"; export { sanitizeArticleHtml }; @@ -25,10 +26,10 @@ export async function extractArticle(url: string) { if (!contentHtml.trim()) throw new Error("No readable article content found."); return { - title: result.title || null, - author: result.author || null, + title: decodeHtmlEntities(result.title || null), + author: decodeHtmlEntities(result.author || null), canonicalUrl: result.schemaOrgData?.url || url, - excerpt: result.description || null, + excerpt: decodeHtmlEntities(result.description || null), leadImageUrl: result.image || null, language: result.language || null, contentHtml, diff --git a/tests/html-entities.test.ts b/tests/html-entities.test.ts new file mode 100644 index 0000000..7785872 --- /dev/null +++ b/tests/html-entities.test.ts @@ -0,0 +1,10 @@ +import { describe, expect, it } from "vitest"; +import { decodeHtmlEntities } from "~/lib/html-entities"; + +describe("decodeHtmlEntities", () => { + it("decodes numeric and common named HTML entities", () => { + expect(decodeHtmlEntities("Don’t Call It Art & Design — now")) + .toBe("Don’t Call It Art & Design — now"); + expect(decodeHtmlEntities("Мы выбираем новости")).toBe("Мы выбираем новости"); + }); +}); diff --git a/tests/rss.test.ts b/tests/rss.test.ts index 8e32c6b..e7020bb 100644 --- a/tests/rss.test.ts +++ b/tests/rss.test.ts @@ -43,6 +43,30 @@ describe("RSS parsing", () => { expect(feed.items[0].summaryHtml).toBe("

Hello

"); }); + it("decodes HTML entities in feed text", () => { + const feed = parseFeedXml( + ` + + + Медиа + https://example.com/ + Мы выбираем новости + + Новость + https://example.com/story + <p>Привет</p> + + + `, + "https://example.com/feed.xml", + ); + + expect(feed.title).toBe("Медиа"); + expect(feed.description).toBe("Мы выбираем новости"); + expect(feed.items[0].title).toBe("Новость"); + expect(feed.items[0].summaryHtml).toBe("

Привет

"); + }); + it("parses OPML subscription files", () => { const feeds = parseOpmlXml( ` -- knotmirror 0.0.1