diff --git a/astro.config.mjs b/astro.config.mjs index b58a652..39d10a0 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -34,12 +34,19 @@ export default defineConfig({ routing: { prefixDefaultLocale: false }, }, - // /stats merged into /community-size. Keep the old URLs working, for every locale this build - // actually has -- hardcoding /nl/stats would 404 on the .eu build, which is English-only. - redirects: Object.fromEntries([ - ['/stats', '/community-size'], - ...scope.locales.slice(1).map((l) => [`/${l}/stats`, `/${l}/community-size`]), - ]), + // Redirects: .eu renamed /countries to /stats; national scopes keep /stats → /community-size. + // Each scope builds separately, so the conditional keeps the right redirect per build. + redirects: Object.fromEntries( + scope.aggregate + ? [ + ['/countries', '/stats'], + ...scope.locales.slice(1).map((l) => [`/${l}/countries`, `/${l}/stats`]), + ] + : [ + ['/stats', '/community-size'], + ...scope.locales.slice(1).map((l) => [`/${l}/stats`, `/${l}/community-size`]), + ], + ), integrations: [ sitemap(), diff --git a/scripts/eu-infra-pds.py b/scripts/eu-infra-pds.py index ee60e69..5fd11ce 100644 --- a/scripts/eu-infra-pds.py +++ b/scripts/eu-infra-pds.py @@ -2,10 +2,10 @@ """Count accounts hosted on European AT Protocol infrastructure, merged into europe-stats.json as `europeanInfra`. -Being hosted on a European sovereign PDS (Eurosky and peers) is a strong signal that an account is -European, but it says nothing about WHICH country, so this is a separate top-level number and is -never folded into the per-country counts or the language estimate. It answers a different question: -how many people run on European infrastructure rather than Bluesky's US hosting. +Being hosted on a European sovereign PDS (Eurosky, w-social, and other EU-operated PDSes) is a strong +signal that an account is European, but it says nothing about WHICH country, so this is a separate +top-level number and is never folded into the per-country counts or the language estimate. It answers +a different question: how many people run on European infrastructure rather than Bluesky's US hosting. Given --placed (the confirmed, country-placed DIDs from eu-stats-aggregate.py --emit-placed) it also reports how much of that overlaps the per-country counts, so the site can publish a European @@ -14,6 +14,7 @@ total that adds up instead of counting a Dutch account on Eurosky twice. python3 scripts/eu-infra-pds.py --stats europe-stats.json --placed eu-placed.dids python3 scripts/eu-infra-pds.py --self-test """ + from __future__ import annotations import argparse @@ -23,10 +24,43 @@ import sys import time import urllib.request -# European-operated PDS hosts. Start with Eurosky; add peers here as they appear. `endpoint` is the -# PDS service URL that answers com.atproto.sync.listRepos. +# European-operated PDS hosts that expose com.atproto.sync.listRepos on the open network. A PDS +# being here means accounts on it are confirmed European — a strong signal, just not which country. +# Only include PDSes whose listRepos endpoint is publicly accessible; private/closed PDSes and bot +# networks (e.g. blacksky.app) are excluded. Unreachable PDSes are harmless here: the script's +# except-Exception swallows the failure so a down host never blocks the deploy. EUROPEAN_PDS = [ - {"host": "eurosky.social", "endpoint": "https://eurosky.social", "label": "Eurosky"}, + { + "host": "eurosky.social", + "endpoint": "https://eurosky.social", + "label": "Eurosky", + }, + { + "host": "pds.wsocial.network", + "endpoint": "https://pds.wsocial.network", + "label": "w-social", + }, + { + "host": "forum-hietzing.at", + "endpoint": "https://forum-hietzing.at", + "label": "Hietzing", + }, + { + "host": "pds.bowverse.fr", + "endpoint": "https://pds.bowverse.fr", + "label": "Bowverse", + }, + {"host": "pds.sved.be", "endpoint": "https://pds.sved.be", "label": "Sved"}, + { + "host": "pds.kelosocial.eu", + "endpoint": "https://pds.kelosocial.eu", + "label": "Kelosocial", + }, + { + "host": "pds.jdkserver.nl", + "endpoint": "https://pds.jdkserver.nl", + "label": "JDK", + }, ] @@ -63,7 +97,12 @@ def build(hosts=EUROPEAN_PDS, placed: set[str] | None = None) -> dict: for h in hosts: try: total, active, placed_active = count_repos(h["endpoint"], placed) - row = {"host": h["host"], "label": h["label"], "accounts": active, "reposTotal": total} + row = { + "host": h["host"], + "label": h["label"], + "accounts": active, + "reposTotal": total, + } if placed: row["placedInCountry"] = placed_active out.append(row) @@ -91,12 +130,14 @@ def read_dids(path: str) -> set[str]: def _self_test(): # Fake one page of listRepos so the de-duplication is tested, not just asserted. - page = {"repos": [ - {"did": "did:plc:a", "active": True}, # placed under a country already - {"did": "did:plc:b", "active": True}, # European infra only - {"did": "did:plc:c", "active": True}, # European infra only - {"did": "did:plc:d", "active": False}, # deactivated: not an account - ]} + page = { + "repos": [ + {"did": "did:plc:a", "active": True}, # placed under a country already + {"did": "did:plc:b", "active": True}, # European infra only + {"did": "did:plc:c", "active": True}, # European infra only + {"did": "did:plc:d", "active": False}, # deactivated: not an account + ] + } import io import contextlib @@ -109,13 +150,19 @@ def _self_test(): try: hosts = [{"host": "x", "label": "X", "endpoint": "https://x"}] plain = build(hosts) - assert plain == {"hosts": [{"host": "x", "label": "X", "accounts": 3, "reposTotal": 4}], - "total": 3}, plain - assert "countryless" not in plain, plain # no --placed: the site must fall back, not guess + assert plain == { + "hosts": [{"host": "x", "label": "X", "accounts": 3, "reposTotal": 4}], + "total": 3, + }, plain + assert "countryless" not in plain, ( + plain + ) # no --placed: the site must fall back, not guess deduped = build(hosts, placed={"did:plc:a", "did:plc:zz"}) assert deduped["total"] == 3, deduped - assert deduped["placedInCountry"] == 1, deduped # only a, and only because it is active - assert deduped["countryless"] == 2, deduped # b + c: the number safe to add + assert deduped["placedInCountry"] == 1, ( + deduped + ) # only a, and only because it is active + assert deduped["countryless"] == 2, deduped # b + c: the number safe to add finally: urllib.request.urlopen = real print("eu-infra self-test OK") @@ -125,9 +172,12 @@ def main(): ap = argparse.ArgumentParser() ap.add_argument("--stats", default="europe-stats.json") ap.add_argument("--out", default=None) - ap.add_argument("--placed", default=None, - help="eu-placed.dids from eu-stats-aggregate.py --emit-placed; without it " - "the overlap is unknown and the site keeps the two numbers separate") + ap.add_argument( + "--placed", + default=None, + help="eu-placed.dids from eu-stats-aggregate.py --emit-placed; without it " + "the overlap is unknown and the site keeps the two numbers separate", + ) ap.add_argument("--self-test", action="store_true") args = ap.parse_args() @@ -144,13 +194,20 @@ def main(): f.write(json.dumps(stats, indent=2) + "\n") os.replace(tmp, out) hosts = ", ".join(f"{h['label']}={h['accounts']:,}" for h in infra["hosts"]) - print(f"europeanInfra: {infra['total']:,} active on {len(infra['hosts'])} European PDS host(s) ({hosts})") + print( + f"europeanInfra: {infra['total']:,} active on {len(infra['hosts'])} European PDS host(s) ({hosts})" + ) if "countryless" in infra: - print(f" of those, {infra['placedInCountry']:,} are already counted under a country; " - f"{infra['countryless']:,} are European infrastructure only") + print( + f" of those, {infra['placedInCountry']:,} are already counted under a country; " + f"{infra['countryless']:,} are European infrastructure only" + ) else: - print(" no --placed given: overlap with the per-country counts unknown, " - "so the site will keep the numbers separate", file=sys.stderr) + print( + " no --placed given: overlap with the per-country counts unknown, " + "so the site will keep the numbers separate", + file=sys.stderr, + ) if __name__ == "__main__": diff --git a/scripts/nl-handle-class.py b/scripts/nl-handle-class.py index 5ee20f5..2b97020 100644 --- a/scripts/nl-handle-class.py +++ b/scripts/nl-handle-class.py @@ -40,6 +40,12 @@ PDS_SUFFIXES = ( ".bsky.team", ".blacksky.community", ".pds.witchcraft.systems", + ".wsocial.eu", + ".kelosocial.eu", + ".pds.bowverse.fr", + ".pds.sved.be", + ".pds.jdkserver.nl", + ".forum-hietzing.at", ) # Bridges: the handle describes an account on another network. @@ -129,7 +135,9 @@ def self_test() -> int: assert is_dutch_domain("smb.phys.tue.nl") assert is_dutch_domain("gemeente.amsterdam") assert not is_dutch_domain("someone.bsky.social") - assert not is_dutch_domain("mastodon.nl.ap.brid.gy"), "bridge domain is not the holder's" + assert not is_dutch_domain("mastodon.nl.ap.brid.gy"), ( + "bridge domain is not the holder's" + ) assert not is_dutch_domain("example.com") print("self-test ok") return 0 diff --git a/src/components/pages/EuropeFacts.astro b/src/components/pages/EuropeFacts.astro index 6ed8372..55ae5bd 100644 --- a/src/components/pages/EuropeFacts.astro +++ b/src/components/pages/EuropeFacts.astro @@ -18,10 +18,18 @@ const lexDetail = europeFacts.lexicons.byNsid.slice(0, 30); const beyondPct = confirmedTotal ? Math.round((europeFacts.lexicons.accountsUsingSomething / confirmedTotal) * 100) : 0; -// Eurosky is shown in full as the European-infrastructure stat, so drop it from this "other hosts" -// list — otherwise the page shows two different Eurosky numbers (the full count vs the small -// country-attributed subset that happens to fall in this scanned set). -const otherHosts = europeFacts.pds.topOtherHosts.filter((h) => h.host !== 'eurosky.social'); +// European PDSes are counted in full in the European-infrastructure stat, so drop them from this +// "other hosts" list — otherwise the page shows two different numbers for the same hosts. +const EU_PDS_HOSTS = new Set([ + 'eurosky.social', + 'pds.wsocial.network', + 'forum-hietzing.at', + 'pds.bowverse.fr', + 'pds.sved.be', + 'pds.kelosocial.eu', + 'pds.jdkserver.nl', +]); +const otherHosts = europeFacts.pds.topOtherHosts.filter((h) => !EU_PDS_HOSTS.has(h.host)); const pdsTotal = europeFacts.pds.blueskyHosted + europeFacts.pds.elsewhere; const elsewherePct = pdsTotal ? Math.round((europeFacts.pds.elsewhere / pdsTotal) * 1000) / 10 : 0; const bskyPct = pdsTotal ? Math.round((europeFacts.pds.blueskyHosted / pdsTotal) * 1000) / 10 : 0; @@ -69,7 +77,7 @@ const bskyPct = pdsTotal ? Math.round((europeFacts.pds.blueskyHosted / pdsTotal) Bluesky hosted {bskyPct}% elsewhere {elsewherePct}% -
{nf(europeFacts.pds.elsewhere)} accounts ({elsewherePct}%) live on a PDS outside Bluesky's own hosting. Eurosky is counted in full in the European-infrastructure figure above, so it is left out of this list to avoid showing two different numbers for it.
+{nf(europeFacts.pds.elsewhere)} accounts ({elsewherePct}%) live on a PDS outside Bluesky's own hosting. European PDSes are counted in full in the European-infrastructure figure above, so they are left out of this list to avoid showing two different numbers.
{otherHosts.length > 0 && (Redirecting to /stats/.
+ + diff --git a/src/pages/[...lang]/countries/[cc].astro b/src/pages/[...lang]/countries/[cc].astro index 97b4f9a..9b37f6f 100644 --- a/src/pages/[...lang]/countries/[cc].astro +++ b/src/pages/[...lang]/countries/[cc].astro @@ -1,13 +1,9 @@ --- -import Base from '../../../layouts/Base.astro'; -import { LOCALES, DEFAULT_LOCALE, localeHref } from '../../../i18n/paths'; +// Former /countries/City communities, run by their own people. They cover a city, not the whole country.
-- {community.name} — {community.note} -
- {!community.ours && ( -Run by its own people, not by us. We link to it and stay out of the way.
- )} -- Would you like to run one? There are {nf(stat.confirmed)} accounts here already, and this - page is the only thing standing in for a community that should be run by people who live there. -
-- We can hand over the data behind this page, the collectors that produce it, and the site - itself, which is open source. What we cannot do is be from {name}. -
-Interested? Ping @gui.do.
-Redirecting to /stats/{cc}/.
+ + diff --git a/src/pages/[...lang]/stats.astro b/src/pages/[...lang]/stats.astro new file mode 100644 index 0000000..d631ca7 --- /dev/null +++ b/src/pages/[...lang]/stats.astro @@ -0,0 +1,347 @@ +--- +import Base from '../../layouts/Base.astro'; +import { localePaths, localeHref } from '../../i18n/paths'; +import { getScope } from '../../config/scopes'; +import { + europeStats, byConfirmed, confirmedTotal, isWeakTldCountry, countryName, communityFor, citiesFor, + europeanInfra, infraCountryless, europeanConfirmedTotal, +} from '../../data/europe-stats'; +import { existingCommunities } from '../../data/communities'; +import EuropeFacts from '../../components/pages/EuropeFacts.astro'; +import EuroskyCta from '../../components/EuroskyCta.astro'; + +// Aggregate-only. On .nl and .be getStaticPaths returns nothing, so the route is not built at +// all — no dead link, no empty page, no redirect. +export const getStaticPaths = () => + getScope(import.meta.env.PUBLIC_SITE_SCOPE).aggregate ? localePaths() : []; + +const { lang } = Astro.props; +const nf = (n: number) => n.toLocaleString('en-GB').replace(/,/g, '.'); + +const withData = byConfirmed.filter((c) => c.total > 0); +const estimate = europeStats.peopleEstimate ?? 0; + +// Accounts on a European PDS are confirmed European; they just have no country. Counting them +// only in a side note understated Europe by tens of thousands, so they are a number of their own +// — but only once the pipeline can tell us how many of them a country already claims. Until then +// (infraCountryless === null) the page keeps the old two-number split rather than guess an +// overlap of zero and publish a total that double-counts. +const infraOnly = infraCountryless; +const hasInfra = !!europeanInfra && europeanInfra.total > 0; +const deduped = hasInfra && infraOnly !== null; +const euTotal = deduped ? europeanConfirmedTotal : confirmedTotal; + +// Confirmed floor as a share of the modelled estimate, for the proportion bar (small by design). +const estPct = estimate > 0 ? Math.round((euTotal / estimate) * 100) : 0; +const pctCountry = estimate > 0 ? (confirmedTotal / estimate) * 100 : 0; +const pctInfra = estimate > 0 && deduped ? ((infraOnly ?? 0) / estimate) * 100 : 0; + +// The directory lists EU members first, then the substantive non-EU European communities. The rest +// of the non-EU tail — Russia/Belarus/Ukraine and micro-states, almost all weak-TLD noise — is left +// off the table (matching what the homepage map already excludes), with a footnote for honesty. +const EU27 = new Set([ + 'at', 'be', 'bg', 'hr', 'cy', 'cz', 'dk', 'ee', 'fi', 'fr', 'de', 'gr', 'hu', 'ie', 'it', 'lv', + 'lt', 'lu', 'mt', 'nl', 'pl', 'pt', 'ro', 'sk', 'si', 'es', 'se', +]); +const ALSO_IN_EUROPE = new Set(['gb', 'no', 'ch', 'is', 'li']); +const euRows = withData.filter((c) => EU27.has(c.country)); +const alsoRows = withData.filter((c) => ALSO_IN_EUROPE.has(c.country)); +const otherRows = withData.filter((c) => !EU27.has(c.country) && !ALSO_IN_EUROPE.has(c.country)); +const otherConfirmed = otherRows.reduce((a, c) => a + c.confirmed, 0); +const shownCount = euRows.length + alsoRows.length; + +// Split the directory into country-wide and single-city communities so the two never blur (a city +// community must never read as standing in for its whole country). +const nationalCommunities = existingCommunities.filter((c) => c.scope === 'national'); +const cityCommunities = existingCommunities.filter((c) => c.scope === 'city'); +--- + +The AT Protocol community across {nf(shownCount)} European countries, each + account counted once.
++ The first two are separate evidence for the same thing, and they do not overlap: together + {' '}{nf(euTotal)} accounts are confirmed European. Of the + {' '}{nf(europeanInfra!.total)} accounts on European PDSes, + {' '}{nf(europeanInfra!.placedInCountry ?? 0)} already appear under a country above and are + counted there, not twice. +
+ )} + {estimate > 0 && ( +
+ Running on a European PDS is a deliberate choice for European data sovereignty. It tells
+ us an account is European, but not which country, so these accounts appear under no
+ country in the table below.
+ {!deduped && ' They are counted here rather than under any country above.'}
+ {' '}Most accounts are still on Bluesky's US hosting.
+
+ AT Protocol has no country field, so we only count people who + left a trace we can read: a country domain in their handle, a location in their profile, a + community verifier, a curated list. Each account is counted once. Where signals disagree we + place it nowhere rather than guess, so these numbers add up instead of overlapping. Language + is never evidence: most European languages cross borders, so posting in German says nothing + about living in Germany. +
+| Country | +Accounts | +Community | +
|---|---|---|
| + {countryName(c.country, lang)} + | ++ {nf(c.confirmed)} + {weak && ( + ⚠ + )} + | ++ {community ? ( + {community.name} + ) : cities.length ? ( + cities.map((x) => {x.name}) + ) : ( + none yet + )} + | +
+ Plus {nf(otherConfirmed)} accounts in {nf(otherRows.length)} other European countries with + only weak or very small signals (and non-EU states like Russia and Belarus), not listed + here. They are still counted in the totals above. +
+ )} +
+ ⚠ marks countries whose domain is widely registered for English word-play rather than as a
+ statement about the country (.es, .it, .at,
+ .se, .is). Those matches are counted but ranked low, so a country
+ can show a small number here and a much larger raw match count. Spain is the clearest case:
+ about 9.800 .es handles, almost none of them Spanish.
+
+ Where a community already exists we link to it, so you can find your people. Some cover a + whole country, others a single city. +
+ ++ Updated {europeStats.generatedAt.slice(0, 10)}. Method: {europeStats.method.attribution}. + {' '}{nf(europeStats.contested)} accounts were contested and are counted in no country. +
+ + + diff --git a/src/pages/[...lang]/stats/[cc].astro b/src/pages/[...lang]/stats/[cc].astro new file mode 100644 index 0000000..bc57a2d --- /dev/null +++ b/src/pages/[...lang]/stats/[cc].astro @@ -0,0 +1,104 @@ +--- +import Base from '../../../layouts/Base.astro'; +import { LOCALES, DEFAULT_LOCALE, localeHref } from '../../../i18n/paths'; +import { getScope } from '../../../config/scopes'; +import { europeStats, countryName, communityFor, citiesFor } from '../../../data/europe-stats'; +import NationalEstimatePage from '../../../components/pages/NationalEstimatePage.astro'; + +// One page per country that has any data, per locale. Aggregate scope only. The body is the shared +// community-size layout (estimate, two numbers, sources, facts), with this country's community / +// invite call-to-action slotted in. +export function getStaticPaths() { + const scope = getScope(import.meta.env.PUBLIC_SITE_SCOPE); + if (!scope.aggregate) return []; + const paths = []; + for (const locale of LOCALES) { + for (const c of europeStats.countries) { + if (c.total > 0) { + paths.push({ + params: { lang: locale === DEFAULT_LOCALE ? undefined : locale, cc: c.country }, + props: { lang: locale, stat: c }, + }); + } + } + } + return paths; +} + +const { lang, stat } = Astro.props; +const cc = stat.country; +const name = countryName(cc, lang); +const nf = (n: number) => n.toLocaleString('en-GB').replace(/,/g, '.'); +const community = communityFor(cc); +const cities = citiesFor(cc); +--- + +City communities, run by their own people. They cover a city, not the whole country.
++ {community.name} — {community.note} +
+ {!community.ours && ( +Run by its own people, not by us. We link to it and stay out of the way.
+ )} ++ Would you like to run one? There are {nf(stat.confirmed)} accounts here already, and this + page is the only thing standing in for a community that should be run by people who live there. +
++ We can hand over the data behind this page, the collectors that produce it, and the site + itself, which is open source. What we cannot do is be from {name}. +
+Interested? Ping @gui.do.
+