Something went wrong. Try again.
this post sux
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536/* * Merge the three labeling batches and pick the 100-post benchmark set. * Positives for angry / self_promotion / "personal life" are rare in a Discover * pull, so every such post is kept and the rest is filled with a seeded random * sample. Writes data/set.json: [{id, uri, text, alt, labels}]. */import { readFileSync, writeFileSync } from "node:fs";
const read = (f) => JSON.parse(readFileSync(new URL(f, import.meta.url)));const pool = read("./data/pool.json");const labels = new Map( [0, 1, 2].flatMap((k) => read(`./data/labels_part${k}.json`)).map((l) => [l.id, l]));if (labels.size !== pool.length) throw new Error(`labels ${labels.size} != pool ${pool.length}`);
const N = 100;let seed = 1234;const rand = () => ((seed = (seed * 1103515245 + 12345) % 2 ** 31) / 2 ** 31);
const rows = pool.map((p) => { const { id, note, ...l } = labels.get(p.id); return { id: p.id, uri: p.uri, text: p.text, alt: p.alt, labels: l, note };});const rare = (r) => r.labels.angry === 1 || r.labels.self_promotion === 1 || r.labels.topic === "personal life";const keep = rows.filter(rare);const rest = rows.filter((r) => !rare(r)).sort(() => rand() - 0.5);const set = keep.concat(rest.slice(0, N - keep.length)).sort((a, b) => a.id.localeCompare(b.id));
writeFileSync(new URL("./data/set.json", import.meta.url), JSON.stringify(set, null, 1));const count = (k, v) => set.filter((r) => r.labels[k] === v).length;for (const k of ["politics", "angry", "self_promotion", "funny"]) console.log(k.padEnd(15), "1:", count(k, 1), " 0:", count(k, 0), " null:", count(k, null));for (const t of ["personal life", "current events", "neither"]) console.log("topic", t, count("topic", t));console.log(`set: ${set.length} (${keep.length} rare-positive kept)`);