Something went wrong. Try again.
vibescrobble! every agent, live on stage. vibescrobble.com
atproto agent llm did
Something went wrong. Try again.
3.3 kB · 70 lines
JavaScript
at main
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071// The Intelligence Index table, read off the identifiers harnesses report:// `node --test 'web/dev/test/*.test.mjs'`.import { test } from "node:test";import assert from "node:assert/strict";import { readFileSync } from "node:fs";import vm from "node:vm";
const source = readFileSync(new URL("../../intelligence.js", import.meta.url), "utf8");const context = {};vm.createContext(context);vm.runInContext(`${source}\nthis.Intelligence = Intelligence;`, context);const { Intelligence } = context;
test("every spelling of one model is one entry: routers, dates, snapshots, tags and quantisations", () => { const cases = { // What live scrobbles name. "claude-opus-5-5": "claude-opus-5-5", "deepseek-flash": "deepseek-v4-1-flash", // A context-window suffix, a router, a dated snapshot, a Vertex snapshot. "claude-opus-5-5[1m]": "claude-opus-5-5", "anthropic/claude-haiku-4.5-20251001": "claude-haiku-4-5", "claude-haiku-4-5-20251001": "claude-haiku-4-5", "claude-sonnet-4-5@20250929": "claude-sonnet-4-5", // Artificial Analysis's own word order. "claude-4-5-haiku": "claude-haiku-4-5", "deepseek/deepseek-v4.1-flash": "deepseek-v4-1-flash", "gpt-5.3-codex": "gpt-5-3-codex", "openai/gpt-5": "gpt-5", "gpt-5-mini-2025-08-07": "gpt-5-mini", "gemini-3-pro-preview": "gemini-3-pro", "models/gemini-2.5-pro": "gemini-2-5-pro", // A local runner's tag, and a quantised file. "qwen3:4b": "qwen3-4b", "Qwen3-4B-Instruct-2507-Q4_K_M.gguf": "qwen3-4b", }; for (const [model, entry] of Object.entries(cases)) { assert.equal(Intelligence.score(model), Intelligence.score(entry), model); assert.notEqual(Intelligence.score(model), null, model); }});
test("a model the table does not know has no score, and a near name is not borrowed", () => { for (const model of ["", null, undefined, "mystery-1", "gpt-5.1-codex-max", "claude-3-5-sonnet-20241022", "deepseek-chat"]) { assert.equal(Intelligence.score(model), null, String(model)); assert.equal(Intelligence.rank(model), null, String(model)); }});
test("rank runs from the weakest model in the table to the strongest", () => { assert.equal(Intelligence.rank("claude-opus-5-5"), 1); assert.equal(Intelligence.rank("qwen3:4b"), 0); const mid = Intelligence.rank("deepseek-flash"); assert.ok(mid > 0 && mid < 1); assert.ok(Intelligence.rank("claude-sonnet-5") > Intelligence.rank("claude-haiku-4-5"));});
test("the table says where it came from and when, and keys each name once", () => { assert.match(source, /https:\/\/artificialanalysis\.ai\/models/); assert.match(source, /pulled \d{4}-\d{2}-\d{2}/); const names = [...source.matchAll(/^\s*\["([a-z0-9-]+)", [\d.]+(?:, \[([^\]]*)\])?\]/gm)].flatMap((m) => [m[1], ...(m[2] ? m[2].match(/[a-z0-9-]+/g) : [])]); assert.ok(names.length > 40); assert.equal(new Set(names).size, names.length); for (const name of names) assert.equal(Intelligence.key(name), name, `${name} is its own key`);});
test("the board credits the index wherever it draws sizes from it", () => { const page = readFileSync(new URL("../../index.html", import.meta.url), "utf8"); assert.match(page, /<script src="intelligence\.js"><\/script>/); assert.match(page, /model sizes from the <a [^>]*>Artificial Analysis Intelligence Index<\/a>/);});