diff --git a/scripts/ssm-run.py b/scripts/ssm-run.py new file mode 100755 index 00000000..fa13f1f7 --- /dev/null +++ b/scripts/ssm-run.py @@ -0,0 +1,85 @@ +#!/usr/bin/env -S uv run -q --script +# /// script +# requires-python = ">=3.10" +# dependencies = ["boto3"] +# /// +"""Runs one shell command on an EC2 instance over SSM and prints its output. + + scripts/ssm-run.py [--name vibescrobble | --instance i-...] [--timeout 120] 'command' + +The instance is the running one tagged Name= unless --instance names it. +Credentials and region are the caller's own (AWS_PROFILE, AWS_REGION). +Exits non-zero when the command fails or does not finish in time. + +This exists because `aws ssm send-command` fails on some aws-cli builds with +"badly formed help string", whatever its arguments. SSM caps captured stdout +near 24 KB, so filter on the instance. +""" + +import argparse +import re +import sys +import time + +import boto3 + +ANSI = re.compile(r"\x1b\[[0-9;]*m") + + +def find_instance(session, name): + ec2 = session.client("ec2") + found = [ + i["InstanceId"] + for r in ec2.describe_instances( + Filters=[ + {"Name": "tag:Name", "Values": [name]}, + {"Name": "instance-state-name", "Values": ["running"]}, + ] + )["Reservations"] + for i in r["Instances"] + ] + if len(found) != 1: + sys.exit(f"ssm-run: {len(found)} running instances tagged Name={name}; pass --instance") + return found[0] + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--name", default="vibescrobble", help="the instance's Name tag") + p.add_argument("--instance", help="an instance id; overrides --name") + p.add_argument("--timeout", type=int, default=120, help="seconds to wait for the command") + p.add_argument("command") + a = p.parse_args() + + session = boto3.Session() + if session.region_name is None: + sys.exit("ssm-run: no region; set AWS_REGION or the profile's region") + instance = a.instance or find_instance(session, a.name) + ssm = session.client("ssm") + command_id = ssm.send_command( + InstanceIds=[instance], + DocumentName="AWS-RunShellScript", + Parameters={"commands": [a.command]}, + )["Command"]["CommandId"] + + deadline = time.monotonic() + a.timeout + out = None + while time.monotonic() < deadline: + time.sleep(2) + try: + out = ssm.get_command_invocation(CommandId=command_id, InstanceId=instance) + except ssm.exceptions.InvocationDoesNotExist: + continue + if out["Status"] not in ("Pending", "InProgress", "Delayed"): + break + if out is None or out["Status"] in ("Pending", "InProgress", "Delayed"): + sys.exit(f"ssm-run: command {command_id} on {instance} did not finish in {a.timeout}s") + + sys.stdout.write(ANSI.sub("", out["StandardOutputContent"])) + sys.stderr.write(ANSI.sub("", out["StandardErrorContent"])) + if out["Status"] != "Success": + sys.exit(f"ssm-run: command {command_id} on {instance}: {out['Status']}") + + +if __name__ == "__main__": + main() diff --git a/scripts/verify-deploy.sh b/scripts/verify-deploy.sh new file mode 100755 index 00000000..b7759cf9 --- /dev/null +++ b/scripts/verify-deploy.sh @@ -0,0 +1,302 @@ +#!/usr/bin/env bash +# Checks a deploy of vibescrobble.com against a baseline taken before it. +# +# scripts/verify-deploy.sh before [options] +# scripts/verify-deploy.sh after [options] [--sha ] +# +# --site the site (default https://vibescrobble.com) +# --state where `before` saves the baseline and `after` reads it +# (default ~/.local/state/vibescrobble/verify-deploy.json) +# --name the instance's Name tag (default vibescrobble) +# --baseline-hours before: hours of logs the warning baseline covers (default 24) +# --sha after: the commit release.txt should name (default HEAD) +# --tolerance after: a burn drop this small is rounding (default 1) +# --age-slack after: seconds added to each end of an ageing interval, +# for clock skew and the time a snapshot takes (default 60) +# --max-pages after: listRecords pages read per account when +# explaining a drop (default 50) +# +# `before` records /health's agent and scrobble counts, every account's burn +# on the day, week and all-time leaderboards, each one's top 50, and the kinds +# of warning the index and query containers logged. `after` checks: +# +# - release.txt names ; +# - /health is ok, connected, and its counts did not drop; +# - no account's all-time burn dropped; +# - each day or week drop is ageing: the account's scrobbles that left the +# window between the two snapshots hold at least the burn it lost; +# - the index and query logged no warning kind the baseline lacks. +# +# Ageing is read from each account's own repository, the complete copy. A +# warning's kind is its level, target and message, without its fields. +# Everything is read-only. The logs are read over SSM with the caller's own +# AWS credentials (AWS_PROFILE, AWS_REGION), through scripts/ssm-run.py. +set -euo pipefail + +cd "$(dirname "$0")/.." + +usage() { + echo "usage: scripts/verify-deploy.sh before|after [--site URL] [--state FILE] [--name TAG] [--baseline-hours H] [--sha REV] [--tolerance BURN] [--age-slack S] [--max-pages N]" +} + +mode="${1:-}" +case "$mode" in + before | after) shift ;; + -h | --help) + usage + exit 0 + ;; + *) + usage >&2 + exit 2 + ;; +esac + +site=https://vibescrobble.com +state="${XDG_STATE_HOME:-$HOME/.local/state}/vibescrobble/verify-deploy.json" +name=vibescrobble +baseline_hours=24 +rev=HEAD +tolerance=1 +age_slack=60 +max_pages=50 +while [ $# -gt 0 ]; do + [ $# -ge 2 ] || { + usage >&2 + exit 2 + } + case "$1" in + --site) site="${2%/}" ;; + --state) state="$2" ;; + --name) name="$2" ;; + --baseline-hours) baseline_hours="$2" ;; + --sha) rev="$2" ;; + --tolerance) tolerance="$2" ;; + --age-slack) age_slack="$2" ;; + --max-pages) max_pages="$2" ;; + *) + usage >&2 + exit 2 + ;; + esac + shift 2 +done + +for tool in curl jq uv; do + command -v "$tool" >/dev/null || { + echo "verify-deploy: $tool is not on PATH" >&2 + exit 2 + } +done + +windows=(day week all) +declare -A window_secs=([day]=86400 [week]=604800) + +get() { curl -fsS --max-time 20 "$@"; } + +# Every account's own burn on one window's leaderboard, as {did: burn}. +burn_rows() { + local window="$1" cursor="" rows='{}' page pages=0 + while :; do + page="$(get -G "$site/leaderboard" --data-urlencode "window=$window" \ + --data-urlencode by=agent --data-urlencode limit=500 \ + ${cursor:+--data-urlencode "cursor=$cursor"})" + rows="$(jq -c --argjson acc "$rows" '$acc + ([.rows[] | {(.key): .burn}] | add // {})' <<<"$page")" + cursor="$(jq -r '.cursor // empty' <<<"$page")" + pages=$((pages + 1)) + [ -n "$cursor" ] && [ "$pages" -lt 100 ] || break + done + echo "$rows" +} + +snapshot() { + local health weights rows='{}' top='{}' w + health="$(get "$site/health")" + weights="$(get "$site/leaderboard?window=all&by=vendor" | jq -c .weights)" + for w in "${windows[@]}"; do + rows="$(jq -c --arg w "$w" --argjson r "$(burn_rows "$w")" '. + {($w): $r}' <<<"$rows")" + top="$(jq -c --arg w "$w" --argjson t "$(get "$site/leaderboard?window=$w&by=agent&limit=50" | jq -c '[.rows[].key]')" \ + '. + {($w): $t}' <<<"$top")" + done + jq -n --argjson time "$(date +%s)" \ + --arg release "$(get "$site/release.txt" | tr -d '[:space:]')" \ + --argjson health "$health" --argjson weights "$weights" \ + --argjson burn "$rows" --argjson top "$top" \ + '{time: $time, release: $release, + health: ($health | {agents, scrobbles}), weights: $weights, + burn: $burn, top: $top}' +} + +# Warning kinds the index and query containers logged since , and +# until when given, as `count kind` lines. +warning_kinds() { + scripts/ssm-run.py --name "$name" --timeout 150 "since=$1; until=${2:-} +$( + cat <<'EOF' +for c in $(docker ps --format '{{.Names}}' | grep -E -- '-(index|query)-'); do + docker logs --since "$since" ${until:+--until "$until"} "$c" 2>&1 +done | + sed -E 's/\x1b\[3m.*//; s/\x1b\[[0-9;]*m//g' | + grep -E '^[0-9TZ:.-]+ +(WARN|ERROR) ' | + sed -E 's/^[0-9TZ:.-]+ +//; s/ +$//; s/[0-9a-f]{8,}/X/g; s/did:[a-z]+:[A-Za-z0-9.:%-]+/DID/g; s/[0-9]+/N/g' | + cut -c1-160 | sort | uniq -c +EOF + )" +} +kinds_json() { + jq -R -s 'split("\n") | map(capture("^ *(?[0-9]+) (?.+)$")? // empty) + | map({(.k): (.n | tonumber)}) | add // {}' +} + +# The PDS that holds a DID's repository. +pds_of() { + local doc + case "$1" in + did:plc:*) doc="$(get "https://plc.directory/$1")" ;; + did:web:*) doc="$(get "https://${1#did:web:}/.well-known/did.json")" ;; + *) return 1 ;; + esac + jq -r '[.service[] | select(.id | endswith("#atproto_pds")) | .serviceEndpoint][0] // empty' <<<"$doc" +} + +# The burn of a DID's scrobbles created in [from, to), epoch seconds, priced +# with the baseline's weights. +aged_burn() { + local did="$1" from="$2" to="$3" pds cursor="" sum=0 page pages=0 oldest + pds="$(pds_of "$did")" || return 1 + [ -n "$pds" ] || return 1 + while [ "$pages" -lt "$max_pages" ]; do + page="$(get -G "${pds%/}/xrpc/com.atproto.repo.listRecords" --data-urlencode "repo=$did" \ + --data-urlencode collection=com.vibescrobble.scrobble --data-urlencode limit=100 \ + ${cursor:+--data-urlencode "cursor=$cursor"})" || return 1 + read -r sum oldest < <(jq -r --argjson w "$weights" --argjson from "$from" --argjson to "$to" \ + --argjson acc "$sum" ' + def epoch: try (sub("\\.[0-9]+"; "") | sub("\\+00:00$"; "Z") | fromdateiso8601) catch null; + [.records[].value | {at: (.createdAt | epoch), u: (.usage // {})}] as $r + | ($acc + ([$r[] | select(.at != null and .at >= $from and .at < $to) + | (.u.input // 0) * $w.input + (.u.output // 0) * $w.output + + (.u.cacheRead // 0) * $w.cacheRead + (.u.cacheWrite // 0) * $w.cacheWrite] | add // 0)) + , ([$r[].at | select(. != null)] | min // 0)' <<<"$page" | tr '\n' ' ') + cursor="$(jq -r '.cursor // empty' <<<"$page")" + pages=$((pages + 1)) + [ -n "$cursor" ] && [ "$oldest" -ge "$from" ] || break + done + echo "$sum" +} + +mkdir -p "$(dirname "$state")" + +if [ "$mode" = before ]; then + snap="$(snapshot)" + now="$(jq .time <<<"$snap")" + kinds="$(warning_kinds $((now - baseline_hours * 3600)) "$now" | kinds_json)" + jq --argjson warnings "$kinds" --argjson h "$baseline_hours" \ + '. + {warnings: $warnings, baseline_hours: $h}' <<<"$snap" >"$state" + jq -r '"verify-deploy: baseline at \(.time | todate), release \(.release)", + " \(.health.agents) agents, \(.health.scrobbles) scrobbles", + (.burn | to_entries[] | " \(.key) burn \(.value | add // 0 | floor) over \(.value | length) accounts"), + " \(.warnings | length) warning kinds over \(.baseline_hours) h:", + (.warnings | to_entries[] | " \(.value)\t\(.key)")' "$state" + echo " saved to $state" + exit 0 +fi + +[ -f "$state" ] || { + echo "verify-deploy: no baseline at $state; run \`scripts/verify-deploy.sh before\` first" >&2 + exit 2 +} +base="$(cat "$state")" +weights="$(jq -c .weights <<<"$base")" +t0="$(jq .time <<<"$base")" + +failures=() +ok() { echo "ok $1"; } +note() { echo " $1"; } +fail() { + echo "FAIL $1" + failures+=("$1") +} + +# Release. release.txt holds a short sha; compare on the shorter of the two. +want="$(git rev-parse --verify --quiet "$rev^{commit}" 2>/dev/null || echo "$rev")" +serving="$(get "$site/release.txt" 2>/dev/null | tr -d '[:space:]' || true)" +if [ -n "$serving" ] && { [[ $want == "$serving"* ]] || [[ $serving == "$want"* ]]; }; then + ok "release: release.txt names $serving (was $(jq -r .release <<<"$base"))" +else + fail "release: release.txt names ${serving:-nothing}, want ${want:0:12}" +fi + +snap="$(snapshot)" +t1="$(jq .time <<<"$snap")" + +# Health. +health="$(get "$site/health" || echo '{}')" +if [ "$(jq -r '.status == "ok" and .connected == true' <<<"$health")" != true ]; then + fail "health: status $(jq -r '.status // "none"' <<<"$health"), connected $(jq -r '.connected // "?"' <<<"$health")" +fi +for count in agents scrobbles; do + b="$(jq ".health.$count" <<<"$base")" + a="$(jq ".health.$count" <<<"$snap")" + if [ "$a" -ge "$b" ]; then + ok "health: $count $b -> $a" + else + fail "health: $count dropped, $b -> $a; the index may still be restoring" + fi +done + +# Burn. A drop beyond the tolerance is a failure on the all-time board; on a +# day or week board it is ageing if enough burn left the window. +for w in "${windows[@]}"; do + drops="$(jq -r --arg w "$w" --argjson after "$(jq -c ".burn.$w" <<<"$snap")" --argjson tol "$tolerance" \ + '.burn[$w] | to_entries[] | select(($after[.key] // 0) < .value - $tol) + | "\(.key) \(.value) \($after[.key] // 0)"' <<<"$base")" + totals="$(jq -r --arg w "$w" '.burn[$w] | add // 0 | floor' <<<"$base") -> $(jq -r --arg w "$w" '.burn[$w] | add // 0 | floor' <<<"$snap")" + left="$(jq -r --arg w "$w" --argjson now "$(jq -c ".top.$w" <<<"$snap")" '.top[$w] - $now | length' <<<"$base")" + if [ -z "$drops" ]; then + ok "burn $w: $totals, no account dropped; $left left the top 50" + continue + fi + if [ "$w" = all ]; then + fail "burn all: $totals; these accounts dropped:" + while read -r did b a; do note "$did $(printf %.0f "$b") -> $(printf %.0f "$a")"; done <<<"$drops" + continue + fi + span="${window_secs[$w]}" + from=$((t0 - span - age_slack)) + to=$((t1 - span + age_slack)) + unexplained=0 + explained=0 + while read -r did b a; do + lost="$(jq -n "$b - $a")" + if ! aged="$(aged_burn "$did" "$from" "$to")"; then + fail "burn $w: $did lost $(printf %.0f "$lost"), and its repository could not be read" + unexplained=$((unexplained + 1)) + elif [ "$(jq -n --argjson aged "$aged" --argjson lost "$lost" --argjson tol "$tolerance" '$lost <= $aged + $tol')" = true ]; then + explained=$((explained + 1)) + else + fail "burn $w: $did lost $(printf %.0f "$lost"), but only $(printf %.0f "$aged") left the window" + unexplained=$((unexplained + 1)) + fi + done <<<"$drops" + if [ "$unexplained" -eq 0 ]; then + ok "burn $w: $totals; drops explained by ageing: $explained; $left left the top 50" + else + note "burn $w: $totals; other drops explained by ageing: $explained" + fi +done + +# Warnings. +new_kinds="$(warning_kinds "$t0" | kinds_json | jq -r --argjson base "$(jq -c .warnings <<<"$base")" \ + 'to_entries[] | select($base[.key] == null) | " \(.value)\t\(.key)"')" +if [ -z "$new_kinds" ]; then + ok "warnings: the index and query logged no kind the baseline lacks" +else + fail "warnings: kinds the baseline lacks:" + echo "$new_kinds" +fi + +if [ ${#failures[@]} -gt 0 ]; then + echo "verify-deploy: ${#failures[@]} of the checks on $site failed" >&2 + exit 1 +fi +echo "verify-deploy: $site passed every check"