Something went wrong. Try again.
vibescrobble! every agent, live on stage. vibescrobble.com
atproto agent llm did
Something went wrong. Try again.
12 kB · 302 lines
Shell
at main
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303#!/usr/bin/env bash# Checks a deploy of vibescrobble.com against a baseline taken before it.## scripts/verify-deploy.sh before [options]# scripts/verify-deploy.sh after [options] [--sha <rev>]## --site <url> the site (default https://vibescrobble.com)# --state <file> where `before` saves the baseline and `after` reads it# (default ~/.local/state/vibescrobble/verify-deploy.json)# --name <tag> the instance's Name tag (default vibescrobble)# --baseline-hours <h> before: hours of logs the warning baseline covers (default 24)# --sha <rev> after: the commit release.txt should name (default HEAD)# --tolerance <burn> after: a burn drop this small is rounding (default 1)# --age-slack <s> after: seconds added to each end of an ageing interval,# for clock skew and the time a snapshot takes (default 60)# --max-pages <n> after: listRecords pages read per account when# explaining a drop (default 50)## `before` records /health's agent and scrobble counts, every account's burn# on the day, week and all-time leaderboards, each one's top 50, and the kinds# of warning the index and query containers logged. `after` checks:## - release.txt names <rev>;# - /health is ok, connected, and its counts did not drop;# - no account's all-time burn dropped;# - each day or week drop is ageing: the account's scrobbles that left the# window between the two snapshots hold at least the burn it lost;# - the index and query logged no warning kind the baseline lacks.## Ageing is read from each account's own repository, the complete copy. A# warning's kind is its level, target and message, without its fields.# Everything is read-only. The logs are read over SSM with the caller's own# AWS credentials (AWS_PROFILE, AWS_REGION), through scripts/ssm-run.py.set -euo pipefail
cd "$(dirname "$0")/.."
usage() { echo "usage: scripts/verify-deploy.sh before|after [--site URL] [--state FILE] [--name TAG] [--baseline-hours H] [--sha REV] [--tolerance BURN] [--age-slack S] [--max-pages N]"}
mode="${1:-}"case "$mode" in before | after) shift ;; -h | --help) usage exit 0 ;; *) usage >&2 exit 2 ;;esac
site=https://vibescrobble.comstate="${XDG_STATE_HOME:-$HOME/.local/state}/vibescrobble/verify-deploy.json"name=vibescrobblebaseline_hours=24rev=HEADtolerance=1age_slack=60max_pages=50while [ $# -gt 0 ]; do [ $# -ge 2 ] || { usage >&2 exit 2 } case "$1" in --site) site="${2%/}" ;; --state) state="$2" ;; --name) name="$2" ;; --baseline-hours) baseline_hours="$2" ;; --sha) rev="$2" ;; --tolerance) tolerance="$2" ;; --age-slack) age_slack="$2" ;; --max-pages) max_pages="$2" ;; *) usage >&2 exit 2 ;; esac shift 2done
for tool in curl jq uv; do command -v "$tool" >/dev/null || { echo "verify-deploy: $tool is not on PATH" >&2 exit 2 }done
windows=(day week all)declare -A window_secs=([day]=86400 [week]=604800)
get() { curl -fsS --max-time 20 "$@"; }
# Every account's own burn on one window's leaderboard, as {did: burn}.burn_rows() { local window="$1" cursor="" rows='{}' page pages=0 while :; do page="$(get -G "$site/leaderboard" --data-urlencode "window=$window" \ --data-urlencode by=agent --data-urlencode limit=500 \ ${cursor:+--data-urlencode "cursor=$cursor"})" rows="$(jq -c --argjson acc "$rows" '$acc + ([.rows[] | {(.key): .burn}] | add // {})' <<<"$page")" cursor="$(jq -r '.cursor // empty' <<<"$page")" pages=$((pages + 1)) [ -n "$cursor" ] && [ "$pages" -lt 100 ] || break done echo "$rows"}
snapshot() { local health weights rows='{}' top='{}' w health="$(get "$site/health")" weights="$(get "$site/leaderboard?window=all&by=vendor" | jq -c .weights)" for w in "${windows[@]}"; do rows="$(jq -c --arg w "$w" --argjson r "$(burn_rows "$w")" '. + {($w): $r}' <<<"$rows")" top="$(jq -c --arg w "$w" --argjson t "$(get "$site/leaderboard?window=$w&by=agent&limit=50" | jq -c '[.rows[].key]')" \ '. + {($w): $t}' <<<"$top")" done jq -n --argjson time "$(date +%s)" \ --arg release "$(get "$site/release.txt" | tr -d '[:space:]')" \ --argjson health "$health" --argjson weights "$weights" \ --argjson burn "$rows" --argjson top "$top" \ '{time: $time, release: $release, health: ($health | {agents, scrobbles}), weights: $weights, burn: $burn, top: $top}'}
# Warning kinds the index and query containers logged since <epoch>, and# until <epoch> when given, as `count kind` lines.warning_kinds() { scripts/ssm-run.py --name "$name" --timeout 150 "since=$1; until=${2:-}$( cat <<'EOF'for c in $(docker ps --format '{{.Names}}' | grep -E -- '-(index|query)-'); do docker logs --since "$since" ${until:+--until "$until"} "$c" 2>&1done | sed -E 's/\x1b\[3m.*//; s/\x1b\[[0-9;]*m//g' | grep -E '^[0-9TZ:.-]+ +(WARN|ERROR) ' | sed -E 's/^[0-9TZ:.-]+ +//; s/ +$//; s/[0-9a-f]{8,}/X/g; s/did:[a-z]+:[A-Za-z0-9.:%-]+/DID/g; s/[0-9]+/N/g' | cut -c1-160 | sort | uniq -cEOF )"}kinds_json() { jq -R -s 'split("\n") | map(capture("^ *(?<n>[0-9]+) (?<k>.+)$")? // empty) | map({(.k): (.n | tonumber)}) | add // {}'}
# The PDS that holds a DID's repository.pds_of() { local doc case "$1" in did:plc:*) doc="$(get "https://plc.directory/$1")" ;; did:web:*) doc="$(get "https://${1#did:web:}/.well-known/did.json")" ;; *) return 1 ;; esac jq -r '[.service[] | select(.id | endswith("#atproto_pds")) | .serviceEndpoint][0] // empty' <<<"$doc"}
# The burn of a DID's scrobbles created in [from, to), epoch seconds, priced# with the baseline's weights.aged_burn() { local did="$1" from="$2" to="$3" pds cursor="" sum=0 page pages=0 oldest pds="$(pds_of "$did")" || return 1 [ -n "$pds" ] || return 1 while [ "$pages" -lt "$max_pages" ]; do page="$(get -G "${pds%/}/xrpc/com.atproto.repo.listRecords" --data-urlencode "repo=$did" \ --data-urlencode collection=com.vibescrobble.scrobble --data-urlencode limit=100 \ ${cursor:+--data-urlencode "cursor=$cursor"})" || return 1 read -r sum oldest < <(jq -r --argjson w "$weights" --argjson from "$from" --argjson to "$to" \ --argjson acc "$sum" ' def epoch: try (sub("\\.[0-9]+"; "") | sub("\\+00:00$"; "Z") | fromdateiso8601) catch null; [.records[].value | {at: (.createdAt | epoch), u: (.usage // {})}] as $r | ($acc + ([$r[] | select(.at != null and .at >= $from and .at < $to) | (.u.input // 0) * $w.input + (.u.output // 0) * $w.output + (.u.cacheRead // 0) * $w.cacheRead + (.u.cacheWrite // 0) * $w.cacheWrite] | add // 0)) , ([$r[].at | select(. != null)] | min // 0)' <<<"$page" | tr '\n' ' ') cursor="$(jq -r '.cursor // empty' <<<"$page")" pages=$((pages + 1)) [ -n "$cursor" ] && [ "$oldest" -ge "$from" ] || break done echo "$sum"}
mkdir -p "$(dirname "$state")"
if [ "$mode" = before ]; then snap="$(snapshot)" now="$(jq .time <<<"$snap")" kinds="$(warning_kinds $((now - baseline_hours * 3600)) "$now" | kinds_json)" jq --argjson warnings "$kinds" --argjson h "$baseline_hours" \ '. + {warnings: $warnings, baseline_hours: $h}' <<<"$snap" >"$state" jq -r '"verify-deploy: baseline at \(.time | todate), release \(.release)", " \(.health.agents) agents, \(.health.scrobbles) scrobbles", (.burn | to_entries[] | " \(.key) burn \(.value | add // 0 | floor) over \(.value | length) accounts"), " \(.warnings | length) warning kinds over \(.baseline_hours) h:", (.warnings | to_entries[] | " \(.value)\t\(.key)")' "$state" echo " saved to $state" exit 0fi
[ -f "$state" ] || { echo "verify-deploy: no baseline at $state; run \`scripts/verify-deploy.sh before\` first" >&2 exit 2}base="$(cat "$state")"weights="$(jq -c .weights <<<"$base")"t0="$(jq .time <<<"$base")"
failures=()ok() { echo "ok $1"; }note() { echo " $1"; }fail() { echo "FAIL $1" failures+=("$1")}
# Release. release.txt holds a short sha; compare on the shorter of the two.want="$(git rev-parse --verify --quiet "$rev^{commit}" 2>/dev/null || echo "$rev")"serving="$(get "$site/release.txt" 2>/dev/null | tr -d '[:space:]' || true)"if [ -n "$serving" ] && { [[ $want == "$serving"* ]] || [[ $serving == "$want"* ]]; }; then ok "release: release.txt names $serving (was $(jq -r .release <<<"$base"))"else fail "release: release.txt names ${serving:-nothing}, want ${want:0:12}"fi
snap="$(snapshot)"t1="$(jq .time <<<"$snap")"
# Health.health="$(get "$site/health" || echo '{}')"if [ "$(jq -r '.status == "ok" and .connected == true' <<<"$health")" != true ]; then fail "health: status $(jq -r '.status // "none"' <<<"$health"), connected $(jq -r '.connected // "?"' <<<"$health")"fifor count in agents scrobbles; do b="$(jq ".health.$count" <<<"$base")" a="$(jq ".health.$count" <<<"$snap")" if [ "$a" -ge "$b" ]; then ok "health: $count $b -> $a" else fail "health: $count dropped, $b -> $a; the index may still be restoring" fidone
# Burn. A drop beyond the tolerance is a failure on the all-time board; on a# day or week board it is ageing if enough burn left the window.for w in "${windows[@]}"; do drops="$(jq -r --arg w "$w" --argjson after "$(jq -c ".burn.$w" <<<"$snap")" --argjson tol "$tolerance" \ '.burn[$w] | to_entries[] | select(($after[.key] // 0) < .value - $tol) | "\(.key) \(.value) \($after[.key] // 0)"' <<<"$base")" totals="$(jq -r --arg w "$w" '.burn[$w] | add // 0 | floor' <<<"$base") -> $(jq -r --arg w "$w" '.burn[$w] | add // 0 | floor' <<<"$snap")" left="$(jq -r --arg w "$w" --argjson now "$(jq -c ".top.$w" <<<"$snap")" '.top[$w] - $now | length' <<<"$base")" if [ -z "$drops" ]; then ok "burn $w: $totals, no account dropped; $left left the top 50" continue fi if [ "$w" = all ]; then fail "burn all: $totals; these accounts dropped:" while read -r did b a; do note "$did $(printf %.0f "$b") -> $(printf %.0f "$a")"; done <<<"$drops" continue fi span="${window_secs[$w]}" from=$((t0 - span - age_slack)) to=$((t1 - span + age_slack)) unexplained=0 explained=0 while read -r did b a; do lost="$(jq -n "$b - $a")" if ! aged="$(aged_burn "$did" "$from" "$to")"; then fail "burn $w: $did lost $(printf %.0f "$lost"), and its repository could not be read" unexplained=$((unexplained + 1)) elif [ "$(jq -n --argjson aged "$aged" --argjson lost "$lost" --argjson tol "$tolerance" '$lost <= $aged + $tol')" = true ]; then explained=$((explained + 1)) else fail "burn $w: $did lost $(printf %.0f "$lost"), but only $(printf %.0f "$aged") left the window" unexplained=$((unexplained + 1)) fi done <<<"$drops" if [ "$unexplained" -eq 0 ]; then ok "burn $w: $totals; drops explained by ageing: $explained; $left left the top 50" else note "burn $w: $totals; other drops explained by ageing: $explained" fidone
# Warnings.new_kinds="$(warning_kinds "$t0" | kinds_json | jq -r --argjson base "$(jq -c .warnings <<<"$base")" \ 'to_entries[] | select($base[.key] == null) | " \(.value)\t\(.key)"')"if [ -z "$new_kinds" ]; then ok "warnings: the index and query logged no kind the baseline lacks"else fail "warnings: kinds the baseline lacks:" echo "$new_kinds"fi
if [ ${#failures[@]} -gt 0 ]; then echo "verify-deploy: ${#failures[@]} of the checks on $site failed" >&2 exit 1fiecho "verify-deploy: $site passed every check"