Something went wrong. Try again.
vibescrobble! every agent, live on stage. vibescrobble.com
atproto agent llm did
Something went wrong. Try again.
12 kB · 281 lines
Shell
at main
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282#!/usr/bin/env bash# Build and publish one commit of this repository as a release, without# changing what is live.## scripts/release.sh # HEAD# scripts/release.sh <rev> # any commit-ish: a sha, a tag, a branch## A release is a short sha with three things behind it, and this makes all# three exist:## 1. vibescrobble/vibescrobble:<sha> in the registry — the index and the# query service, cross-compiled for arm64 from that commit's tree and# nothing else. The build context is `git archive`, so a dirty working# tree cannot leak into an image that carries a commit's name.# 2. The relay and Jetstream images, tagged by the refs that commit pins in# infra/variables.tf, which are the ones scripts/deploy.sh applies.# Upstream, Go, and built under emulation, so slow — but once per ref,# not once per release.# 3. releases/<sha>/ in the artifacts bucket: the canvas, the compose file,# the Caddyfile, data-disk.sh, and activate.sh, which is what the# instance runs to bring the release up. activate.sh goes last, so its presence means the# tree is whole.## A commit that is the tag v<version> also publishes the CLI, once: the# tarballs and SHA256SUMS scripts/release-cli.sh builds, to cli/v<version>/,# which the instance serves at /releases/cli/ for web/install.sh.## Everything it would produce is checked for before it is built, so calling# it for a sha that exists costs a few API calls. scripts/deploy.sh calls it# and then applies; call it alone to build ahead of a deploy.## Credentials are the caller's, and have to be the did.bot account's: the# profile is jmm-did-bot-admin, the same one didbot deploys with.set -euo pipefail
cd "$(dirname "$0")/.."
stack=infraaccount=226086920337profile=jmm-did-bot-adminregion=us-east-1
usage() { echo "usage: scripts/release.sh [<rev>]"}
rev=""for arg in "$@"; do case "$arg" in -h | --help) usage exit 0 ;; -*) usage >&2 exit 2 ;; *) if [ -n "$rev" ]; then usage >&2 exit 2 fi rev="$arg" ;; esacdonerev="${rev:-HEAD}"
say() { printf '\033[1m==> %s\033[0m\n' "$*"; }
for tool in tofu aws git docker jq; do command -v "$tool" >/dev/null || { echo "release: $tool is required and was not found on PATH" >&2 exit 1 }done
# The release is the commit. Twelve characters, the same as `release_sha`'s# validation in infra/variables.tf and what a tree in the bucket is named.sha="$(git rev-parse --verify --quiet --short=12 "$rev^{commit}")" || { echo "release: $rev is not a commit" >&2 exit 1}
# Every file this ships has to exist at that commit, and a commit from# before any of them did cannot be released as it stands.for path in web infra/files/compose.yml infra/files/Caddyfile \ infra/files/activate.sh infra/files/data-disk.sh infra/files/vibescrobble.Dockerfile; do git cat-file -e "$sha:$path" 2>/dev/null || { echo "release: $path is not in $sha; only commits carrying it can be released" >&2 exit 1 }done
# The upstream refs to build are the commit's, not the live state's: a deploy# that moves a pin applies the new ref, and state holds the old one until that# apply has run.indigo_ref="$(scripts/pin.sh "$sha" indigo_ref)"jetstream_ref="$(scripts/pin.sh "$sha" jetstream_ref)"
# Fail before anything is built, and in the right account: an expired session# and a different profile both stop here.read -r caller arn _ < <(aws sts get-caller-identity --query '[Account,Arn]' --output text 2>/dev/null) || { echo "release: no working AWS credentials; aws sso login --profile $profile" >&2 exit 1}if [ "$caller" != "$account" ]; then echo "release: signed in to account $caller, not $account (did.bot); AWS_PROFILE=$profile" >&2 exit 1fisay "releasing $sha as $arn"
# Where things go, read out of state in one call. Nothing here creates any of# it: the first deploy applies the stack before it can release into it.outputs="$(tofu -chdir="$stack" output -json)"bucket="$(jq -r '.artifacts_bucket.value // empty' <<<"$outputs")"registry="$(jq -r '.registry.value // empty' <<<"$outputs")"if [ -z "$bucket" ] || [ -z "$registry" ]; then echo "release: the stack has no outputs yet; scripts/deploy.sh applies it first, and calls this" >&2 exit 1fi
work="$(mktemp -d -t vibescrobble-release-XXXXXX)"trap 'rm -rf "$work"' EXIT
# ------------------------------------------------------------------ images
aws ecr get-login-password --region "$region" \ | docker login --username AWS --password-stdin "$registry" >/dev/null
# A builder of its own, named, and never made the default: that is a global# setting on somebody else's laptop. Every build below names it.builder=vibescrobbledocker buildx inspect "$builder" >/dev/null 2>&1 || docker buildx create --name "$builder" >/dev/null
# The two upstreams build arm64 under emulation, so the builder has to be able# to run it before either is built. Nothing else does: the vibescrobble image# runs no arm64 instruction, so a release whose pins have not moved needs no# emulator, and a machine that lost its registration at a reboot can still# make one. Asked once, and only when an upstream is about to be built.## A buildkitd answers before it has finished probing for emulators, so the# first inspect of one that was just booted can leave linux/arm64 out of a# list it will have a second later. Ask again: telling somebody to install# what they already have is worse than waiting.arm64=""require_arm64() { [ -z "$arm64" ] || return 0 local attempt info platforms for attempt in 1 2 3 4 5; do [ "$attempt" = 1 ] || sleep 1 if ! info="$(docker buildx inspect --bootstrap "$builder" 2>&1)"; then echo "release: the '$builder' builder would not start:" >&2 sed 's/^/ /' <<<"$info" >&2 exit 1 fi platforms=",$(sed -n 's/^Platforms: *//p' <<<"$info" | tr -d ' ' | tr '\n' ',')," # Some daemons name the variant and some do not; both run the image. case "$platforms" in *,linux/arm64,* | *,linux/arm64/*) arm64=yes return 0 ;; esac done echo "release: no arm64 emulator on this machine, which building $1 at $2 needs." >&2 echo " Register it: docker run --privileged --rm tonistiigi/binfmt --install arm64" >&2 echo " A registration lasts until the machine reboots, so having done this before is not" >&2 echo " having it now; infra/README.md says how to make it survive one. The builder picks a" >&2 echo " new registration up on its own, so run this again once that command has printed OK." >&2 exit 1}
already_pushed() { aws ecr describe-images --region "$region" \ --repository-name "vibescrobble/$1" --image-ids "imageTag=$2" >/dev/null 2>&1}
if already_pushed vibescrobble "$sha"; then say "vibescrobble:$sha is already in the registry"else # The compiler runs natively here and only emits arm64, so this is a couple # of minutes rather than most of an hour under emulation; see the # Dockerfile. The context is the commit and not the working tree. The # caller's CARGO_BUILD_JOBS, if it has one, goes in with it. say "building vibescrobble:$sha for arm64" mkdir "$work/context" git archive "$sha" | tar -x -C "$work/context" docker buildx build \ --builder "$builder" \ --platform linux/arm64 \ --file "$work/context/infra/files/vibescrobble.Dockerfile" \ --build-arg CARGO_BUILD_JOBS \ --tag "$registry/vibescrobble/vibescrobble:$sha" \ --cache-from "type=registry,ref=$registry/vibescrobble/vibescrobble:cache" \ --cache-to "type=registry,ref=$registry/vibescrobble/vibescrobble:cache,mode=max" \ --push "$work/context"fi
# The two upstreams build their own Dockerfiles, which are not cross-aware, so# these go through emulation. They are Go, they are pinned, and they are built# only when the ref moves — slow once rather than every release.build_upstream() { local image="$1" ref="$2" url="$3" dockerfile="$4" shift 4 if already_pushed "$image" "$ref"; then say "$image:$ref is already in the registry" return fi require_arm64 "$image" "$ref" say "building $image at $ref (emulated, and therefore slow)" docker buildx build \ --builder "$builder" \ --platform linux/arm64 \ --file "$dockerfile" \ --tag "$registry/vibescrobble/$image:$ref" \ "$@" \ --push "$url#$ref"}
# A git URL is a context BuildKit clones itself, and what it hands the build is# the checkout without the `.git` beneath it. The relay's Dockerfile opens with# `git describe`, so for that one the directory has to be asked for by name.build_upstream relay "$indigo_ref" \ "https://github.com/bluesky-social/indigo.git" cmd/relay/Dockerfile \ --build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=1# Jetstream takes its version as build arguments precisely so that it does not# need one, so it is not sent one.build_upstream jetstream "$jetstream_ref" \ "https://github.com/bluesky-social/jetstream.git" Dockerfile
# ---------------------------------------------------------------- the tree
# The canvas and the three files the instance runs a release from do not go# in an image: they are small, and a rebuild per typo in a Caddyfile is a bad# trade. They travel by the bucket, under the sha, read out of the commit.prefix="s3://$bucket/releases/$sha"if aws s3 ls "$prefix/activate.sh" >/dev/null 2>&1; then say "releases/$sha/ is already in the bucket"else say "publishing releases/$sha/" mkdir "$work/tree" # web/dev/ holds the mock and its fixtures, which only the dev page loads. # The READMEs beside the assets are for people reading the repository. git archive --format=tar "$sha" web ":(exclude)web/dev" ":(exclude,glob)web/**/*.md" | gzip -n > "$work/tree/web.tar.gz" for file in compose.yml Caddyfile data-disk.sh activate.sh; do git show "$sha:infra/files/$file" > "$work/tree/$file" done # activate.sh last: the instance takes its presence as the tree being whole. for file in web.tar.gz compose.yml Caddyfile data-disk.sh activate.sh; do aws s3 cp --region "$region" --no-progress "$work/tree/$file" "$prefix/$file" donefi
# ----------------------------------------------------------------- the CLI
# Only a tag is published, and a version only once: the URLs install.sh reads# are cached for good, so what is under one must never change. Any other# commit's CLI is scripts/release-cli.sh's to build and nobody's to download.cli="$(scripts/release-cli.sh --name "$sha")"case "$cli" in *-dev.*) say "$sha is not a tagged version; the CLI is not published" ;; *) if aws s3 ls "s3://$bucket/cli/$cli/SHA256SUMS" >/dev/null 2>&1; then say "cli/$cli/ is already in the bucket" else scripts/release-cli.sh "$sha" say "publishing cli/$cli/" # SHA256SUMS after the tarballs, and `latest` after both: a reader that # finds either finds everything it names. for file in "dist/cli/$cli"/*.tar.gz "dist/cli/$cli/SHA256SUMS"; do aws s3 cp --region "$region" --no-progress "$file" "s3://$bucket/cli/$cli/${file##*/}" done newest="$(aws s3 cp --region "$region" "s3://$bucket/cli/latest" - 2>/dev/null || true)" if [ "$(printf '%s\n%s\n' "$newest" "$cli" | sort -V | tail -n 1)" = "$cli" ]; then printf '%s\n' "$cli" | aws s3 cp --region "$region" - "s3://$bucket/cli/latest" fi fi ;;esac
say "release $sha is ready: scripts/deploy.sh $sha"