#!/usr/bin/env bash # Build and publish one commit of this repository as a release, without # changing what is live. # # scripts/release.sh # HEAD # scripts/release.sh # any commit-ish: a sha, a tag, a branch # # A release is a short sha with three things behind it, and this makes all # three exist: # # 1. vibescrobble/vibescrobble: in the registry — the index and the # query service, cross-compiled for arm64 from that commit's tree and # nothing else. The build context is `git archive`, so a dirty working # tree cannot leak into an image that carries a commit's name. # 2. The relay and Jetstream images, tagged by the refs that commit pins in # infra/variables.tf, which are the ones scripts/deploy.sh applies. # Upstream, Go, and built under emulation, so slow — but once per ref, # not once per release. # 3. releases// in the artifacts bucket: the canvas, the compose file, # the Caddyfile, data-disk.sh, and activate.sh, which is what the # instance runs to bring the release up. activate.sh goes last, so its presence means the # tree is whole. # # A commit that is the tag v also publishes the CLI, once: the # tarballs and SHA256SUMS scripts/release-cli.sh builds, to cli/v/, # which the instance serves at /releases/cli/ for web/install.sh. # # Everything it would produce is checked for before it is built, so calling # it for a sha that exists costs a few API calls. scripts/deploy.sh calls it # and then applies; call it alone to build ahead of a deploy. # # Credentials are the caller's, and have to be the did.bot account's: the # profile is jmm-did-bot-admin, the same one didbot deploys with. set -euo pipefail cd "$(dirname "$0")/.." stack=infra account=226086920337 profile=jmm-did-bot-admin region=us-east-1 usage() { echo "usage: scripts/release.sh []" } rev="" for arg in "$@"; do case "$arg" in -h | --help) usage exit 0 ;; -*) usage >&2 exit 2 ;; *) if [ -n "$rev" ]; then usage >&2 exit 2 fi rev="$arg" ;; esac done rev="${rev:-HEAD}" say() { printf '\033[1m==> %s\033[0m\n' "$*"; } for tool in tofu aws git docker jq; do command -v "$tool" >/dev/null || { echo "release: $tool is required and was not found on PATH" >&2 exit 1 } done # The release is the commit. Twelve characters, the same as `release_sha`'s # validation in infra/variables.tf and what a tree in the bucket is named. sha="$(git rev-parse --verify --quiet --short=12 "$rev^{commit}")" || { echo "release: $rev is not a commit" >&2 exit 1 } # Every file this ships has to exist at that commit, and a commit from # before any of them did cannot be released as it stands. for path in web infra/files/compose.yml infra/files/Caddyfile \ infra/files/activate.sh infra/files/data-disk.sh infra/files/vibescrobble.Dockerfile; do git cat-file -e "$sha:$path" 2>/dev/null || { echo "release: $path is not in $sha; only commits carrying it can be released" >&2 exit 1 } done # The upstream refs to build are the commit's, not the live state's: a deploy # that moves a pin applies the new ref, and state holds the old one until that # apply has run. indigo_ref="$(scripts/pin.sh "$sha" indigo_ref)" jetstream_ref="$(scripts/pin.sh "$sha" jetstream_ref)" # Fail before anything is built, and in the right account: an expired session # and a different profile both stop here. read -r caller arn _ < <(aws sts get-caller-identity --query '[Account,Arn]' --output text 2>/dev/null) || { echo "release: no working AWS credentials; aws sso login --profile $profile" >&2 exit 1 } if [ "$caller" != "$account" ]; then echo "release: signed in to account $caller, not $account (did.bot); AWS_PROFILE=$profile" >&2 exit 1 fi say "releasing $sha as $arn" # Where things go, read out of state in one call. Nothing here creates any of # it: the first deploy applies the stack before it can release into it. outputs="$(tofu -chdir="$stack" output -json)" bucket="$(jq -r '.artifacts_bucket.value // empty' <<<"$outputs")" registry="$(jq -r '.registry.value // empty' <<<"$outputs")" if [ -z "$bucket" ] || [ -z "$registry" ]; then echo "release: the stack has no outputs yet; scripts/deploy.sh applies it first, and calls this" >&2 exit 1 fi work="$(mktemp -d -t vibescrobble-release-XXXXXX)" trap 'rm -rf "$work"' EXIT # ------------------------------------------------------------------ images aws ecr get-login-password --region "$region" \ | docker login --username AWS --password-stdin "$registry" >/dev/null # A builder of its own, named, and never made the default: that is a global # setting on somebody else's laptop. Every build below names it. builder=vibescrobble docker buildx inspect "$builder" >/dev/null 2>&1 || docker buildx create --name "$builder" >/dev/null # The two upstreams build arm64 under emulation, so the builder has to be able # to run it before either is built. Nothing else does: the vibescrobble image # runs no arm64 instruction, so a release whose pins have not moved needs no # emulator, and a machine that lost its registration at a reboot can still # make one. Asked once, and only when an upstream is about to be built. # # A buildkitd answers before it has finished probing for emulators, so the # first inspect of one that was just booted can leave linux/arm64 out of a # list it will have a second later. Ask again: telling somebody to install # what they already have is worse than waiting. arm64="" require_arm64() { [ -z "$arm64" ] || return 0 local attempt info platforms for attempt in 1 2 3 4 5; do [ "$attempt" = 1 ] || sleep 1 if ! info="$(docker buildx inspect --bootstrap "$builder" 2>&1)"; then echo "release: the '$builder' builder would not start:" >&2 sed 's/^/ /' <<<"$info" >&2 exit 1 fi platforms=",$(sed -n 's/^Platforms: *//p' <<<"$info" | tr -d ' ' | tr '\n' ',')," # Some daemons name the variant and some do not; both run the image. case "$platforms" in *,linux/arm64,* | *,linux/arm64/*) arm64=yes return 0 ;; esac done echo "release: no arm64 emulator on this machine, which building $1 at $2 needs." >&2 echo " Register it: docker run --privileged --rm tonistiigi/binfmt --install arm64" >&2 echo " A registration lasts until the machine reboots, so having done this before is not" >&2 echo " having it now; infra/README.md says how to make it survive one. The builder picks a" >&2 echo " new registration up on its own, so run this again once that command has printed OK." >&2 exit 1 } already_pushed() { aws ecr describe-images --region "$region" \ --repository-name "vibescrobble/$1" --image-ids "imageTag=$2" >/dev/null 2>&1 } if already_pushed vibescrobble "$sha"; then say "vibescrobble:$sha is already in the registry" else # The compiler runs natively here and only emits arm64, so this is a couple # of minutes rather than most of an hour under emulation; see the # Dockerfile. The context is the commit and not the working tree. The # caller's CARGO_BUILD_JOBS, if it has one, goes in with it. say "building vibescrobble:$sha for arm64" mkdir "$work/context" git archive "$sha" | tar -x -C "$work/context" docker buildx build \ --builder "$builder" \ --platform linux/arm64 \ --file "$work/context/infra/files/vibescrobble.Dockerfile" \ --build-arg CARGO_BUILD_JOBS \ --tag "$registry/vibescrobble/vibescrobble:$sha" \ --cache-from "type=registry,ref=$registry/vibescrobble/vibescrobble:cache" \ --cache-to "type=registry,ref=$registry/vibescrobble/vibescrobble:cache,mode=max" \ --push "$work/context" fi # The two upstreams build their own Dockerfiles, which are not cross-aware, so # these go through emulation. They are Go, they are pinned, and they are built # only when the ref moves — slow once rather than every release. build_upstream() { local image="$1" ref="$2" url="$3" dockerfile="$4" shift 4 if already_pushed "$image" "$ref"; then say "$image:$ref is already in the registry" return fi require_arm64 "$image" "$ref" say "building $image at $ref (emulated, and therefore slow)" docker buildx build \ --builder "$builder" \ --platform linux/arm64 \ --file "$dockerfile" \ --tag "$registry/vibescrobble/$image:$ref" \ "$@" \ --push "$url#$ref" } # A git URL is a context BuildKit clones itself, and what it hands the build is # the checkout without the `.git` beneath it. The relay's Dockerfile opens with # `git describe`, so for that one the directory has to be asked for by name. build_upstream relay "$indigo_ref" \ "https://github.com/bluesky-social/indigo.git" cmd/relay/Dockerfile \ --build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=1 # Jetstream takes its version as build arguments precisely so that it does not # need one, so it is not sent one. build_upstream jetstream "$jetstream_ref" \ "https://github.com/bluesky-social/jetstream.git" Dockerfile # ---------------------------------------------------------------- the tree # The canvas and the three files the instance runs a release from do not go # in an image: they are small, and a rebuild per typo in a Caddyfile is a bad # trade. They travel by the bucket, under the sha, read out of the commit. prefix="s3://$bucket/releases/$sha" if aws s3 ls "$prefix/activate.sh" >/dev/null 2>&1; then say "releases/$sha/ is already in the bucket" else say "publishing releases/$sha/" mkdir "$work/tree" # web/dev/ holds the mock and its fixtures, which only the dev page loads. # The READMEs beside the assets are for people reading the repository. git archive --format=tar "$sha" web ":(exclude)web/dev" ":(exclude,glob)web/**/*.md" | gzip -n > "$work/tree/web.tar.gz" for file in compose.yml Caddyfile data-disk.sh activate.sh; do git show "$sha:infra/files/$file" > "$work/tree/$file" done # activate.sh last: the instance takes its presence as the tree being whole. for file in web.tar.gz compose.yml Caddyfile data-disk.sh activate.sh; do aws s3 cp --region "$region" --no-progress "$work/tree/$file" "$prefix/$file" done fi # ----------------------------------------------------------------- the CLI # Only a tag is published, and a version only once: the URLs install.sh reads # are cached for good, so what is under one must never change. Any other # commit's CLI is scripts/release-cli.sh's to build and nobody's to download. cli="$(scripts/release-cli.sh --name "$sha")" case "$cli" in *-dev.*) say "$sha is not a tagged version; the CLI is not published" ;; *) if aws s3 ls "s3://$bucket/cli/$cli/SHA256SUMS" >/dev/null 2>&1; then say "cli/$cli/ is already in the bucket" else scripts/release-cli.sh "$sha" say "publishing cli/$cli/" # SHA256SUMS after the tarballs, and `latest` after both: a reader that # finds either finds everything it names. for file in "dist/cli/$cli"/*.tar.gz "dist/cli/$cli/SHA256SUMS"; do aws s3 cp --region "$region" --no-progress "$file" "s3://$bucket/cli/$cli/${file##*/}" done newest="$(aws s3 cp --region "$region" "s3://$bucket/cli/latest" - 2>/dev/null || true)" if [ "$(printf '%s\n%s\n' "$newest" "$cli" | sort -V | tail -n 1)" = "$cli" ]; then printf '%s\n' "$cli" | aws s3 cp --region "$region" - "s3://$bucket/cli/latest" fi fi ;; esac say "release $sha is ready: scripts/deploy.sh $sha"