Something went wrong. Try again.
Identities for entities did.bot
atproto agent llm did
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271#!/usr/bin/env bash# Makes the infra/pds deployment run a commit of this repository, in place.## scripts/deploy-pds.sh --var-file <tfvars> [<rev>]# scripts/deploy-pds.sh --var-file <tfvars> --dry-run [<rev>]## --dry-run plan only; nothing is backed up or applied. The image# must already be published, or the plan fails on it# --yes apply without asking# --no-backup skip the on-demand backup of the data volume# --allow-replace apply a plan that replaces or destroys a resource## The var file is the operator's, the same one a hand-run apply uses; it can# also come from DIDBOT_PDS_TFVARS. <rev> defaults to HEAD.## A deploy is an apply that changes the release parameter (release.tf in# terraform-aws-didbot-pds, the module infra/pds calls). In order: publish the image (scripts/publish-image.sh skips# one already pushed), plan with container_image_tag set to the commit's# short sha, refuse a plan that replaces anything, take an on-demand backup# of the data volume, apply, and write the tag back into the var file so the# next apply does not move it back. Then run the instance's converger at once# over SSM, print what it said, and wait for /health to report the commit.## A release that fails its health check on the instance is rolled back there,# and this exits non-zero saying so. The backup is what reverses a release# that changed the data directory's layout, which the earlier image refuses:# see docs/operations.md, "Rolling back".## Credentials are the caller's own, as for scripts/publish-image.sh.set -euo pipefail
cd "$(dirname "$0")/.."
stack=infra/pds
usage() { echo "usage: scripts/deploy-pds.sh --var-file <tfvars> [--dry-run] [--yes] [--no-backup] [--allow-replace] [<rev>]"}
var_file="${DIDBOT_PDS_TFVARS:-}"dry_run=""assume_yes=""backup=1allow_replace=""rev=""while [ $# -gt 0 ]; do case "$1" in --var-file) [ $# -ge 2 ] || { usage >&2 exit 2 } var_file="$2" shift ;; -n | --dry-run) dry_run=1 ;; -y | --yes) assume_yes=1 ;; --no-backup) backup="" ;; --allow-replace) allow_replace=1 ;; -h | --help) usage exit 0 ;; -*) usage >&2 exit 2 ;; *) if [ -n "$rev" ]; then usage >&2 exit 2 fi rev="$1" ;; esac shiftdonerev="${rev:-HEAD}"
say() { printf '\033[1m==> %s\033[0m\n' "$*"; }warn() { printf '\033[1;33m %s\033[0m\n' "$*" >&2; }fail() { printf '\033[1;31mdeploy-pds: %s\033[0m\n' "$*" >&2 exit 1}
if [ -z "$var_file" ]; then usage >&2 fail "no var file: pass --var-file or set DIDBOT_PDS_TFVARS"fi[ -f "$var_file" ] || fail "$var_file is not a file"var_file="$(cd "$(dirname "$var_file")" && pwd)/$(basename "$var_file")"
for tool in tofu aws git jq curl; do command -v "$tool" >/dev/null || fail "$tool is required and was not found on PATH"done
identity="$(aws sts get-caller-identity --query Arn --output text 2>&1)" || fail "no working AWS credentials: $identity"say "deploying as $identity"
sha="$(git rev-parse --verify --quiet --short=12 "$rev^{commit}")" || fail "$rev is not a commit"
say "initialising $stack"tofu -chdir="$stack" init -input=false -backend-config=backend.hcl >/dev/null
outputs="$(tofu -chdir="$stack" output -json)"output() { jq -r --arg name "$1" '.[$name].value // empty' <<<"$outputs"; }live="$(output container_image_tag)"
if [ -z "$dry_run" ]; then scripts/publish-image.sh "$sha"fi
# A saved plan, so what is applied is what was read. It embeds resolved# values, so it lives outside the repository and not past this run.plan="$(mktemp -t didbot-pds-XXXXXX.tfplan)"trap 'rm -f "$plan"' EXIT
say "planning $sha${live:+ (live: $live)}"rc=0tofu -chdir="$stack" plan -input=false -detailed-exitcode -out="$plan" \ -var-file="$var_file" -var "container_image_tag=$sha" || rc=$?case "$rc" in 0 | 2) ;; *) exit "$rc" ;;esac
# A deploy changes the release parameter. One that also replaces the# instance is an outage of minutes rather than seconds, and one that# destroys anything else is not a deploy at all.planned="$(tofu -chdir="$stack" show -json "$plan")"replaced="$(jq -r '.resource_changes[]? | select(.change.actions | index("delete")) | .address' <<<"$planned")"if [ -n "$replaced" ]; then warn "this plan replaces or destroys:" printf '%s\n' "$replaced" | sed 's/^/ /' >&2 [ -n "$allow_replace" ] || fail "refusing a plan that replaces anything; --allow-replace applies it"fi
if [ -n "$dry_run" ]; then say "dry run: nothing backed up, nothing applied" exit 0fiif [ "$rc" -eq 0 ]; then say "no changes: the stack already names $sha"else if [ -z "$assume_yes" ]; then printf '\napply this plan? [y/N] ' # Nothing on stdin is a no. read -r reply || reply="" case "$reply" in y | Y | yes | Yes) ;; *) fail "nothing applied" ;; esac fi
if [ -n "$backup" ]; then vault="$(output backup_vault_name)" role="$(output backup_role_arn)" volume="$(output data_volume_id)" if [ -z "$vault" ] || [ -z "$role" ] || [ -z "$volume" ]; then fail "the state has no backup_vault_name, backup_role_arn or data_volume_id output yet; take a snapshot of the data volume by hand and rerun with --no-backup" fi # Kept as long as the daily snapshots the module takes. retention="$(jq -r '.variables.backup_retention_days.value' <<<"$planned")" region="$(cut -d. -f4 <<<"$(output ecr_repository_url)")" account="$(aws sts get-caller-identity --query Account --output text)" say "backing up $volume into $vault" job="$(aws backup start-backup-job --region "$region" \ --backup-vault-name "$vault" \ --resource-arn "arn:aws:ec2:$region:$account:volume/$volume" \ --iam-role-arn "$role" \ --lifecycle "DeleteAfterDays=$retention" \ --start-window-minutes 60 \ --query BackupJobId --output text)" # An EBS snapshot holds the volume as it was when it started, so the # deploy waits for the start rather than for the copy to finish. for _ in $(seq 90); do state="$(aws backup describe-backup-job --region "$region" --backup-job-id "$job" \ --query State --output text)" case "$state" in RUNNING | COMPLETED) break ;; CREATED | PENDING) sleep 10 ;; *) fail "backup job $job is $state" ;; esac done case "$state" in RUNNING | COMPLETED) say "backup $job is $state" ;; *) fail "backup job $job had not started after fifteen minutes" ;; esac fi
tofu -chdir="$stack" apply -input=false "$plan"
# The var file names what is live, so a later hand-run apply keeps it. if grep -q '^[[:space:]]*container_image_tag[[:space:]]*=' "$var_file"; then sed -i -E "s|^([[:space:]]*container_image_tag[[:space:]]*=[[:space:]]*).*|\\1\"$sha\"|" "$var_file" say "$var_file now names $sha" else warn "$var_file names no container_image_tag; pass -var container_image_tag=$sha to every later apply" fi outputs="$(tofu -chdir="$stack" output -json)"fi
# The timer converges within thirty seconds on its own. Asking directly is# sooner, and brings the converger's own account back here. `systemctl# start` on a oneshot waits for the run, joining one already going. A new# instance takes a minute or two to register with Session Manager, so the# ask is retried.instance="$(output instance_id)"say "converging $instance on $sha"command_id=""for _ in $(seq 20); do command_id="$(aws ssm send-command \ --instance-ids "$instance" \ --document-name AWS-RunShellScript \ --comment "didbot deploy $sha" \ --parameters '{"commands":["until [ -x /usr/local/bin/didbot-converge ]; do sleep 5; done","systemctl start didbot-converge.service; rc=$?; journalctl -u didbot-converge.service --since -30min -o cat --no-pager | tail -20; exit $rc"]}' \ --query Command.CommandId --output text 2>/dev/null || true)" [ -n "$command_id" ] && break printf '.' sleep 15doneecho
converged=""if [ -z "$command_id" ]; then warn "the instance is not reachable over Session Manager; leaving it to the timer"else while :; do status="$(aws ssm get-command-invocation --command-id "$command_id" --instance-id "$instance" \ --query Status --output text 2>/dev/null || echo Pending)" case "$status" in Success | Failed | Cancelled | TimedOut) break ;; *) printf '.' sleep 5 ;; esac done echo aws ssm get-command-invocation --command-id "$command_id" --instance-id "$instance" \ --query '[StandardOutputContent, StandardErrorContent]' --output text | sed 's/^/ /' [ "$status" = Success ] && converged=1fi
zone="$(output root_zone)"serving=""for _ in $(seq 30); do serving="$(curl -fs --max-time 10 "https://$zone/health" | jq -r '.revision // empty' 2>/dev/null || true)" [ "$serving" = "$sha" ] && break [ -n "$command_id" ] && [ -z "$converged" ] && break printf '.' sleep 10doneecho
if [ "$serving" != "$sha" ]; then printf '\033[1;31m' >&2 echo "deploy-pds: $sha IS NOT RUNNING. https://$zone/health reports ${serving:-no revision}." >&2 [ -n "$serving" ] || curl -fsS --max-time 10 -o /dev/null "https://$zone/health" 2>&1 | head -1 | sed 's/^/ /' >&2 || true if [ -n "$command_id" ] && [ -z "$converged" ]; then echo "deploy-pds: the converger refused it and rolled back; its output is above." >&2 fi printf '\033[0m' >&2 echo " To look: aws ssm start-session --target $instance" >&2 echo " sudo journalctl -u didbot-converge -u didbot-pds --since -30min" >&2 exit 1fisay "up: https://$zone runs $sha"