#!/usr/bin/env bash
# Makes the infra/pds deployment run a commit of this repository, in place.
#
#   scripts/deploy-pds.sh --var-file <tfvars> [<rev>]
#   scripts/deploy-pds.sh --var-file <tfvars> --dry-run [<rev>]
#
#   --dry-run        plan only; nothing is backed up or applied. The image
#                    must already be published, or the plan fails on it
#   --yes            apply without asking
#   --no-backup      skip the on-demand backup of the data volume
#   --allow-replace  apply a plan that replaces or destroys a resource
#
# The var file is the operator's, the same one a hand-run apply uses; it can
# also come from DIDBOT_PDS_TFVARS. <rev> defaults to HEAD.
#
# A deploy is an apply that changes the release parameter (release.tf in
# terraform-aws-didbot-pds, the module infra/pds calls). In order: publish the image (scripts/publish-image.sh skips
# one already pushed), plan with container_image_tag set to the commit's
# short sha, refuse a plan that replaces anything, take an on-demand backup
# of the data volume, apply, and write the tag back into the var file so the
# next apply does not move it back. Then run the instance's converger at once
# over SSM, print what it said, and wait for /health to report the commit.
#
# A release that fails its health check on the instance is rolled back there,
# and this exits non-zero saying so. The backup is what reverses a release
# that changed the data directory's layout, which the earlier image refuses:
# see docs/operations.md, "Rolling back".
#
# Credentials are the caller's own, as for scripts/publish-image.sh.
set -euo pipefail

cd "$(dirname "$0")/.."

stack=infra/pds

usage() {
  echo "usage: scripts/deploy-pds.sh --var-file <tfvars> [--dry-run] [--yes] [--no-backup] [--allow-replace] [<rev>]"
}

var_file="${DIDBOT_PDS_TFVARS:-}"
dry_run=""
assume_yes=""
backup=1
allow_replace=""
rev=""
while [ $# -gt 0 ]; do
  case "$1" in
    --var-file)
      [ $# -ge 2 ] || {
        usage >&2
        exit 2
      }
      var_file="$2"
      shift
      ;;
    -n | --dry-run) dry_run=1 ;;
    -y | --yes) assume_yes=1 ;;
    --no-backup) backup="" ;;
    --allow-replace) allow_replace=1 ;;
    -h | --help)
      usage
      exit 0
      ;;
    -*)
      usage >&2
      exit 2
      ;;
    *)
      if [ -n "$rev" ]; then
        usage >&2
        exit 2
      fi
      rev="$1"
      ;;
  esac
  shift
done
rev="${rev:-HEAD}"

say() { printf '\033[1m==> %s\033[0m\n' "$*"; }
warn() { printf '\033[1;33m    %s\033[0m\n' "$*" >&2; }
fail() {
  printf '\033[1;31mdeploy-pds: %s\033[0m\n' "$*" >&2
  exit 1
}

if [ -z "$var_file" ]; then
  usage >&2
  fail "no var file: pass --var-file or set DIDBOT_PDS_TFVARS"
fi
[ -f "$var_file" ] || fail "$var_file is not a file"
var_file="$(cd "$(dirname "$var_file")" && pwd)/$(basename "$var_file")"

for tool in tofu aws git jq curl; do
  command -v "$tool" >/dev/null || fail "$tool is required and was not found on PATH"
done

identity="$(aws sts get-caller-identity --query Arn --output text 2>&1)" ||
  fail "no working AWS credentials: $identity"
say "deploying as $identity"

sha="$(git rev-parse --verify --quiet --short=12 "$rev^{commit}")" || fail "$rev is not a commit"

say "initialising $stack"
tofu -chdir="$stack" init -input=false -backend-config=backend.hcl >/dev/null

outputs="$(tofu -chdir="$stack" output -json)"
output() { jq -r --arg name "$1" '.[$name].value // empty' <<<"$outputs"; }
live="$(output container_image_tag)"

if [ -z "$dry_run" ]; then
  scripts/publish-image.sh "$sha"
fi

# A saved plan, so what is applied is what was read. It embeds resolved
# values, so it lives outside the repository and not past this run.
plan="$(mktemp -t didbot-pds-XXXXXX.tfplan)"
trap 'rm -f "$plan"' EXIT

say "planning $sha${live:+ (live: $live)}"
rc=0
tofu -chdir="$stack" plan -input=false -detailed-exitcode -out="$plan" \
  -var-file="$var_file" -var "container_image_tag=$sha" || rc=$?
case "$rc" in
  0 | 2) ;;
  *) exit "$rc" ;;
esac

# A deploy changes the release parameter. One that also replaces the
# instance is an outage of minutes rather than seconds, and one that
# destroys anything else is not a deploy at all.
planned="$(tofu -chdir="$stack" show -json "$plan")"
replaced="$(jq -r '.resource_changes[]? | select(.change.actions | index("delete")) | .address' <<<"$planned")"
if [ -n "$replaced" ]; then
  warn "this plan replaces or destroys:"
  printf '%s\n' "$replaced" | sed 's/^/      /' >&2
  [ -n "$allow_replace" ] || fail "refusing a plan that replaces anything; --allow-replace applies it"
fi

if [ -n "$dry_run" ]; then
  say "dry run: nothing backed up, nothing applied"
  exit 0
fi
if [ "$rc" -eq 0 ]; then
  say "no changes: the stack already names $sha"
else
  if [ -z "$assume_yes" ]; then
    printf '\napply this plan? [y/N] '
    # Nothing on stdin is a no.
    read -r reply || reply=""
    case "$reply" in
      y | Y | yes | Yes) ;;
      *) fail "nothing applied" ;;
    esac
  fi

  if [ -n "$backup" ]; then
    vault="$(output backup_vault_name)"
    role="$(output backup_role_arn)"
    volume="$(output data_volume_id)"
    if [ -z "$vault" ] || [ -z "$role" ] || [ -z "$volume" ]; then
      fail "the state has no backup_vault_name, backup_role_arn or data_volume_id output yet; take a snapshot of the data volume by hand and rerun with --no-backup"
    fi
    # Kept as long as the daily snapshots the module takes.
    retention="$(jq -r '.variables.backup_retention_days.value' <<<"$planned")"
    region="$(cut -d. -f4 <<<"$(output ecr_repository_url)")"
    account="$(aws sts get-caller-identity --query Account --output text)"
    say "backing up $volume into $vault"
    job="$(aws backup start-backup-job --region "$region" \
      --backup-vault-name "$vault" \
      --resource-arn "arn:aws:ec2:$region:$account:volume/$volume" \
      --iam-role-arn "$role" \
      --lifecycle "DeleteAfterDays=$retention" \
      --start-window-minutes 60 \
      --query BackupJobId --output text)"
    # An EBS snapshot holds the volume as it was when it started, so the
    # deploy waits for the start rather than for the copy to finish.
    for _ in $(seq 90); do
      state="$(aws backup describe-backup-job --region "$region" --backup-job-id "$job" \
        --query State --output text)"
      case "$state" in
        RUNNING | COMPLETED) break ;;
        CREATED | PENDING) sleep 10 ;;
        *) fail "backup job $job is $state" ;;
      esac
    done
    case "$state" in
      RUNNING | COMPLETED) say "backup $job is $state" ;;
      *) fail "backup job $job had not started after fifteen minutes" ;;
    esac
  fi

  tofu -chdir="$stack" apply -input=false "$plan"

  # The var file names what is live, so a later hand-run apply keeps it.
  if grep -q '^[[:space:]]*container_image_tag[[:space:]]*=' "$var_file"; then
    sed -i -E "s|^([[:space:]]*container_image_tag[[:space:]]*=[[:space:]]*).*|\\1\"$sha\"|" "$var_file"
    say "$var_file now names $sha"
  else
    warn "$var_file names no container_image_tag; pass -var container_image_tag=$sha to every later apply"
  fi
  outputs="$(tofu -chdir="$stack" output -json)"
fi

# The timer converges within thirty seconds on its own. Asking directly is
# sooner, and brings the converger's own account back here. `systemctl
# start` on a oneshot waits for the run, joining one already going. A new
# instance takes a minute or two to register with Session Manager, so the
# ask is retried.
instance="$(output instance_id)"
say "converging $instance on $sha"
command_id=""
for _ in $(seq 20); do
  command_id="$(aws ssm send-command \
    --instance-ids "$instance" \
    --document-name AWS-RunShellScript \
    --comment "didbot deploy $sha" \
    --parameters '{"commands":["until [ -x /usr/local/bin/didbot-converge ]; do sleep 5; done","systemctl start didbot-converge.service; rc=$?; journalctl -u didbot-converge.service --since -30min -o cat --no-pager | tail -20; exit $rc"]}' \
    --query Command.CommandId --output text 2>/dev/null || true)"
  [ -n "$command_id" ] && break
  printf '.'
  sleep 15
done
echo

converged=""
if [ -z "$command_id" ]; then
  warn "the instance is not reachable over Session Manager; leaving it to the timer"
else
  while :; do
    status="$(aws ssm get-command-invocation --command-id "$command_id" --instance-id "$instance" \
      --query Status --output text 2>/dev/null || echo Pending)"
    case "$status" in
      Success | Failed | Cancelled | TimedOut) break ;;
      *)
        printf '.'
        sleep 5
        ;;
    esac
  done
  echo
  aws ssm get-command-invocation --command-id "$command_id" --instance-id "$instance" \
    --query '[StandardOutputContent, StandardErrorContent]' --output text | sed 's/^/    /'
  [ "$status" = Success ] && converged=1
fi

zone="$(output root_zone)"
serving=""
for _ in $(seq 30); do
  serving="$(curl -fs --max-time 10 "https://$zone/health" | jq -r '.revision // empty' 2>/dev/null || true)"
  [ "$serving" = "$sha" ] && break
  [ -n "$command_id" ] && [ -z "$converged" ] && break
  printf '.'
  sleep 10
done
echo

if [ "$serving" != "$sha" ]; then
  printf '\033[1;31m' >&2
  echo "deploy-pds: $sha IS NOT RUNNING. https://$zone/health reports ${serving:-no revision}." >&2
  [ -n "$serving" ] || curl -fsS --max-time 10 -o /dev/null "https://$zone/health" 2>&1 | head -1 | sed 's/^/  /' >&2 || true
  if [ -n "$command_id" ] && [ -z "$converged" ]; then
    echo "deploy-pds: the converger refused it and rolled back; its output is above." >&2
  fi
  printf '\033[0m' >&2
  echo "  To look:   aws ssm start-session --target $instance" >&2
  echo "             sudo journalctl -u didbot-converge -u didbot-pds --since -30min" >&2
  exit 1
fi
say "up: https://$zone runs $sha"