diff --git a/docs/operations.md b/docs/operations.md index 3711acf0..9668c3b5 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -185,7 +185,10 @@ The instance boots with no volume, so docker does not start and nothing is brought up empty. The deploy attaches the new volume and writes its id into the release parameter; activation starts docker, which mounts the volume, a copy of the old one down to its filesystem's id. It is already copied to, so -nothing is copied, and the stack starts from the snapshot. The old volume is +nothing is copied, and the stack starts from the snapshot. If the deploy's +converge fails because the attachment has not finished, the poller, which +outlives a docker that could not start, activates within thirty seconds of it +finishing. The old volume is no longer in state, so `prevent_destroy` does not cover it: delete it by hand once the board is back. To restore in another zone, set `availability_zone` to it as well, which replaces the subnet and the instance. diff --git a/infra/README.md b/infra/README.md index 7a2625f9..1c33507d 100644 --- a/infra/README.md +++ b/infra/README.md @@ -89,6 +89,10 @@ bucket and runs the `activate.sh` inside it: unpack the canvas into a tree of its own, write the compose environment, pull, `up -d`, and finally move the `/srv/web/current` symlink. The link moves last, so `release.txt` answering with the sha means everything before it succeeded. +Docker requires the data volume's mounts, so a boot without the volume leaves +docker stopped; the poller only wants docker (`files/data-disk.sh` rewrites +its unit to say so), so it keeps polling, and retries a release it has not +activated until the volume is attached. `scripts/deploy.sh` also asks the instance to converge directly, over Session Manager, so the output comes back to the laptop instead of thirty seconds later in a journal. diff --git a/infra/files/data-disk.sh b/infra/files/data-disk.sh index 7c992ce3..7332538c 100755 --- a/infra/files/data-disk.sh +++ b/infra/files/data-disk.sh @@ -15,7 +15,9 @@ # - each directory the stack writes is bind-mounted from it, by fstab too, # so a reboot puts them back before docker starts; # - docker itself is made to require those mounts, so after a reboot that -# came up without the disk, no container starts on an empty directory. +# came up without the disk, no container starts on an empty directory; +# - the converge poller only wants docker, so it outlives a docker that +# failed for want of the disk, and keeps retrying the activation. # # Exits non-zero, with nothing started, when the volume is not attached yet # or cannot be used: converge retries the activation every thirty seconds. @@ -29,10 +31,38 @@ home="$root/opt/vibescrobble" disk="$root/var/lib/vibescrobble" fstab="$root/etc/fstab" dropin="$root/etc/systemd/system/docker.service.d/vibescrobble-data.conf" +converge="$root/etc/systemd/system/vibescrobble-converge.service" # What was copied onto the disk, and when. Its presence is what says the # disk holds the board, whatever the root volume beside it holds. copied="$disk/.copied" +# The poller's unit, as user-data.sh writes it but for one line: it wants +# docker rather than requiring it. Docker requires the disk's mounts, so a +# boot without the disk fails docker, and a unit that required docker would +# fail with it and never be started again, leaving nothing to retry the +# activation that mounts the disk. A drop-in can add a dependency and never +# remove one, so the whole unit is replaced. Written first, before anything +# here can fail, and the running poller is not restarted by it. +mkdir -p "$(dirname "$converge")" +cat > "$converge.next" <<'UNIT' +# Written by data-disk.sh: the poller outlives a docker that cannot start. +[Unit] +Description=Converge vibescrobble on its release parameter +After=docker.service network-online.target +Wants=docker.service network-online.target +[Service] +ExecStart=/bin/bash -c 'while true; do /usr/local/bin/vibescrobble-converge || true; sleep 30; done' +Restart=always +[Install] +WantedBy=multi-user.target +UNIT +if ! cmp -s "$converge.next" "$converge"; then + mv "$converge.next" "$converge" + systemctl daemon-reload +else + rm "$converge.next" +fi + volume="$(sed -n 's/^DATA_VOLUME=//p' "$env_file")" if [ -z "$volume" ]; then echo "data-disk: the release names no DATA_VOLUME; nothing is started without one" >&2 diff --git a/scripts/test-data-disk.sh b/scripts/test-data-disk.sh index 916ee268..c7f11f42 100755 --- a/scripts/test-data-disk.sh +++ b/scripts/test-data-disk.sh @@ -4,7 +4,8 @@ # file, `mount` stands a directory in for a disk or a bind mount by a symlink, # and `docker` and `systemctl` say where docker's volumes are and note what # was asked of them. A blank disk, a disk that already holds the board, a -# disk not attached, and one holding something else. +# disk not attached, and one holding something else. Each machine starts with +# the converge poller's unit as user-data.sh writes it. set -euo pipefail here="$(cd "$(dirname "$0")" && pwd)" @@ -74,6 +75,11 @@ fail() { exit 1 } +user_data="$here/../infra/files/user-data.sh" +poller_unit="$(awk '/^cat > \/etc\/systemd\/system\/vibescrobble-converge.service/ {on = 1; next} + on && /^UNIT$/ {exit} on' "$user_data")" +[ -n "$poller_unit" ] || fail "user-data.sh writes the converge poller's unit" + # A machine: a root with a release env, and a disk, blank unless told. machine() { local name="$1" @@ -87,6 +93,8 @@ machine() { : >"$FAKE_LOG" printf 'VERSION=0123456789ab\nDATA_VOLUME=%s\n' "$volume" >"$T/$name/release.env" touch "$VIBESCROBBLE_ROOT/opt/vibescrobble/compose.yml" + mkdir -p "$VIBESCROBBLE_ROOT/etc/systemd/system" + printf '%s\n' "$poller_unit" >"$VIBESCROBBLE_ROOT/etc/systemd/system/vibescrobble-converge.service" } attach() { printf '%s' "${1:-}" >"$VIBESCROBBLE_ROOT/dev/disk/by-id/$device_name" @@ -100,6 +108,20 @@ log_has() { data() { cat "$VIBESCROBBLE_ROOT/opt/vibescrobble/data/$1" } +# Docker requires the disk's mounts, so after a boot without the disk docker +# fails, and so does every unit that requires it. The poller must not be one: +# it is what retries the activation once the disk is attached. +poller_outlives_docker() { + local unit="$VIBESCROBBLE_ROOT/etc/systemd/system/vibescrobble-converge.service" + if grep -E '^(Requires|Requisite|BindsTo|PartOf|Upholds)=.*docker\.service' "$unit"; then + fail "the poller fails with docker: $unit" + fi + grep -qxF 'Wants=docker.service network-online.target' "$unit" || fail "the poller still pulls docker in" + grep -qxF 'After=docker.service network-online.target' "$unit" || fail "and starts after it" + [ "$(grep '^ExecStart=' "$unit")" = "$(grep '^ExecStart=' <<<"$poller_unit")" ] || + fail "the poller runs the loop user-data.sh gave it" + grep -qxF 'Restart=always' "$unit" || fail "and is restarted when it exits" +} # A blank disk beside a machine that has been running: formatted, the stack # stopped, what the root held copied across, and every directory mounted @@ -123,6 +145,7 @@ grep -qxF '/var/lib/vibescrobble/data /opt/vibescrobble/data none bind,nofail,x- dropin="$VIBESCROBBLE_ROOT/etc/systemd/system/docker.service.d/vibescrobble-data.conf" grep -qxF 'RequiresMountsFor=/opt/vibescrobble/data' "$dropin" || fail "docker requires the mounts" log_has "systemctl daemon-reload" || fail "systemd reads the drop-in" +poller_outlives_docker # Again, as every activation runs it: nothing is formatted, stopped, copied # or written again. @@ -147,19 +170,21 @@ log_has "mkfs $VIBESCROBBLE_ROOT/dev/disk/by-id/$device_name" && fail "a populat [ "$(cat "$FAKE_DISK/volumes/relay-data/relay.sqlite")" = relay-db ] || fail "the relay's database survives" # Not attached yet: nothing is formatted, mounted or stopped, and the -# activation fails, so nothing is brought up. +# activation fails, so nothing is brought up. The poller is freed from docker +# all the same, so it is still running to try again. machine missing if run missing 2>"$T/missing.err"; then fail "a missing disk stops the activation"; fi grep -q 'not attached yet' "$T/missing.err" || fail "and says why: $(cat "$T/missing.err")" -[ ! -s "$FAKE_LOG" ] || fail "a missing disk touches nothing: $(cat "$FAKE_LOG")" +[ "$(cat "$FAKE_LOG")" = "systemctl daemon-reload" ] || fail "a missing disk touches only the poller: $(cat "$FAKE_LOG")" [ ! -e "$VIBESCROBBLE_ROOT/etc/fstab" ] || fail "nor fstab" +poller_outlives_docker # A disk holding another filesystem is neither formatted nor used. machine foreign attach "ext4 0ther-uuid" if run foreign 2>"$T/foreign.err"; then fail "another filesystem stops the activation"; fi grep -q 'holds ext4' "$T/foreign.err" || fail "and says why: $(cat "$T/foreign.err")" -[ ! -s "$FAKE_LOG" ] || fail "nothing is formatted or mounted: $(cat "$FAKE_LOG")" +[ "$(cat "$FAKE_LOG")" = "systemctl daemon-reload" ] || fail "nothing is formatted or mounted: $(cat "$FAKE_LOG")" # A release that names no volume brings nothing up. machine unnamed