#!/usr/bin/env bash # Runs infra/files/data-disk.sh under a fake root, against fakes of what it # asks the machine: `blkid` and `mkfs.xfs` read and write a device that is a # file, `mount` stands a directory in for a disk or a bind mount by a symlink, # and `docker` and `systemctl` say where docker's volumes are and note what # was asked of them. A blank disk, a disk that already holds the board, a # disk not attached, and one holding something else. Each machine starts with # the converge poller's unit as user-data.sh writes it. set -euo pipefail here="$(cd "$(dirname "$0")" && pwd)" script="$here/../infra/files/data-disk.sh" T="$(mktemp -d)" trap 'rm -rf "$T"' EXIT mkdir "$T/bin" volume=vol-0abc123def4567890 device_name="nvme-Amazon_Elastic_Block_Store_${volume//-/}" cat >"$T/bin/blkid" <<'FAKE' #!/usr/bin/env bash # blkid -p -o value -s TYPE|UUID ; the device file holds "TYPE UUID", # and is empty when blank, which blkid says by exiting 2. set -euo pipefail field="$5" device="$6" read -r type uuid <"$device" || true [ -n "${type:-}" ] || exit 2 case "$field" in TYPE) echo "$type" ;; UUID) echo "$uuid" ;; esac FAKE cat >"$T/bin/mkfs.xfs" <<'FAKE' #!/usr/bin/env bash set -euo pipefail for device; do :; done echo "xfs 5a1e-uuid" >"$device" echo "mkfs $device" >>"$FAKE_LOG" FAKE cat >"$T/bin/mountpoint" <<'FAKE' #!/usr/bin/env bash grep -qxF "$2" "$FAKE_MOUNTED" 2>/dev/null FAKE cat >"$T/bin/mount" <<'FAKE' #!/usr/bin/env bash # mount , from the fstab line naming it. The disk is the directory # FAKE_DISK; a bind mount is a symlink to its source, with what the point held # moved aside, as a real one hides it. set -euo pipefail point="$1" line="$(awk -v p="${point#"$VIBESCROBBLE_ROOT"}" '$2 == p' "$VIBESCROBBLE_ROOT/etc/fstab")" [ -n "$line" ] || { echo "mount: $point is not in fstab" >&2; exit 32; } case "$line" in UUID=*) source="$FAKE_DISK" ;; *) source="$VIBESCROBBLE_ROOT${line%% *}" ;; esac mv "$point" "$point.hidden" ln -s "$source" "$point" echo "$point" >>"$FAKE_MOUNTED" echo "mount $point" >>"$FAKE_LOG" FAKE cat >"$T/bin/docker" <<'FAKE' #!/usr/bin/env bash set -euo pipefail case "$1" in volume) for name; do :; done; echo "$VIBESCROBBLE_ROOT/var/lib/docker/volumes/$name/_data" ;; compose) echo "compose $2" >>"$FAKE_LOG" ;; esac FAKE cat >"$T/bin/systemctl" <<'FAKE' #!/usr/bin/env bash echo "systemctl $*" >>"$FAKE_LOG" FAKE chmod +x "$T/bin"/* export PATH="$T/bin:$PATH" fail() { echo "FAIL: $*" >&2 exit 1 } user_data="$here/../infra/files/user-data.sh" poller_unit="$(awk '/^cat > \/etc\/systemd\/system\/vibescrobble-converge.service/ {on = 1; next} on && /^UNIT$/ {exit} on' "$user_data")" [ -n "$poller_unit" ] || fail "user-data.sh writes the converge poller's unit" # A machine: a root with a release env, and a disk, blank unless told. machine() { local name="$1" export VIBESCROBBLE_ROOT="$T/$name/root" FAKE_DISK="$T/$name/disk" export FAKE_LOG="$T/$name/log" FAKE_MOUNTED="$T/$name/mounted" mkdir -p "$VIBESCROBBLE_ROOT/opt/vibescrobble/data" "$VIBESCROBBLE_ROOT/etc" \ "$VIBESCROBBLE_ROOT/dev/disk/by-id" "$FAKE_DISK" for v in caddy-data caddy-config relay-data; do mkdir -p "$VIBESCROBBLE_ROOT/var/lib/docker/volumes/vibescrobble_$v/_data" done : >"$FAKE_LOG" printf 'VERSION=0123456789ab\nDATA_VOLUME=%s\n' "$volume" >"$T/$name/release.env" touch "$VIBESCROBBLE_ROOT/opt/vibescrobble/compose.yml" mkdir -p "$VIBESCROBBLE_ROOT/etc/systemd/system" printf '%s\n' "$poller_unit" >"$VIBESCROBBLE_ROOT/etc/systemd/system/vibescrobble-converge.service" } attach() { printf '%s' "${1:-}" >"$VIBESCROBBLE_ROOT/dev/disk/by-id/$device_name" } run() { bash "$script" "$T/$1/release.env" } log_has() { grep -qxF "$1" "$FAKE_LOG" } data() { cat "$VIBESCROBBLE_ROOT/opt/vibescrobble/data/$1" } # Docker requires the disk's mounts, so after a boot without the disk docker # fails, and so does every unit that requires it. The poller must not be one: # it is what retries the activation once the disk is attached. poller_outlives_docker() { local unit="$VIBESCROBBLE_ROOT/etc/systemd/system/vibescrobble-converge.service" if grep -E '^(Requires|Requisite|BindsTo|PartOf|Upholds)=.*docker\.service' "$unit"; then fail "the poller fails with docker: $unit" fi grep -qxF 'Wants=docker.service network-online.target' "$unit" || fail "the poller still pulls docker in" grep -qxF 'After=docker.service network-online.target' "$unit" || fail "and starts after it" [ "$(grep '^ExecStart=' "$unit")" = "$(grep '^ExecStart=' <<<"$poller_unit")" ] || fail "the poller runs the loop user-data.sh gave it" grep -qxF 'Restart=always' "$unit" || fail "and is restarted when it exits" } # A blank disk beside a machine that has been running: formatted, the stack # stopped, what the root held copied across, and every directory mounted # from it. machine running attach echo root-state >"$VIBESCROBBLE_ROOT/opt/vibescrobble/data/state.json" echo relay-db >"$VIBESCROBBLE_ROOT/var/lib/docker/volumes/vibescrobble_relay-data/_data/relay.sqlite" run running >/dev/null || fail "a blank disk is taken" log_has "mkfs $VIBESCROBBLE_ROOT/dev/disk/by-id/$device_name" || fail "a blank disk is formatted" log_has "compose stop" || fail "the stack stops before it is moved" [ "$(data state.json)" = root-state ] || fail "the index's state is on the disk" [ "$(cat "$FAKE_DISK/data/state.json")" = root-state ] || fail "copied onto the disk" [ "$(cat "$FAKE_DISK/volumes/relay-data/relay.sqlite")" = relay-db ] || fail "the relay's database is on the disk" [ -f "$FAKE_DISK/.copied" ] || fail "the copy is noted on the disk" [ "$(grep -c . "$VIBESCROBBLE_ROOT/etc/fstab")" = 5 ] || fail "the disk and four binds in fstab: $(cat "$VIBESCROBBLE_ROOT/etc/fstab")" grep -q '^UUID=5a1e-uuid /var/lib/vibescrobble xfs defaults,nofail' "$VIBESCROBBLE_ROOT/etc/fstab" || fail "the disk by its UUID, nofail" grep -qxF '/var/lib/vibescrobble/data /opt/vibescrobble/data none bind,nofail,x-systemd.requires-mounts-for=/var/lib/vibescrobble 0 0' \ "$VIBESCROBBLE_ROOT/etc/fstab" || fail "the index's directory, bound from the disk" dropin="$VIBESCROBBLE_ROOT/etc/systemd/system/docker.service.d/vibescrobble-data.conf" grep -qxF 'RequiresMountsFor=/opt/vibescrobble/data' "$dropin" || fail "docker requires the mounts" log_has "systemctl daemon-reload" || fail "systemd reads the drop-in" poller_outlives_docker # Again, as every activation runs it: nothing is formatted, stopped, copied # or written again. fstab_before="$(cat "$VIBESCROBBLE_ROOT/etc/fstab")" : >"$FAKE_LOG" echo later-state >"$FAKE_DISK/data/state.json" run running >/dev/null || fail "a second run is fine" [ ! -s "$FAKE_LOG" ] || fail "a second run does nothing: $(cat "$FAKE_LOG")" [ "$(cat "$VIBESCROBBLE_ROOT/etc/fstab")" = "$fstab_before" ] || fail "fstab is not written twice" [ "$(data state.json)" = later-state ] || fail "what the disk holds stays" # The replacement machine is handed the same disk: it is not formatted, and # the new root's empty directories are not copied over what it holds. disk_dir="$FAKE_DISK" machine replaced export FAKE_DISK="$disk_dir" attach "xfs 5a1e-uuid" echo fresh-empty >"$VIBESCROBBLE_ROOT/opt/vibescrobble/data/state.json" run replaced >/dev/null || fail "a populated disk is taken" log_has "mkfs $VIBESCROBBLE_ROOT/dev/disk/by-id/$device_name" && fail "a populated disk is never formatted" [ "$(data state.json)" = later-state ] || fail "the board comes back from the disk, not the new root" [ "$(cat "$FAKE_DISK/volumes/relay-data/relay.sqlite")" = relay-db ] || fail "the relay's database survives" # Not attached yet: nothing is formatted, mounted or stopped, and the # activation fails, so nothing is brought up. The poller is freed from docker # all the same, so it is still running to try again. machine missing if run missing 2>"$T/missing.err"; then fail "a missing disk stops the activation"; fi grep -q 'not attached yet' "$T/missing.err" || fail "and says why: $(cat "$T/missing.err")" [ "$(cat "$FAKE_LOG")" = "systemctl daemon-reload" ] || fail "a missing disk touches only the poller: $(cat "$FAKE_LOG")" [ ! -e "$VIBESCROBBLE_ROOT/etc/fstab" ] || fail "nor fstab" poller_outlives_docker # A disk holding another filesystem is neither formatted nor used. machine foreign attach "ext4 0ther-uuid" if run foreign 2>"$T/foreign.err"; then fail "another filesystem stops the activation"; fi grep -q 'holds ext4' "$T/foreign.err" || fail "and says why: $(cat "$T/foreign.err")" [ "$(cat "$FAKE_LOG")" = "systemctl daemon-reload" ] || fail "nothing is formatted or mounted: $(cat "$FAKE_LOG")" # A release that names no volume brings nothing up. machine unnamed attach sed -i '/^DATA_VOLUME=/d' "$T/unnamed/release.env" if run unnamed 2>/dev/null; then fail "a release without DATA_VOLUME stops the activation"; fi # And activate.sh asks for the disk before it brings anything up. activate="$here/../infra/files/activate.sh" disk_line="$(grep -n 'data-disk.sh' "$activate" | head -1 | cut -d: -f1)" up_line="$(grep -n 'docker compose up' "$activate" | head -1 | cut -d: -f1)" [ -n "$disk_line" ] && [ "$disk_line" -lt "$up_line" ] || fail "activate.sh runs data-disk.sh before compose up" echo "data-disk: ok"