Queue-North-Website/scripts/deploy.sh

#!/usr/bin/env bash
#
# Move the running stack to an image that is already published.
#
#   bash scripts/deploy.sh                  # newest published vX.Y.Z
#   bash scripts/deploy.sh --tag v0.9.4     # a specific published version
#   bash scripts/deploy.sh --allow-floating --tag dev   # chase a pointer, loudly
#   bash scripts/deploy.sh --dry-run        # show the change, touch nothing
#   bash scripts/deploy.sh --no-backup      # skip the pre-deploy dump (say why)
#
# Exit codes: 0 deployed and healthy. 1 something failed and the message says
# what state it left behind. 2 nothing was attempted.
#
# ===========================================================================
# THIS DEPLOYS. IT DOES NOT BUILD.
# ===========================================================================
#
# `scripts/release.sh` publishes an image. This points the stack at one. They
# are separate decisions and separate commands, which is the rule both the
# template and PrivacyLLC-Web's release script state and the reason neither of
# them deploys.
#
# The template ships `deploy.py`, which does build, push AND deploy. It was not
# adopted here for exactly that reason: with `release.sh` already publishing a
# versioned image, a deploy tool that rebuilds would produce a second, different
# image for the same code and two answers to "what is running". This does the
# half that was missing and nothing else.
#
# ## What it costs, stated up front
#
# Redeploying stack 58 RECREATES the container. Both public front doors —
# queuenorth.com through nginx-proxy-manager, qn.isnull.dev through Cloudflare —
# reach the same instance, so they go down together for the few seconds it takes.
# There is no non-production environment to rehearse against.
#
# So it takes a verified backup first. The database is a named volume and a
# stack update does not remove one, but "does not normally" is not a reason to
# skip the cheap thing before the irreversible one.
#
# ## THE POLICY: production always runs a numbered version
#
# Set 2026-08-18. The stack pins `:vX.Y.Z` and never a floating tag.
#
# `:dev` and `:latest` are pointers. A stack that follows one has no answer to
# "what is running" that survives the next push — the tag in the stack file says
# `dev` today and will still say `dev` after the image underneath it changes, so
# the file records a preference rather than a fact. Rollback is worse: there is
# nothing to roll back *to*, because the pointer only ever names now.
#
# So this refuses a floating tag. `--allow-floating` exists for the one case
# that is not a mistake — reproducing a fault on whatever `:dev` happens to be —
# and it says so loudly every time.
#
# **Stack 58 pins `:dev` as this is written.** Correcting that is the first
# deploy made under this policy, and it is why the default below resolves a
# version rather than reusing whatever the stack already names.
#
# The digest is reported before and after either way, because that is the only
# thing that cannot be moved underneath you.

set -uo pipefail

cd "$(git rev-parse --show-toplevel)" || exit 1

IMAGE="${DEPLOY_IMAGE:-dream.scheller.ltd/null/queue-north-website}"
STACK_ID="${DEPLOY_STACK_ID:-58}"
ENDPOINT_ID="${DEPLOY_ENDPOINT_ID:-3}"
CONTAINER="${DEPLOY_CONTAINER:-qn-website-dev}"
HOST="${DEPLOY_HOST:-nebula}"
PORTAINER_URL="${PORTAINER_API_URL:-https://192.168.1.11:9443}"
PORTAINER_CREDS="${PORTAINER_CREDS_FILE:-$HOME/.openclaw/credentials/portainer.md}"
ORIGINS=("${DEPLOY_ORIGIN_PRIMARY:-https://queuenorth.com}" "${DEPLOY_ORIGIN_SECONDARY:-https://qn.isnull.dev}")

say()  { printf '\033[1mdeploy:\033[0m %s\n' "$*" >&2; }
die()  { printf '\033[1mdeploy:\033[0m %s\n' "$*" >&2; exit 1; }
stop() { printf '\033[1mdeploy:\033[0m %s\n' "$*" >&2; exit 2; }

TAG=""
DRY_RUN=""
NO_BACKUP=""
ALLOW_FLOATING=""
ROLLBACK=""
FIX_CORS=""
WATCHTOWER_OFF=""
NO_ROLLBACK=""

while [ "$#" -gt 0 ]; do
  case "$1" in
    --tag)       shift; [ "$#" -gt 0 ] || stop "--tag needs a value."; TAG="$1" ;;
    --tag=*)     TAG="${1#--tag=}" ;;
    --dry-run)   DRY_RUN="yes" ;;
    --no-backup) NO_BACKUP="yes" ;;
    --allow-floating) ALLOW_FLOATING="yes" ;;
    --rollback)       ROLLBACK="yes" ;;
    --fix-cors)       FIX_CORS="yes" ;;
    --watchtower-off) WATCHTOWER_OFF="yes" ;;
    --no-rollback)    NO_ROLLBACK="yes" ;;
    -h|--help)   say "usage: bash scripts/deploy.sh [--tag vX.Y.Z] [--dry-run] [--no-backup]"
                 say "       [--fix-cors] [--watchtower-off] [--rollback] [--allow-floating] [--no-rollback]"; exit 0 ;;
    *)           stop "unknown argument '$1'. Run --help." ;;
  esac
  shift
done

for t in curl python3; do
  command -v "$t" >/dev/null 2>&1 || stop "$t is not on PATH. Nothing was attempted."
done

KEY="${PORTAINER_API_KEY:-}"
if [ -z "$KEY" ] && [ -r "$PORTAINER_CREDS" ]; then
  KEY=$(grep -m1 'API Key' "$PORTAINER_CREDS" | grep -oE 'ptr_[^`]+')
fi
[ -n "$KEY" ] || stop "no Portainer API key. Set PORTAINER_API_KEY, or put it in
      $PORTAINER_CREDS — the file named after Portainer, not the one named after
      this project. Nothing was attempted."

api() { curl -sk --max-time 30 -H "X-API-Key: $KEY" "$@"; }

# ---------------------------------------------------------------------------
# Refuse to deploy a tag that is not published. Portainer would accept the
# stack file, fail to pull, and leave the container stopped — an outage caused
# by a typo, discovered by the site going down.
# ---------------------------------------------------------------------------
REG_ENV="${RELEASE_REGISTRY_ENV:-$HOME/.openclaw/docker-registry.env}"
if [ -r "$REG_ENV" ]; then set -a; . "$REG_ENV"; set +a; fi

tags_json=""
if [ -n "${FORGEJO_REGISTRY:-}" ] && [ -n "${FORGEJO_REGISTRY_TOKEN:-}" ]; then
  repo="${IMAGE#*/}"
  tags_json=$(curl -sS --max-time 20 -u "${FORGEJO_REGISTRY_USER}:${FORGEJO_REGISTRY_TOKEN}" \
    "https://${FORGEJO_REGISTRY}/v2/${repo}/tags/list" 2>/dev/null)
fi

# ---------------------------------------------------------------------------
# Resolve the tag. With none named, this is the newest published NUMBERED
# version — never `:dev`, and never "whatever the stack already says", because
# both of those are how a deployment stops being able to name itself.
# ---------------------------------------------------------------------------
if [ -z "$TAG" ]; then
  [ -n "$tags_json" ] || stop "no tag given and the registry tag list could not be read, so
      the newest published version is unknown. Name one explicitly:
        bash scripts/deploy.sh --tag vX.Y.Z
      Nothing was attempted."

  TAG=$(printf '%s' "$tags_json" | python3 -c "
import sys, json, re
tags = json.load(sys.stdin).get('tags') or []
def key(t):
    m = re.fullmatch(r'v(\d+)\.(\d+)\.(\d+)', t)
    return tuple(int(x) for x in m.groups()) if m else None
vers = sorted([t for t in tags if key(t)], key=key)
print(vers[-1] if vers else '')
")
  [ -n "$TAG" ] || stop "the registry holds no numbered vX.Y.Z tag to deploy — only
      floating ones. Publish a version first: npm run release. Nothing was attempted."
  say "no tag given; newest published version is ${TAG}"
fi

# ---------------------------------------------------------------------------
# Refuse a floating tag. See THE POLICY at the top.
# ---------------------------------------------------------------------------
if ! printf '%s' "$TAG" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+$'; then
  if [ -z "$ALLOW_FLOATING" ]; then
    die "'${TAG}' is not a numbered version, and production always runs one.
      A stack pinned to a moving tag cannot say what it is running and cannot be
      rolled back to anything, because the pointer only ever names now.
      Deploy a version:      bash scripts/deploy.sh --tag vX.Y.Z
      Publish one first:     npm run release
      If you genuinely mean to chase a pointer, --allow-floating."
  fi
  say "WARNING: --allow-floating. Deploying the moving tag '${TAG}'."
  say "         After this, the stack file will not record what is running."
fi

# ---------------------------------------------------------------------------
# Refuse to deploy a tag that is not published. Portainer would accept the
# stack file, fail to pull, and leave the container stopped — an outage caused
# by a typo, discovered by the site going down.
# ---------------------------------------------------------------------------
if [ -n "$tags_json" ]; then
  printf '%s' "$tags_json" | grep -q "\"${TAG}\"" \
    || die "${IMAGE}:${TAG} is not published. Nothing was changed.
      Published tags: $(printf '%s' "$tags_json" | python3 -c 'import sys,json;print(", ".join(json.load(sys.stdin).get("tags") or []))' 2>/dev/null)"
else
  say "WARNING: could not read the registry tag list, so it is not known whether"
  say "         ${TAG} is published. A missing tag becomes an outage here."
fi

# ---------------------------------------------------------------------------
# Refuse to go backwards.
#
# The newest published NUMBERED tag is not necessarily newer than what is
# running: on 2026-08-18 it was v0.8.3, built 2026-05-28, while the running
# :dev image was built 2026-08-01. Deploying it would have removed the privacy
# policy and every prerendered route — two months of regressions, applied by
# the default path with no warning.
#
# So compare creation dates. The target's is read from the registry WITHOUT
# pulling it: manifest -> config digest -> config blob -> .created.
# ---------------------------------------------------------------------------
registry_created() {
  local tag="$1" tok cfg
  tok=$(curl -sS --max-time 20 -u "${FORGEJO_REGISTRY_USER:-}:${FORGEJO_REGISTRY_TOKEN:-}" \
    "https://${FORGEJO_REGISTRY}/v2/token?scope=repository:${IMAGE#*/}:pull&service=container_registry" \
    2>/dev/null | python3 -c 'import sys,json;print(json.load(sys.stdin).get("token",""))' 2>/dev/null)
  [ -n "$tok" ] || return 1
  cfg=$(curl -sS --max-time 20 -H "Authorization: Bearer $tok" \
    -H 'Accept: application/vnd.oci.image.manifest.v1+json,application/vnd.docker.distribution.manifest.v2+json' \
    "https://${FORGEJO_REGISTRY}/v2/${IMAGE#*/}/manifests/${tag}" 2>/dev/null \
    | python3 -c 'import sys,json;print(json.load(sys.stdin).get("config",{}).get("digest",""))' 2>/dev/null)
  [ -n "$cfg" ] || return 1
  curl -sSL --max-time 30 -H "Authorization: Bearer $tok" \
    "https://${FORGEJO_REGISTRY}/v2/${IMAGE#*/}/blobs/${cfg}" 2>/dev/null \
    | python3 -c 'import sys,json;print(json.load(sys.stdin).get("created",""))' 2>/dev/null
}

# ---------------------------------------------------------------------------
# What is running now. Reported before anything changes, so the two halves of
# "before and after" come from the same run.
# ---------------------------------------------------------------------------
before_digest=$(ssh -o BatchMode=yes -o ConnectTimeout=10 "$HOST" \
  "docker image inspect \$(docker inspect '$CONTAINER' --format '{{.Image}}') --format '{{index .RepoDigests 0}}'" 2>/dev/null)
before_version=$(ssh -o BatchMode=yes -o ConnectTimeout=10 "$HOST" \
  "docker inspect '$CONTAINER' --format '{{index .Config.Labels \"org.opencontainers.image.version\"}}'" 2>/dev/null)

target_created=$(registry_created "$TAG")
running_created=$(ssh -o BatchMode=yes -o ConnectTimeout=10 "$HOST" \
  "docker image inspect \$(docker inspect '$CONTAINER' --format '{{.Image}}') --format '{{.Created}}'" 2>/dev/null)

if [ -n "$target_created" ] && [ -n "$running_created" ]; then
  if [ "$target_created" \< "$running_created" ] && [ -z "$ROLLBACK" ]; then
    die "${TAG} was built ${target_created}, and what is running was built ${running_created}.
      That is BACKWARDS. Deploying it would undo everything built in between —
      on this project that has meant losing the privacy policy and every
      prerendered route, silently, on the default path.
      If you mean to roll back, say so:  --rollback
      Nothing was changed."
  fi
  [ -n "$ROLLBACK" ] && say "--rollback: deploying an image built ${target_created}"
else
  say "WARNING: could not compare build dates (target='${target_created:-?}'"
  say "         running='${running_created:-?}'), so the going-backwards check"
  say "         did not run. That is not a pass."
fi

say "stack      ${STACK_ID} on ${HOST} (container ${CONTAINER})"
say "deploying  ${IMAGE}:${TAG}"
say "running    ${before_digest:-unknown}"
say "version    ${before_version:-<no version label on the running image>}"

stack=$(api "${PORTAINER_URL}/api/stacks/${STACK_ID}") \
  || die "could not reach Portainer at ${PORTAINER_URL}. Nothing was changed."
printf '%s' "$stack" | grep -q '"Id"' \
  || die "Portainer did not return stack ${STACK_ID}. Nothing was changed."

# Written to a file rather than captured in $( ), which strips trailing
# newlines. The first version of this did, so every PUT silently removed the
# file's final newline — harmless to YAML, and still a change beyond the lines
# this script says it touches. "Only the image line" has to be true.
STACK_ORIG="$(mktemp)"; STACK_NEW="$(mktemp)"
trap 'rm -f "$STACK_ORIG" "$STACK_NEW"' EXIT
api "${PORTAINER_URL}/api/stacks/${STACK_ID}/file" \
  | python3 -c 'import sys,json;sys.stdout.write(json.load(sys.stdin)["StackFileContent"])' > "$STACK_ORIG" \
  || die "could not read the stack file. Nothing was changed."
[ -s "$STACK_ORIG" ] || die "the stack file came back empty. Nothing was changed."
file=$(cat "$STACK_ORIG")

# ---------------------------------------------------------------------------
# Build the new stack file from the original bytes. Every edit is opt-in and
# every one is printed as a diff before anything is sent.
#
# python3 with the script as an ARGUMENT and the file on stdin: a heredoc would
# claim stdin too. The first version of this did exactly that, and python tried
# to execute the YAML.
# ---------------------------------------------------------------------------
python3 -c "
import re, sys
img, tag, fix_cors, wt_off = sys.argv[1], sys.argv[2], sys.argv[3] == 'yes', sys.argv[4] == 'yes'
s = sys.stdin.read()

pat = re.compile(r'^(\s*image:\s*)' + re.escape(img) + r':\S+[ \t]*\$', re.M)
if not pat.search(s):
    sys.exit(3)
s = pat.sub(lambda m: m.group(1) + img + ':' + tag, s)

if fix_cors:
    c = re.compile(r'^(\s*-\s*CORS_ORIGIN=)(\S+?)/+[ \t]*\$', re.M)
    if not c.search(s):
        sys.stderr.write('NOCORS\n')
        sys.exit(4)
    s = c.sub(lambda m: m.group(1) + m.group(2), s)

if wt_off:
    lab = 'com.centurylinklabs.watchtower.enable=false'
    if lab not in s:
        m = re.search(r'^(\s*)restart:\s*\S+[ \t]*\$', s, re.M)
        if not m:
            sys.stderr.write('NOANCHOR\n')
            sys.exit(5)
        ind = m.group(1)
        block = f'{ind}labels:\n{ind}  - "{lab}"\n'
        s = s[:m.start()] + block + s[m.start():]

sys.stdout.write(s)
" "$IMAGE" "$TAG" "${FIX_CORS:-no}" "${WATCHTOWER_OFF:-no}" < "$STACK_ORIG" > "$STACK_NEW"
rc=$?
case "$rc" in
  0) ;;
  3) die "the stack file has no 'image: ${IMAGE}:<tag>' line to update. Nothing was changed." ;;
  4) die "--fix-cors: no CORS_ORIGIN line with a trailing slash to fix. Either it is
      already correct or the line moved. Nothing was changed." ;;
  5) die "--watchtower-off: could not find a 'restart:' line to anchor the labels
      block to. Nothing was changed." ;;
  *) die "could not build the new stack file (exit $rc). Nothing was changed." ;;
esac

[ -s "$STACK_NEW" ] || die "the new stack file came out empty. Nothing was changed."

# Every env line must survive. A dropped one comes back healthy and stops
# capturing leads, which is the worst shape of failure available here.
orig_env=$(grep -cE '^\s*-\s*[A-Z_]+=' "$STACK_ORIG")
new_env=$(grep -cE '^\s*-\s*[A-Z_]+=' "$STACK_NEW")
[ "$orig_env" -eq "$new_env" ] || die "the rewrite changed the number of environment lines
      (${orig_env} -> ${new_env}). Refusing to send it. Nothing was changed."

say ""
say "the exact change to the stack file:"
diff -u "$STACK_ORIG" "$STACK_NEW" \
  | sed -E 's/^([-+].*(SECRET|XNQSJSDP|XMIWTLD)=).+/\1<redacted>/' \
  | sed 's/^/    /' >&2
say ""

new_file=$(cat "$STACK_NEW")

if cmp -s "$STACK_ORIG" "$STACK_NEW"; then
  say "the stack file is already exactly as requested — redeploying it to pull the newest digest."
fi

if [ -n "$DRY_RUN" ]; then
  say ""
  say "--dry-run: nothing was changed. It would have:"
  [ -z "$NO_BACKUP" ] && say "  bash scripts/backup.sh          (verified dump, before the irreversible part)"
  say "  PUT ${PORTAINER_URL}/api/stacks/${STACK_ID}?endpointId=${ENDPOINT_ID}  (pullImage: true)"
  say "  waited for ${CONTAINER} to report healthy on ${HOST}"
  say "  checked ${ORIGINS[*]}"
  say "  checked the live bundle still carries the reCAPTCHA site key"
  say "  rolled the whole stack file back automatically if any of that failed"
  say ""
  say "That PUT recreates the container. Both front doors go down together."
  exit 0
fi

# ---------------------------------------------------------------------------
# Backup before the irreversible part. backup.sh verifies its own dump.
# ---------------------------------------------------------------------------
if [ -z "$NO_BACKUP" ]; then
  say "taking a verified backup first…"
  bash scripts/backup.sh >&2 \
    || die "the backup failed, so NOTHING was deployed. Fix it, or re-run with
      --no-backup if you have a copy elsewhere and can say where."
else
  say "WARNING: --no-backup. Deploying without a pre-deploy dump of the only copy"
  say "         of every lead this site has taken."
fi

# ---------------------------------------------------------------------------
# The irreversible part.
# ---------------------------------------------------------------------------
say "redeploying stack ${STACK_ID} — the container is being recreated…"

# The stack's existing Env array must be sent back with the update. Portainer
# treats a PUT as the whole desired state, so omitting it would silently strip
# twelve variables — including the reCAPTCHA secret and the Zoho form tokens —
# and the container would come up healthy and quietly stop capturing leads.
payload=$(printf '%s' "$stack" | python3 -c "
import sys, json
stack = json.load(sys.stdin)
env = stack.get('Env') or []
if not env:
    sys.stderr.write('NOENV\\n')
    sys.exit(3)
print(json.dumps({
    'stackFileContent': sys.argv[1],
    'env': env,
    'prune': False,
    'pullImage': True,
}))
" "$new_file") || die "could not build the update payload, or the stack reported no
      environment variables at all. Sending an empty env would strip the Zoho and
      reCAPTCHA credentials from the running container. Nothing was changed."

env_count=$(printf '%s' "$payload" | python3 -c 'import sys,json;print(len(json.load(sys.stdin)["env"]))' 2>/dev/null)
say "preserving ${env_count:-?} stack environment variables"

resp=$(api -X PUT -H "Content-Type: application/json" --data-binary "$payload" \
  "${PORTAINER_URL}/api/stacks/${STACK_ID}?endpointId=${ENDPOINT_ID}")

printf '%s' "$resp" | grep -q '"Id"' || die "the redeploy was refused by Portainer:
$(printf '%s' "$resp" | head -c 400)
      The stack may or may not have changed — check it before retrying."

say "stack updated. waiting for ${CONTAINER}…"

# ---------------------------------------------------------------------------
# Verify, and undo it if the verification fails.
#
# Three things are checked, and the third is the one that would otherwise be
# found by a customer: the reCAPTCHA site key is frozen into the bundle at
# build time, and an image built without it renders "Security verification is
# not configured." and rejects every submission. Health checks pass happily
# while that is true, so the live bundle is asked directly.
# ---------------------------------------------------------------------------
EXPECTED_KEY=$(printf '%s' "$stack" | python3 -c "
import sys, json
env = {e['name']: e['value'] for e in (json.load(sys.stdin).get('Env') or [])}
print(env.get('VITE_RECAPTCHA_SITE_KEY', ''))
" 2>/dev/null)

wait_healthy() {
  local i state
  for i in $(seq 1 30); do
    state=$(ssh -o BatchMode=yes -o ConnectTimeout=10 "$HOST" \
      "docker inspect '$CONTAINER' --format '{{.State.Status}}:{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}'" 2>/dev/null)
    case "$state" in running:healthy|running:none) return 0 ;; esac
    sleep 4
  done
  return 1
}

check_live() {
  local fail=0 o body js
  for o in "${ORIGINS[@]}"; do
    body=$(curl -s --max-time 15 "$o/api/health")
    if printf '%s' "$body" | grep -q '"status":"ok"' && printf '%s' "$body" | grep -q '"db":"ok"'; then
      say "  ok    $o"
    else
      say "  FAIL  $o -> ${body:-<no response>}"; fail=1
    fi
  done

  if [ -n "$EXPECTED_KEY" ]; then
    js=$(curl -s --max-time 15 "${ORIGINS[0]}/" | grep -oE 'assets/index-[A-Za-z0-9_-]+\.js' | head -1)
    if [ -z "$js" ]; then
      say "  FAIL  could not find the bundle on ${ORIGINS[0]}"; fail=1
    elif curl -s --max-time 20 "${ORIGINS[0]}/${js}" | grep -qF "$EXPECTED_KEY"; then
      say "  ok    the live bundle carries the reCAPTCHA site key"
    else
      say "  FAIL  the live bundle does NOT carry the reCAPTCHA site key."
      say "        The contact form will read \"Security verification is not"
      say "        configured.\" and reject every submission."
      fail=1
    fi
  else
    say "  note  the stack declares no VITE_RECAPTCHA_SITE_KEY, so the bundle"
    say "        check was skipped. That is not a pass."
  fi
  return $fail
}

deploy_failed=""
wait_healthy || deploy_failed="the container did not come back healthy"
if [ -z "$deploy_failed" ]; then
  say "checking the live site…"
  check_live || deploy_failed="the container is healthy but the live checks failed"
fi

# ---------------------------------------------------------------------------
# Auto-rollback. Once, and only once.
#
# The original bytes are still in $STACK_ORIG, so this restores the state that
# was known to work — not just the image line, but the CORS and label edits
# too. A half-applied change is not a rollback.
#
# The previous image is already on the host, so this does not depend on the
# registry being reachable at the worst possible moment.
# ---------------------------------------------------------------------------
if [ -n "$deploy_failed" ]; then
  say ""
  say "DEPLOY FAILED: ${deploy_failed}."

  if [ -n "$NO_ROLLBACK" ]; then
    die "--no-rollback: leaving it exactly as it is so you can inspect it.
      Logs:     bash scripts/status.sh --logs 200
      Restore:  bash scripts/deploy.sh --tag <previous> --rollback
      Backup:   \$HOME/backups/queue-north-website"
  fi

  say "rolling back to the previous stack file…"
  rb_payload=$(printf '%s' "$stack" | python3 -c "
import sys, json
stack = json.load(sys.stdin)
print(json.dumps({'stackFileContent': sys.argv[1], 'env': stack.get('Env') or [],
                  'prune': False, 'pullImage': True}))
" "$(cat "$STACK_ORIG")")

  if api -X PUT -H "Content-Type: application/json" --data-binary "$rb_payload" \
       "${PORTAINER_URL}/api/stacks/${STACK_ID}?endpointId=${ENDPOINT_ID}" | grep -q '"Id"'; then
    if wait_healthy && check_live; then
      say ""
      die "ROLLED BACK successfully. The site is serving the previous image again.
      The deploy of ${TAG} failed: ${deploy_failed}.
      Nothing is lost; the pre-deploy backup is in \$HOME/backups/queue-north-website."
    fi
    die "THE ROLLBACK ALSO FAILED. THE SITE IS LIKELY DOWN. Stopping rather than
      trying again — a script retrying an outage is how a short one becomes long.
      bash scripts/status.sh --logs 200
      Portainer: ${PORTAINER_URL}  stack ${STACK_ID}"
  fi

  die "THE ROLLBACK PUT WAS REFUSED. THE SITE IS LIKELY DOWN.
      bash scripts/status.sh --logs 200
      Portainer: ${PORTAINER_URL}  stack ${STACK_ID}"
fi

after_digest=$(ssh -o BatchMode=yes -o ConnectTimeout=10 "$HOST" \
  "docker image inspect \$(docker inspect '$CONTAINER' --format '{{.Image}}') --format '{{index .RepoDigests 0}}'" 2>/dev/null)
after_version=$(ssh -o BatchMode=yes -o ConnectTimeout=10 "$HOST" \
  "docker inspect '$CONTAINER' --format '{{index .Config.Labels \"org.opencontainers.image.version\"}}'" 2>/dev/null)

say ""
say "digest   ${before_digest:-unknown}"
say "      -> ${after_digest:-unknown}"
say "version  ${before_version:-none}  ->  ${after_version:-none}"

say ""
say "deployed. Record it: docs/history/DEVELOPMENT_LOG.md, and close whatever"
say "issue this shipped with the evidence above."