Queue-North-Website/scripts/deploy.sh

#!/usr/bin/env bash
#
# Move the running stack to an image that is already published.
#
#   bash scripts/deploy.sh                  # newest published vX.Y.Z
#   bash scripts/deploy.sh --tag v0.9.4     # a specific published version
#   bash scripts/deploy.sh --allow-floating --tag dev   # chase a pointer, loudly
#   bash scripts/deploy.sh --dry-run        # show the change, touch nothing
#   bash scripts/deploy.sh --no-backup      # skip the pre-deploy dump (say why)
#
# Exit codes: 0 deployed and healthy. 1 something failed and the message says
# what state it left behind. 2 nothing was attempted.
#
# ===========================================================================
# THIS DEPLOYS. IT DOES NOT BUILD.
# ===========================================================================
#
# `scripts/release.sh` publishes an image. This points the stack at one. They
# are separate decisions and separate commands, which is the rule both the
# template and PrivacyLLC-Web's release script state and the reason neither of
# them deploys.
#
# The template ships `deploy.py`, which does build, push AND deploy. It was not
# adopted here for exactly that reason: with `release.sh` already publishing a
# versioned image, a deploy tool that rebuilds would produce a second, different
# image for the same code and two answers to "what is running". This does the
# half that was missing and nothing else.
#
# ## What it costs, stated up front
#
# Redeploying stack 58 RECREATES the container. Both public front doors —
# queuenorth.com through nginx-proxy-manager, qn.isnull.dev through Cloudflare —
# reach the same instance, so they go down together for the few seconds it takes.
# There is no non-production environment to rehearse against.
#
# So it takes a verified backup first. The database is a named volume and a
# stack update does not remove one, but "does not normally" is not a reason to
# skip the cheap thing before the irreversible one.
#
# ## THE POLICY: production always runs a numbered version
#
# Set 2026-08-18. The stack pins `:vX.Y.Z` and never a floating tag.
#
# `:dev` and `:latest` are pointers. A stack that follows one has no answer to
# "what is running" that survives the next push — the tag in the stack file says
# `dev` today and will still say `dev` after the image underneath it changes, so
# the file records a preference rather than a fact. Rollback is worse: there is
# nothing to roll back *to*, because the pointer only ever names now.
#
# So this refuses a floating tag. `--allow-floating` exists for the one case
# that is not a mistake — reproducing a fault on whatever `:dev` happens to be —
# and it says so loudly every time.
#
# **Stack 58 pins `:dev` as this is written.** Correcting that is the first
# deploy made under this policy, and it is why the default below resolves a
# version rather than reusing whatever the stack already names.
#
# The digest is reported before and after either way, because that is the only
# thing that cannot be moved underneath you.

set -uo pipefail

cd "$(git rev-parse --show-toplevel)" || exit 1

IMAGE="${DEPLOY_IMAGE:-dream.scheller.ltd/null/queue-north-website}"
STACK_ID="${DEPLOY_STACK_ID:-58}"
ENDPOINT_ID="${DEPLOY_ENDPOINT_ID:-3}"
CONTAINER="${DEPLOY_CONTAINER:-qn-website-dev}"
HOST="${DEPLOY_HOST:-nebula}"
PORTAINER_URL="${PORTAINER_API_URL:-https://192.168.1.11:9443}"
PORTAINER_CREDS="${PORTAINER_CREDS_FILE:-$HOME/.openclaw/credentials/portainer.md}"
ORIGINS=("${DEPLOY_ORIGIN_PRIMARY:-https://queuenorth.com}" "${DEPLOY_ORIGIN_SECONDARY:-https://qn.isnull.dev}")

say()  { printf '\033[1mdeploy:\033[0m %s\n' "$*" >&2; }
die()  { printf '\033[1mdeploy:\033[0m %s\n' "$*" >&2; exit 1; }
stop() { printf '\033[1mdeploy:\033[0m %s\n' "$*" >&2; exit 2; }

TAG=""
DRY_RUN=""
NO_BACKUP=""
ALLOW_FLOATING=""

while [ "$#" -gt 0 ]; do
  case "$1" in
    --tag)       shift; [ "$#" -gt 0 ] || stop "--tag needs a value."; TAG="$1" ;;
    --tag=*)     TAG="${1#--tag=}" ;;
    --dry-run)   DRY_RUN="yes" ;;
    --no-backup) NO_BACKUP="yes" ;;
    --allow-floating) ALLOW_FLOATING="yes" ;;
    -h|--help)   say "usage: bash scripts/deploy.sh [--tag vX.Y.Z] [--dry-run] [--no-backup] [--allow-floating]"; exit 0 ;;
    *)           stop "unknown argument '$1'. Run --help." ;;
  esac
  shift
done

for t in curl python3; do
  command -v "$t" >/dev/null 2>&1 || stop "$t is not on PATH. Nothing was attempted."
done

KEY="${PORTAINER_API_KEY:-}"
if [ -z "$KEY" ] && [ -r "$PORTAINER_CREDS" ]; then
  KEY=$(grep -m1 'API Key' "$PORTAINER_CREDS" | grep -oE 'ptr_[^`]+')
fi
[ -n "$KEY" ] || stop "no Portainer API key. Set PORTAINER_API_KEY, or put it in
      $PORTAINER_CREDS — the file named after Portainer, not the one named after
      this project. Nothing was attempted."

api() { curl -sk --max-time 30 -H "X-API-Key: $KEY" "$@"; }

# ---------------------------------------------------------------------------
# Refuse to deploy a tag that is not published. Portainer would accept the
# stack file, fail to pull, and leave the container stopped — an outage caused
# by a typo, discovered by the site going down.
# ---------------------------------------------------------------------------
REG_ENV="${RELEASE_REGISTRY_ENV:-$HOME/.openclaw/docker-registry.env}"
if [ -r "$REG_ENV" ]; then set -a; . "$REG_ENV"; set +a; fi

tags_json=""
if [ -n "${FORGEJO_REGISTRY:-}" ] && [ -n "${FORGEJO_REGISTRY_TOKEN:-}" ]; then
  repo="${IMAGE#*/}"
  tags_json=$(curl -sS --max-time 20 -u "${FORGEJO_REGISTRY_USER}:${FORGEJO_REGISTRY_TOKEN}" \
    "https://${FORGEJO_REGISTRY}/v2/${repo}/tags/list" 2>/dev/null)
fi

# ---------------------------------------------------------------------------
# Resolve the tag. With none named, this is the newest published NUMBERED
# version — never `:dev`, and never "whatever the stack already says", because
# both of those are how a deployment stops being able to name itself.
# ---------------------------------------------------------------------------
if [ -z "$TAG" ]; then
  [ -n "$tags_json" ] || stop "no tag given and the registry tag list could not be read, so
      the newest published version is unknown. Name one explicitly:
        bash scripts/deploy.sh --tag vX.Y.Z
      Nothing was attempted."

  TAG=$(printf '%s' "$tags_json" | python3 -c "
import sys, json, re
tags = json.load(sys.stdin).get('tags') or []
def key(t):
    m = re.fullmatch(r'v(\d+)\.(\d+)\.(\d+)', t)
    return tuple(int(x) for x in m.groups()) if m else None
vers = sorted([t for t in tags if key(t)], key=key)
print(vers[-1] if vers else '')
")
  [ -n "$TAG" ] || stop "the registry holds no numbered vX.Y.Z tag to deploy — only
      floating ones. Publish a version first: npm run release. Nothing was attempted."
  say "no tag given; newest published version is ${TAG}"
fi

# ---------------------------------------------------------------------------
# Refuse a floating tag. See THE POLICY at the top.
# ---------------------------------------------------------------------------
if ! printf '%s' "$TAG" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+$'; then
  if [ -z "$ALLOW_FLOATING" ]; then
    die "'${TAG}' is not a numbered version, and production always runs one.
      A stack pinned to a moving tag cannot say what it is running and cannot be
      rolled back to anything, because the pointer only ever names now.
      Deploy a version:      bash scripts/deploy.sh --tag vX.Y.Z
      Publish one first:     npm run release
      If you genuinely mean to chase a pointer, --allow-floating."
  fi
  say "WARNING: --allow-floating. Deploying the moving tag '${TAG}'."
  say "         After this, the stack file will not record what is running."
fi

# ---------------------------------------------------------------------------
# Refuse to deploy a tag that is not published. Portainer would accept the
# stack file, fail to pull, and leave the container stopped — an outage caused
# by a typo, discovered by the site going down.
# ---------------------------------------------------------------------------
if [ -n "$tags_json" ]; then
  printf '%s' "$tags_json" | grep -q "\"${TAG}\"" \
    || die "${IMAGE}:${TAG} is not published. Nothing was changed.
      Published tags: $(printf '%s' "$tags_json" | python3 -c 'import sys,json;print(", ".join(json.load(sys.stdin).get("tags") or []))' 2>/dev/null)"
else
  say "WARNING: could not read the registry tag list, so it is not known whether"
  say "         ${TAG} is published. A missing tag becomes an outage here."
fi

# ---------------------------------------------------------------------------
# What is running now. Reported before anything changes, so the two halves of
# "before and after" come from the same run.
# ---------------------------------------------------------------------------
before_digest=$(ssh -o BatchMode=yes -o ConnectTimeout=10 "$HOST" \
  "docker image inspect \$(docker inspect '$CONTAINER' --format '{{.Image}}') --format '{{index .RepoDigests 0}}'" 2>/dev/null)
before_version=$(ssh -o BatchMode=yes -o ConnectTimeout=10 "$HOST" \
  "docker inspect '$CONTAINER' --format '{{index .Config.Labels \"org.opencontainers.image.version\"}}'" 2>/dev/null)

say "stack      ${STACK_ID} on ${HOST} (container ${CONTAINER})"
say "deploying  ${IMAGE}:${TAG}"
say "running    ${before_digest:-unknown}"
say "version    ${before_version:-<no version label on the running image>}"

stack=$(api "${PORTAINER_URL}/api/stacks/${STACK_ID}") \
  || die "could not reach Portainer at ${PORTAINER_URL}. Nothing was changed."
printf '%s' "$stack" | grep -q '"Id"' \
  || die "Portainer did not return stack ${STACK_ID}. Nothing was changed."

file=$(api "${PORTAINER_URL}/api/stacks/${STACK_ID}/file" \
  | python3 -c 'import sys,json;sys.stdout.write(json.load(sys.stdin)["StackFileContent"])') \
  || die "could not read the stack file. Nothing was changed."

# Rewrite the image line only if it does not already name the tag we want.
#
# python3 -c with the script as an ARGUMENT, not `python3 -` with a heredoc:
# the stack file arrives on stdin, and a heredoc would claim stdin too. The
# first version of this did exactly that, and python tried to execute the YAML.
new_file=$(printf '%s' "$file" | python3 -c "
import re, sys
img, tag = sys.argv[1], sys.argv[2]
s = sys.stdin.read()
pat = re.compile(r'^(\s*image:\s*)' + re.escape(img) + r':\S+[ \t]*\$', re.M)
if not pat.search(s):
    sys.exit(3)
sys.stdout.write(pat.sub(lambda m: m.group(1) + img + ':' + tag, s))
" "$IMAGE" "$TAG") || die "the stack file has no 'image: ${IMAGE}:<tag>' line to update.
      Nothing was changed. Check the stack file at ${PORTAINER_URL} — if the image
      name changed, DEPLOY_IMAGE here is stale."

if [ "$new_file" = "$file" ]; then
  say "the stack already names ${IMAGE}:${TAG} — redeploying it to pull the newest digest."
else
  say "stack image line: $(printf '%s' "$file" | grep -oE "image: ${IMAGE}:\S+" | head -1)"
  say "              -> image: ${IMAGE}:${TAG}"
fi

if [ -n "$DRY_RUN" ]; then
  say ""
  say "--dry-run: nothing was changed. It would have:"
  [ -z "$NO_BACKUP" ] && say "  bash scripts/backup.sh          (verified dump, before the irreversible part)"
  say "  PUT ${PORTAINER_URL}/api/stacks/${STACK_ID}?endpointId=${ENDPOINT_ID}  (pullImage: true)"
  say "  waited for ${CONTAINER} to report healthy on ${HOST}"
  say "  checked ${ORIGINS[*]}"
  say ""
  say "That PUT recreates the container. Both front doors go down together."
  exit 0
fi

# ---------------------------------------------------------------------------
# Backup before the irreversible part. backup.sh verifies its own dump.
# ---------------------------------------------------------------------------
if [ -z "$NO_BACKUP" ]; then
  say "taking a verified backup first…"
  bash scripts/backup.sh >&2 \
    || die "the backup failed, so NOTHING was deployed. Fix it, or re-run with
      --no-backup if you have a copy elsewhere and can say where."
else
  say "WARNING: --no-backup. Deploying without a pre-deploy dump of the only copy"
  say "         of every lead this site has taken."
fi

# ---------------------------------------------------------------------------
# The irreversible part.
# ---------------------------------------------------------------------------
say "redeploying stack ${STACK_ID} — the container is being recreated…"

# The stack's existing Env array must be sent back with the update. Portainer
# treats a PUT as the whole desired state, so omitting it would silently strip
# twelve variables — including the reCAPTCHA secret and the Zoho form tokens —
# and the container would come up healthy and quietly stop capturing leads.
payload=$(printf '%s' "$stack" | python3 -c "
import sys, json
stack = json.load(sys.stdin)
env = stack.get('Env') or []
if not env:
    sys.stderr.write('NOENV\\n')
    sys.exit(3)
print(json.dumps({
    'stackFileContent': sys.argv[1],
    'env': env,
    'prune': False,
    'pullImage': True,
}))
" "$new_file") || die "could not build the update payload, or the stack reported no
      environment variables at all. Sending an empty env would strip the Zoho and
      reCAPTCHA credentials from the running container. Nothing was changed."

env_count=$(printf '%s' "$payload" | python3 -c 'import sys,json;print(len(json.load(sys.stdin)["env"]))' 2>/dev/null)
say "preserving ${env_count:-?} stack environment variables"

resp=$(api -X PUT -H "Content-Type: application/json" --data-binary "$payload" \
  "${PORTAINER_URL}/api/stacks/${STACK_ID}?endpointId=${ENDPOINT_ID}")

printf '%s' "$resp" | grep -q '"Id"' || die "the redeploy was refused by Portainer:
$(printf '%s' "$resp" | head -c 400)
      The stack may or may not have changed — check it before retrying."

say "stack updated. waiting for ${CONTAINER}…"

# ---------------------------------------------------------------------------
# Verify. A deploy that is not checked is a deploy you find out about later.
# ---------------------------------------------------------------------------
healthy=""
for _ in $(seq 1 30); do
  state=$(ssh -o BatchMode=yes -o ConnectTimeout=10 "$HOST" \
    "docker inspect '$CONTAINER' --format '{{.State.Status}}:{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}'" 2>/dev/null)
  case "$state" in
    running:healthy|running:none) healthy="yes"; break ;;
    running:starting)             ;;
    "")                           ;;
    *)                            ;;
  esac
  sleep 4
done

[ -n "$healthy" ] || die "${CONTAINER} did not come back healthy. THE SITE MAY BE DOWN.
      Check: bash scripts/status.sh --logs 200
      Roll back: bash scripts/deploy.sh --tag <the previous version>
      The pre-deploy backup is in \$HOME/backups/queue-north-website."

after_digest=$(ssh -o BatchMode=yes -o ConnectTimeout=10 "$HOST" \
  "docker image inspect \$(docker inspect '$CONTAINER' --format '{{.Image}}') --format '{{index .RepoDigests 0}}'" 2>/dev/null)
after_version=$(ssh -o BatchMode=yes -o ConnectTimeout=10 "$HOST" \
  "docker inspect '$CONTAINER' --format '{{index .Config.Labels \"org.opencontainers.image.version\"}}'" 2>/dev/null)

fail=0
for o in "${ORIGINS[@]}"; do
  body=$(curl -s --max-time 15 "$o/api/health")
  if printf '%s' "$body" | grep -q '"status":"ok"' && printf '%s' "$body" | grep -q '"db":"ok"'; then
    say "ok    $o"
  else
    say "FAIL  $o -> ${body:-<no response>}"
    fail=1
  fi
done

say ""
say "digest   ${before_digest:-unknown}"
say "      -> ${after_digest:-unknown}"
say "version  ${before_version:-none}  ->  ${after_version:-none}"

if [ "$fail" -ne 0 ]; then
  die "the container is healthy but at least one public origin is not answering.
      That is the ingress, not the app — see docs/OPERATIONS.md."
fi

say ""
say "deployed. Record it: docs/history/DEVELOPMENT_LOG.md, and close whatever"
say "issue this shipped with the evidence above."