340 lines
16 KiB
Bash
Executable File
340 lines
16 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# Move the running stack to an image that is already published.
|
|
#
|
|
# bash scripts/deploy.sh # newest published vX.Y.Z
|
|
# bash scripts/deploy.sh --tag v0.9.4 # a specific published version
|
|
# bash scripts/deploy.sh --allow-floating --tag dev # chase a pointer, loudly
|
|
# bash scripts/deploy.sh --dry-run # show the change, touch nothing
|
|
# bash scripts/deploy.sh --no-backup # skip the pre-deploy dump (say why)
|
|
#
|
|
# Exit codes: 0 deployed and healthy. 1 something failed and the message says
|
|
# what state it left behind. 2 nothing was attempted.
|
|
#
|
|
# ===========================================================================
|
|
# THIS DEPLOYS. IT DOES NOT BUILD.
|
|
# ===========================================================================
|
|
#
|
|
# `scripts/release.sh` publishes an image. This points the stack at one. They
|
|
# are separate decisions and separate commands, which is the rule both the
|
|
# template and PrivacyLLC-Web's release script state and the reason neither of
|
|
# them deploys.
|
|
#
|
|
# The template ships `deploy.py`, which does build, push AND deploy. It was not
|
|
# adopted here for exactly that reason: with `release.sh` already publishing a
|
|
# versioned image, a deploy tool that rebuilds would produce a second, different
|
|
# image for the same code and two answers to "what is running". This does the
|
|
# half that was missing and nothing else.
|
|
#
|
|
# ## What it costs, stated up front
|
|
#
|
|
# Redeploying stack 58 RECREATES the container. Both public front doors —
|
|
# queuenorth.com through nginx-proxy-manager, qn.isnull.dev through Cloudflare —
|
|
# reach the same instance, so they go down together for the few seconds it takes.
|
|
# There is no non-production environment to rehearse against.
|
|
#
|
|
# So it takes a verified backup first. The database is a named volume and a
|
|
# stack update does not remove one, but "does not normally" is not a reason to
|
|
# skip the cheap thing before the irreversible one.
|
|
#
|
|
# ## THE POLICY: production always runs a numbered version
|
|
#
|
|
# Set 2026-08-18. The stack pins `:vX.Y.Z` and never a floating tag.
|
|
#
|
|
# `:dev` and `:latest` are pointers. A stack that follows one has no answer to
|
|
# "what is running" that survives the next push — the tag in the stack file says
|
|
# `dev` today and will still say `dev` after the image underneath it changes, so
|
|
# the file records a preference rather than a fact. Rollback is worse: there is
|
|
# nothing to roll back *to*, because the pointer only ever names now.
|
|
#
|
|
# So this refuses a floating tag. `--allow-floating` exists for the one case
|
|
# that is not a mistake — reproducing a fault on whatever `:dev` happens to be —
|
|
# and it says so loudly every time.
|
|
#
|
|
# **Stack 58 pins `:dev` as this is written.** Correcting that is the first
|
|
# deploy made under this policy, and it is why the default below resolves a
|
|
# version rather than reusing whatever the stack already names.
|
|
#
|
|
# The digest is reported before and after either way, because that is the only
|
|
# thing that cannot be moved underneath you.
|
|
|
|
set -uo pipefail
|
|
|
|
cd "$(git rev-parse --show-toplevel)" || exit 1
|
|
|
|
IMAGE="${DEPLOY_IMAGE:-dream.scheller.ltd/null/queue-north-website}"
|
|
STACK_ID="${DEPLOY_STACK_ID:-58}"
|
|
ENDPOINT_ID="${DEPLOY_ENDPOINT_ID:-3}"
|
|
CONTAINER="${DEPLOY_CONTAINER:-qn-website-dev}"
|
|
HOST="${DEPLOY_HOST:-nebula}"
|
|
PORTAINER_URL="${PORTAINER_API_URL:-https://192.168.1.11:9443}"
|
|
PORTAINER_CREDS="${PORTAINER_CREDS_FILE:-$HOME/.openclaw/credentials/portainer.md}"
|
|
ORIGINS=("${DEPLOY_ORIGIN_PRIMARY:-https://queuenorth.com}" "${DEPLOY_ORIGIN_SECONDARY:-https://qn.isnull.dev}")
|
|
|
|
say() { printf '\033[1mdeploy:\033[0m %s\n' "$*" >&2; }
|
|
die() { printf '\033[1mdeploy:\033[0m %s\n' "$*" >&2; exit 1; }
|
|
stop() { printf '\033[1mdeploy:\033[0m %s\n' "$*" >&2; exit 2; }
|
|
|
|
TAG=""
|
|
DRY_RUN=""
|
|
NO_BACKUP=""
|
|
ALLOW_FLOATING=""
|
|
|
|
while [ "$#" -gt 0 ]; do
|
|
case "$1" in
|
|
--tag) shift; [ "$#" -gt 0 ] || stop "--tag needs a value."; TAG="$1" ;;
|
|
--tag=*) TAG="${1#--tag=}" ;;
|
|
--dry-run) DRY_RUN="yes" ;;
|
|
--no-backup) NO_BACKUP="yes" ;;
|
|
--allow-floating) ALLOW_FLOATING="yes" ;;
|
|
-h|--help) say "usage: bash scripts/deploy.sh [--tag vX.Y.Z] [--dry-run] [--no-backup] [--allow-floating]"; exit 0 ;;
|
|
*) stop "unknown argument '$1'. Run --help." ;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
for t in curl python3; do
|
|
command -v "$t" >/dev/null 2>&1 || stop "$t is not on PATH. Nothing was attempted."
|
|
done
|
|
|
|
KEY="${PORTAINER_API_KEY:-}"
|
|
if [ -z "$KEY" ] && [ -r "$PORTAINER_CREDS" ]; then
|
|
KEY=$(grep -m1 'API Key' "$PORTAINER_CREDS" | grep -oE 'ptr_[^`]+')
|
|
fi
|
|
[ -n "$KEY" ] || stop "no Portainer API key. Set PORTAINER_API_KEY, or put it in
|
|
$PORTAINER_CREDS — the file named after Portainer, not the one named after
|
|
this project. Nothing was attempted."
|
|
|
|
api() { curl -sk --max-time 30 -H "X-API-Key: $KEY" "$@"; }
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Refuse to deploy a tag that is not published. Portainer would accept the
|
|
# stack file, fail to pull, and leave the container stopped — an outage caused
|
|
# by a typo, discovered by the site going down.
|
|
# ---------------------------------------------------------------------------
|
|
REG_ENV="${RELEASE_REGISTRY_ENV:-$HOME/.openclaw/docker-registry.env}"
|
|
if [ -r "$REG_ENV" ]; then set -a; . "$REG_ENV"; set +a; fi
|
|
|
|
tags_json=""
|
|
if [ -n "${FORGEJO_REGISTRY:-}" ] && [ -n "${FORGEJO_REGISTRY_TOKEN:-}" ]; then
|
|
repo="${IMAGE#*/}"
|
|
tags_json=$(curl -sS --max-time 20 -u "${FORGEJO_REGISTRY_USER}:${FORGEJO_REGISTRY_TOKEN}" \
|
|
"https://${FORGEJO_REGISTRY}/v2/${repo}/tags/list" 2>/dev/null)
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Resolve the tag. With none named, this is the newest published NUMBERED
|
|
# version — never `:dev`, and never "whatever the stack already says", because
|
|
# both of those are how a deployment stops being able to name itself.
|
|
# ---------------------------------------------------------------------------
|
|
if [ -z "$TAG" ]; then
|
|
[ -n "$tags_json" ] || stop "no tag given and the registry tag list could not be read, so
|
|
the newest published version is unknown. Name one explicitly:
|
|
bash scripts/deploy.sh --tag vX.Y.Z
|
|
Nothing was attempted."
|
|
|
|
TAG=$(printf '%s' "$tags_json" | python3 -c "
|
|
import sys, json, re
|
|
tags = json.load(sys.stdin).get('tags') or []
|
|
def key(t):
|
|
m = re.fullmatch(r'v(\d+)\.(\d+)\.(\d+)', t)
|
|
return tuple(int(x) for x in m.groups()) if m else None
|
|
vers = sorted([t for t in tags if key(t)], key=key)
|
|
print(vers[-1] if vers else '')
|
|
")
|
|
[ -n "$TAG" ] || stop "the registry holds no numbered vX.Y.Z tag to deploy — only
|
|
floating ones. Publish a version first: npm run release. Nothing was attempted."
|
|
say "no tag given; newest published version is ${TAG}"
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Refuse a floating tag. See THE POLICY at the top.
|
|
# ---------------------------------------------------------------------------
|
|
if ! printf '%s' "$TAG" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+$'; then
|
|
if [ -z "$ALLOW_FLOATING" ]; then
|
|
die "'${TAG}' is not a numbered version, and production always runs one.
|
|
A stack pinned to a moving tag cannot say what it is running and cannot be
|
|
rolled back to anything, because the pointer only ever names now.
|
|
Deploy a version: bash scripts/deploy.sh --tag vX.Y.Z
|
|
Publish one first: npm run release
|
|
If you genuinely mean to chase a pointer, --allow-floating."
|
|
fi
|
|
say "WARNING: --allow-floating. Deploying the moving tag '${TAG}'."
|
|
say " After this, the stack file will not record what is running."
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Refuse to deploy a tag that is not published. Portainer would accept the
|
|
# stack file, fail to pull, and leave the container stopped — an outage caused
|
|
# by a typo, discovered by the site going down.
|
|
# ---------------------------------------------------------------------------
|
|
if [ -n "$tags_json" ]; then
|
|
printf '%s' "$tags_json" | grep -q "\"${TAG}\"" \
|
|
|| die "${IMAGE}:${TAG} is not published. Nothing was changed.
|
|
Published tags: $(printf '%s' "$tags_json" | python3 -c 'import sys,json;print(", ".join(json.load(sys.stdin).get("tags") or []))' 2>/dev/null)"
|
|
else
|
|
say "WARNING: could not read the registry tag list, so it is not known whether"
|
|
say " ${TAG} is published. A missing tag becomes an outage here."
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# What is running now. Reported before anything changes, so the two halves of
|
|
# "before and after" come from the same run.
|
|
# ---------------------------------------------------------------------------
|
|
before_digest=$(ssh -o BatchMode=yes -o ConnectTimeout=10 "$HOST" \
|
|
"docker image inspect \$(docker inspect '$CONTAINER' --format '{{.Image}}') --format '{{index .RepoDigests 0}}'" 2>/dev/null)
|
|
before_version=$(ssh -o BatchMode=yes -o ConnectTimeout=10 "$HOST" \
|
|
"docker inspect '$CONTAINER' --format '{{index .Config.Labels \"org.opencontainers.image.version\"}}'" 2>/dev/null)
|
|
|
|
say "stack ${STACK_ID} on ${HOST} (container ${CONTAINER})"
|
|
say "deploying ${IMAGE}:${TAG}"
|
|
say "running ${before_digest:-unknown}"
|
|
say "version ${before_version:-<no version label on the running image>}"
|
|
|
|
stack=$(api "${PORTAINER_URL}/api/stacks/${STACK_ID}") \
|
|
|| die "could not reach Portainer at ${PORTAINER_URL}. Nothing was changed."
|
|
printf '%s' "$stack" | grep -q '"Id"' \
|
|
|| die "Portainer did not return stack ${STACK_ID}. Nothing was changed."
|
|
|
|
file=$(api "${PORTAINER_URL}/api/stacks/${STACK_ID}/file" \
|
|
| python3 -c 'import sys,json;sys.stdout.write(json.load(sys.stdin)["StackFileContent"])') \
|
|
|| die "could not read the stack file. Nothing was changed."
|
|
|
|
# Rewrite the image line only if it does not already name the tag we want.
|
|
#
|
|
# python3 -c with the script as an ARGUMENT, not `python3 -` with a heredoc:
|
|
# the stack file arrives on stdin, and a heredoc would claim stdin too. The
|
|
# first version of this did exactly that, and python tried to execute the YAML.
|
|
new_file=$(printf '%s' "$file" | python3 -c "
|
|
import re, sys
|
|
img, tag = sys.argv[1], sys.argv[2]
|
|
s = sys.stdin.read()
|
|
pat = re.compile(r'^(\s*image:\s*)' + re.escape(img) + r':\S+[ \t]*\$', re.M)
|
|
if not pat.search(s):
|
|
sys.exit(3)
|
|
sys.stdout.write(pat.sub(lambda m: m.group(1) + img + ':' + tag, s))
|
|
" "$IMAGE" "$TAG") || die "the stack file has no 'image: ${IMAGE}:<tag>' line to update.
|
|
Nothing was changed. Check the stack file at ${PORTAINER_URL} — if the image
|
|
name changed, DEPLOY_IMAGE here is stale."
|
|
|
|
if [ "$new_file" = "$file" ]; then
|
|
say "the stack already names ${IMAGE}:${TAG} — redeploying it to pull the newest digest."
|
|
else
|
|
say "stack image line: $(printf '%s' "$file" | grep -oE "image: ${IMAGE}:\S+" | head -1)"
|
|
say " -> image: ${IMAGE}:${TAG}"
|
|
fi
|
|
|
|
if [ -n "$DRY_RUN" ]; then
|
|
say ""
|
|
say "--dry-run: nothing was changed. It would have:"
|
|
[ -z "$NO_BACKUP" ] && say " bash scripts/backup.sh (verified dump, before the irreversible part)"
|
|
say " PUT ${PORTAINER_URL}/api/stacks/${STACK_ID}?endpointId=${ENDPOINT_ID} (pullImage: true)"
|
|
say " waited for ${CONTAINER} to report healthy on ${HOST}"
|
|
say " checked ${ORIGINS[*]}"
|
|
say ""
|
|
say "That PUT recreates the container. Both front doors go down together."
|
|
exit 0
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Backup before the irreversible part. backup.sh verifies its own dump.
|
|
# ---------------------------------------------------------------------------
|
|
if [ -z "$NO_BACKUP" ]; then
|
|
say "taking a verified backup first…"
|
|
bash scripts/backup.sh >&2 \
|
|
|| die "the backup failed, so NOTHING was deployed. Fix it, or re-run with
|
|
--no-backup if you have a copy elsewhere and can say where."
|
|
else
|
|
say "WARNING: --no-backup. Deploying without a pre-deploy dump of the only copy"
|
|
say " of every lead this site has taken."
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# The irreversible part.
|
|
# ---------------------------------------------------------------------------
|
|
say "redeploying stack ${STACK_ID} — the container is being recreated…"
|
|
|
|
# The stack's existing Env array must be sent back with the update. Portainer
|
|
# treats a PUT as the whole desired state, so omitting it would silently strip
|
|
# twelve variables — including the reCAPTCHA secret and the Zoho form tokens —
|
|
# and the container would come up healthy and quietly stop capturing leads.
|
|
payload=$(printf '%s' "$stack" | python3 -c "
|
|
import sys, json
|
|
stack = json.load(sys.stdin)
|
|
env = stack.get('Env') or []
|
|
if not env:
|
|
sys.stderr.write('NOENV\\n')
|
|
sys.exit(3)
|
|
print(json.dumps({
|
|
'stackFileContent': sys.argv[1],
|
|
'env': env,
|
|
'prune': False,
|
|
'pullImage': True,
|
|
}))
|
|
" "$new_file") || die "could not build the update payload, or the stack reported no
|
|
environment variables at all. Sending an empty env would strip the Zoho and
|
|
reCAPTCHA credentials from the running container. Nothing was changed."
|
|
|
|
env_count=$(printf '%s' "$payload" | python3 -c 'import sys,json;print(len(json.load(sys.stdin)["env"]))' 2>/dev/null)
|
|
say "preserving ${env_count:-?} stack environment variables"
|
|
|
|
resp=$(api -X PUT -H "Content-Type: application/json" --data-binary "$payload" \
|
|
"${PORTAINER_URL}/api/stacks/${STACK_ID}?endpointId=${ENDPOINT_ID}")
|
|
|
|
printf '%s' "$resp" | grep -q '"Id"' || die "the redeploy was refused by Portainer:
|
|
$(printf '%s' "$resp" | head -c 400)
|
|
The stack may or may not have changed — check it before retrying."
|
|
|
|
say "stack updated. waiting for ${CONTAINER}…"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Verify. A deploy that is not checked is a deploy you find out about later.
|
|
# ---------------------------------------------------------------------------
|
|
healthy=""
|
|
for _ in $(seq 1 30); do
|
|
state=$(ssh -o BatchMode=yes -o ConnectTimeout=10 "$HOST" \
|
|
"docker inspect '$CONTAINER' --format '{{.State.Status}}:{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}'" 2>/dev/null)
|
|
case "$state" in
|
|
running:healthy|running:none) healthy="yes"; break ;;
|
|
running:starting) ;;
|
|
"") ;;
|
|
*) ;;
|
|
esac
|
|
sleep 4
|
|
done
|
|
|
|
[ -n "$healthy" ] || die "${CONTAINER} did not come back healthy. THE SITE MAY BE DOWN.
|
|
Check: bash scripts/status.sh --logs 200
|
|
Roll back: bash scripts/deploy.sh --tag <the previous version>
|
|
The pre-deploy backup is in \$HOME/backups/queue-north-website."
|
|
|
|
after_digest=$(ssh -o BatchMode=yes -o ConnectTimeout=10 "$HOST" \
|
|
"docker image inspect \$(docker inspect '$CONTAINER' --format '{{.Image}}') --format '{{index .RepoDigests 0}}'" 2>/dev/null)
|
|
after_version=$(ssh -o BatchMode=yes -o ConnectTimeout=10 "$HOST" \
|
|
"docker inspect '$CONTAINER' --format '{{index .Config.Labels \"org.opencontainers.image.version\"}}'" 2>/dev/null)
|
|
|
|
fail=0
|
|
for o in "${ORIGINS[@]}"; do
|
|
body=$(curl -s --max-time 15 "$o/api/health")
|
|
if printf '%s' "$body" | grep -q '"status":"ok"' && printf '%s' "$body" | grep -q '"db":"ok"'; then
|
|
say "ok $o"
|
|
else
|
|
say "FAIL $o -> ${body:-<no response>}"
|
|
fail=1
|
|
fi
|
|
done
|
|
|
|
say ""
|
|
say "digest ${before_digest:-unknown}"
|
|
say " -> ${after_digest:-unknown}"
|
|
say "version ${before_version:-none} -> ${after_version:-none}"
|
|
|
|
if [ "$fail" -ne 0 ]; then
|
|
die "the container is healthy but at least one public origin is not answering.
|
|
That is the ingress, not the app — see docs/OPERATIONS.md."
|
|
fi
|
|
|
|
say ""
|
|
say "deployed. Record it: docs/history/DEVELOPMENT_LOG.md, and close whatever"
|
|
say "issue this shipped with the evidence above."
|