diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 912da59..e4c8aa1 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -79,6 +79,10 @@ TAG="" DRY_RUN="" NO_BACKUP="" ALLOW_FLOATING="" +ROLLBACK="" +FIX_CORS="" +WATCHTOWER_OFF="" +NO_ROLLBACK="" while [ "$#" -gt 0 ]; do case "$1" in @@ -87,7 +91,12 @@ while [ "$#" -gt 0 ]; do --dry-run) DRY_RUN="yes" ;; --no-backup) NO_BACKUP="yes" ;; --allow-floating) ALLOW_FLOATING="yes" ;; - -h|--help) say "usage: bash scripts/deploy.sh [--tag vX.Y.Z] [--dry-run] [--no-backup] [--allow-floating]"; exit 0 ;; + --rollback) ROLLBACK="yes" ;; + --fix-cors) FIX_CORS="yes" ;; + --watchtower-off) WATCHTOWER_OFF="yes" ;; + --no-rollback) NO_ROLLBACK="yes" ;; + -h|--help) say "usage: bash scripts/deploy.sh [--tag vX.Y.Z] [--dry-run] [--no-backup]" + say " [--fix-cors] [--watchtower-off] [--rollback] [--allow-floating] [--no-rollback]"; exit 0 ;; *) stop "unknown argument '$1'. Run --help." ;; esac shift @@ -177,6 +186,34 @@ else say " ${TAG} is published. A missing tag becomes an outage here." fi +# --------------------------------------------------------------------------- +# Refuse to go backwards. +# +# The newest published NUMBERED tag is not necessarily newer than what is +# running: on 2026-08-18 it was v0.8.3, built 2026-05-28, while the running +# :dev image was built 2026-08-01. Deploying it would have removed the privacy +# policy and every prerendered route — two months of regressions, applied by +# the default path with no warning. +# +# So compare creation dates. The target's is read from the registry WITHOUT +# pulling it: manifest -> config digest -> config blob -> .created. +# --------------------------------------------------------------------------- +registry_created() { + local tag="$1" tok cfg + tok=$(curl -sS --max-time 20 -u "${FORGEJO_REGISTRY_USER:-}:${FORGEJO_REGISTRY_TOKEN:-}" \ + "https://${FORGEJO_REGISTRY}/v2/token?scope=repository:${IMAGE#*/}:pull&service=container_registry" \ + 2>/dev/null | python3 -c 'import sys,json;print(json.load(sys.stdin).get("token",""))' 2>/dev/null) + [ -n "$tok" ] || return 1 + cfg=$(curl -sS --max-time 20 -H "Authorization: Bearer $tok" \ + -H 'Accept: application/vnd.oci.image.manifest.v1+json,application/vnd.docker.distribution.manifest.v2+json' \ + "https://${FORGEJO_REGISTRY}/v2/${IMAGE#*/}/manifests/${tag}" 2>/dev/null \ + | python3 -c 'import sys,json;print(json.load(sys.stdin).get("config",{}).get("digest",""))' 2>/dev/null) + [ -n "$cfg" ] || return 1 + curl -sSL --max-time 30 -H "Authorization: Bearer $tok" \ + "https://${FORGEJO_REGISTRY}/v2/${IMAGE#*/}/blobs/${cfg}" 2>/dev/null \ + | python3 -c 'import sys,json;print(json.load(sys.stdin).get("created",""))' 2>/dev/null +} + # --------------------------------------------------------------------------- # What is running now. Reported before anything changes, so the two halves of # "before and after" come from the same run. @@ -186,6 +223,26 @@ before_digest=$(ssh -o BatchMode=yes -o ConnectTimeout=10 "$HOST" \ before_version=$(ssh -o BatchMode=yes -o ConnectTimeout=10 "$HOST" \ "docker inspect '$CONTAINER' --format '{{index .Config.Labels \"org.opencontainers.image.version\"}}'" 2>/dev/null) +target_created=$(registry_created "$TAG") +running_created=$(ssh -o BatchMode=yes -o ConnectTimeout=10 "$HOST" \ + "docker image inspect \$(docker inspect '$CONTAINER' --format '{{.Image}}') --format '{{.Created}}'" 2>/dev/null) + +if [ -n "$target_created" ] && [ -n "$running_created" ]; then + if [ "$target_created" \< "$running_created" ] && [ -z "$ROLLBACK" ]; then + die "${TAG} was built ${target_created}, and what is running was built ${running_created}. + That is BACKWARDS. Deploying it would undo everything built in between — + on this project that has meant losing the privacy policy and every + prerendered route, silently, on the default path. + If you mean to roll back, say so: --rollback + Nothing was changed." + fi + [ -n "$ROLLBACK" ] && say "--rollback: deploying an image built ${target_created}" +else + say "WARNING: could not compare build dates (target='${target_created:-?}'" + say " running='${running_created:-?}'), so the going-backwards check" + say " did not run. That is not a pass." +fi + say "stack ${STACK_ID} on ${HOST} (container ${CONTAINER})" say "deploying ${IMAGE}:${TAG}" say "running ${before_digest:-unknown}" @@ -196,32 +253,87 @@ stack=$(api "${PORTAINER_URL}/api/stacks/${STACK_ID}") \ printf '%s' "$stack" | grep -q '"Id"' \ || die "Portainer did not return stack ${STACK_ID}. Nothing was changed." -file=$(api "${PORTAINER_URL}/api/stacks/${STACK_ID}/file" \ - | python3 -c 'import sys,json;sys.stdout.write(json.load(sys.stdin)["StackFileContent"])') \ +# Written to a file rather than captured in $( ), which strips trailing +# newlines. The first version of this did, so every PUT silently removed the +# file's final newline — harmless to YAML, and still a change beyond the lines +# this script says it touches. "Only the image line" has to be true. +STACK_ORIG="$(mktemp)"; STACK_NEW="$(mktemp)" +trap 'rm -f "$STACK_ORIG" "$STACK_NEW"' EXIT +api "${PORTAINER_URL}/api/stacks/${STACK_ID}/file" \ + | python3 -c 'import sys,json;sys.stdout.write(json.load(sys.stdin)["StackFileContent"])' > "$STACK_ORIG" \ || die "could not read the stack file. Nothing was changed." +[ -s "$STACK_ORIG" ] || die "the stack file came back empty. Nothing was changed." +file=$(cat "$STACK_ORIG") -# Rewrite the image line only if it does not already name the tag we want. +# --------------------------------------------------------------------------- +# Build the new stack file from the original bytes. Every edit is opt-in and +# every one is printed as a diff before anything is sent. # -# python3 -c with the script as an ARGUMENT, not `python3 -` with a heredoc: -# the stack file arrives on stdin, and a heredoc would claim stdin too. The -# first version of this did exactly that, and python tried to execute the YAML. -new_file=$(printf '%s' "$file" | python3 -c " +# python3 with the script as an ARGUMENT and the file on stdin: a heredoc would +# claim stdin too. The first version of this did exactly that, and python tried +# to execute the YAML. +# --------------------------------------------------------------------------- +python3 -c " import re, sys -img, tag = sys.argv[1], sys.argv[2] +img, tag, fix_cors, wt_off = sys.argv[1], sys.argv[2], sys.argv[3] == 'yes', sys.argv[4] == 'yes' s = sys.stdin.read() + pat = re.compile(r'^(\s*image:\s*)' + re.escape(img) + r':\S+[ \t]*\$', re.M) if not pat.search(s): sys.exit(3) -sys.stdout.write(pat.sub(lambda m: m.group(1) + img + ':' + tag, s)) -" "$IMAGE" "$TAG") || die "the stack file has no 'image: ${IMAGE}:' line to update. - Nothing was changed. Check the stack file at ${PORTAINER_URL} — if the image - name changed, DEPLOY_IMAGE here is stale." +s = pat.sub(lambda m: m.group(1) + img + ':' + tag, s) -if [ "$new_file" = "$file" ]; then - say "the stack already names ${IMAGE}:${TAG} — redeploying it to pull the newest digest." -else - say "stack image line: $(printf '%s' "$file" | grep -oE "image: ${IMAGE}:\S+" | head -1)" - say " -> image: ${IMAGE}:${TAG}" +if fix_cors: + c = re.compile(r'^(\s*-\s*CORS_ORIGIN=)(\S+?)/+[ \t]*\$', re.M) + if not c.search(s): + sys.stderr.write('NOCORS\n') + sys.exit(4) + s = c.sub(lambda m: m.group(1) + m.group(2), s) + +if wt_off: + lab = 'com.centurylinklabs.watchtower.enable=false' + if lab not in s: + m = re.search(r'^(\s*)restart:\s*\S+[ \t]*\$', s, re.M) + if not m: + sys.stderr.write('NOANCHOR\n') + sys.exit(5) + ind = m.group(1) + block = f'{ind}labels:\n{ind} - "{lab}"\n' + s = s[:m.start()] + block + s[m.start():] + +sys.stdout.write(s) +" "$IMAGE" "$TAG" "${FIX_CORS:-no}" "${WATCHTOWER_OFF:-no}" < "$STACK_ORIG" > "$STACK_NEW" +rc=$? +case "$rc" in + 0) ;; + 3) die "the stack file has no 'image: ${IMAGE}:' line to update. Nothing was changed." ;; + 4) die "--fix-cors: no CORS_ORIGIN line with a trailing slash to fix. Either it is + already correct or the line moved. Nothing was changed." ;; + 5) die "--watchtower-off: could not find a 'restart:' line to anchor the labels + block to. Nothing was changed." ;; + *) die "could not build the new stack file (exit $rc). Nothing was changed." ;; +esac + +[ -s "$STACK_NEW" ] || die "the new stack file came out empty. Nothing was changed." + +# Every env line must survive. A dropped one comes back healthy and stops +# capturing leads, which is the worst shape of failure available here. +orig_env=$(grep -cE '^\s*-\s*[A-Z_]+=' "$STACK_ORIG") +new_env=$(grep -cE '^\s*-\s*[A-Z_]+=' "$STACK_NEW") +[ "$orig_env" -eq "$new_env" ] || die "the rewrite changed the number of environment lines + (${orig_env} -> ${new_env}). Refusing to send it. Nothing was changed." + +say "" +say "the exact change to the stack file:" +diff -u "$STACK_ORIG" "$STACK_NEW" \ + | sed -E 's/^([-+].*(SECRET|XNQSJSDP|XMIWTLD)=).+/\1/' \ + | sed 's/^/ /' >&2 +say "" + +new_file=$(cat "$STACK_NEW") + +if cmp -s "$STACK_ORIG" "$STACK_NEW"; then + say "the stack file is already exactly as requested — redeploying it to pull the newest digest." fi if [ -n "$DRY_RUN" ]; then @@ -231,6 +343,8 @@ if [ -n "$DRY_RUN" ]; then say " PUT ${PORTAINER_URL}/api/stacks/${STACK_ID}?endpointId=${ENDPOINT_ID} (pullImage: true)" say " waited for ${CONTAINER} to report healthy on ${HOST}" say " checked ${ORIGINS[*]}" + say " checked the live bundle still carries the reCAPTCHA site key" + say " rolled the whole stack file back automatically if any of that failed" say "" say "That PUT recreates the container. Both front doors go down together." exit 0 @@ -288,52 +402,126 @@ $(printf '%s' "$resp" | head -c 400) say "stack updated. waiting for ${CONTAINER}…" # --------------------------------------------------------------------------- -# Verify. A deploy that is not checked is a deploy you find out about later. +# Verify, and undo it if the verification fails. +# +# Three things are checked, and the third is the one that would otherwise be +# found by a customer: the reCAPTCHA site key is frozen into the bundle at +# build time, and an image built without it renders "Security verification is +# not configured." and rejects every submission. Health checks pass happily +# while that is true, so the live bundle is asked directly. # --------------------------------------------------------------------------- -healthy="" -for _ in $(seq 1 30); do - state=$(ssh -o BatchMode=yes -o ConnectTimeout=10 "$HOST" \ - "docker inspect '$CONTAINER' --format '{{.State.Status}}:{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}'" 2>/dev/null) - case "$state" in - running:healthy|running:none) healthy="yes"; break ;; - running:starting) ;; - "") ;; - *) ;; - esac - sleep 4 -done +EXPECTED_KEY=$(printf '%s' "$stack" | python3 -c " +import sys, json +env = {e['name']: e['value'] for e in (json.load(sys.stdin).get('Env') or [])} +print(env.get('VITE_RECAPTCHA_SITE_KEY', '')) +" 2>/dev/null) -[ -n "$healthy" ] || die "${CONTAINER} did not come back healthy. THE SITE MAY BE DOWN. - Check: bash scripts/status.sh --logs 200 - Roll back: bash scripts/deploy.sh --tag - The pre-deploy backup is in \$HOME/backups/queue-north-website." +wait_healthy() { + local i state + for i in $(seq 1 30); do + state=$(ssh -o BatchMode=yes -o ConnectTimeout=10 "$HOST" \ + "docker inspect '$CONTAINER' --format '{{.State.Status}}:{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}'" 2>/dev/null) + case "$state" in running:healthy|running:none) return 0 ;; esac + sleep 4 + done + return 1 +} + +check_live() { + local fail=0 o body js + for o in "${ORIGINS[@]}"; do + body=$(curl -s --max-time 15 "$o/api/health") + if printf '%s' "$body" | grep -q '"status":"ok"' && printf '%s' "$body" | grep -q '"db":"ok"'; then + say " ok $o" + else + say " FAIL $o -> ${body:-}"; fail=1 + fi + done + + if [ -n "$EXPECTED_KEY" ]; then + js=$(curl -s --max-time 15 "${ORIGINS[0]}/" | grep -oE 'assets/index-[A-Za-z0-9_-]+\.js' | head -1) + if [ -z "$js" ]; then + say " FAIL could not find the bundle on ${ORIGINS[0]}"; fail=1 + elif curl -s --max-time 20 "${ORIGINS[0]}/${js}" | grep -qF "$EXPECTED_KEY"; then + say " ok the live bundle carries the reCAPTCHA site key" + else + say " FAIL the live bundle does NOT carry the reCAPTCHA site key." + say " The contact form will read \"Security verification is not" + say " configured.\" and reject every submission." + fail=1 + fi + else + say " note the stack declares no VITE_RECAPTCHA_SITE_KEY, so the bundle" + say " check was skipped. That is not a pass." + fi + return $fail +} + +deploy_failed="" +wait_healthy || deploy_failed="the container did not come back healthy" +if [ -z "$deploy_failed" ]; then + say "checking the live site…" + check_live || deploy_failed="the container is healthy but the live checks failed" +fi + +# --------------------------------------------------------------------------- +# Auto-rollback. Once, and only once. +# +# The original bytes are still in $STACK_ORIG, so this restores the state that +# was known to work — not just the image line, but the CORS and label edits +# too. A half-applied change is not a rollback. +# +# The previous image is already on the host, so this does not depend on the +# registry being reachable at the worst possible moment. +# --------------------------------------------------------------------------- +if [ -n "$deploy_failed" ]; then + say "" + say "DEPLOY FAILED: ${deploy_failed}." + + if [ -n "$NO_ROLLBACK" ]; then + die "--no-rollback: leaving it exactly as it is so you can inspect it. + Logs: bash scripts/status.sh --logs 200 + Restore: bash scripts/deploy.sh --tag --rollback + Backup: \$HOME/backups/queue-north-website" + fi + + say "rolling back to the previous stack file…" + rb_payload=$(printf '%s' "$stack" | python3 -c " +import sys, json +stack = json.load(sys.stdin) +print(json.dumps({'stackFileContent': sys.argv[1], 'env': stack.get('Env') or [], + 'prune': False, 'pullImage': True})) +" "$(cat "$STACK_ORIG")") + + if api -X PUT -H "Content-Type: application/json" --data-binary "$rb_payload" \ + "${PORTAINER_URL}/api/stacks/${STACK_ID}?endpointId=${ENDPOINT_ID}" | grep -q '"Id"'; then + if wait_healthy && check_live; then + say "" + die "ROLLED BACK successfully. The site is serving the previous image again. + The deploy of ${TAG} failed: ${deploy_failed}. + Nothing is lost; the pre-deploy backup is in \$HOME/backups/queue-north-website." + fi + die "THE ROLLBACK ALSO FAILED. THE SITE IS LIKELY DOWN. Stopping rather than + trying again — a script retrying an outage is how a short one becomes long. + bash scripts/status.sh --logs 200 + Portainer: ${PORTAINER_URL} stack ${STACK_ID}" + fi + + die "THE ROLLBACK PUT WAS REFUSED. THE SITE IS LIKELY DOWN. + bash scripts/status.sh --logs 200 + Portainer: ${PORTAINER_URL} stack ${STACK_ID}" +fi after_digest=$(ssh -o BatchMode=yes -o ConnectTimeout=10 "$HOST" \ "docker image inspect \$(docker inspect '$CONTAINER' --format '{{.Image}}') --format '{{index .RepoDigests 0}}'" 2>/dev/null) after_version=$(ssh -o BatchMode=yes -o ConnectTimeout=10 "$HOST" \ "docker inspect '$CONTAINER' --format '{{index .Config.Labels \"org.opencontainers.image.version\"}}'" 2>/dev/null) -fail=0 -for o in "${ORIGINS[@]}"; do - body=$(curl -s --max-time 15 "$o/api/health") - if printf '%s' "$body" | grep -q '"status":"ok"' && printf '%s' "$body" | grep -q '"db":"ok"'; then - say "ok $o" - else - say "FAIL $o -> ${body:-}" - fail=1 - fi -done - say "" say "digest ${before_digest:-unknown}" say " -> ${after_digest:-unknown}" say "version ${before_version:-none} -> ${after_version:-none}" -if [ "$fail" -ne 0 ]; then - die "the container is healthy but at least one public origin is not answering. - That is the ingress, not the app — see docs/OPERATIONS.md." -fi - say "" say "deployed. Record it: docs/history/DEVELOPMENT_LOG.md, and close whatever" say "issue this shipped with the evidence above." diff --git a/scripts/release.sh b/scripts/release.sh index 3dc6cad..b07f2c8 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -92,6 +92,9 @@ IMAGE="${RELEASE_IMAGE:-dream.scheller.ltd/null/queue-north-website}" SITE_URL="${RELEASE_SITE_URL:-https://queuenorth.com}" REGISTRY_ENV="${RELEASE_REGISTRY_ENV:-$HOME/.openclaw/docker-registry.env}" +# Public values Vite freezes into the bundle at build time. See the guard below. +BUILD_ENV_FILE="${RELEASE_BUILD_ENV:-.env}" + # Files carrying a version string. The Dockerfile is here because its # `ARG APP_VERSION=` becomes the image's org.opencontainers.image.version label, # which `status.sh --deployed-version` reads — left unbumped, every future @@ -142,6 +145,49 @@ for f in "${ORIGIN_FILES[@]}"; do URLs, and it is not what you asked for. Fix it before releasing." done +# --------------------------------------------------------------------------- +# The reCAPTCHA site key, which is frozen into the bundle exactly like the +# origin above and is the one that stops the product working. +# +# `.dockerignore` excludes .env from the build context, so Vite inside the +# container cannot read it; the value has to arrive as a build arg. An earlier +# version of this script passed `${VITE_RECAPTCHA_SITE_KEY:-}` and never loaded +# .env, so on a shell where it was unset — which is every shell — it would have +# built an image with an EMPTY key. +# +# That is not a degraded build. src/components/RecaptchaPlaceholder.jsx renders +# "Security verification is not configured." in place of the widget and never +# produces a token, and the server has RECAPTCHA_ENABLED=true, so every contact +# submission is rejected with "Security verification is required". The form +# visibly breaks and lead capture stops — the entire purpose of the site — and +# nothing about the build says so. +# +# So: load it, and refuse when it is empty. No `:-` fallback, because empty is +# the failure rather than a default. +# --------------------------------------------------------------------------- +if [ -r "$BUILD_ENV_FILE" ]; then + # Only the VITE_ keys, and only when not already set in the environment. + # Sourcing the whole file would drag the server's secrets into a build that + # has no use for them. + while IFS='=' read -r k v; do + case "$k" in + VITE_*) [ -n "${!k:-}" ] || export "$k=$v" ;; + esac + done < <(grep -E '^VITE_[A-Z0-9_]+=' "$BUILD_ENV_FILE" 2>/dev/null) +fi + +[ -n "${VITE_RECAPTCHA_SITE_KEY:-}" ] || die "VITE_RECAPTCHA_SITE_KEY is empty, and it is baked into the bundle. + An empty one does not degrade the form, it breaks it: the widget is + replaced by \"Security verification is not configured.\", no token is + produced, and the server rejects every submission because + RECAPTCHA_ENABLED=true. Lead capture stops. + It lives in ${BUILD_ENV_FILE}, and in stack 58's environment. + Nothing was built." + +# The site key is public by design — it ships to every visitor — so echoing a +# prefix is not a leak, and seeing it is how you catch the wrong one. +say "recaptcha site key ${VITE_RECAPTCHA_SITE_KEY:0:14}… (public, baked into the bundle)" + BUMP="patch" DRY_RUN="" @@ -221,6 +267,7 @@ if [ -n "$DRY_RUN" ]; then say " bash scripts/verify.sh" say " docker build --build-arg APP_VERSION=${next} -t ${IMAGE}:${TAG} ." say " verify the image's org.opencontainers.image.version label reads ${next}" + say " verify the built bundle contains the reCAPTCHA site key" say " docker push ${IMAGE}:${TAG}" say " git commit -m 'chore(release): ${TAG}' (post-commit then pushes)" say " git tag ${TAG} && git push origin ${TAG}" @@ -307,6 +354,23 @@ if [ "$baked" != "$next" ]; then exit 1 fi +# Ask the artifact, not the wiring — the same move the version check above +# makes, for the value that actually stops the product working. This survives +# somebody editing the Dockerfile's ARG/ENV pair or adding dist to +# .dockerignore, neither of which would fail the build. +say "verifying the bundle carries the reCAPTCHA site key…" + +if ! docker run --rm --entrypoint sh "${IMAGE}:${TAG}" -c \ + "grep -lqF '${VITE_RECAPTCHA_SITE_KEY}' /app/dist/assets/*.js" >/dev/null 2>&1; then + docker rmi "${IMAGE}:${TAG}" >/dev/null 2>&1 + say "the built bundle does NOT contain the reCAPTCHA site key." + say " This image would render \"Security verification is not configured.\"" + say " on the contact form and reject every submission. Check ARG/ENV" + say " VITE_RECAPTCHA_SITE_KEY in the builder stage of the Dockerfile." + say " Nothing was published or committed; the local image was removed." + exit 1 +fi + say "pushing ${IMAGE}:${TAG}…" if ! docker push "${IMAGE}:${TAG}"; then say "push failed. The image exists locally and the bump is in your working"