Queue-North-Website/scripts/healthcheck.sh

#!/usr/bin/env bash
#
# Liveness tick for a deployed service.
#
# ## Why this exists
#
# The five-minute healthcheck it replaced was prose handed to a model: "GET
# <origin>/healthz. Do NOT use /api/internal/v1/healthz." It duly reported a 404
# on `/api/internal/v1/health` — a third path, neither the one it was told to
# use nor the one it was told to avoid, and one that had never existed. The site
# was healthy throughout.
#
# An explicit prohibition constrained one wrong URL and left every other wrong
# URL open, because the address was being re-derived on every run rather than
# read. So it is written here once, as a string in version control, and the
# whole class of failure goes with it.
#
# That failure is the expensive kind. The job's own state read `lastRunStatus:
# ok, consecutiveErrors: 0` while a red alert went to a DM — so the monitor was
# reporting itself healthy and crying wolf at the same time, and a monitor
# nobody believes is a monitor nobody has.
#
# ## It holds no credential, and that is the point
#
# `/healthz` is unauthenticated by design — `SECURITY_CHECKLIST.md` makes it a
# checklist item, and the container's own HEALTHCHECK uses it. So unlike
# `reconcile.sh` and `analyze.sh`, this script reads no token, sources no env
# file, and has nothing to leak. At 288 runs a day that is worth more than the
# extra assurance an authenticated probe would buy.
#
# The authenticated sibling, `/api/internal/v1/agent/health`, reports more and
# needs a token. It is deliberately not used here: this asks "is the site up",
# which is a question with a public answer.
#
# ## Both halves are checked
#
# 503 is a real answer, not an outage — `/api/health` returns it when the
# `SELECT 1` against SQLite fails, so the container is marked unhealthy while
# the marketing pages keep serving perfectly well. Status alone is therefore
# not a verdict; the check asserts HTTP 200 *and* `"status":"ok"` *and*
# `"db":"ok"`.
#
# ## The path, written down rather than re-derived
#
# This project's endpoint is `/api/health`. It is NOT `/healthz`, which is what
# the template's copy of this script probed and what a plausible guess produces.
# The whole argument above is about exactly this line, so it is stated once,
# here, and nowhere else.
#
# ## Installing it
#
#   install -m 0755 healthcheck.sh ~/bin/healthcheck.sh
#   # then, in the crontab — every five minutes
#   */5 * * * * $HOME/bin/healthcheck.sh \
#     >> $HOME/.healthcheck.log 2>&1
#
# On this deployment it is run by an OpenClaw cron job instead, which is the
# same thing with a scheduler that can also deliver the alert.

set -euo pipefail

# `-`, not `:-`. Unset means "no opinion, use production". Set-and-empty means
# a config is wrong, and substituting production for it would report the health
# of a site nobody asked about — quietly, and only where somebody was trying to
# point this somewhere else.
# The PRODUCTION origin — the one customers use. qn.isnull.dev is the same
# container through a different ingress, and is worth checking separately when
# you are trying to tell the app apart from what is in front of it:
#
#   HEALTHCHECK_BASE_URL=https://qn.isnull.dev bash scripts/healthcheck.sh
BASE_URL="${HEALTHCHECK_BASE_URL:-https://queuenorth.com}"
TIMEOUT="${PRIVACY_TIMEOUT:-15}"

stamp() { date -Is; }
say() { printf '%s healthcheck: %s\n' "$(stamp)" "$*"; }

if [ -z "$BASE_URL" ]; then
  say "FAIL no base URL. HEALTHCHECK_BASE_URL is set but empty; unset it for the default."
  exit 78 # EX_CONFIG
fi

body=$(mktemp)
trap 'rm -f "$body"' EXIT

# Assigned in the `if`, not with `|| echo "000"` appended. On a connection
# failure curl *already* prints "000" via --write-out and then exits non-zero,
# so appending a fallback produces "000000", which matches no branch below and
# reports "unexpected HTTP" for the one failure this script names explicitly.
#
# No -v and no --trace, ever. This request carries no credential, but the habit
# is the rule SECURITY.md states: a request URL or header must not reach a log.
if ! status=$(
  curl --silent --show-error --output "$body" --write-out '%{http_code}' \
    --max-time "$TIMEOUT" \
    "$BASE_URL/api/health"
); then
  status="000"
fi

case "$status" in
  200)
    # The status got us here; the body decides. Both fields are asserted rather
    # than assumed because a 200 from Cloudflare, a cached page or an error page
    # is still a 200, and none of them are this application answering.
    #
    # Both, not either: the app can answer `"status":"ok"` while its database
    # handle is gone, and that is the state where the site looks fine and every
    # form submission is being lost.
    if grep -q '"status":"ok"' "$body" && grep -q '"db":"ok"' "$body"; then
      say "ok"
    elif grep -q '"status":"ok"' "$body"; then
      say "FAIL 200 and the app is up, but the database is not answering."
      say "     Every form submission is failing. See docs/OPERATIONS.md."
      exit 70 # EX_SOFTWARE
    else
      say "FAIL 200 but not this application. The response carried neither"
      say "     \"status\":\"ok\" nor a recognisable health body — check whether"
      say "     something in front of the origin answered instead."
      exit 70 # EX_SOFTWARE
    fi
    ;;
  503)
    # Named rather than left to the catch-all, because it is the one unhealthy
    # answer this route is designed to give: migrations failed, the container is
    # marked unhealthy, and the public pages are still being served. That is a
    # different thing from the site being down and reads differently at 3am.
    say "FAIL the database is not reachable (HTTP 503). The site is serving"
    say "     static pages; every lead and support submission is being lost."
    exit 70 # EX_SOFTWARE
    ;;
  000)
    say "FAIL could not reach $BASE_URL"
    exit 69 # EX_UNAVAILABLE
    ;;
  *)
    # Includes 404. If this ever fires on a path this script wrote itself, the
    # route moved — which is a thing to fix in one place rather than a thing for
    # a caller to guess around.
    say "FAIL unexpected HTTP $status from $BASE_URL/api/health"
    exit 1
    ;;
esac