Project-Template/docs/architecture/scripts/healthcheck.sh

#!/usr/bin/env bash
#
# Liveness tick for a deployed service.
#
# ## Why this exists
#
# The five-minute healthcheck it replaced was prose handed to a model: "GET
# <origin>/healthz. Do NOT use /api/internal/v1/healthz." It duly reported a 404
# on `/api/internal/v1/health` — a third path, neither the one it was told to
# use nor the one it was told to avoid, and one that had never existed. The site
# was healthy throughout.
#
# An explicit prohibition constrained one wrong URL and left every other wrong
# URL open, because the address was being re-derived on every run rather than
# read. So it is written here once, as a string in version control, and the
# whole class of failure goes with it.
#
# That failure is the expensive kind. The job's own state read `lastRunStatus:
# ok, consecutiveErrors: 0` while a red alert went to a DM — so the monitor was
# reporting itself healthy and crying wolf at the same time, and a monitor
# nobody believes is a monitor nobody has.
#
# ## It holds no credential, and that is the point
#
# `/healthz` is unauthenticated by design — `SECURITY_CHECKLIST.md` makes it a
# checklist item, and the container's own HEALTHCHECK uses it. So unlike
# `reconcile.sh` and `analyze.sh`, this script reads no token, sources no env
# file, and has nothing to leak. At 288 runs a day that is worth more than the
# extra assurance an authenticated probe would buy.
#
# The authenticated sibling, `/api/internal/v1/agent/health`, reports more and
# needs a token. It is deliberately not used here: this asks "is the site up",
# which is a question with a public answer.
#
# ## Both halves are checked
#
# 503 is a real answer, not an outage — the route returns it when migrations
# failed, so the container is marked unhealthy while the marketing pages keep
# serving. Its body still contains `"service"`. Status alone is therefore not a
# verdict, and neither is a grep for the service name; the check asserts HTTP
# 200 *and* `"ok":true`.
#
# ## Installing it
#
#   install -m 0755 healthcheck.sh ~/bin/healthcheck.sh
#   # then, in the crontab — every five minutes
#   */5 * * * * $HOME/bin/healthcheck.sh \
#     >> $HOME/.healthcheck.log 2>&1
#
# On this deployment it is run by an OpenClaw cron job instead, which is the
# same thing with a scheduler that can also deliver the alert.

set -euo pipefail

# `-`, not `:-`. Unset means "no opinion, use production". Set-and-empty means
# a config is wrong, and substituting production for it would report the health
# of a site nobody asked about — quietly, and only where somebody was trying to
# point this somewhere else.
BASE_URL="${HEALTHCHECK_BASE_URL:?set HEALTHCHECK_BASE_URL to the deployed origin, e.g. https://example.com}"
TIMEOUT="${PRIVACY_TIMEOUT:-15}"

stamp() { date -Is; }
say() { printf '%s healthcheck: %s\n' "$(stamp)" "$*"; }

if [ -z "$BASE_URL" ]; then
  say "FAIL no base URL. HEALTHCHECK_BASE_URL is set but empty; unset it for the default."
  exit 78 # EX_CONFIG
fi

body=$(mktemp)
trap 'rm -f "$body"' EXIT

# Assigned in the `if`, not with `|| echo "000"` appended. On a connection
# failure curl *already* prints "000" via --write-out and then exits non-zero,
# so appending a fallback produces "000000", which matches no branch below and
# reports "unexpected HTTP" for the one failure this script names explicitly.
#
# No -v and no --trace, ever. This request carries no credential, but the habit
# is the rule SECURITY.md states: a request URL or header must not reach a log.
if ! status=$(
  curl --silent --show-error --output "$body" --write-out '%{http_code}' \
    --max-time "$TIMEOUT" \
    "$BASE_URL/healthz"
); then
  status="000"
fi

case "$status" in
  200)
    # The status got us here; the body decides. `"ok":true` is asserted rather
    # than assumed because a 200 from a proxy, a cached page or a captive portal
    # is still a 200, and none of them are this application answering.
    if grep -q '"ok":true' "$body"; then
      say "ok"
    else
      say "FAIL 200 but not ok. The response did not contain \"ok\":true."
      exit 70 # EX_SOFTWARE
    fi
    ;;
  503)
    # Named rather than left to the catch-all, because it is the one unhealthy
    # answer this route is designed to give: migrations failed, the container is
    # marked unhealthy, and the public pages are still being served. That is a
    # different thing from the site being down and reads differently at 3am.
    say "FAIL migrations are not ok (HTTP 503). The site is serving; the schema is not."
    exit 70 # EX_SOFTWARE
    ;;
  000)
    say "FAIL could not reach $BASE_URL"
    exit 69 # EX_UNAVAILABLE
    ;;
  *)
    # Includes 404. If this ever fires on a path this script wrote itself, the
    # route moved — which is a thing to fix in one place rather than a thing for
    # a caller to guess around.
    say "FAIL unexpected HTTP $status from $BASE_URL/healthz"
    exit 1
    ;;
esac