120 lines
4.8 KiB
Bash
Executable File
120 lines
4.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# Liveness tick for a deployed service.
|
|
#
|
|
# ## Why this exists
|
|
#
|
|
# The five-minute healthcheck it replaced was prose handed to a model: "GET
|
|
# <origin>/healthz. Do NOT use /api/internal/v1/healthz." It duly reported a 404
|
|
# on `/api/internal/v1/health` — a third path, neither the one it was told to
|
|
# use nor the one it was told to avoid, and one that had never existed. The site
|
|
# was healthy throughout.
|
|
#
|
|
# An explicit prohibition constrained one wrong URL and left every other wrong
|
|
# URL open, because the address was being re-derived on every run rather than
|
|
# read. So it is written here once, as a string in version control, and the
|
|
# whole class of failure goes with it.
|
|
#
|
|
# That failure is the expensive kind. The job's own state read `lastRunStatus:
|
|
# ok, consecutiveErrors: 0` while a red alert went to a DM — so the monitor was
|
|
# reporting itself healthy and crying wolf at the same time, and a monitor
|
|
# nobody believes is a monitor nobody has.
|
|
#
|
|
# ## It holds no credential, and that is the point
|
|
#
|
|
# `/healthz` is unauthenticated by design — `SECURITY_CHECKLIST.md` makes it a
|
|
# checklist item, and the container's own HEALTHCHECK uses it. So unlike
|
|
# `reconcile.sh` and `analyze.sh`, this script reads no token, sources no env
|
|
# file, and has nothing to leak. At 288 runs a day that is worth more than the
|
|
# extra assurance an authenticated probe would buy.
|
|
#
|
|
# The authenticated sibling, `/api/internal/v1/agent/health`, reports more and
|
|
# needs a token. It is deliberately not used here: this asks "is the site up",
|
|
# which is a question with a public answer.
|
|
#
|
|
# ## Both halves are checked
|
|
#
|
|
# 503 is a real answer, not an outage — the route returns it when migrations
|
|
# failed, so the container is marked unhealthy while the marketing pages keep
|
|
# serving. Its body still contains `"service"`. Status alone is therefore not a
|
|
# verdict, and neither is a grep for the service name; the check asserts HTTP
|
|
# 200 *and* `"ok":true`.
|
|
#
|
|
# ## Installing it
|
|
#
|
|
# install -m 0755 healthcheck.sh ~/bin/healthcheck.sh
|
|
# # then, in the crontab — every five minutes
|
|
# */5 * * * * $HOME/bin/healthcheck.sh \
|
|
# >> $HOME/.healthcheck.log 2>&1
|
|
#
|
|
# On this deployment it is run by an OpenClaw cron job instead, which is the
|
|
# same thing with a scheduler that can also deliver the alert.
|
|
|
|
set -euo pipefail
|
|
|
|
# `-`, not `:-`. Unset means "no opinion, use production". Set-and-empty means
|
|
# a config is wrong, and substituting production for it would report the health
|
|
# of a site nobody asked about — quietly, and only where somebody was trying to
|
|
# point this somewhere else.
|
|
BASE_URL="${HEALTHCHECK_BASE_URL:?set HEALTHCHECK_BASE_URL to the deployed origin, e.g. https://example.com}"
|
|
TIMEOUT="${PRIVACY_TIMEOUT:-15}"
|
|
|
|
stamp() { date -Is; }
|
|
say() { printf '%s healthcheck: %s\n' "$(stamp)" "$*"; }
|
|
|
|
if [ -z "$BASE_URL" ]; then
|
|
say "FAIL no base URL. HEALTHCHECK_BASE_URL is set but empty; unset it for the default."
|
|
exit 78 # EX_CONFIG
|
|
fi
|
|
|
|
body=$(mktemp)
|
|
trap 'rm -f "$body"' EXIT
|
|
|
|
# Assigned in the `if`, not with `|| echo "000"` appended. On a connection
|
|
# failure curl *already* prints "000" via --write-out and then exits non-zero,
|
|
# so appending a fallback produces "000000", which matches no branch below and
|
|
# reports "unexpected HTTP" for the one failure this script names explicitly.
|
|
#
|
|
# No -v and no --trace, ever. This request carries no credential, but the habit
|
|
# is the rule SECURITY.md states: a request URL or header must not reach a log.
|
|
if ! status=$(
|
|
curl --silent --show-error --output "$body" --write-out '%{http_code}' \
|
|
--max-time "$TIMEOUT" \
|
|
"$BASE_URL/healthz"
|
|
); then
|
|
status="000"
|
|
fi
|
|
|
|
case "$status" in
|
|
200)
|
|
# The status got us here; the body decides. `"ok":true` is asserted rather
|
|
# than assumed because a 200 from a proxy, a cached page or a captive portal
|
|
# is still a 200, and none of them are this application answering.
|
|
if grep -q '"ok":true' "$body"; then
|
|
say "ok"
|
|
else
|
|
say "FAIL 200 but not ok. The response did not contain \"ok\":true."
|
|
exit 70 # EX_SOFTWARE
|
|
fi
|
|
;;
|
|
503)
|
|
# Named rather than left to the catch-all, because it is the one unhealthy
|
|
# answer this route is designed to give: migrations failed, the container is
|
|
# marked unhealthy, and the public pages are still being served. That is a
|
|
# different thing from the site being down and reads differently at 3am.
|
|
say "FAIL migrations are not ok (HTTP 503). The site is serving; the schema is not."
|
|
exit 70 # EX_SOFTWARE
|
|
;;
|
|
000)
|
|
say "FAIL could not reach $BASE_URL"
|
|
exit 69 # EX_UNAVAILABLE
|
|
;;
|
|
*)
|
|
# Includes 404. If this ever fires on a path this script wrote itself, the
|
|
# route moved — which is a thing to fix in one place rather than a thing for
|
|
# a caller to guess around.
|
|
say "FAIL unexpected HTTP $status from $BASE_URL/healthz"
|
|
exit 1
|
|
;;
|
|
esac
|