#!/usr/bin/env bash # # Liveness tick for a deployed service. # # ## Why this exists # # The five-minute healthcheck it replaced was prose handed to a model: "GET # /healthz. Do NOT use /api/internal/v1/healthz." It duly reported a 404 # on `/api/internal/v1/health` — a third path, neither the one it was told to # use nor the one it was told to avoid, and one that had never existed. The site # was healthy throughout. # # An explicit prohibition constrained one wrong URL and left every other wrong # URL open, because the address was being re-derived on every run rather than # read. So it is written here once, as a string in version control, and the # whole class of failure goes with it. # # That failure is the expensive kind. The job's own state read `lastRunStatus: # ok, consecutiveErrors: 0` while a red alert went to a DM — so the monitor was # reporting itself healthy and crying wolf at the same time, and a monitor # nobody believes is a monitor nobody has. # # ## It holds no credential, and that is the point # # `/healthz` is unauthenticated by design — `SECURITY_CHECKLIST.md` makes it a # checklist item, and the container's own HEALTHCHECK uses it. So unlike # `reconcile.sh` and `analyze.sh`, this script reads no token, sources no env # file, and has nothing to leak. At 288 runs a day that is worth more than the # extra assurance an authenticated probe would buy. # # The authenticated sibling, `/api/internal/v1/agent/health`, reports more and # needs a token. It is deliberately not used here: this asks "is the site up", # which is a question with a public answer. # # ## Both halves are checked # # 503 is a real answer, not an outage — the route returns it when migrations # failed, so the container is marked unhealthy while the marketing pages keep # serving. Its body still contains `"service"`. Status alone is therefore not a # verdict, and neither is a grep for the service name; the check asserts HTTP # 200 *and* `"ok":true`. # # ## Installing it # # install -m 0755 healthcheck.sh ~/bin/healthcheck.sh # # then, in the crontab — every five minutes # */5 * * * * /home/kaspa/bin/healthcheck.sh \ # >> /home/kaspa/.healthcheck.log 2>&1 # # On this deployment it is run by an OpenClaw cron job instead, which is the # same thing with a scheduler that can also deliver the alert. set -euo pipefail # `-`, not `:-`. Unset means "no opinion, use production". Set-and-empty means # a config is wrong, and substituting production for it would report the health # of a site nobody asked about — quietly, and only where somebody was trying to # point this somewhere else. BASE_URL="${HEALTHCHECK_BASE_URL:?set HEALTHCHECK_BASE_URL to the deployed origin, e.g. https://example.com}" TIMEOUT="${PRIVACY_TIMEOUT:-15}" stamp() { date -Is; } say() { printf '%s healthcheck: %s\n' "$(stamp)" "$*"; } if [ -z "$BASE_URL" ]; then say "FAIL no base URL. HEALTHCHECK_BASE_URL is set but empty; unset it for the default." exit 78 # EX_CONFIG fi body=$(mktemp) trap 'rm -f "$body"' EXIT # Assigned in the `if`, not with `|| echo "000"` appended. On a connection # failure curl *already* prints "000" via --write-out and then exits non-zero, # so appending a fallback produces "000000", which matches no branch below and # reports "unexpected HTTP" for the one failure this script names explicitly. # # No -v and no --trace, ever. This request carries no credential, but the habit # is the rule SECURITY.md states: a request URL or header must not reach a log. if ! status=$( curl --silent --show-error --output "$body" --write-out '%{http_code}' \ --max-time "$TIMEOUT" \ "$BASE_URL/healthz" ); then status="000" fi case "$status" in 200) # The status got us here; the body decides. `"ok":true` is asserted rather # than assumed because a 200 from a proxy, a cached page or a captive portal # is still a 200, and none of them are this application answering. if grep -q '"ok":true' "$body"; then say "ok" else say "FAIL 200 but not ok. The response did not contain \"ok\":true." exit 70 # EX_SOFTWARE fi ;; 503) # Named rather than left to the catch-all, because it is the one unhealthy # answer this route is designed to give: migrations failed, the container is # marked unhealthy, and the public pages are still being served. That is a # different thing from the site being down and reads differently at 3am. say "FAIL migrations are not ok (HTTP 503). The site is serving; the schema is not." exit 70 # EX_SOFTWARE ;; 000) say "FAIL could not reach $BASE_URL" exit 69 # EX_UNAVAILABLE ;; *) # Includes 404. If this ever fires on a path this script wrote itself, the # route moved — which is a thing to fix in one place rather than a thing for # a caller to guess around. say "FAIL unexpected HTTP $status from $BASE_URL/healthz" exit 1 ;; esac