#!/usr/bin/env bash # # Which operational controls does this project actually have, and how do we know? # # bash scripts/controls.sh # the table # bash scripts/controls.sh --quiet # only the rows that are not fine # # Exit codes: # # 0 every expected control is present # 1 at least one expected control is absent or could not be determined # 2 NOTHING WAS CHECKED — no control was declared expected. Not a pass. # # =========================================================================== # TEMPLATE COPY — configure this before the first run # =========================================================================== # # Set CONTROLS_EXPECTED to the controls this project is supposed to have, then # the variables for each. Everything not named there reports `n/a`, and the # defaults are empty on purpose: a script that assumes a library needs a # healthcheck reports a gap that is not one, and a report full of findings # nobody can act on is a report nobody reads twice. # # Assumes: bash, coreutils, `date`. # # ## Why this exists # # The question that decides whether a system can be depended on is not "does it # work" — a demo answers that. It is **which controls are present**, asked # before something goes wrong rather than after. # # Nothing answers it here. `verify.sh` reports which checks ran just now. # `backup.sh` proves one dump. The Command Center reports on documents and # tracker labels. None of them says whether this project has backups AND has # ever restored one AND has somewhere errors go AND has an environment that is # not production. That list is the actual answer to "is this safe to depend on", # and until now it lived in whoever remembered. # # ## Measured is not the same as declared, and both beat unknown # # Every row carries how it is known, because the four states mean genuinely # different things and flattening them is how a report starts lying: # # measured this script observed the fact — a file with a date on it, a # variable that is set, a command that answered # declared the operator asserted it in configuration. Checked for shape, # not for truth: `CONTROLS_AUDIT=activity_events table` is a claim # this script cannot verify and will not pretend to. # n/a the project declared the control does not apply. A library has no # uptime; saying so is an answer, not an omission. # unknown expected, and could not be determined. **Never rendered as # absent**, because "I could not tell" and "it is not there" send # people to different places. # # An unknown is not a failure of the project, it is a failure of this script to # find out, and it exits non-zero for the same reason `verify.sh` exits 2 on an # empty run: a check that did not happen must not look like one that passed. # # ## Wiring it into verify.sh # # `verify.sh` runs every executable file in VERIFY_GUARD_DIR, and refuses to # accept the scripts directory itself as that directory -- because release.sh, # backup.sh and migrate.sh live there and would all be RUN. So this file is not # dropped into verify.d. A three-line wrapper is: # # # scripts/verify.d/controls # #!/usr/bin/env bash # export CONTROLS_EXPECTED="backup restore errors healthcheck" # exec bash "$(dirname "$0")/../controls.sh" --quiet # # The wrapper is where this project's expectations live, which is the right # place for them: they are a fact about the project, not about the check. # # ## It writes nothing # # There is no CONTROLS.md and there will not be one. A committed file saying # "backups: ok" is a description of current state living in a document, which is # what the batch ledger was and why it was archived: it is right on the day it # is written and silently wrong afterwards. The live answer comes from running # this. What belongs in git is the *intent* — `docs/OPERATIONS.md` — which # changes rarely because it is a decision rather than a measurement. set -uo pipefail say() { printf '\033[1mcontrols:\033[0m %s\n' "$*" >&2; } # --------------------------------------------------------------------------- # CONFIGURATION # # Space-separated, from this set: # backup restore errors healthcheck ratelimit audit testenv deploy # --------------------------------------------------------------------------- EXPECTED="${CONTROLS_EXPECTED:-}" # Measured controls read the same variables the scripts that own them use, so # the two cannot disagree about which project this is. BACKUP_DIR="${BACKUP_DIR:-}" BACKUP_NAME="${BACKUP_NAME:-}" BACKUP_MAX_AGE="${CONTROLS_BACKUP_MAX_AGE_DAYS:-2}" RESTORE_MAX_AGE="${CONTROLS_RESTORE_MAX_AGE_DAYS:-120}" LAST_RESTORE="${CONTROLS_LAST_RESTORE:-}" ERROR_VAR="${CONTROLS_ERROR_TRACKING_VAR:-}" HEALTHCHECK_BASE_URL="${HEALTHCHECK_BASE_URL:-}" RATELIMIT="${CONTROLS_RATELIMIT:-}" AUDIT="${CONTROLS_AUDIT:-}" TEST_ENV="${CONTROLS_TEST_ENV:-}" STATUS_HOST="${STATUS_HOST:-}" STATUS_CONTAINER="${STATUS_CONTAINER:-}" QUIET="" while [ $# -gt 0 ]; do case "$1" in --quiet) QUIET="yes"; shift ;; -h|--help) say "usage: bash scripts/controls.sh [--quiet]"; say "set CONTROLS_EXPECTED to the controls this project should have."; exit 0 ;; *) say "unknown argument '$1'."; exit 2 ;; esac done ALL="backup restore errors healthcheck ratelimit audit testenv deploy" for want in $EXPECTED; do case " $ALL " in *" $want "*) ;; *) say "CONTROLS_EXPECTED names '$want', which is not a control this knows about." say "Known: $ALL" exit 2 ;; esac done expects() { case " $EXPECTED " in *" $1 "*) return 0 ;; *) return 1 ;; esac; } # days_since — prints whole days, or nothing if it cannot be read. days_since() { local then now # The shape is checked before `date` sees it. GNU date parses relative English # -- `date -d "last tuesday"` succeeds and returns a real timestamp -- so a # CONTROLS_LAST_RESTORE somebody typed as prose would come back as a plausible # age instead of an unreadable value. A restore date this script invented is # worse than one it admits it cannot read. case "$1" in [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]) ;; *) return 1 ;; esac then=$(date -u -d "$1" +%s 2>/dev/null) || return 1 [ -n "$then" ] || return 1 now=$(date -u +%s) || return 1 printf '%d' $(( (now - then) / 86400 )) } ROWS="" missing=0 unknowns=0 row() { # ROWS+="$(printf '%-12s %-9s %s' "$1" "$2" "$3")"$'\n' case "$2" in absent) missing=$((missing + 1)) ;; unknown) unknowns=$((unknowns + 1)) ;; esac } check_backup() { if [ -z "$BACKUP_DIR" ] || [ -z "$BACKUP_NAME" ]; then row backup unknown "BACKUP_DIR/BACKUP_NAME not set — cannot look" return fi if [ ! -d "$BACKUP_DIR" ]; then row backup unknown "$BACKUP_DIR is not a directory" return fi local newest="" f shopt -s nullglob for f in "$BACKUP_DIR/$BACKUP_NAME"*; do [ -f "$f" ] || continue { [ -z "$newest" ] || [ "$f" -nt "$newest" ]; } && newest="$f" done shopt -u nullglob if [ -z "$newest" ]; then row backup absent "no dump matching $BACKUP_NAME* in $BACKUP_DIR" return fi local age age=$(( ( $(date -u +%s) - $(date -u -r "$newest" +%s 2>/dev/null || echo 0) ) / 86400 )) if [ "$age" -gt "$BACKUP_MAX_AGE" ]; then row backup absent "newest dump is ${age}d old, limit ${BACKUP_MAX_AGE}d" else row backup measured "newest dump ${age}d old" fi } check_restore() { if [ -z "$LAST_RESTORE" ]; then row restore absent "CONTROLS_LAST_RESTORE unset — no restore has been recorded" return fi local age if ! age=$(days_since "$LAST_RESTORE"); then row restore unknown "CONTROLS_LAST_RESTORE='$LAST_RESTORE' is not a date this can read" return fi if [ "$age" -gt "$RESTORE_MAX_AGE" ]; then row restore absent "last restore ${age}d ago, limit ${RESTORE_MAX_AGE}d" else row restore declared "last restore ${age}d ago (asserted, not observed)" fi } check_errors() { if [ -z "$ERROR_VAR" ]; then row errors unknown "CONTROLS_ERROR_TRACKING_VAR unset — name the variable that carries the DSN" return fi if [ -n "${!ERROR_VAR:-}" ]; then row errors measured "$ERROR_VAR is set" else row errors absent "$ERROR_VAR is empty — errors go nowhere" fi } check_healthcheck() { if [ -n "$HEALTHCHECK_BASE_URL" ]; then row healthcheck measured "HEALTHCHECK_BASE_URL is set" else row healthcheck absent "HEALTHCHECK_BASE_URL unset — nothing ticks" fi } check_deploy() { if [ -z "$STATUS_HOST" ] || [ -z "$STATUS_CONTAINER" ]; then row deploy unknown "STATUS_HOST/STATUS_CONTAINER unset — cannot ask what is deployed" return fi local dir version dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" if [ ! -x "$dir/status.sh" ] && [ ! -f "$dir/status.sh" ]; then row deploy unknown "status.sh is not beside this script" return fi if version=$(bash "$dir/status.sh" --deployed-version 2>/dev/null) && [ -n "$version" ]; then row deploy measured "running $version" else # status.sh distinguishes stopped from unreachable and says so on stderr; # this only needs to know it did not get an answer. row deploy unknown "status.sh could not read the deployed version — run it directly for the reason" fi } declared_or_absent() { # if [ -n "$2" ]; then row "$1" declared "$2"; else row "$1" absent "$3"; fi } for c in $ALL; do if ! expects "$c"; then row "$c" n/a "not expected for this project" continue fi case "$c" in backup) check_backup ;; restore) check_restore ;; errors) check_errors ;; healthcheck) check_healthcheck ;; deploy) check_deploy ;; ratelimit) declared_or_absent ratelimit "$RATELIMIT" "CONTROLS_RATELIMIT unset — nothing bounds an expensive endpoint" ;; audit) declared_or_absent audit "$AUDIT" "CONTROLS_AUDIT unset — no record of who changed what" ;; testenv) declared_or_absent testenv "$TEST_ENV" "CONTROLS_TEST_ENV unset — production is the only environment" ;; esac done if [ -z "$EXPECTED" ]; then say "CONTROLS_EXPECTED is empty, so no control was checked." say "That is not a pass. Name what this project is supposed to have:" say " CONTROLS_EXPECTED='$ALL'" say "and drop the ones that genuinely do not apply — a library has no uptime." exit 2 fi if [ -n "$QUIET" ]; then printf '%s' "$ROWS" | grep -vE '^\S+ +(measured|declared|n/a) ' >&2 || true else printf '%s' "$ROWS" >&2 fi say "" if [ "$missing" -gt 0 ] || [ "$unknowns" -gt 0 ]; then say "$missing absent, $unknowns unknown, of $(printf '%s' "$EXPECTED" | wc -w) expected." [ "$unknowns" -gt 0 ] && say "An unknown is this script failing to find out, not the control failing to exist. Both are non-zero; only one is the project's fault." exit 1 fi say "every expected control present ($(printf '%s' "$EXPECTED" | wc -w) checked)."