292 lines
11 KiB
Bash
292 lines
11 KiB
Bash
|
#!/usr/bin/env bash
|
||
|
|
#
|
||
|
|
# Which operational controls does this project actually have, and how do we know?
|
||
|
|
#
|
||
|
|
# bash scripts/controls.sh # the table
|
||
|
|
# bash scripts/controls.sh --quiet # only the rows that are not fine
|
||
|
|
#
|
||
|
|
# Exit codes:
|
||
|
|
#
|
||
|
|
# 0 every expected control is present
|
||
|
|
# 1 at least one expected control is absent or could not be determined
|
||
|
|
# 2 NOTHING WAS CHECKED — no control was declared expected. Not a pass.
|
||
|
|
#
|
||
|
|
# ===========================================================================
|
||
|
|
# TEMPLATE COPY — configure this before the first run
|
||
|
|
# ===========================================================================
|
||
|
|
#
|
||
|
|
# Set CONTROLS_EXPECTED to the controls this project is supposed to have, then
|
||
|
|
# the variables for each. Everything not named there reports `n/a`, and the
|
||
|
|
# defaults are empty on purpose: a script that assumes a library needs a
|
||
|
|
# healthcheck reports a gap that is not one, and a report full of findings
|
||
|
|
# nobody can act on is a report nobody reads twice.
|
||
|
|
#
|
||
|
|
# Assumes: bash, coreutils, `date`.
|
||
|
|
#
|
||
|
|
# ## Why this exists
|
||
|
|
#
|
||
|
|
# The question that decides whether a system can be depended on is not "does it
|
||
|
|
# work" — a demo answers that. It is **which controls are present**, asked
|
||
|
|
# before something goes wrong rather than after.
|
||
|
|
#
|
||
|
|
# Nothing answers it here. `verify.sh` reports which checks ran just now.
|
||
|
|
# `backup.sh` proves one dump. The Command Center reports on documents and
|
||
|
|
# tracker labels. None of them says whether this project has backups AND has
|
||
|
|
# ever restored one AND has somewhere errors go AND has an environment that is
|
||
|
|
# not production. That list is the actual answer to "is this safe to depend on",
|
||
|
|
# and until now it lived in whoever remembered.
|
||
|
|
#
|
||
|
|
# ## Measured is not the same as declared, and both beat unknown
|
||
|
|
#
|
||
|
|
# Every row carries how it is known, because the four states mean genuinely
|
||
|
|
# different things and flattening them is how a report starts lying:
|
||
|
|
#
|
||
|
|
# measured this script observed the fact — a file with a date on it, a
|
||
|
|
# variable that is set, a command that answered
|
||
|
|
# declared the operator asserted it in configuration. Checked for shape,
|
||
|
|
# not for truth: `CONTROLS_AUDIT=activity_events table` is a claim
|
||
|
|
# this script cannot verify and will not pretend to.
|
||
|
|
# n/a the project declared the control does not apply. A library has no
|
||
|
|
# uptime; saying so is an answer, not an omission.
|
||
|
|
# unknown expected, and could not be determined. **Never rendered as
|
||
|
|
# absent**, because "I could not tell" and "it is not there" send
|
||
|
|
# people to different places.
|
||
|
|
#
|
||
|
|
# An unknown is not a failure of the project, it is a failure of this script to
|
||
|
|
# find out, and it exits non-zero for the same reason `verify.sh` exits 2 on an
|
||
|
|
# empty run: a check that did not happen must not look like one that passed.
|
||
|
|
#
|
||
|
|
# ## Wiring it into verify.sh
|
||
|
|
#
|
||
|
|
# `verify.sh` runs every executable file in VERIFY_GUARD_DIR, and refuses to
|
||
|
|
# accept the scripts directory itself as that directory -- because release.sh,
|
||
|
|
# backup.sh and migrate.sh live there and would all be RUN. So this file is not
|
||
|
|
# dropped into verify.d. A three-line wrapper is:
|
||
|
|
#
|
||
|
|
# # scripts/verify.d/controls
|
||
|
|
# #!/usr/bin/env bash
|
||
|
|
# export CONTROLS_EXPECTED="backup restore errors healthcheck"
|
||
|
|
# exec bash "$(dirname "$0")/../controls.sh" --quiet
|
||
|
|
#
|
||
|
|
# The wrapper is where this project's expectations live, which is the right
|
||
|
|
# place for them: they are a fact about the project, not about the check.
|
||
|
|
#
|
||
|
|
# ## It writes nothing
|
||
|
|
#
|
||
|
|
# There is no CONTROLS.md and there will not be one. A committed file saying
|
||
|
|
# "backups: ok" is a description of current state living in a document, which is
|
||
|
|
# what the batch ledger was and why it was archived: it is right on the day it
|
||
|
|
# is written and silently wrong afterwards. The live answer comes from running
|
||
|
|
# this. What belongs in git is the *intent* — `docs/OPERATIONS.md` — which
|
||
|
|
# changes rarely because it is a decision rather than a measurement.
|
||
|
|
|
||
|
|
set -uo pipefail
|
||
|
|
|
||
|
|
say() { printf '\033[1mcontrols:\033[0m %s\n' "$*" >&2; }
|
||
|
|
|
||
|
|
# ---------------------------------------------------------------------------
|
||
|
|
# CONFIGURATION
|
||
|
|
#
|
||
|
|
# Space-separated, from this set:
|
||
|
|
# backup restore errors healthcheck ratelimit audit testenv deploy
|
||
|
|
# ---------------------------------------------------------------------------
|
||
|
|
EXPECTED="${CONTROLS_EXPECTED:-}"
|
||
|
|
|
||
|
|
# Measured controls read the same variables the scripts that own them use, so
|
||
|
|
# the two cannot disagree about which project this is.
|
||
|
|
BACKUP_DIR="${BACKUP_DIR:-}"
|
||
|
|
BACKUP_NAME="${BACKUP_NAME:-}"
|
||
|
|
BACKUP_MAX_AGE="${CONTROLS_BACKUP_MAX_AGE_DAYS:-2}"
|
||
|
|
RESTORE_MAX_AGE="${CONTROLS_RESTORE_MAX_AGE_DAYS:-120}"
|
||
|
|
LAST_RESTORE="${CONTROLS_LAST_RESTORE:-}"
|
||
|
|
ERROR_VAR="${CONTROLS_ERROR_TRACKING_VAR:-}"
|
||
|
|
HEALTHCHECK_BASE_URL="${HEALTHCHECK_BASE_URL:-}"
|
||
|
|
RATELIMIT="${CONTROLS_RATELIMIT:-}"
|
||
|
|
AUDIT="${CONTROLS_AUDIT:-}"
|
||
|
|
TEST_ENV="${CONTROLS_TEST_ENV:-}"
|
||
|
|
STATUS_HOST="${STATUS_HOST:-}"
|
||
|
|
STATUS_CONTAINER="${STATUS_CONTAINER:-}"
|
||
|
|
|
||
|
|
QUIET=""
|
||
|
|
while [ $# -gt 0 ]; do
|
||
|
|
case "$1" in
|
||
|
|
--quiet) QUIET="yes"; shift ;;
|
||
|
|
-h|--help) say "usage: bash scripts/controls.sh [--quiet]"; say "set CONTROLS_EXPECTED to the controls this project should have."; exit 0 ;;
|
||
|
|
*) say "unknown argument '$1'."; exit 2 ;;
|
||
|
|
esac
|
||
|
|
done
|
||
|
|
|
||
|
|
ALL="backup restore errors healthcheck ratelimit audit testenv deploy"
|
||
|
|
|
||
|
|
for want in $EXPECTED; do
|
||
|
|
case " $ALL " in
|
||
|
|
*" $want "*) ;;
|
||
|
|
*) say "CONTROLS_EXPECTED names '$want', which is not a control this knows about."
|
||
|
|
say "Known: $ALL"
|
||
|
|
exit 2 ;;
|
||
|
|
esac
|
||
|
|
done
|
||
|
|
|
||
|
|
expects() { case " $EXPECTED " in *" $1 "*) return 0 ;; *) return 1 ;; esac; }
|
||
|
|
|
||
|
|
# days_since <YYYY-MM-DD> — prints whole days, or nothing if it cannot be read.
|
||
|
|
days_since() {
|
||
|
|
local then now
|
||
|
|
# The shape is checked before `date` sees it. GNU date parses relative English
|
||
|
|
# -- `date -d "last tuesday"` succeeds and returns a real timestamp -- so a
|
||
|
|
# CONTROLS_LAST_RESTORE somebody typed as prose would come back as a plausible
|
||
|
|
# age instead of an unreadable value. A restore date this script invented is
|
||
|
|
# worse than one it admits it cannot read.
|
||
|
|
case "$1" in
|
||
|
|
[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]) ;;
|
||
|
|
*) return 1 ;;
|
||
|
|
esac
|
||
|
|
then=$(date -u -d "$1" +%s 2>/dev/null) || return 1
|
||
|
|
[ -n "$then" ] || return 1
|
||
|
|
now=$(date -u +%s) || return 1
|
||
|
|
printf '%d' $(( (now - then) / 86400 ))
|
||
|
|
}
|
||
|
|
|
||
|
|
ROWS=""
|
||
|
|
missing=0
|
||
|
|
unknowns=0
|
||
|
|
|
||
|
|
row() {
|
||
|
|
# <control> <state> <detail>
|
||
|
|
ROWS+="$(printf '%-12s %-9s %s' "$1" "$2" "$3")"$'\n'
|
||
|
|
case "$2" in
|
||
|
|
absent) missing=$((missing + 1)) ;;
|
||
|
|
unknown) unknowns=$((unknowns + 1)) ;;
|
||
|
|
esac
|
||
|
|
}
|
||
|
|
|
||
|
|
check_backup() {
|
||
|
|
if [ -z "$BACKUP_DIR" ] || [ -z "$BACKUP_NAME" ]; then
|
||
|
|
row backup unknown "BACKUP_DIR/BACKUP_NAME not set — cannot look"
|
||
|
|
return
|
||
|
|
fi
|
||
|
|
if [ ! -d "$BACKUP_DIR" ]; then
|
||
|
|
row backup unknown "$BACKUP_DIR is not a directory"
|
||
|
|
return
|
||
|
|
fi
|
||
|
|
local newest="" f
|
||
|
|
shopt -s nullglob
|
||
|
|
for f in "$BACKUP_DIR/$BACKUP_NAME"*; do
|
||
|
|
[ -f "$f" ] || continue
|
||
|
|
{ [ -z "$newest" ] || [ "$f" -nt "$newest" ]; } && newest="$f"
|
||
|
|
done
|
||
|
|
shopt -u nullglob
|
||
|
|
if [ -z "$newest" ]; then
|
||
|
|
row backup absent "no dump matching $BACKUP_NAME* in $BACKUP_DIR"
|
||
|
|
return
|
||
|
|
fi
|
||
|
|
local age
|
||
|
|
age=$(( ( $(date -u +%s) - $(date -u -r "$newest" +%s 2>/dev/null || echo 0) ) / 86400 ))
|
||
|
|
if [ "$age" -gt "$BACKUP_MAX_AGE" ]; then
|
||
|
|
row backup absent "newest dump is ${age}d old, limit ${BACKUP_MAX_AGE}d"
|
||
|
|
else
|
||
|
|
row backup measured "newest dump ${age}d old"
|
||
|
|
fi
|
||
|
|
}
|
||
|
|
|
||
|
|
check_restore() {
|
||
|
|
if [ -z "$LAST_RESTORE" ]; then
|
||
|
|
row restore absent "CONTROLS_LAST_RESTORE unset — no restore has been recorded"
|
||
|
|
return
|
||
|
|
fi
|
||
|
|
local age
|
||
|
|
if ! age=$(days_since "$LAST_RESTORE"); then
|
||
|
|
row restore unknown "CONTROLS_LAST_RESTORE='$LAST_RESTORE' is not a date this can read"
|
||
|
|
return
|
||
|
|
fi
|
||
|
|
if [ "$age" -gt "$RESTORE_MAX_AGE" ]; then
|
||
|
|
row restore absent "last restore ${age}d ago, limit ${RESTORE_MAX_AGE}d"
|
||
|
|
else
|
||
|
|
row restore declared "last restore ${age}d ago (asserted, not observed)"
|
||
|
|
fi
|
||
|
|
}
|
||
|
|
|
||
|
|
check_errors() {
|
||
|
|
if [ -z "$ERROR_VAR" ]; then
|
||
|
|
row errors unknown "CONTROLS_ERROR_TRACKING_VAR unset — name the variable that carries the DSN"
|
||
|
|
return
|
||
|
|
fi
|
||
|
|
if [ -n "${!ERROR_VAR:-}" ]; then
|
||
|
|
row errors measured "$ERROR_VAR is set"
|
||
|
|
else
|
||
|
|
row errors absent "$ERROR_VAR is empty — errors go nowhere"
|
||
|
|
fi
|
||
|
|
}
|
||
|
|
|
||
|
|
check_healthcheck() {
|
||
|
|
if [ -n "$HEALTHCHECK_BASE_URL" ]; then
|
||
|
|
row healthcheck measured "HEALTHCHECK_BASE_URL is set"
|
||
|
|
else
|
||
|
|
row healthcheck absent "HEALTHCHECK_BASE_URL unset — nothing ticks"
|
||
|
|
fi
|
||
|
|
}
|
||
|
|
|
||
|
|
check_deploy() {
|
||
|
|
if [ -z "$STATUS_HOST" ] || [ -z "$STATUS_CONTAINER" ]; then
|
||
|
|
row deploy unknown "STATUS_HOST/STATUS_CONTAINER unset — cannot ask what is deployed"
|
||
|
|
return
|
||
|
|
fi
|
||
|
|
local dir version
|
||
|
|
dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
|
|
if [ ! -x "$dir/status.sh" ] && [ ! -f "$dir/status.sh" ]; then
|
||
|
|
row deploy unknown "status.sh is not beside this script"
|
||
|
|
return
|
||
|
|
fi
|
||
|
|
if version=$(bash "$dir/status.sh" --deployed-version 2>/dev/null) && [ -n "$version" ]; then
|
||
|
|
row deploy measured "running $version"
|
||
|
|
else
|
||
|
|
# status.sh distinguishes stopped from unreachable and says so on stderr;
|
||
|
|
# this only needs to know it did not get an answer.
|
||
|
|
row deploy unknown "status.sh could not read the deployed version — run it directly for the reason"
|
||
|
|
fi
|
||
|
|
}
|
||
|
|
|
||
|
|
declared_or_absent() { # <control> <value> <hint>
|
||
|
|
if [ -n "$2" ]; then row "$1" declared "$2"; else row "$1" absent "$3"; fi
|
||
|
|
}
|
||
|
|
|
||
|
|
for c in $ALL; do
|
||
|
|
if ! expects "$c"; then
|
||
|
|
row "$c" n/a "not expected for this project"
|
||
|
|
continue
|
||
|
|
fi
|
||
|
|
case "$c" in
|
||
|
|
backup) check_backup ;;
|
||
|
|
restore) check_restore ;;
|
||
|
|
errors) check_errors ;;
|
||
|
|
healthcheck) check_healthcheck ;;
|
||
|
|
deploy) check_deploy ;;
|
||
|
|
ratelimit) declared_or_absent ratelimit "$RATELIMIT" "CONTROLS_RATELIMIT unset — nothing bounds an expensive endpoint" ;;
|
||
|
|
audit) declared_or_absent audit "$AUDIT" "CONTROLS_AUDIT unset — no record of who changed what" ;;
|
||
|
|
testenv) declared_or_absent testenv "$TEST_ENV" "CONTROLS_TEST_ENV unset — production is the only environment" ;;
|
||
|
|
esac
|
||
|
|
done
|
||
|
|
|
||
|
|
if [ -z "$EXPECTED" ]; then
|
||
|
|
say "CONTROLS_EXPECTED is empty, so no control was checked."
|
||
|
|
say "That is not a pass. Name what this project is supposed to have:"
|
||
|
|
say " CONTROLS_EXPECTED='$ALL'"
|
||
|
|
say "and drop the ones that genuinely do not apply — a library has no uptime."
|
||
|
|
exit 2
|
||
|
|
fi
|
||
|
|
|
||
|
|
if [ -n "$QUIET" ]; then
|
||
|
|
printf '%s' "$ROWS" | grep -vE '^\S+ +(measured|declared|n/a) ' >&2 || true
|
||
|
|
else
|
||
|
|
printf '%s' "$ROWS" >&2
|
||
|
|
fi
|
||
|
|
|
||
|
|
say ""
|
||
|
|
if [ "$missing" -gt 0 ] || [ "$unknowns" -gt 0 ]; then
|
||
|
|
say "$missing absent, $unknowns unknown, of $(printf '%s' "$EXPECTED" | wc -w) expected."
|
||
|
|
[ "$unknowns" -gt 0 ] && say "An unknown is this script failing to find out, not the control failing to exist. Both are non-zero; only one is the project's fault."
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
|
||
|
|
say "every expected control present ($(printf '%s' "$EXPECTED" | wc -w) checked)."
|