diff --git a/docs/architecture/README.md b/docs/architecture/README.md index 0e72459..19f7d11 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -45,6 +45,7 @@ can stop you, and where to start in a fresh clone. | `scripts/check-env.sh` | which variables are set, which are missing, before anything reads them | | `scripts/migrate.sh` | apply and report migrations, including the ones that run outside a transaction | | `scripts/backup.sh` | a dump that is verified before it is trusted | +| `scripts/restore-check.sh` | the other half of `backup.sh`: restores the newest dump into a scratch database it creates and drops, counts the tables, and **times it** — the number an incident actually needs. Never accepts a target, because naming one is the mistake `--clean` punishes. | | `scripts/healthcheck.sh` | a liveness tick with the URL written down rather than re-derived each run | | `scripts/status.sh` | what is deployed, and whether it matches this checkout | | `scripts/dev.sh` | bring the local stack up | diff --git a/docs/architecture/scripts/restore-check.sh b/docs/architecture/scripts/restore-check.sh new file mode 100755 index 0000000..6113777 --- /dev/null +++ b/docs/architecture/scripts/restore-check.sh @@ -0,0 +1,244 @@ +#!/usr/bin/env bash +# +# Restore the newest dump into a scratch database, count what arrived, and time +# it. Then throw the scratch database away. +# +# bash scripts/restore-check.sh # restore the newest dump +# bash scripts/restore-check.sh --dry-run # print the plan, touch nothing +# bash scripts/restore-check.sh --file X # a specific dump, not the newest +# +# Exit codes, because a caller must be able to act on the answer: +# +# 0 restored, and what came back is plausible +# 1 the restore failed, or the result holds fewer tables than the minimum +# 2 NOTHING WAS CHECKED — unconfigured, no dump found, or a missing tool. +# Not a pass. A restore check that did not run and one that succeeded must +# never exit the same way. +# +# =========================================================================== +# TEMPLATE COPY — configure this before the first run +# =========================================================================== +# +# Copy to `scripts/restore-check.sh` and set RESTORE_ADMIN_URL below. It shares +# BACKUP_DIR, BACKUP_NAME and BACKUP_MIN_TABLES with `backup.sh` and reads them +# from the environment, so the two cannot disagree about which series belongs to +# this project — a restore check pointed at another project's dumps answers +# confidently about the wrong database, which is worse than not running. +# +# Assumes: bash, coreutils, `pg_restore` and `psql`. +# +# ## Why this exists +# +# `backup.sh` says it plainly: it verifies the artefact, and only a restore +# verifies the backup. Its header describes this script as "the other half" and +# names the command; this is that command, with the parts that stop it being +# dangerous. +# +# A dump that `pg_restore --list` can read is a file with a table of contents. +# It is not yet a database. Between those two facts sit every reason a restore +# fails on the day it is needed: an extension the target does not have, an owner +# that does not exist, a version skew, a dump of the wrong database that reads +# perfectly and holds somebody else's rows. +# +# And there is a number nobody has that they will want badly: **how long it +# takes**. During an incident that number decides whether you restore or fail +# over, and it is unknowable from the file size. This prints it every run. +# +# ## The dangerous part, and what is done about it +# +# `pg_restore --clean` issues DROP statements. Pointed at the production +# database it does exactly what it is told, immediately and irreversibly. That +# is the entire risk surface of this script and it is handled by never accepting +# a target at all: +# +# - There is no --database flag and no RESTORE_TARGET_URL. You cannot name the +# database to restore into, because naming it is the mistake. +# - This script CREATES a database with a name it generates, restores into +# that, and drops it. The name is `restorecheck__`, which no +# project's real database is called. +# - RESTORE_ADMIN_URL is a connection used only to create and drop that +# scratch database. If it points at a database rather than a server, the +# create still happens beside it, not in it. +# - The drop runs from a trap, so an interrupted run does not leave a full +# copy of the production data sitting on the server. +# +# The same argument `status.sh` makes about having no --host flag: a flag would +# make it one keystroke to point a destructive command at the wrong place, and +# the environment form is at least self-documenting in shell history. +# +# ## What it deliberately does not do +# +# It does not check that the *contents* are correct — that the rows are the +# right rows, that the newest order is present. It counts tables, because that +# is what can be counted without knowing the schema. A dump of the wrong +# database passes this check. Restoring is necessary and not sufficient, and the +# thing that makes it sufficient is a person looking at the result once. +# +# It does not delete or rotate dumps. `backup.sh` owns retention. + +set -uo pipefail + +unset LC_ALL +export LC_COLLATE=C +shopt -s nullglob + +say() { printf '\033[1mrestore-check:\033[0m %s\n' "$*" >&2; } +die() { printf '\033[1mrestore-check:\033[0m %s\n' "$*" >&2; exit 2; } +fail() { printf '\033[1mrestore-check:\033[0m %s\n' "$*" >&2; exit 1; } + +# --------------------------------------------------------------------------- +# CONFIGURATION — set this one, then delete this banner. +# +# A connection to the SERVER holding the scratch database, e.g. +# postgres://user@host:5432/postgres. Used only for CREATE DATABASE and DROP +# DATABASE on a name this script generates. +# +# Empty on purpose, like every other target in this tree. A default here would +# be a credentialed connection to somebody's database, inherited silently by a +# copy of this file. +# --------------------------------------------------------------------------- +RESTORE_ADMIN_URL="${RESTORE_ADMIN_URL:-}" + +# Shared with backup.sh, and read from the environment for that reason. +BACKUP_DIR="${BACKUP_DIR:-}" +BACKUP_NAME="${BACKUP_NAME:-}" +MIN_TABLES="${BACKUP_MIN_TABLES:-1}" + +DUMP_SUFFIX=".dump" + +DRY_RUN="" +ONE_FILE="" + +while [ $# -gt 0 ]; do + case "$1" in + --dry-run) DRY_RUN="yes"; shift ;; + --file) + ONE_FILE="${2:-}" + [ -n "$ONE_FILE" ] || die "--file needs a path." + shift 2 ;; + -h|--help) + say "usage: bash scripts/restore-check.sh [--dry-run] [--file ]" + say " set RESTORE_ADMIN_URL, BACKUP_DIR and BACKUP_NAME." + exit 0 ;; + *) die "unknown argument '$1'." ;; + esac +done + +# --------------------------------------------------------------------------- +# Refuse to run half-configured, before anything is contacted, naming the +# missing value one at a time so the message says which. +# --------------------------------------------------------------------------- + +[ -n "$RESTORE_ADMIN_URL" ] || die "set RESTORE_ADMIN_URL — a connection to the server that will host the scratch database. See the CONFIGURATION block." +[ -n "$BACKUP_DIR" ] || die "set BACKUP_DIR — the same directory backup.sh writes to." +[ -n "$BACKUP_NAME" ] || die "set BACKUP_NAME — the same filename prefix backup.sh uses." +[ -d "$BACKUP_DIR" ] || die "BACKUP_DIR '$BACKUP_DIR' is not a directory. Nothing was checked." + +for tool in pg_restore psql; do + command -v "$tool" >/dev/null 2>&1 || die "$tool is not on PATH, so nothing could be restored." +done + +case "$MIN_TABLES" in + ''|*[!0-9]*) die "BACKUP_MIN_TABLES must be a whole number, got '$MIN_TABLES'." ;; +esac + +# --------------------------------------------------------------------------- +# Choose the dump. +# --------------------------------------------------------------------------- + +if [ -n "$ONE_FILE" ]; then + DUMP="$ONE_FILE" + [ -f "$DUMP" ] || die "no such file: $DUMP" +else + CANDIDATES=("$BACKUP_DIR/$BACKUP_NAME"*"$DUMP_SUFFIX") + [ "${#CANDIDATES[@]}" -gt 0 ] \ + || die "no dump matching '$BACKUP_NAME*$DUMP_SUFFIX' in $BACKUP_DIR. Nothing was checked — this is not a report that the backups are bad, it is a report that none were found." + DUMP="" + for f in "${CANDIDATES[@]}"; do + [ -z "$DUMP" ] && DUMP="$f" + [ "$f" -nt "$DUMP" ] && DUMP="$f" + done +fi + + +SCRATCH="restorecheck_$(date -u +%s)_$$" + +say "dump $DUMP" +say "size $(du -h "$DUMP" | cut -f1)" +say "scratch $SCRATCH" + +# After the dry-run block below, deliberately: --dry-run contacts nothing and +# reports the plan, and every other --dry-run in this tree exits 0. A mode whose +# exit code depends on the state of the data is not a dry run. +if [ -n "$DRY_RUN" ]; then + say "--dry-run: nothing was created, restored or dropped. It would have run:" + printf ' psql "" -c %s\n' "'CREATE DATABASE $SCRATCH'" >&2 + printf ' pg_restore --clean --if-exists --no-owner --dbname "" %s\n' "$DUMP" >&2 + printf ' psql "" -tAc "select count(*) from information_schema.tables where table_schema not in (…)"\n' >&2 + printf ' psql "" -c %s\n' "'DROP DATABASE $SCRATCH'" >&2 + [ -s "$DUMP" ] || say "note: that dump is zero bytes, so a real run would fail." + exit 0 +fi + +[ -s "$DUMP" ] || fail "the dump '$DUMP' is empty. Nothing to restore from, and that IS a finding." + +# The scratch database is dropped from a trap rather than at the end, so an +# interrupted run does not leave a full copy of production data on the server. +# `|| true` because a drop that fails must not mask the exit code of the check. +DROPPED="" +cleanup() { + [ -n "$DROPPED" ] && return 0 + DROPPED="yes" + psql "$RESTORE_ADMIN_URL" -v ON_ERROR_STOP=1 -q \ + -c "DROP DATABASE IF EXISTS \"$SCRATCH\"" >/dev/null 2>&1 || { + say "WARNING: could not drop the scratch database '$SCRATCH'. It holds a" + say " full copy of the data in that dump. Remove it by hand." + } + return 0 +} +trap cleanup EXIT INT TERM + +psql "$RESTORE_ADMIN_URL" -v ON_ERROR_STOP=1 -q -c "CREATE DATABASE \"$SCRATCH\"" >/dev/null 2>&1 \ + || die "could not create the scratch database on that server. Nothing was checked — check RESTORE_ADMIN_URL and that the role may CREATE DATABASE." + +# The admin URL points at a server; the scratch URL is the same server, other +# database. Substituting the path rather than asking the operator for a second +# URL keeps the two from disagreeing. +SCRATCH_URL="${RESTORE_ADMIN_URL%%\?*}" +QUERY="" +case "$RESTORE_ADMIN_URL" in *\?*) QUERY="?${RESTORE_ADMIN_URL#*\?}" ;; esac +SCRATCH_URL="${SCRATCH_URL%/*}/$SCRATCH$QUERY" + +START=$(date -u +%s) + +ERRFILE=$(mktemp) || die "cannot create a temporary file." +trap 'rm -f -- "$ERRFILE"; cleanup' EXIT INT TERM + +if ! pg_restore --clean --if-exists --no-owner --dbname "$SCRATCH_URL" "$DUMP" >/dev/null 2>"$ERRFILE"; then + say "pg_restore reported errors restoring $DUMP:" + head -n 5 "$ERRFILE" >&2 + fail "the newest dump did not restore. This is the finding this script exists to produce — do not wait for an incident to see it again." +fi + +ELAPSED=$(( $(date -u +%s) - START )) + +# Counted, and a count that did not happen is not a count of zero — the +# argument backup.sh makes at length about awk applies here to psql. +COUNT=$(psql "$SCRATCH_URL" -tAc \ + "select count(*) from information_schema.tables where table_schema not in ('pg_catalog','information_schema')" \ + 2>/dev/null | tr -d '[:space:]') + +case "$COUNT" in + ''|*[!0-9]*) + die "restored in ${ELAPSED}s, but the table count could not be read. Nothing is known about what arrived, which is not the same as nothing arriving." ;; +esac + +say "restored in ${ELAPSED}s — ${COUNT} table(s)" + +if [ "$COUNT" -lt "$MIN_TABLES" ]; then + fail "only $COUNT table(s), and BACKUP_MIN_TABLES is $MIN_TABLES. A dump that restores into almost nothing is the shape of a backup taken against the wrong database, or after a failed migration." +fi + +say "the newest dump restores, and holds $COUNT table(s)." +say "Record the date and the ${ELAPSED}s in docs/OPERATIONS.md — during an" +say "incident that number decides whether you restore or fail over."