250 lines
11 KiB
Bash
Executable File
250 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# Replay the newest dump into a scratch database, count what arrived, and time
|
|
# it. Then throw the scratch database away.
|
|
#
|
|
# bash scripts/restore-check.sh # the newest dump
|
|
# bash scripts/restore-check.sh --dry-run # print the plan, touch nothing
|
|
# bash scripts/restore-check.sh --file X # a specific dump, not the newest
|
|
#
|
|
# Exit codes, because a caller must be able to act on the answer:
|
|
#
|
|
# 0 restored, and what came back is plausible
|
|
# 1 the restore failed, or the result holds fewer tables than the minimum
|
|
# 2 NOTHING WAS CHECKED — unconfigured, no dump found, or a missing tool.
|
|
# Not a pass. A restore check that did not run and one that succeeded must
|
|
# never exit the same way.
|
|
#
|
|
# ===========================================================================
|
|
# THIS IS A REWRITE, NOT A CONFIGURED COPY
|
|
# ===========================================================================
|
|
#
|
|
# The template ships this as PostgreSQL — `pg_restore --clean` into a scratch
|
|
# database created and dropped over `psql`, with RESTORE_ADMIN_URL naming the
|
|
# server. Unlike `backup.sh`, which isolates its engine in one block precisely
|
|
# so it can be swapped, this script is engine-specific end to end: there was no
|
|
# seam to configure. So the argument was kept and the mechanism replaced.
|
|
#
|
|
# Everything below that reads like the original is deliberate. The exit codes,
|
|
# the refusal to accept a target, the timing, and the rule that a dump which
|
|
# merely *reads* is not a backup are all the template's and all still true.
|
|
#
|
|
# Assumes: bash, coreutils, `sqlite3`.
|
|
#
|
|
# ## Why this exists
|
|
#
|
|
# `backup.sh` says it plainly: it verifies the artefact, and only a restore
|
|
# verifies the backup. Its header describes this script as "the other half".
|
|
#
|
|
# A SQLite file that answers `PRAGMA integrity_check` is a well-formed
|
|
# database. It is not yet *your* database. Between those two facts sit the
|
|
# reasons a restore disappoints on the day it is needed: a snapshot of an empty
|
|
# database taken after the volume was recreated, a snapshot of the wrong
|
|
# container that reads perfectly and holds somebody else's rows, a schema from
|
|
# before a migration, or a table that exists with nothing in it.
|
|
#
|
|
# So this does not repeat the integrity check. It does what a recovery actually
|
|
# does: serialises the snapshot to SQL and replays it into an empty database.
|
|
# That exercises every CREATE and every INSERT, which is the part that fails.
|
|
#
|
|
# And there is a number nobody has that they will want badly: **how long it
|
|
# takes**. During an incident that number decides whether you restore or start
|
|
# apologising, and it is unknowable from the file size. This prints it every run.
|
|
#
|
|
# ## The dangerous part, and what is done about it
|
|
#
|
|
# A restore writes. Pointed at the live database it would overwrite every lead
|
|
# the site has ever taken, immediately and irreversibly. That is the entire risk
|
|
# surface of this script, and it is handled the way the template handles it —
|
|
# by never accepting a target at all:
|
|
#
|
|
# - There is no --into flag and no RESTORE_TARGET path. You cannot name the
|
|
# database to restore into, because naming it is the mistake.
|
|
# - The scratch database is created inside a fresh `mktemp -d`, under a name
|
|
# this script generates, and the whole directory is removed by a trap on
|
|
# every exit path including the failures.
|
|
# - The source dump is opened READ-ONLY, over a `file:…?mode=ro` URI, so a
|
|
# mistyped path cannot damage the thing it was pointed at either.
|
|
#
|
|
# ## What it does not prove
|
|
#
|
|
# That the application runs against the restored file. It proves the schema and
|
|
# the rows come back, which is the half that is mechanically checkable. Standing
|
|
# the container up against a restored volume is a person's job, and belongs in
|
|
# `docs/OPERATIONS.md` the day somebody does it.
|
|
|
|
set -uo pipefail
|
|
|
|
say() { printf '\033[1mrestore-check:\033[0m %s\n' "$*" >&2; }
|
|
die() { printf '\033[1mrestore-check:\033[0m %s\n' "$*" >&2; exit 2; }
|
|
fail() { printf '\033[1mrestore-check:\033[0m %s\n' "$*" >&2; exit 1; }
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Configuration — shared with backup.sh, and read from the environment for that
|
|
# reason. A restore check pointed at another series answers confidently about
|
|
# the wrong database, which is worse than not running.
|
|
# ---------------------------------------------------------------------------
|
|
BACKUP_DIR="${BACKUP_DIR:-$HOME/backups/queue-north-website}"
|
|
BACKUP_NAME="${BACKUP_NAME:-queuenorth-leads}"
|
|
MIN_TABLES="${BACKUP_MIN_TABLES:-1}"
|
|
|
|
# Refuse a restore that comes back with fewer rows than this ACROSS ALL TABLES.
|
|
# Zero is allowed and is the default, because a brand-new deployment legitimately
|
|
# has no leads yet — but set it once there are rows, and the check starts
|
|
# catching the snapshot-of-an-empty-volume case that no structural test can see.
|
|
MIN_ROWS="${RESTORE_MIN_ROWS:-0}"
|
|
|
|
DUMP_SUFFIX=".sqlite"
|
|
|
|
DRY_RUN=""
|
|
ONE_FILE=""
|
|
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--dry-run) DRY_RUN="yes"; shift ;;
|
|
--file)
|
|
ONE_FILE="${2:-}"
|
|
[ -n "$ONE_FILE" ] || die "--file needs a path."
|
|
shift 2 ;;
|
|
-h|--help)
|
|
say "usage: bash scripts/restore-check.sh [--dry-run] [--file <dump>]"
|
|
say " BACKUP_DIR and BACKUP_NAME are shared with backup.sh."
|
|
exit 0 ;;
|
|
*) die "unknown argument '$1'. There is deliberately no flag naming a restore target." ;;
|
|
esac
|
|
done
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Refuse to run half-configured, before anything is touched, naming the missing
|
|
# value one at a time so the message says which.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
[ -n "$BACKUP_DIR" ] || die "set BACKUP_DIR — the same directory backup.sh writes to."
|
|
[ -n "$BACKUP_NAME" ] || die "set BACKUP_NAME — the same filename prefix backup.sh uses."
|
|
[ -d "$BACKUP_DIR" ] || die "BACKUP_DIR '$BACKUP_DIR' is not a directory. Nothing was checked."
|
|
|
|
command -v sqlite3 >/dev/null 2>&1 \
|
|
|| die "sqlite3 is not on PATH, so nothing could be restored. Nothing was checked."
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Choose the dump. Newest by modification time, within this series only.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
if [ -n "$ONE_FILE" ]; then
|
|
DUMP="$ONE_FILE"
|
|
[ -f "$DUMP" ] || die "'$DUMP' is not a file. Nothing was checked."
|
|
else
|
|
# `.part` files are half-written by definition and must never be selected;
|
|
# backup.sh only renames into place after it has verified, so the glob below
|
|
# matching the final suffix exactly is what keeps those out.
|
|
DUMP=$(find "$BACKUP_DIR" -maxdepth 1 -type f \
|
|
-name "${BACKUP_NAME}-*${DUMP_SUFFIX}" -printf '%T@ %p\n' 2>/dev/null \
|
|
| sort -rn | head -1 | cut -d' ' -f2-)
|
|
|
|
[ -n "$DUMP" ] || die "no dump matching '${BACKUP_NAME}-*${DUMP_SUFFIX}' in $BACKUP_DIR.
|
|
Nothing was checked — which is not the same as a backup that failed, and
|
|
not the same as one that passed."
|
|
fi
|
|
|
|
DUMP_SIZE=$(du -h "$DUMP" 2>/dev/null | cut -f1)
|
|
|
|
if [ -n "$DRY_RUN" ]; then
|
|
say "--dry-run: nothing was created, written or removed. It would have:"
|
|
say " read $DUMP (${DUMP_SIZE:-size unknown}), read-only"
|
|
say " created a scratch database in a fresh mktemp -d"
|
|
say " restored sqlite3 '<scratch>' < (sqlite3 'file:<dump>?mode=ro' .dump)"
|
|
say " counted tables and rows, requiring at least $MIN_TABLES table(s) and $MIN_ROWS row(s)"
|
|
say " removed the scratch directory, on every exit path"
|
|
exit 0
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# The scratch database. Created inside a temporary directory that the trap
|
|
# removes, so there is no path here that outlives the run.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
SCRATCH_DIR=$(mktemp -d) || die "could not create a temporary directory. Nothing was checked."
|
|
trap 'rm -rf "$SCRATCH_DIR"' EXIT
|
|
|
|
SCRATCH="$SCRATCH_DIR/restorecheck_$$.db"
|
|
ERRFILE="$SCRATCH_DIR/err"
|
|
|
|
say "restoring $DUMP (${DUMP_SIZE:-size unknown})"
|
|
say " -> $SCRATCH"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# The restore, timed.
|
|
#
|
|
# `.dump` serialises schema and data to SQL; replaying it builds the database
|
|
# from nothing. That is the point — a file copy would prove only that `cp`
|
|
# works, which is the check this script exists to be better than.
|
|
#
|
|
# SECONDS is bash's own counter and needs no external date arithmetic.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
START=$SECONDS
|
|
|
|
if ! sqlite3 "file:${DUMP}?mode=ro" .dump 2>"$ERRFILE" | sqlite3 "$SCRATCH" 2>>"$ERRFILE"; then
|
|
say "the restore failed. sqlite3 said:"
|
|
sed 's/^/ /' "$ERRFILE" >&2
|
|
fail "$DUMP did not restore. This dump is not a backup."
|
|
fi
|
|
|
|
ELAPSED=$(( SECONDS - START ))
|
|
|
|
# A non-empty stderr with a zero exit is the case worth catching: sqlite3 will
|
|
# report a constraint or a duplicate and carry on, leaving a database that is
|
|
# missing exactly the rows it complained about.
|
|
if [ -s "$ERRFILE" ]; then
|
|
say "WARNING: the restore reported problems while still exiting zero:"
|
|
sed 's/^/ /' "$ERRFILE" >&2
|
|
say " What follows counted whatever survived that."
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# What came back. Counted, not assumed.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
TABLES=$(sqlite3 "$SCRATCH" \
|
|
"SELECT count(*) FROM sqlite_master
|
|
WHERE type='table' AND name NOT LIKE 'sqlite\_%' ESCAPE '\';" 2>/dev/null)
|
|
|
|
[ -n "$TABLES" ] || fail "the restored database could not be counted, so nothing about it is known."
|
|
|
|
if [ "$TABLES" -lt "$MIN_TABLES" ]; then
|
|
fail "the restore produced $TABLES table(s), fewer than the $MIN_TABLES required.
|
|
A dump of an empty or wrong database restores perfectly and looks like this."
|
|
fi
|
|
|
|
# Per-table row counts, built as SQL and then run — sqlite3 has no built-in for
|
|
# it. Printed per table rather than only as a total, because "leads: 0" beside
|
|
# "support_requests: 40" is a different incident from both being zero.
|
|
ROW_SQL=$(sqlite3 "$SCRATCH" \
|
|
"SELECT 'SELECT ''' || name || ''', count(*) FROM \"' || name || '\";'
|
|
FROM sqlite_master
|
|
WHERE type='table' AND name NOT LIKE 'sqlite\_%' ESCAPE '\'
|
|
ORDER BY name;" 2>/dev/null)
|
|
|
|
TOTAL_ROWS=0
|
|
if [ -n "$ROW_SQL" ]; then
|
|
while IFS='|' read -r tname tcount; do
|
|
[ -n "$tname" ] || continue
|
|
printf ' %-24s %s row(s)\n' "$tname" "$tcount" >&2
|
|
TOTAL_ROWS=$(( TOTAL_ROWS + tcount ))
|
|
done < <(printf '%s\n' "$ROW_SQL" | sqlite3 "$SCRATCH" 2>/dev/null)
|
|
fi
|
|
|
|
if [ "$TOTAL_ROWS" -lt "$MIN_ROWS" ]; then
|
|
fail "the restore produced $TOTAL_ROWS row(s) across $TABLES table(s), fewer than the
|
|
$MIN_ROWS required by RESTORE_MIN_ROWS. Either the snapshot is of an empty
|
|
volume, or the threshold is stale."
|
|
fi
|
|
|
|
say ""
|
|
say "restored in ${ELAPSED}s — $TABLES table(s), $TOTAL_ROWS row(s) total."
|
|
say ""
|
|
say "That number is the one an incident needs: it is how long this takes with"
|
|
say "the data as it is today, measured rather than guessed. Record it in"
|
|
say "docs/OPERATIONS.md beside the date, and update the 'Last verified restore'"
|
|
say "row — that row is the only thing separating a backup from a file."
|