#!/usr/bin/env bash # # Replay the newest dump into a scratch database, count what arrived, and time # it. Then throw the scratch database away. # # bash scripts/restore-check.sh # the newest dump # bash scripts/restore-check.sh --dry-run # print the plan, touch nothing # bash scripts/restore-check.sh --file X # a specific dump, not the newest # # Exit codes, because a caller must be able to act on the answer: # # 0 restored, and what came back is plausible # 1 the restore failed, or the result holds fewer tables than the minimum # 2 NOTHING WAS CHECKED — unconfigured, no dump found, or a missing tool. # Not a pass. A restore check that did not run and one that succeeded must # never exit the same way. # # =========================================================================== # THIS IS A REWRITE, NOT A CONFIGURED COPY # =========================================================================== # # The template ships this as PostgreSQL — `pg_restore --clean` into a scratch # database created and dropped over `psql`, with RESTORE_ADMIN_URL naming the # server. Unlike `backup.sh`, which isolates its engine in one block precisely # so it can be swapped, this script is engine-specific end to end: there was no # seam to configure. So the argument was kept and the mechanism replaced. # # Everything below that reads like the original is deliberate. The exit codes, # the refusal to accept a target, the timing, and the rule that a dump which # merely *reads* is not a backup are all the template's and all still true. # # Assumes: bash, coreutils, `sqlite3`. # # ## Why this exists # # `backup.sh` says it plainly: it verifies the artefact, and only a restore # verifies the backup. Its header describes this script as "the other half". # # A SQLite file that answers `PRAGMA integrity_check` is a well-formed # database. It is not yet *your* database. Between those two facts sit the # reasons a restore disappoints on the day it is needed: a snapshot of an empty # database taken after the volume was recreated, a snapshot of the wrong # container that reads perfectly and holds somebody else's rows, a schema from # before a migration, or a table that exists with nothing in it. # # So this does not repeat the integrity check. It does what a recovery actually # does: serialises the snapshot to SQL and replays it into an empty database. # That exercises every CREATE and every INSERT, which is the part that fails. # # And there is a number nobody has that they will want badly: **how long it # takes**. During an incident that number decides whether you restore or start # apologising, and it is unknowable from the file size. This prints it every run. # # ## The dangerous part, and what is done about it # # A restore writes. Pointed at the live database it would overwrite every lead # the site has ever taken, immediately and irreversibly. That is the entire risk # surface of this script, and it is handled the way the template handles it — # by never accepting a target at all: # # - There is no --into flag and no RESTORE_TARGET path. You cannot name the # database to restore into, because naming it is the mistake. # - The scratch database is created inside a fresh `mktemp -d`, under a name # this script generates, and the whole directory is removed by a trap on # every exit path including the failures. # - The source dump is opened READ-ONLY, over a `file:…?mode=ro` URI, so a # mistyped path cannot damage the thing it was pointed at either. # # ## What it does not prove # # That the application runs against the restored file. It proves the schema and # the rows come back, which is the half that is mechanically checkable. Standing # the container up against a restored volume is a person's job, and belongs in # `docs/OPERATIONS.md` the day somebody does it. set -uo pipefail say() { printf '\033[1mrestore-check:\033[0m %s\n' "$*" >&2; } die() { printf '\033[1mrestore-check:\033[0m %s\n' "$*" >&2; exit 2; } fail() { printf '\033[1mrestore-check:\033[0m %s\n' "$*" >&2; exit 1; } # --------------------------------------------------------------------------- # Configuration — shared with backup.sh, and read from the environment for that # reason. A restore check pointed at another series answers confidently about # the wrong database, which is worse than not running. # --------------------------------------------------------------------------- BACKUP_DIR="${BACKUP_DIR:-$HOME/backups/queue-north-website}" BACKUP_NAME="${BACKUP_NAME:-queuenorth-leads}" MIN_TABLES="${BACKUP_MIN_TABLES:-1}" # Refuse a restore that comes back with fewer rows than this ACROSS ALL TABLES. # Zero is allowed and is the default, because a brand-new deployment legitimately # has no leads yet — but set it once there are rows, and the check starts # catching the snapshot-of-an-empty-volume case that no structural test can see. MIN_ROWS="${RESTORE_MIN_ROWS:-0}" DUMP_SUFFIX=".sqlite" DRY_RUN="" ONE_FILE="" while [ $# -gt 0 ]; do case "$1" in --dry-run) DRY_RUN="yes"; shift ;; --file) ONE_FILE="${2:-}" [ -n "$ONE_FILE" ] || die "--file needs a path." shift 2 ;; -h|--help) say "usage: bash scripts/restore-check.sh [--dry-run] [--file ]" say " BACKUP_DIR and BACKUP_NAME are shared with backup.sh." exit 0 ;; *) die "unknown argument '$1'. There is deliberately no flag naming a restore target." ;; esac done # --------------------------------------------------------------------------- # Refuse to run half-configured, before anything is touched, naming the missing # value one at a time so the message says which. # --------------------------------------------------------------------------- [ -n "$BACKUP_DIR" ] || die "set BACKUP_DIR — the same directory backup.sh writes to." [ -n "$BACKUP_NAME" ] || die "set BACKUP_NAME — the same filename prefix backup.sh uses." [ -d "$BACKUP_DIR" ] || die "BACKUP_DIR '$BACKUP_DIR' is not a directory. Nothing was checked." command -v sqlite3 >/dev/null 2>&1 \ || die "sqlite3 is not on PATH, so nothing could be restored. Nothing was checked." # --------------------------------------------------------------------------- # Choose the dump. Newest by modification time, within this series only. # --------------------------------------------------------------------------- if [ -n "$ONE_FILE" ]; then DUMP="$ONE_FILE" [ -f "$DUMP" ] || die "'$DUMP' is not a file. Nothing was checked." else # `.part` files are half-written by definition and must never be selected; # backup.sh only renames into place after it has verified, so the glob below # matching the final suffix exactly is what keeps those out. DUMP=$(find "$BACKUP_DIR" -maxdepth 1 -type f \ -name "${BACKUP_NAME}-*${DUMP_SUFFIX}" -printf '%T@ %p\n' 2>/dev/null \ | sort -rn | head -1 | cut -d' ' -f2-) [ -n "$DUMP" ] || die "no dump matching '${BACKUP_NAME}-*${DUMP_SUFFIX}' in $BACKUP_DIR. Nothing was checked — which is not the same as a backup that failed, and not the same as one that passed." fi DUMP_SIZE=$(du -h "$DUMP" 2>/dev/null | cut -f1) if [ -n "$DRY_RUN" ]; then say "--dry-run: nothing was created, written or removed. It would have:" say " read $DUMP (${DUMP_SIZE:-size unknown}), read-only" say " created a scratch database in a fresh mktemp -d" say " restored sqlite3 '' < (sqlite3 'file:?mode=ro' .dump)" say " counted tables and rows, requiring at least $MIN_TABLES table(s) and $MIN_ROWS row(s)" say " removed the scratch directory, on every exit path" exit 0 fi # --------------------------------------------------------------------------- # The scratch database. Created inside a temporary directory that the trap # removes, so there is no path here that outlives the run. # --------------------------------------------------------------------------- SCRATCH_DIR=$(mktemp -d) || die "could not create a temporary directory. Nothing was checked." trap 'rm -rf "$SCRATCH_DIR"' EXIT SCRATCH="$SCRATCH_DIR/restorecheck_$$.db" ERRFILE="$SCRATCH_DIR/err" say "restoring $DUMP (${DUMP_SIZE:-size unknown})" say " -> $SCRATCH" # --------------------------------------------------------------------------- # The restore, timed. # # `.dump` serialises schema and data to SQL; replaying it builds the database # from nothing. That is the point — a file copy would prove only that `cp` # works, which is the check this script exists to be better than. # # SECONDS is bash's own counter and needs no external date arithmetic. # --------------------------------------------------------------------------- START=$SECONDS if ! sqlite3 "file:${DUMP}?mode=ro" .dump 2>"$ERRFILE" | sqlite3 "$SCRATCH" 2>>"$ERRFILE"; then say "the restore failed. sqlite3 said:" sed 's/^/ /' "$ERRFILE" >&2 fail "$DUMP did not restore. This dump is not a backup." fi ELAPSED=$(( SECONDS - START )) # A non-empty stderr with a zero exit is the case worth catching: sqlite3 will # report a constraint or a duplicate and carry on, leaving a database that is # missing exactly the rows it complained about. if [ -s "$ERRFILE" ]; then say "WARNING: the restore reported problems while still exiting zero:" sed 's/^/ /' "$ERRFILE" >&2 say " What follows counted whatever survived that." fi # --------------------------------------------------------------------------- # What came back. Counted, not assumed. # --------------------------------------------------------------------------- TABLES=$(sqlite3 "$SCRATCH" \ "SELECT count(*) FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite\_%' ESCAPE '\';" 2>/dev/null) [ -n "$TABLES" ] || fail "the restored database could not be counted, so nothing about it is known." if [ "$TABLES" -lt "$MIN_TABLES" ]; then fail "the restore produced $TABLES table(s), fewer than the $MIN_TABLES required. A dump of an empty or wrong database restores perfectly and looks like this." fi # Per-table row counts, built as SQL and then run — sqlite3 has no built-in for # it. Printed per table rather than only as a total, because "leads: 0" beside # "support_requests: 40" is a different incident from both being zero. ROW_SQL=$(sqlite3 "$SCRATCH" \ "SELECT 'SELECT ''' || name || ''', count(*) FROM \"' || name || '\";' FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite\_%' ESCAPE '\' ORDER BY name;" 2>/dev/null) TOTAL_ROWS=0 if [ -n "$ROW_SQL" ]; then while IFS='|' read -r tname tcount; do [ -n "$tname" ] || continue printf ' %-24s %s row(s)\n' "$tname" "$tcount" >&2 TOTAL_ROWS=$(( TOTAL_ROWS + tcount )) done < <(printf '%s\n' "$ROW_SQL" | sqlite3 "$SCRATCH" 2>/dev/null) fi if [ "$TOTAL_ROWS" -lt "$MIN_ROWS" ]; then fail "the restore produced $TOTAL_ROWS row(s) across $TABLES table(s), fewer than the $MIN_ROWS required by RESTORE_MIN_ROWS. Either the snapshot is of an empty volume, or the threshold is stale." fi say "" say "restored in ${ELAPSED}s — $TABLES table(s), $TOTAL_ROWS row(s) total." say "" say "That number is the one an incident needs: it is how long this takes with" say "the data as it is today, measured rather than guessed. Record it in" say "docs/OPERATIONS.md beside the date, and update the 'Last verified restore'" say "row — that row is the only thing separating a backup from a file."