Queue-North-Website/scripts/restore-check.sh

#!/usr/bin/env bash
#
# Replay the newest dump into a scratch database, count what arrived, and time
# it. Then throw the scratch database away.
#
#   bash scripts/restore-check.sh              # the newest dump
#   bash scripts/restore-check.sh --dry-run    # print the plan, touch nothing
#   bash scripts/restore-check.sh --file X     # a specific dump, not the newest
#
# Exit codes, because a caller must be able to act on the answer:
#
#   0  restored, and what came back is plausible
#   1  the restore failed, or the result holds fewer tables than the minimum
#   2  NOTHING WAS CHECKED — unconfigured, no dump found, or a missing tool.
#      Not a pass. A restore check that did not run and one that succeeded must
#      never exit the same way.
#
# ===========================================================================
# THIS IS A REWRITE, NOT A CONFIGURED COPY
# ===========================================================================
#
# The template ships this as PostgreSQL — `pg_restore --clean` into a scratch
# database created and dropped over `psql`, with RESTORE_ADMIN_URL naming the
# server. Unlike `backup.sh`, which isolates its engine in one block precisely
# so it can be swapped, this script is engine-specific end to end: there was no
# seam to configure. So the argument was kept and the mechanism replaced.
#
# Everything below that reads like the original is deliberate. The exit codes,
# the refusal to accept a target, the timing, and the rule that a dump which
# merely *reads* is not a backup are all the template's and all still true.
#
# Assumes: bash, coreutils, `sqlite3`.
#
# ## Why this exists
#
# `backup.sh` says it plainly: it verifies the artefact, and only a restore
# verifies the backup. Its header describes this script as "the other half".
#
# A SQLite file that answers `PRAGMA integrity_check` is a well-formed
# database. It is not yet *your* database. Between those two facts sit the
# reasons a restore disappoints on the day it is needed: a snapshot of an empty
# database taken after the volume was recreated, a snapshot of the wrong
# container that reads perfectly and holds somebody else's rows, a schema from
# before a migration, or a table that exists with nothing in it.
#
# So this does not repeat the integrity check. It does what a recovery actually
# does: serialises the snapshot to SQL and replays it into an empty database.
# That exercises every CREATE and every INSERT, which is the part that fails.
#
# And there is a number nobody has that they will want badly: **how long it
# takes**. During an incident that number decides whether you restore or start
# apologising, and it is unknowable from the file size. This prints it every run.
#
# ## The dangerous part, and what is done about it
#
# A restore writes. Pointed at the live database it would overwrite every lead
# the site has ever taken, immediately and irreversibly. That is the entire risk
# surface of this script, and it is handled the way the template handles it —
# by never accepting a target at all:
#
#   - There is no --into flag and no RESTORE_TARGET path. You cannot name the
#     database to restore into, because naming it is the mistake.
#   - The scratch database is created inside a fresh `mktemp -d`, under a name
#     this script generates, and the whole directory is removed by a trap on
#     every exit path including the failures.
#   - The source dump is opened READ-ONLY, over a `file:…?mode=ro` URI, so a
#     mistyped path cannot damage the thing it was pointed at either.
#
# ## What it does not prove
#
# That the application runs against the restored file. It proves the schema and
# the rows come back, which is the half that is mechanically checkable. Standing
# the container up against a restored volume is a person's job, and belongs in
# `docs/OPERATIONS.md` the day somebody does it.

set -uo pipefail

say()  { printf '\033[1mrestore-check:\033[0m %s\n' "$*" >&2; }
die()  { printf '\033[1mrestore-check:\033[0m %s\n' "$*" >&2; exit 2; }
fail() { printf '\033[1mrestore-check:\033[0m %s\n' "$*" >&2; exit 1; }

# ---------------------------------------------------------------------------
# Configuration — shared with backup.sh, and read from the environment for that
# reason. A restore check pointed at another series answers confidently about
# the wrong database, which is worse than not running.
# ---------------------------------------------------------------------------
BACKUP_DIR="${BACKUP_DIR:-$HOME/backups/queue-north-website}"
BACKUP_NAME="${BACKUP_NAME:-queuenorth-leads}"
MIN_TABLES="${BACKUP_MIN_TABLES:-1}"

# Refuse a restore that comes back with fewer rows than this ACROSS ALL TABLES.
# Zero is allowed and is the default, because a brand-new deployment legitimately
# has no leads yet — but set it once there are rows, and the check starts
# catching the snapshot-of-an-empty-volume case that no structural test can see.
MIN_ROWS="${RESTORE_MIN_ROWS:-0}"

DUMP_SUFFIX=".sqlite"

DRY_RUN=""
ONE_FILE=""

while [ $# -gt 0 ]; do
  case "$1" in
    --dry-run) DRY_RUN="yes"; shift ;;
    --file)
      ONE_FILE="${2:-}"
      [ -n "$ONE_FILE" ] || die "--file needs a path."
      shift 2 ;;
    -h|--help)
      say "usage: bash scripts/restore-check.sh [--dry-run] [--file <dump>]"
      say "       BACKUP_DIR and BACKUP_NAME are shared with backup.sh."
      exit 0 ;;
    *) die "unknown argument '$1'. There is deliberately no flag naming a restore target." ;;
  esac
done

# ---------------------------------------------------------------------------
# Refuse to run half-configured, before anything is touched, naming the missing
# value one at a time so the message says which.
# ---------------------------------------------------------------------------

[ -n "$BACKUP_DIR" ]  || die "set BACKUP_DIR — the same directory backup.sh writes to."
[ -n "$BACKUP_NAME" ] || die "set BACKUP_NAME — the same filename prefix backup.sh uses."
[ -d "$BACKUP_DIR" ]  || die "BACKUP_DIR '$BACKUP_DIR' is not a directory. Nothing was checked."

command -v sqlite3 >/dev/null 2>&1 \
  || die "sqlite3 is not on PATH, so nothing could be restored. Nothing was checked."

# ---------------------------------------------------------------------------
# Choose the dump. Newest by modification time, within this series only.
# ---------------------------------------------------------------------------

if [ -n "$ONE_FILE" ]; then
  DUMP="$ONE_FILE"
  [ -f "$DUMP" ] || die "'$DUMP' is not a file. Nothing was checked."
else
  # `.part` files are half-written by definition and must never be selected;
  # backup.sh only renames into place after it has verified, so the glob below
  # matching the final suffix exactly is what keeps those out.
  DUMP=$(find "$BACKUP_DIR" -maxdepth 1 -type f \
           -name "${BACKUP_NAME}-*${DUMP_SUFFIX}" -printf '%T@ %p\n' 2>/dev/null \
         | sort -rn | head -1 | cut -d' ' -f2-)

  [ -n "$DUMP" ] || die "no dump matching '${BACKUP_NAME}-*${DUMP_SUFFIX}' in $BACKUP_DIR.
      Nothing was checked — which is not the same as a backup that failed, and
      not the same as one that passed."
fi

DUMP_SIZE=$(du -h "$DUMP" 2>/dev/null | cut -f1)

if [ -n "$DRY_RUN" ]; then
  say "--dry-run: nothing was created, written or removed. It would have:"
  say "  read      $DUMP  (${DUMP_SIZE:-size unknown}), read-only"
  say "  created   a scratch database in a fresh mktemp -d"
  say "  restored  sqlite3 '<scratch>' < (sqlite3 'file:<dump>?mode=ro' .dump)"
  say "  counted   tables and rows, requiring at least $MIN_TABLES table(s) and $MIN_ROWS row(s)"
  say "  removed   the scratch directory, on every exit path"
  exit 0
fi

# ---------------------------------------------------------------------------
# The scratch database. Created inside a temporary directory that the trap
# removes, so there is no path here that outlives the run.
# ---------------------------------------------------------------------------

SCRATCH_DIR=$(mktemp -d) || die "could not create a temporary directory. Nothing was checked."
trap 'rm -rf "$SCRATCH_DIR"' EXIT

SCRATCH="$SCRATCH_DIR/restorecheck_$$.db"
ERRFILE="$SCRATCH_DIR/err"

say "restoring $DUMP (${DUMP_SIZE:-size unknown})"
say "       -> $SCRATCH"

# ---------------------------------------------------------------------------
# The restore, timed.
#
# `.dump` serialises schema and data to SQL; replaying it builds the database
# from nothing. That is the point — a file copy would prove only that `cp`
# works, which is the check this script exists to be better than.
#
# SECONDS is bash's own counter and needs no external date arithmetic.
# ---------------------------------------------------------------------------

START=$SECONDS

if ! sqlite3 "file:${DUMP}?mode=ro" .dump 2>"$ERRFILE" | sqlite3 "$SCRATCH" 2>>"$ERRFILE"; then
  say "the restore failed. sqlite3 said:"
  sed 's/^/  /' "$ERRFILE" >&2
  fail "$DUMP did not restore. This dump is not a backup."
fi

ELAPSED=$(( SECONDS - START ))

# A non-empty stderr with a zero exit is the case worth catching: sqlite3 will
# report a constraint or a duplicate and carry on, leaving a database that is
# missing exactly the rows it complained about.
if [ -s "$ERRFILE" ]; then
  say "WARNING: the restore reported problems while still exiting zero:"
  sed 's/^/  /' "$ERRFILE" >&2
  say "         What follows counted whatever survived that."
fi

# ---------------------------------------------------------------------------
# What came back. Counted, not assumed.
# ---------------------------------------------------------------------------

TABLES=$(sqlite3 "$SCRATCH" \
  "SELECT count(*) FROM sqlite_master
    WHERE type='table' AND name NOT LIKE 'sqlite\_%' ESCAPE '\';" 2>/dev/null)

[ -n "$TABLES" ] || fail "the restored database could not be counted, so nothing about it is known."

if [ "$TABLES" -lt "$MIN_TABLES" ]; then
  fail "the restore produced $TABLES table(s), fewer than the $MIN_TABLES required.
      A dump of an empty or wrong database restores perfectly and looks like this."
fi

# Per-table row counts, built as SQL and then run — sqlite3 has no built-in for
# it. Printed per table rather than only as a total, because "leads: 0" beside
# "support_requests: 40" is a different incident from both being zero.
ROW_SQL=$(sqlite3 "$SCRATCH" \
  "SELECT 'SELECT ''' || name || ''', count(*) FROM \"' || name || '\";'
     FROM sqlite_master
    WHERE type='table' AND name NOT LIKE 'sqlite\_%' ESCAPE '\'
    ORDER BY name;" 2>/dev/null)

TOTAL_ROWS=0
if [ -n "$ROW_SQL" ]; then
  while IFS='|' read -r tname tcount; do
    [ -n "$tname" ] || continue
    printf '  %-24s %s row(s)\n' "$tname" "$tcount" >&2
    TOTAL_ROWS=$(( TOTAL_ROWS + tcount ))
  done < <(printf '%s\n' "$ROW_SQL" | sqlite3 "$SCRATCH" 2>/dev/null)
fi

if [ "$TOTAL_ROWS" -lt "$MIN_ROWS" ]; then
  fail "the restore produced $TOTAL_ROWS row(s) across $TABLES table(s), fewer than the
      $MIN_ROWS required by RESTORE_MIN_ROWS. Either the snapshot is of an empty
      volume, or the threshold is stale."
fi

say ""
say "restored in ${ELAPSED}s — $TABLES table(s), $TOTAL_ROWS row(s) total."
say ""
say "That number is the one an incident needs: it is how long this takes with"
say "the data as it is today, measured rather than guessed. Record it in"
say "docs/OPERATIONS.md beside the date, and update the 'Last verified restore'"
say "row — that row is the only thing separating a backup from a file."