245 lines
11 KiB
Bash
Executable File
245 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# Restore the newest dump into a scratch database, count what arrived, and time
|
|
# it. Then throw the scratch database away.
|
|
#
|
|
# bash scripts/restore-check.sh # restore the newest dump
|
|
# bash scripts/restore-check.sh --dry-run # print the plan, touch nothing
|
|
# bash scripts/restore-check.sh --file X # a specific dump, not the newest
|
|
#
|
|
# Exit codes, because a caller must be able to act on the answer:
|
|
#
|
|
# 0 restored, and what came back is plausible
|
|
# 1 the restore failed, or the result holds fewer tables than the minimum
|
|
# 2 NOTHING WAS CHECKED — unconfigured, no dump found, or a missing tool.
|
|
# Not a pass. A restore check that did not run and one that succeeded must
|
|
# never exit the same way.
|
|
#
|
|
# ===========================================================================
|
|
# TEMPLATE COPY — configure this before the first run
|
|
# ===========================================================================
|
|
#
|
|
# Copy to `scripts/restore-check.sh` and set RESTORE_ADMIN_URL below. It shares
|
|
# BACKUP_DIR, BACKUP_NAME and BACKUP_MIN_TABLES with `backup.sh` and reads them
|
|
# from the environment, so the two cannot disagree about which series belongs to
|
|
# this project — a restore check pointed at another project's dumps answers
|
|
# confidently about the wrong database, which is worse than not running.
|
|
#
|
|
# Assumes: bash, coreutils, `pg_restore` and `psql`.
|
|
#
|
|
# ## Why this exists
|
|
#
|
|
# `backup.sh` says it plainly: it verifies the artefact, and only a restore
|
|
# verifies the backup. Its header describes this script as "the other half" and
|
|
# names the command; this is that command, with the parts that stop it being
|
|
# dangerous.
|
|
#
|
|
# A dump that `pg_restore --list` can read is a file with a table of contents.
|
|
# It is not yet a database. Between those two facts sit every reason a restore
|
|
# fails on the day it is needed: an extension the target does not have, an owner
|
|
# that does not exist, a version skew, a dump of the wrong database that reads
|
|
# perfectly and holds somebody else's rows.
|
|
#
|
|
# And there is a number nobody has that they will want badly: **how long it
|
|
# takes**. During an incident that number decides whether you restore or fail
|
|
# over, and it is unknowable from the file size. This prints it every run.
|
|
#
|
|
# ## The dangerous part, and what is done about it
|
|
#
|
|
# `pg_restore --clean` issues DROP statements. Pointed at the production
|
|
# database it does exactly what it is told, immediately and irreversibly. That
|
|
# is the entire risk surface of this script and it is handled by never accepting
|
|
# a target at all:
|
|
#
|
|
# - There is no --database flag and no RESTORE_TARGET_URL. You cannot name the
|
|
# database to restore into, because naming it is the mistake.
|
|
# - This script CREATES a database with a name it generates, restores into
|
|
# that, and drops it. The name is `restorecheck_<epoch>_<pid>`, which no
|
|
# project's real database is called.
|
|
# - RESTORE_ADMIN_URL is a connection used only to create and drop that
|
|
# scratch database. If it points at a database rather than a server, the
|
|
# create still happens beside it, not in it.
|
|
# - The drop runs from a trap, so an interrupted run does not leave a full
|
|
# copy of the production data sitting on the server.
|
|
#
|
|
# The same argument `status.sh` makes about having no --host flag: a flag would
|
|
# make it one keystroke to point a destructive command at the wrong place, and
|
|
# the environment form is at least self-documenting in shell history.
|
|
#
|
|
# ## What it deliberately does not do
|
|
#
|
|
# It does not check that the *contents* are correct — that the rows are the
|
|
# right rows, that the newest order is present. It counts tables, because that
|
|
# is what can be counted without knowing the schema. A dump of the wrong
|
|
# database passes this check. Restoring is necessary and not sufficient, and the
|
|
# thing that makes it sufficient is a person looking at the result once.
|
|
#
|
|
# It does not delete or rotate dumps. `backup.sh` owns retention.
|
|
|
|
set -uo pipefail
|
|
|
|
unset LC_ALL
|
|
export LC_COLLATE=C
|
|
shopt -s nullglob
|
|
|
|
say() { printf '\033[1mrestore-check:\033[0m %s\n' "$*" >&2; }
|
|
die() { printf '\033[1mrestore-check:\033[0m %s\n' "$*" >&2; exit 2; }
|
|
fail() { printf '\033[1mrestore-check:\033[0m %s\n' "$*" >&2; exit 1; }
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# CONFIGURATION — set this one, then delete this banner.
|
|
#
|
|
# A connection to the SERVER holding the scratch database, e.g.
|
|
# postgres://user@host:5432/postgres. Used only for CREATE DATABASE and DROP
|
|
# DATABASE on a name this script generates.
|
|
#
|
|
# Empty on purpose, like every other target in this tree. A default here would
|
|
# be a credentialed connection to somebody's database, inherited silently by a
|
|
# copy of this file.
|
|
# ---------------------------------------------------------------------------
|
|
RESTORE_ADMIN_URL="${RESTORE_ADMIN_URL:-}"
|
|
|
|
# Shared with backup.sh, and read from the environment for that reason.
|
|
BACKUP_DIR="${BACKUP_DIR:-}"
|
|
BACKUP_NAME="${BACKUP_NAME:-}"
|
|
MIN_TABLES="${BACKUP_MIN_TABLES:-1}"
|
|
|
|
DUMP_SUFFIX=".dump"
|
|
|
|
DRY_RUN=""
|
|
ONE_FILE=""
|
|
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--dry-run) DRY_RUN="yes"; shift ;;
|
|
--file)
|
|
ONE_FILE="${2:-}"
|
|
[ -n "$ONE_FILE" ] || die "--file needs a path."
|
|
shift 2 ;;
|
|
-h|--help)
|
|
say "usage: bash scripts/restore-check.sh [--dry-run] [--file <dump>]"
|
|
say " set RESTORE_ADMIN_URL, BACKUP_DIR and BACKUP_NAME."
|
|
exit 0 ;;
|
|
*) die "unknown argument '$1'." ;;
|
|
esac
|
|
done
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Refuse to run half-configured, before anything is contacted, naming the
|
|
# missing value one at a time so the message says which.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
[ -n "$RESTORE_ADMIN_URL" ] || die "set RESTORE_ADMIN_URL — a connection to the server that will host the scratch database. See the CONFIGURATION block."
|
|
[ -n "$BACKUP_DIR" ] || die "set BACKUP_DIR — the same directory backup.sh writes to."
|
|
[ -n "$BACKUP_NAME" ] || die "set BACKUP_NAME — the same filename prefix backup.sh uses."
|
|
[ -d "$BACKUP_DIR" ] || die "BACKUP_DIR '$BACKUP_DIR' is not a directory. Nothing was checked."
|
|
|
|
for tool in pg_restore psql; do
|
|
command -v "$tool" >/dev/null 2>&1 || die "$tool is not on PATH, so nothing could be restored."
|
|
done
|
|
|
|
case "$MIN_TABLES" in
|
|
''|*[!0-9]*) die "BACKUP_MIN_TABLES must be a whole number, got '$MIN_TABLES'." ;;
|
|
esac
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Choose the dump.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
if [ -n "$ONE_FILE" ]; then
|
|
DUMP="$ONE_FILE"
|
|
[ -f "$DUMP" ] || die "no such file: $DUMP"
|
|
else
|
|
CANDIDATES=("$BACKUP_DIR/$BACKUP_NAME"*"$DUMP_SUFFIX")
|
|
[ "${#CANDIDATES[@]}" -gt 0 ] \
|
|
|| die "no dump matching '$BACKUP_NAME*$DUMP_SUFFIX' in $BACKUP_DIR. Nothing was checked — this is not a report that the backups are bad, it is a report that none were found."
|
|
DUMP=""
|
|
for f in "${CANDIDATES[@]}"; do
|
|
[ -z "$DUMP" ] && DUMP="$f"
|
|
[ "$f" -nt "$DUMP" ] && DUMP="$f"
|
|
done
|
|
fi
|
|
|
|
|
|
SCRATCH="restorecheck_$(date -u +%s)_$$"
|
|
|
|
say "dump $DUMP"
|
|
say "size $(du -h "$DUMP" | cut -f1)"
|
|
say "scratch $SCRATCH"
|
|
|
|
# After the dry-run block below, deliberately: --dry-run contacts nothing and
|
|
# reports the plan, and every other --dry-run in this tree exits 0. A mode whose
|
|
# exit code depends on the state of the data is not a dry run.
|
|
if [ -n "$DRY_RUN" ]; then
|
|
say "--dry-run: nothing was created, restored or dropped. It would have run:"
|
|
printf ' psql "<RESTORE_ADMIN_URL>" -c %s\n' "'CREATE DATABASE $SCRATCH'" >&2
|
|
printf ' pg_restore --clean --if-exists --no-owner --dbname "<scratch>" %s\n' "$DUMP" >&2
|
|
printf ' psql "<scratch>" -tAc "select count(*) from information_schema.tables where table_schema not in (…)"\n' >&2
|
|
printf ' psql "<RESTORE_ADMIN_URL>" -c %s\n' "'DROP DATABASE $SCRATCH'" >&2
|
|
[ -s "$DUMP" ] || say "note: that dump is zero bytes, so a real run would fail."
|
|
exit 0
|
|
fi
|
|
|
|
[ -s "$DUMP" ] || fail "the dump '$DUMP' is empty. Nothing to restore from, and that IS a finding."
|
|
|
|
# The scratch database is dropped from a trap rather than at the end, so an
|
|
# interrupted run does not leave a full copy of production data on the server.
|
|
# `|| true` because a drop that fails must not mask the exit code of the check.
|
|
DROPPED=""
|
|
cleanup() {
|
|
[ -n "$DROPPED" ] && return 0
|
|
DROPPED="yes"
|
|
psql "$RESTORE_ADMIN_URL" -v ON_ERROR_STOP=1 -q \
|
|
-c "DROP DATABASE IF EXISTS \"$SCRATCH\"" >/dev/null 2>&1 || {
|
|
say "WARNING: could not drop the scratch database '$SCRATCH'. It holds a"
|
|
say " full copy of the data in that dump. Remove it by hand."
|
|
}
|
|
return 0
|
|
}
|
|
trap cleanup EXIT INT TERM
|
|
|
|
psql "$RESTORE_ADMIN_URL" -v ON_ERROR_STOP=1 -q -c "CREATE DATABASE \"$SCRATCH\"" >/dev/null 2>&1 \
|
|
|| die "could not create the scratch database on that server. Nothing was checked — check RESTORE_ADMIN_URL and that the role may CREATE DATABASE."
|
|
|
|
# The admin URL points at a server; the scratch URL is the same server, other
|
|
# database. Substituting the path rather than asking the operator for a second
|
|
# URL keeps the two from disagreeing.
|
|
SCRATCH_URL="${RESTORE_ADMIN_URL%%\?*}"
|
|
QUERY=""
|
|
case "$RESTORE_ADMIN_URL" in *\?*) QUERY="?${RESTORE_ADMIN_URL#*\?}" ;; esac
|
|
SCRATCH_URL="${SCRATCH_URL%/*}/$SCRATCH$QUERY"
|
|
|
|
START=$(date -u +%s)
|
|
|
|
ERRFILE=$(mktemp) || die "cannot create a temporary file."
|
|
trap 'rm -f -- "$ERRFILE"; cleanup' EXIT INT TERM
|
|
|
|
if ! pg_restore --clean --if-exists --no-owner --dbname "$SCRATCH_URL" "$DUMP" >/dev/null 2>"$ERRFILE"; then
|
|
say "pg_restore reported errors restoring $DUMP:"
|
|
head -n 5 "$ERRFILE" >&2
|
|
fail "the newest dump did not restore. This is the finding this script exists to produce — do not wait for an incident to see it again."
|
|
fi
|
|
|
|
ELAPSED=$(( $(date -u +%s) - START ))
|
|
|
|
# Counted, and a count that did not happen is not a count of zero — the
|
|
# argument backup.sh makes at length about awk applies here to psql.
|
|
COUNT=$(psql "$SCRATCH_URL" -tAc \
|
|
"select count(*) from information_schema.tables where table_schema not in ('pg_catalog','information_schema')" \
|
|
2>/dev/null | tr -d '[:space:]')
|
|
|
|
case "$COUNT" in
|
|
''|*[!0-9]*)
|
|
die "restored in ${ELAPSED}s, but the table count could not be read. Nothing is known about what arrived, which is not the same as nothing arriving." ;;
|
|
esac
|
|
|
|
say "restored in ${ELAPSED}s — ${COUNT} table(s)"
|
|
|
|
if [ "$COUNT" -lt "$MIN_TABLES" ]; then
|
|
fail "only $COUNT table(s), and BACKUP_MIN_TABLES is $MIN_TABLES. A dump that restores into almost nothing is the shape of a backup taken against the wrong database, or after a failed migration."
|
|
fi
|
|
|
|
say "the newest dump restores, and holds $COUNT table(s)."
|
|
say "Record the date and the ${ELAPSED}s in docs/OPERATIONS.md — during an"
|
|
say "incident that number decides whether you restore or fail over."
|