Privacy-Period-Tracker/scripts/prove-guard.sh

#!/usr/bin/env bash
#
# Prove a guard fails before you believe it passes.
#
# ## The failure this catches
#
# A guard that cannot fail is worse than no guard, because it is trusted.
# `docs/architecture/GUARDS.md` opens with that sentence and its first rule is
# this procedure, written out as a manual recipe: back the file up, break exactly
# the thing the guard protects, run the guard, expect one failure, restore.
#
# The recipe is thirty seconds and it is skipped anyway, for two reasons this
# script removes:
#
#   - **Restoring is a step you can forget**, and forgetting is silent. The tests
#     pass again once the mutation is undone in your head but not on disk, so the
#     reverted code ships looking green. Here the restore is a `trap`, which runs
#     on success, on failure, and on Ctrl-C.
#   - **Counting the failures is the part people skip.** GUARDS.md §1: "If
#     breaking the guard's target fails three tests, two of them are coincidental
#     and will mask a real regression later." A human doing this by hand sees red
#     and stops reading.
#
# ## Usage
#
#   bash scripts/prove-guard.sh <file> <find> <replace> <test command…>
#
#   bash scripts/prove-guard.sh src/lib/thing.ts \
#     'if (body.error)' 'if (false)' \
#     npx vitest run tests/thing.test.ts
#
# Everything after the third argument is the command that runs the guard, so any
# runner works. `$PROVE_GUARD_CMD` is used when no command is given.
#
# ## Counting the failures
#
# "Exactly one" is a claim about test *cases*, and counting matching log lines
# does not measure that: Gradle reports a single failing test on six lines — the
# task, the test, its assertion, the summary, and twice more for the build — and
# a naive count calls that six coincidental failures. Tried that first; it fired
# on the very first run against a guard that was behaving perfectly.
#
# So the summary line is preferred, because almost every runner prints one and it
# is the runner's own count. Runners disagree about which side of the word the
# number goes on, so both orders are read: `1 failed` from vitest, `1 failed, 5
# passed` from pytest, `6 tests completed, 1 failed` from Gradle — and `fail 1`
# from node --test, `Failures: 2` from Maven and JUnit, `failures=2` from python
# unittest, `# fail 1` from TAP. The **last** such line wins, and only if none is
# found does it fall back to counting lines matching `$PROVE_GUARD_FAIL_PATTERN`
# — saying so, because an approximate count presented as an exact one is the kind
# of thing this script exists to object to.
#
# ## Exit codes
#
#   0  the guard caught it, and nothing else did — the outcome you want
#   1  the guard stayed GREEN with its target broken. It is not testing what you
#      think it is, and you have just learned that for the price of one edit
#   2  nothing was proven: bad arguments, missing file, or a find-string that is
#      absent or ambiguous. **Two is not a pass**
#   3  the guard caught it, but so did something else. Red for more than one
#      reason hides the next regression behind a failure you have learned to
#      expect — narrow the guard, or the mutation
#
# The file is restored in every one of those cases.
set -euo pipefail

FAIL_PATTERN="${PROVE_GUARD_FAIL_PATTERN:-(FAIL|✗|[0-9]+ (tests? )?failed|FAILED|AssertionError)}"

if [ "$#" -lt 3 ]; then
  sed -n '2,30p' "$0" >&2
  exit 2
fi

FILE="$1"; FIND="$2"; REPLACE="$3"; shift 3

if [ "$#" -gt 0 ]; then
  CMD=("$@")
elif [ -n "${PROVE_GUARD_CMD:-}" ]; then
  # shellcheck disable=SC2206
  CMD=($PROVE_GUARD_CMD)
else
  echo "prove-guard: no test command given and PROVE_GUARD_CMD is unset." >&2
  echo "Nothing was proven, which is not the same as nothing being wrong." >&2
  exit 2
fi

[ -f "$FILE" ] || { echo "prove-guard: no such file: $FILE" >&2; exit 2; }

BACKUP="$(mktemp)"
cp "$FILE" "$BACKUP"
restore() {
  cp "$BACKUP" "$FILE"
  rm -f "$BACKUP"
  echo "prove-guard: restored $FILE"
}
trap restore EXIT INT TERM

# Exact-string replacement, and it must be unique. A mutation that lands in two
# places proves nothing about either, and a regex here would make the mutation
# itself the thing to debug.
#
# Both refusals here exit 2, like every other "nothing was proven" path
# above. `sys.exit("message")` prints it and exits **1** — the code this
# script reserves for "the guard stayed GREEN with its target broken", which
# is a diagnosis about the guard, not a refusal to run. So a mistyped
# find-string accused the guard under test of being broken. `TOOLS.md`
# teaches callers to tell 1 from 2 and that "two is never a pass"; that
# distinction has to survive this block.
python3 - "$FILE" "$FIND" "$REPLACE" <<'PY'
import sys


def refuse(message: str) -> None:
    print(message, file=sys.stderr)
    raise SystemExit(2)


path, find, replace = sys.argv[1], sys.argv[2], sys.argv[3]
text = open(path, encoding="utf-8").read()
count = text.count(find)
if count == 0:
    refuse(f"prove-guard: the string to break is not in {path}")
if count > 1:
    refuse(
        f"prove-guard: {count} occurrences of that string; a mutation in "
        "two places proves neither. Pick a longer, unique one."
    )
open(path, "w", encoding="utf-8").write(text.replace(find, replace))
PY

LOG="$(mktemp)"
trap 'restore; rm -f "$LOG"' EXIT INT TERM

echo "prove-guard: broke $FILE — expecting '${CMD[*]}' to go red"
echo

if "${CMD[@]}" >"$LOG" 2>&1; then
  echo "prove-guard: FAILED — the guard stayed GREEN with its target broken." >&2
  echo >&2
  echo "It is not checking what you think. Either the assertion does not reach" >&2
  echo "the mutated code, or it would pass without it. Log: $LOG" >&2
  tail -20 "$LOG" >&2
  exit 1
fi

echo "--- what failed ---"
grep -E "$FAIL_PATTERN" "$LOG" | head -12 || true
echo

# The runner's own count, from the last summary line that states one. Preferred
# over counting log lines for the reason in the header: one failing test is
# routinely reported on half a dozen lines.
# Two orders, because runners disagree about which side the number goes on.
# The first pattern reads `1 failed` (vitest, pytest, Gradle); the second reads
# the number on the right: `ℹ fail 1` (node --test), `Failures: 2` (Maven,
# JUnit), `failures=2` (python unittest), `# fail 1` (TAP).
#
# Matching only the first order made every node run fall through to the
# approximate line count, and that fallback is not conservative. A guard over a
# status enum — mutating the string `'FAILED'` — matches FAIL_PATTERN three
# times inside one AssertionError diff, so a single correct guard exited 3 with
# "narrow the guard, or narrow the mutation". The header says that exact false
# fire was tried once and rejected; it was still reachable through the fallback.
# The message compounded it, reporting "this runner printed no summary" about a
# runner that printed one this script could not read.
COUNT="$(grep -oiE '[0-9]+ (tests? )?failed' "$LOG" | tail -1 | grep -oE '^[0-9]+' || true)"
if [ -z "$COUNT" ]; then
  COUNT="$(grep -oiE '\bfail(ure)?s?[:= ]+[0-9]+' "$LOG" | tail -1 | grep -oE '[0-9]+$' || true)"
fi
COUNTED_BY="the runner's summary"
if [ -z "$COUNT" ]; then
  COUNT="$(grep -cE "$FAIL_PATTERN" "$LOG" || true)"
  COUNTED_BY="matching log lines, approximately — this runner printed no summary"
fi

if [ "$COUNT" -gt 1 ]; then
  echo "prove-guard: the guard caught it — but $COUNT failures, by $COUNTED_BY."
  echo
  echo "GUARDS.md §1: if breaking one thing fails three tests, two are"
  echo "coincidental and will mask a real regression later behind a red you have"
  echo "learned to expect. Narrow the guard, or narrow the mutation."
  exit 3
fi

echo "prove-guard: good — the guard caught it, and only it ($COUNTED_BY)."