#!/usr/bin/env bash # # Prove a guard fails before you believe it passes. # # ## The failure this catches # # A guard that cannot fail is worse than no guard, because it is trusted. # `docs/architecture/GUARDS.md` opens with that sentence and its first rule is # this procedure, written out as a manual recipe: back the file up, break exactly # the thing the guard protects, run the guard, expect one failure, restore. # # The recipe is thirty seconds and it is skipped anyway, for two reasons this # script removes: # # - **Restoring is a step you can forget**, and forgetting is silent. The tests # pass again once the mutation is undone in your head but not on disk, so the # reverted code ships looking green. Here the restore is a `trap`, which runs # on success, on failure, and on Ctrl-C. # - **Counting the failures is the part people skip.** GUARDS.md §1: "If # breaking the guard's target fails three tests, two of them are coincidental # and will mask a real regression later." A human doing this by hand sees red # and stops reading. # # ## Usage # # bash scripts/prove-guard.sh # # bash scripts/prove-guard.sh src/lib/thing.ts \ # 'if (body.error)' 'if (false)' \ # npx vitest run tests/thing.test.ts # # Everything after the third argument is the command that runs the guard, so any # runner works. `$PROVE_GUARD_CMD` is used when no command is given. # # ## Counting the failures # # "Exactly one" is a claim about test *cases*, and counting matching log lines # does not measure that: Gradle reports a single failing test on six lines — the # task, the test, its assertion, the summary, and twice more for the build — and # a naive count calls that six coincidental failures. Tried that first; it fired # on the very first run against a guard that was behaving perfectly. # # So the summary line is preferred, because almost every runner prints one and it # is the runner's own count. Runners disagree about which side of the word the # number goes on, so both orders are read: `1 failed` from vitest, `1 failed, 5 # passed` from pytest, `6 tests completed, 1 failed` from Gradle — and `fail 1` # from node --test, `Failures: 2` from Maven and JUnit, `failures=2` from python # unittest, `# fail 1` from TAP. The **last** such line wins, and only if none is # found does it fall back to counting lines matching `$PROVE_GUARD_FAIL_PATTERN` # — saying so, because an approximate count presented as an exact one is the kind # of thing this script exists to object to. # # ## Exit codes # # 0 the guard caught it, and nothing else did — the outcome you want # 1 the guard stayed GREEN with its target broken. It is not testing what you # think it is, and you have just learned that for the price of one edit # 2 nothing was proven: bad arguments, missing file, or a find-string that is # absent or ambiguous. **Two is not a pass** # 3 the guard caught it, but so did something else. Red for more than one # reason hides the next regression behind a failure you have learned to # expect — narrow the guard, or the mutation # # The file is restored in every one of those cases. set -euo pipefail FAIL_PATTERN="${PROVE_GUARD_FAIL_PATTERN:-(FAIL|✗|[0-9]+ (tests? )?failed|FAILED|AssertionError)}" if [ "$#" -lt 3 ]; then sed -n '2,30p' "$0" >&2 exit 2 fi FILE="$1"; FIND="$2"; REPLACE="$3"; shift 3 if [ "$#" -gt 0 ]; then CMD=("$@") elif [ -n "${PROVE_GUARD_CMD:-}" ]; then # shellcheck disable=SC2206 CMD=($PROVE_GUARD_CMD) else echo "prove-guard: no test command given and PROVE_GUARD_CMD is unset." >&2 echo "Nothing was proven, which is not the same as nothing being wrong." >&2 exit 2 fi [ -f "$FILE" ] || { echo "prove-guard: no such file: $FILE" >&2; exit 2; } BACKUP="$(mktemp)" cp "$FILE" "$BACKUP" restore() { cp "$BACKUP" "$FILE" rm -f "$BACKUP" echo "prove-guard: restored $FILE" } trap restore EXIT INT TERM # Exact-string replacement, and it must be unique. A mutation that lands in two # places proves nothing about either, and a regex here would make the mutation # itself the thing to debug. # # Both refusals here exit 2, like every other "nothing was proven" path # above. `sys.exit("message")` prints it and exits **1** — the code this # script reserves for "the guard stayed GREEN with its target broken", which # is a diagnosis about the guard, not a refusal to run. So a mistyped # find-string accused the guard under test of being broken. `TOOLS.md` # teaches callers to tell 1 from 2 and that "two is never a pass"; that # distinction has to survive this block. python3 - "$FILE" "$FIND" "$REPLACE" <<'PY' import sys def refuse(message: str) -> None: print(message, file=sys.stderr) raise SystemExit(2) path, find, replace = sys.argv[1], sys.argv[2], sys.argv[3] text = open(path, encoding="utf-8").read() count = text.count(find) if count == 0: refuse(f"prove-guard: the string to break is not in {path}") if count > 1: refuse( f"prove-guard: {count} occurrences of that string; a mutation in " "two places proves neither. Pick a longer, unique one." ) open(path, "w", encoding="utf-8").write(text.replace(find, replace)) PY LOG="$(mktemp)" trap 'restore; rm -f "$LOG"' EXIT INT TERM echo "prove-guard: broke $FILE — expecting '${CMD[*]}' to go red" echo if "${CMD[@]}" >"$LOG" 2>&1; then echo "prove-guard: FAILED — the guard stayed GREEN with its target broken." >&2 echo >&2 echo "It is not checking what you think. Either the assertion does not reach" >&2 echo "the mutated code, or it would pass without it. Log: $LOG" >&2 tail -20 "$LOG" >&2 exit 1 fi echo "--- what failed ---" grep -E "$FAIL_PATTERN" "$LOG" | head -12 || true echo # The runner's own count, from the last summary line that states one. Preferred # over counting log lines for the reason in the header: one failing test is # routinely reported on half a dozen lines. # Two orders, because runners disagree about which side the number goes on. # The first pattern reads `1 failed` (vitest, pytest, Gradle); the second reads # the number on the right: `ℹ fail 1` (node --test), `Failures: 2` (Maven, # JUnit), `failures=2` (python unittest), `# fail 1` (TAP). # # Matching only the first order made every node run fall through to the # approximate line count, and that fallback is not conservative. A guard over a # status enum — mutating the string `'FAILED'` — matches FAIL_PATTERN three # times inside one AssertionError diff, so a single correct guard exited 3 with # "narrow the guard, or narrow the mutation". The header says that exact false # fire was tried once and rejected; it was still reachable through the fallback. # The message compounded it, reporting "this runner printed no summary" about a # runner that printed one this script could not read. COUNT="$(grep -oiE '[0-9]+ (tests? )?failed' "$LOG" | tail -1 | grep -oE '^[0-9]+' || true)" if [ -z "$COUNT" ]; then COUNT="$(grep -oiE '\bfail(ure)?s?[:= ]+[0-9]+' "$LOG" | tail -1 | grep -oE '[0-9]+$' || true)" fi COUNTED_BY="the runner's summary" if [ -z "$COUNT" ]; then COUNT="$(grep -cE "$FAIL_PATTERN" "$LOG" || true)" COUNTED_BY="matching log lines, approximately — this runner printed no summary" fi if [ "$COUNT" -gt 1 ]; then echo "prove-guard: the guard caught it — but $COUNT failures, by $COUNTED_BY." echo echo "GUARDS.md §1: if breaking one thing fails three tests, two are" echo "coincidental and will mask a real regression later behind a red you have" echo "learned to expect. Narrow the guard, or narrow the mutation." exit 3 fi echo "prove-guard: good — the guard caught it, and only it ($COUNTED_BY)."