diff --git a/docs/architecture/scripts/prove-guard.sh b/docs/architecture/scripts/prove-guard.sh index 4914d05..ec0a3d4 100755 --- a/docs/architecture/scripts/prove-guard.sh +++ b/docs/architecture/scripts/prove-guard.sh @@ -41,11 +41,14 @@ # on the very first run against a guard that was behaving perfectly. # # So the summary line is preferred, because almost every runner prints one and it -# is the runner's own count: `1 failed` from vitest, `6 tests completed, 1 failed` -# from Gradle, `1 failed, 5 passed` from pytest. The **last** such line wins, and -# only if none is found does it fall back to counting lines matching -# `$PROVE_GUARD_FAIL_PATTERN` — saying so, because an approximate count presented -# as an exact one is the kind of thing this script exists to object to. +# is the runner's own count. Runners disagree about which side of the word the +# number goes on, so both orders are read: `1 failed` from vitest, `1 failed, 5 +# passed` from pytest, `6 tests completed, 1 failed` from Gradle — and `fail 1` +# from node --test, `Failures: 2` from Maven and JUnit, `failures=2` from python +# unittest, `# fail 1` from TAP. The **last** such line wins, and only if none is +# found does it fall back to counting lines matching `$PROVE_GUARD_FAIL_PATTERN` +# — saying so, because an approximate count presented as an exact one is the kind +# of thing this script exists to object to. # # ## Exit codes # @@ -95,15 +98,30 @@ trap restore EXIT INT TERM # Exact-string replacement, and it must be unique. A mutation that lands in two # places proves nothing about either, and a regex here would make the mutation # itself the thing to debug. +# +# Both refusals here exit 2, like every other "nothing was proven" path +# above. `sys.exit("message")` prints it and exits **1** — the code this +# script reserves for "the guard stayed GREEN with its target broken", which +# is a diagnosis about the guard, not a refusal to run. So a mistyped +# find-string accused the guard under test of being broken. `TOOLS.md` +# teaches callers to tell 1 from 2 and that "two is never a pass"; that +# distinction has to survive this block. python3 - "$FILE" "$FIND" "$REPLACE" <<'PY' import sys + + +def refuse(message: str) -> None: + print(message, file=sys.stderr) + raise SystemExit(2) + + path, find, replace = sys.argv[1], sys.argv[2], sys.argv[3] text = open(path, encoding="utf-8").read() count = text.count(find) if count == 0: - sys.exit(f"prove-guard: the string to break is not in {path}") + refuse(f"prove-guard: the string to break is not in {path}") if count > 1: - sys.exit( + refuse( f"prove-guard: {count} occurrences of that string; a mutation in " "two places proves neither. Pick a longer, unique one." ) @@ -132,7 +150,23 @@ echo # The runner's own count, from the last summary line that states one. Preferred # over counting log lines for the reason in the header: one failing test is # routinely reported on half a dozen lines. +# Two orders, because runners disagree about which side the number goes on. +# The first pattern reads `1 failed` (vitest, pytest, Gradle); the second reads +# the number on the right: `ℹ fail 1` (node --test), `Failures: 2` (Maven, +# JUnit), `failures=2` (python unittest), `# fail 1` (TAP). +# +# Matching only the first order made every node run fall through to the +# approximate line count, and that fallback is not conservative. A guard over a +# status enum — mutating the string `'FAILED'` — matches FAIL_PATTERN three +# times inside one AssertionError diff, so a single correct guard exited 3 with +# "narrow the guard, or narrow the mutation". The header says that exact false +# fire was tried once and rejected; it was still reachable through the fallback. +# The message compounded it, reporting "this runner printed no summary" about a +# runner that printed one this script could not read. COUNT="$(grep -oiE '[0-9]+ (tests? )?failed' "$LOG" | tail -1 | grep -oE '^[0-9]+' || true)" +if [ -z "$COUNT" ]; then + COUNT="$(grep -oiE '\bfail(ure)?s?[:= ]+[0-9]+' "$LOG" | tail -1 | grep -oE '[0-9]+$' || true)" +fi COUNTED_BY="the runner's summary" if [ -z "$COUNT" ]; then COUNT="$(grep -cE "$FAIL_PATTERN" "$LOG" || true)"