diff --git a/.githooks/commit-msg b/.githooks/commit-msg new file mode 100755 index 0000000..d02ebfc --- /dev/null +++ b/.githooks/commit-msg @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# +# Every commit says what kind of change it is, before it says anything else. +# +# ## Why +# +# Ninety-nine of this repository's commits already carry a conventional type — +# `feat:`, `fix:`, `docs:`, `chore:` — and ninety-six do not, including a run of +# recent ones written as bare sentences. That split is the problem: `git log +# --grep '^fix'` answers "what have we fixed" for half the history and quietly +# omits the other half, which is worse than having no convention at all, because +# the answer looks complete. +# +# So the type is required, and the vocabulary is closed. A closed list is the +# point — `feat`, `feature` and `feat!` as three spellings of one idea is how a +# convention stops being searchable. +# +# ## The vocabulary +# +# feat a new capability somebody can use +# fix a defect. The thing behaved wrongly and now does not +# ui appearance, layout, copy, or interaction, with no change in what +# the software knows or decides +# docs documentation only +# test tests only, with no change to what they test +# refactor same behaviour, different shape. If behaviour changed it is not this +# security hardening, boundaries, secret handling. Kept separate from `fix` +# on purpose: "what have we hardened" is a question worth being able +# to ask on its own, and it is the one an auditor asks first +# perf faster or lighter, same answers +# chore tooling, dependencies, releases. The bucket for work that is not +# about the product +# +# `harden`, `style` and `content` each appear once or twice in the history and +# are deliberately not here — they are `security`, `ui` and `docs` under other +# names, and a synonym is a hole in a closed list. +# +# ## Scope is optional, and lowercase +# +# fix(admin): ... the fifteen existing `admin` scopes, and `release`, +# `integrations`, `db` and the rest, all keep working. +# +# ## What it deliberately does not enforce +# +# Subject length. Thirty-five existing subjects run past 72 characters, several +# of them deliberately, and rejecting a commit for a well-written 80-character +# sentence would teach people to use `--no-verify` — which switches off the +# checks that actually matter. The type is the part a tool reads; the length is +# a matter of taste and stays that way. +# +# ## Escape hatch +# +# SKIP_GUARDS=1 git commit ... skips this too, and says so +# +# The same variable the pre-commit hook uses, because two switches for "I know +# what I am doing" is one more than anybody will remember. + +set -uo pipefail + +say() { printf '\033[1mcommit-msg:\033[0m %s\n' "$*" >&2; } + +message_file="$1" +subject=$(head -1 "$message_file") + +if [ -n "${SKIP_GUARDS:-}" ]; then + say "SKIP_GUARDS set — the commit type was NOT checked." + exit 0 +fi + +# Git writes these itself, or writes them on a human's behalf during a rebase. +# Rejecting them would break `git merge`, `git revert` and autosquash for a +# convention none of them ever agreed to. +case "$subject" in + "Merge "*|"Revert "*|"fixup!"*|"squash!"*|"amend!"*) + exit 0 + ;; +esac + +# A comment-only file is an aborted commit; git handles that itself. +if [ -z "${subject// /}" ]; then + exit 0 +fi + +TYPES="feat|fix|ui|docs|test|refactor|security|perf|chore" + +if printf '%s' "$subject" | grep -qE "^(${TYPES})(\([a-z0-9._-]+\))?!?: .+"; then + exit 0 +fi + +say "the subject line needs a type." +say "" +say " got: ${subject}" +say "" +say " expected: : e.g. fix: stop counting milestones as records" +say " (): ui(admin): mark cloud models on the picker" +say "" +say " types: feat a new capability" +say " fix a defect, now not" +say " ui appearance, layout, copy — no change to what it decides" +say " docs documentation only" +say " test tests only" +say " refactor same behaviour, different shape" +say " security hardening, boundaries, secrets" +say " perf faster or lighter, same answers" +say " chore tooling, dependencies, releases" +say "" +say " Your message is kept. Run 'git commit' again to edit it, or" +say " SKIP_GUARDS=1 git commit ... to bypass this loudly." + +exit 1 diff --git a/.githooks/post-commit b/.githooks/post-commit new file mode 100755 index 0000000..a7bbaa0 --- /dev/null +++ b/.githooks/post-commit @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# +# Every commit goes to Forgejo, without anybody having to remember the push. +# +# ## Why a hook rather than a habit +# +# The commit that added `pre-commit` sat unpushed for a day. Nothing was wrong +# with it; it just never got the second command. That is the whole failure this +# closes — work that exists on one laptop and nowhere else is work that is one +# disk away from gone, and it is invisible to anybody reading the tracker. +# +# ## It runs after the commit, and cannot undo one +# +# git ignores this hook's exit code, which is the right shape for the job: a +# network that is down must not cost somebody a commit they already made. So a +# failed push is **reported loudly and left in place** — the commit stands, the +# branch is simply still ahead, and the next commit tries again. +# +# What it will never do is force. A rejected push means the remote has something +# this checkout has not seen, and the fix for that is a human running a pull, not +# a hook overwriting the difference. +# +# ## When it deliberately stays out of the way +# +# - mid-rebase, mid-cherry-pick, mid-am: every step of a rebase fires this +# hook, and pushing an intermediate commit publishes a history that is about +# to be rewritten. Wait for the rebase to finish. +# - detached HEAD: there is no branch to push, and guessing one is worse than +# doing nothing. +# - no `origin`: a clone with no remote is a legitimate state, not an error. +# +# SKIP_PUSH=1 git commit ... commits without publishing, loudly +# +# The counterpart to `SKIP_GUARDS` in the pre-commit hook, and loud for the same +# reason: an exception that leaves no trace becomes a habit. + +set -uo pipefail + +cd "$(git rev-parse --show-toplevel)" || exit 0 + +say() { printf '\033[1mpost-commit:\033[0m %s\n' "$*" >&2; } + +if [ -n "${SKIP_PUSH:-}" ]; then + say "SKIP_PUSH set — this commit was NOT pushed." + exit 0 +fi + +git_dir=$(git rev-parse --git-dir) + +# A rebase, cherry-pick or `git am` fires this hook once per replayed commit. +# Those commits are provisional by definition. +for marker in rebase-merge rebase-apply CHERRY_PICK_HEAD; do + if [ -e "$git_dir/$marker" ]; then + say "a rebase or cherry-pick is in progress — not pushing until it finishes." + exit 0 + fi +done + +branch=$(git symbolic-ref --quiet --short HEAD) || { + say "detached HEAD — no branch to push." + exit 0 +} + +if ! git remote get-url origin >/dev/null 2>&1; then + say "no 'origin' remote — nothing to push to." + exit 0 +fi + +say "pushing $branch to origin…" + +# --porcelain keeps the output to one parseable line per ref rather than the +# usual banner, and the timeout is here because an unreachable SSH host +# otherwise hangs the terminal well past the point of being useful. +if timeout 60 git push --porcelain origin "$branch"; then + exit 0 +fi + +status=$? + +if [ "$status" -eq 124 ]; then + say "push timed out after 60s. The commit is safe locally; push when the" + say " remote is reachable." +else + say "push was refused. The commit is safe locally and the branch is now ahead." + say " If this is a non-fast-forward, pull and reconcile — this hook will" + say " not force, and should not." +fi + +# Deliberately zero. git ignores it either way, and returning non-zero here reads +# as though the commit failed when it did not. +exit 0 diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 0000000..ceffb10 --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +# +# The repo's own guards, before a commit rather than after it. +# +# ## Why this exists in the repository and not in .git/hooks +# +# `.git/hooks` is not versioned, so a hook living there protects exactly one +# checkout and silently protects nothing anywhere else. This directory is +# committed, and `core.hooksPath` points at it: +# +# git config core.hooksPath .githooks +# +# That one line is the only setup, and it is in the README beside the test +# command. +# +# ## ADAPTED FOR THIS PROJECT +# +# The template's version runs `npx tsc --noEmit` and then `npx vitest run`. This +# project has neither: no TypeScript, no tsconfig, no test runner, and every +# source file is plain .jsx. Installed unchanged, that hook refuses every commit +# on a project where nothing is wrong. +# +# So it runs the two things that exist: +# +# 1. The secret scan, on the staged diff. This is the reason the hook earns its +# place here at all. The Zoho WebToLead form identifiers were hardcoded in +# index.html and later src/pages/Contact.jsx and reached four commits of a +# then-PUBLIC repository before anyone noticed. A secret caught here costs a +# `git reset`; the same secret caught after a push costs a rotation, because +# deleting the line does not remove it from a commit that already exists. +# +# 2. `npm run build`, when source is staged. Three steps — client bundle, SSR +# bundle, then prerender across every route — and it fails on a broken +# import, a syntax error, or a page component that cannot render in Node. +# +# **The second is a build, not a test.** It proves the imports resolve. A form +# that posts to the wrong URL builds perfectly. Do not read a green hook as +# "the change works"; `docs/qa/ClaudeQACoverage.md` is honest about what is +# actually untested here, which is nearly everything. +# +# ## It warns about unstaged changes rather than failing on them +# +# Both commands run against the **working tree**, not against the index. So a +# clean run proves the working tree is good, which is only the same thing as the +# commit being good when nothing is left unstaged. +# +# That distinction matters in this checkout specifically: it is edited by more +# than one person at a time, and commits are staged by explicit path. A green +# hook beside three unstaged files has verified something other than what is +# about to be committed, and the honest thing is to say so rather than imply a +# guarantee that was not made. +# +# ## Escape hatch +# +# SKIP_GUARDS=1 git commit ... skips both, loudly +# git commit --no-verify ... skips the hook entirely, silently +# +# The first is preferred: it leaves a line in the terminal saying the guards did +# not run, which is the difference between a deliberate exception and a habit. + +set -uo pipefail + +cd "$(git rev-parse --show-toplevel)" || exit 1 + +say() { printf '\033[1mpre-commit:\033[0m %s\n' "$*" >&2; } + +if [ -n "${SKIP_GUARDS:-}" ]; then + say "SKIP_GUARDS set — typecheck and tests did NOT run for this commit." + exit 0 +fi + +# Nothing staged is not this hook's problem; git will refuse on its own. +if git diff --cached --quiet; then + exit 0 +fi + +# Only worth running when source or tests changed. A commit that touches docs or +# migrations alone still gets the typecheck, because a migration can be +# referenced from a test, but it should not wait on the whole suite. +# +# The four files after the `|` are not source, and they are here because of what +# `tests/version.test.ts` guards: package.json, the Dockerfile, and the two +# image pins in README.md and docker-compose.example.yml must all name the same +# version. With `src|tests` alone, a commit that hand-edits only the README pin +# — precisely the drift that left those pins six versions stale — would get the +# typecheck and skip the one test that would have caught it. The guard has to +# run on the commits it exists to police. +staged=$(git diff --cached --name-only) + +# What is worth a build. src/ and server/ are the obvious ones; the other four +# are here because each can break the build on its own — index.html is the Vite +# entry, vite.config.js owns the alias resolution and the build inputs, +# package.json can remove a dependency the bundle imports, and prerender.js runs +# as the third build step against every route. +touches_code=$(printf '%s\n' "$staged" \ + | grep -cE '^(src|server|scripts)/|^(package\.json|index\.html|vite\.config\.js|tailwind\.config\.js|postcss\.config\.js)$' || true) + +# Credentials, before the commit exists. FIRST, and cheap: it reads the staged +# diff only, and it runs on every commit including a docs-only one — a key +# pasted into a markdown file is still a key. +if [ -x scripts/secrets.sh ] || [ -f scripts/secrets.sh ]; then + say "secrets…" + if ! bash scripts/secrets.sh; then + say "possible credential in the staged changes — commit refused." + say "If it is real, ROTATE IT FIRST. Deleting the line does not remove it" + say "from a commit that already exists. If it is not, --allow the path or" + say "adjust the patterns in scripts/secrets.sh; do not silence the check." + exit 1 + fi +else + # Said out loud. A missing scanner and a clean scan look identical from here, + # and this hook has exactly two jobs. + say "WARNING: scripts/secrets.sh is missing — NOTHING was scanned for" + say " credentials. That is not a pass." +fi + +if [ "$touches_code" -gt 0 ]; then + say "build…" + + if ! command -v npm >/dev/null 2>&1; then + say "npm is not on PATH, so the build did not run — commit refused." + say "This hook will not pass a commit it could not check." + exit 1 + fi + + if ! npm run build; then + say "build failed — commit refused." + exit 1 + fi +else + say "no source staged — skipping the build." +fi + +# Said last so it is the thing still on screen when the editor opens. +if ! git diff --quiet; then + say "NOTE: unstaged changes are present. The guards ran against the working" + say " tree, so they did not verify this commit in isolation." +fi + +exit 0 diff --git a/package.json b/package.json index 2c9c1a9..f7de6a3 100644 --- a/package.json +++ b/package.json @@ -1,11 +1,12 @@ { "name": "queuenorth-website", "private": true, - "version": "0.8.3", + "version": "0.9.3", "type": "module", "scripts": { "dev": "concurrently \"vite\" \"node server/index.js\"", "build": "vite build && vite build --ssr src/entry-server.jsx --outDir dist-ssr && node scripts/prerender.js", + "verify": "bash scripts/verify.sh", "build:client": "vite build", "preview": "vite preview", "start": "node server/index.js", diff --git a/scripts/backup.sh b/scripts/backup.sh new file mode 100755 index 0000000..0b819cc --- /dev/null +++ b/scripts/backup.sh @@ -0,0 +1,1048 @@ +#!/usr/bin/env bash +# +# Write one database dump, prove the file is readable before trusting it, and +# keep the newest few. +# +# bash scripts/backup.sh # write one verified dump +# bash scripts/backup.sh --dry-run # show what it would write and delete +# bash scripts/backup.sh --no-prune # write, delete nothing +# BACKUP_KEEP=30 bash scripts/backup.sh # keep thirty instead of seven +# +# 0 3 * * * cd /srv/app && bash scripts/backup.sh # from cron, see below +# +# =========================================================================== +# TEMPLATE COPY — configure this before the first run +# =========================================================================== +# +# Copy to `scripts/backup.sh`, add `"backup": "bash scripts/backup.sh"` to +# package.json if this is a Node project, and set the two values in the +# CONFIGURATION block below. The script refuses to run until they are set: it +# has no defaults, deliberately. +# +# A BACKUP_DIR or BACKUP_NAME inherited from another project is not a cosmetic +# mistake. BACKUP_NAME is the filename prefix, and the filename prefix is what +# the retention rule globs on — so the wrong name writes this project's rows +# into that project's series and then deletes that project's oldest dumps to +# make room for them. Both halves are silent, the second is irreversible, and +# the directory afterwards looks exactly like a healthy backup series. Hence: +# no defaults, and a loud failure instead. +# +# Assumes: bash, coreutils, `awk`, `sed`, and the client tools for the engine +# (`pg_dump` and `pg_restore` for the default). No jq, no Node, nothing +# language-specific — unlike the release script beside it, this does not care +# what the project is written in. +# +# awk is named because it is not coreutils, a minimal container image can be +# without it, and it is what counts the tables — so it is checked at startup +# alongside the engine's tools. Left unchecked, its absence arrives as "the +# archive holds 0 tables", which is a sentence about the database and would send +# somebody looking at the wrong thing entirely. +# +# ## Why this exists +# +# A backup nobody has restored is a hypothesis, not a backup. +# +# The failure this is built around is not "the backup did not run". That one is +# loud: the directory is empty and somebody notices. The failure is a job that +# runs every night for a year and writes a file every night for a year, and the +# file is truncated, or is a dump of the wrong database, or is 400 bytes of +# `pg_dump: error:` because the password expired in March. Nothing about the +# directory listing distinguishes that from a working backup. The size column +# is plausible. The timestamps march forward. It is discovered on the one day +# it matters, by somebody who has already lost the database. +# +# So this script does not write a file and call it a backup. It writes the file +# under a temporary name, reads the file back with `pg_restore --list`, counts +# what is in it, refuses the whole run if the archive cannot be read, holds no +# tables, or CANNOT BE COUNTED, and only then renames it into place. +# +# The third of those is not padding. The counting step is one `awk` away from +# returning nothing, and a count that did not happen is not a count of zero: if +# the result were trusted unchecked, `[ "$TABLES" -gt 0 ]` would fail with +# "integer expression expected", `if` would read that failure as false, and an +# archive nobody had counted would be renamed into place and announced as +# verified — with retention then deleting real dumps to make room for it. +# +# ## What --list proves, and what it does not +# +# It proves the file is a real archive: the header parses, the table of +# contents is intact and complete, and the dump contains the tables you can see +# named in the output. Combined with `pg_dump` exiting zero, that eliminates +# the truncated file, the zero-byte file, the error message written where a +# dump should be, and the dump taken against an empty or wrong database. +# +# It does not decompress and check every data block, and it is not a restore. +# An archive can list cleanly and still fail to load — a broken large object, a +# circular constraint order, an extension that is not installed on the machine +# you are restoring onto. Nothing short of an actual restore finds those, which +# is why "the other half" below is not optional. +# +# ## Atomic, because the failure mode is a good backup destroyed by a bad one +# +# Every write goes to `.part` and is renamed only after it verifies. The +# rename is within one directory, so it is rename(2) and not a copy: the final +# name never exists holding half a file, and a reader — a sync job, a human, an +# offsite copy — cannot pick up a dump that is still being written. +# +# Writing directly to the final path would mean a dump that dies at 90% has +# already overwritten last night's, which was fine. That is worse than not +# running at all: it converts a working backup into a broken one and reports +# success while doing it. +# +# A dump that fails verification is KEPT, under its `.part` name, and the path +# is printed. It is evidence — usually the error message is inside it — and +# deleting evidence to keep the directory tidy is how the cause stays unknown. +# +# ## Retention deletes by explicit path, inside one directory, and nowhere else +# +# There is no `find -delete`, no `rm` with a glob, and no recursion. The +# candidate list is a glob of the exact filename shape this script writes, and +# every path is then checked one at a time: its parent directory, resolved with +# `pwd -P` rather than compared as text, must BE the backup directory; it must +# be a regular file and not a symlink; its name must match the pattern. Four +# checks, all of which have to pass, for each file, immediately before the `rm`. +# +# This is deliberately more than is needed for paths the script generated +# itself. The point is that no configuration value, no symlink planted in the +# directory, and no future edit to the glob can produce a deletion outside the +# backup directory — the guard does not trust the list it was handed. +# +# ## Credentials +# +# Connection details come from the environment and are never written here. +# PGHOST/PGPORT/PGUSER/PGDATABASE with a ~/.pgpass is the preferred form; +# DATABASE_URL works and is warned about, because a URL carries the password in +# it and reaches pg_dump as an argument, where `ps` shows it to every user on +# the host. +# +# Nothing prints a password or a connection URL — not to stdout, not to stderr, +# and above all not into a filename, where it would sit in the directory +# listing forever and be copied offsite with the dumps. Credentials in logs is +# the classic leak in backup scripts and it is usually introduced by an +# innocent-looking `say "dumping $DATABASE_URL"`. Do not add one. +# +# ## Changing the engine +# +# PostgreSQL is the supported default. Everything engine-specific is in the +# block marked ENGINE below — four functions and two variables — and nothing +# outside that block knows what a database is. Swapping in MySQL, SQLite or +# anything else is an edit to that block alone. +# +# One constraint carries over: `engine_verify` must READ THE FILE THAT WAS JUST +# WRITTEN. Re-querying the database, checking an exit code again, or trusting +# the file size verifies nothing about the artefact. Engines whose dump is +# plain SQL have no `--list` equivalent; for those the honest verification is a +# restore into a scratch database, and if that is too expensive to do on every +# run then the backup is unverified and the header of this script should be +# edited to stop claiming otherwise. +# +# ## What it deliberately does not do +# +# It does not copy the dump anywhere. A backup on the same disk as the database +# survives `DROP TABLE` and nothing else — not the disk, not the host, not the +# provider account. Getting these files onto different hardware is a separate, +# deliberate act (rsync, restic, object storage) for the same reason the +# release script does not deploy: two decisions, made one at a time. +# +# It does not schedule itself. BACKUP_KEEP is a count of files, not a period — +# seven of these is a week of nightly runs or seven hours of hourly ones. +# +# It does not restore. See below. +# +# ## The other half: a restore you have actually performed +# +# This script verifies the artefact. Only a restore verifies the backup, and a +# restore is also the only way to learn the number that matters during an +# incident, which is how long it takes. +# +# Do it on a schedule you write down — quarterly is a reasonable floor — into a +# scratch database, from the newest file this script produced, and record the +# duration and the command. Until that has happened once, the honest status of +# this directory is "dumps that appear to be readable", and the first restore +# will be attempted by somebody who has already lost the database. +# +# pg_restore --clean --if-exists --no-owner -d "$SCRATCH_URL" + +set -uo pipefail + +# Glob expansion and `sort` follow the collation locale, and a cron job's +# locale is not the one your shell has. C collation makes the ordering of the +# candidate list byte-order and therefore the same everywhere — the ordering +# decides which files retention calls "oldest", so it deciding differently +# under cron than under test would be discovered by deleting the wrong ones. +# LC_COLLATE only: LC_ALL=C would also change LC_CTYPE, and character handling +# is not something a backup script should be quietly redefining. +unset LC_ALL +export LC_COLLATE=C + +# An unmatched glob otherwise expands to the pattern itself, so an empty backup +# directory would produce one "file" literally named `myapp-[0-9][0-9]…` and +# hand it to the retention loop as a real candidate. +shopt -s nullglob + +say() { printf '\033[1mbackup:\033[0m %s\n' "$*" >&2; } +die() { printf '\033[1mbackup:\033[0m %s\n' "$*" >&2; exit 1; } + +# --------------------------------------------------------------------------- +# CONFIGURATION — set these two, then delete this banner. +# +# Both are empty on purpose. See the note at the top: an inherited BACKUP_NAME +# points the retention rule at another project's dumps, and an inherited +# BACKUP_DIR puts this project's rows where that project's offsite copy will +# collect them. +# --------------------------------------------------------------------------- + +# Absolute path to the directory holding the dumps. Must already exist, must be +# outside the repository, and should be readable only by the user running this +# — a dump is every row in the database in one file. +BACKUP_DIR="${BACKUP_DIR:-$HOME/backups/queue-north-website}" + +# Filename prefix identifying this project's series, e.g. `acme-orders`. +# Letters, digits, dot, underscore and hyphen only. This is what retention +# globs on; see the header for why it is not derived from the database name. +BACKUP_NAME="${BACKUP_NAME:-queuenorth-leads}" + +# --------------------------------------------------------------------------- +# Tunables. These have defaults because none of them names anything belonging +# to a particular project. +# --------------------------------------------------------------------------- + +# How many dumps survive, newest first. Seven is a week of nightly runs. +# +# Two is the floor enforced below. Keeping one means every run destroys the +# only copy you have in order to make a copy that has not yet been shown to be +# worth anything — and "worth something" here is more than readable, since a +# dump of an empty or wrong database verifies perfectly. Two means the newest +# dump can turn out to be worthless without that being the end of it. +KEEP="${BACKUP_KEEP:-7}" + +# Refuse a dump containing fewer tables than this. One is the default because +# an archive with no tables in it is not a backup of anything, and the way you +# get one is `pg_dump` connecting somewhere you did not intend — see the +# PGDATABASE note below, where the accident produces a valid, tiny, entirely +# empty archive. Set 0 only if this project's database genuinely has no tables, +# in which case there is nothing here to protect. +MIN_TABLES="${BACKUP_MIN_TABLES:-1}" + +# Warn when the new dump is smaller than this percentage of the previous one. +# A dump that halves overnight is usually a partial dump, a lost permission on +# some schema, or the wrong database — none of which fail loudly on their own. +# +# A warning and never a refusal: a legitimate large delete would otherwise +# block every backup from that moment on, which is a way of losing data by +# being careful about data. +SHRINK_PCT="${BACKUP_SHRINK_PCT:-50}" + +# =========================================================================== +# ENGINE — SQLite, inside the running container on the deploy host. +# +# The template ships this block as PostgreSQL and says to replace this and +# nothing else. This is that replacement. +# +# ## Where the database actually is +# +# Not on this machine. `/app/db/queuenorth.db` lives in the Docker named volume +# `qn-website-dev_queuenorth-db` on nebula, and the only copy of every lead and +# support request the site has ever taken is in it. +# +# ## Why `.backup()` and not `cp` +# +# SQLite is a file, which makes copying it look trivial and makes doing it +# wrong silent. A plain `cp` of a live database can capture a torn page mid +# write, and the result opens, reads, and is corrupt in a way nothing announces +# until the row you need is the missing one. The online backup API takes a +# consistent snapshot of a database that is being written to, which is exactly +# the situation here — the site is live and took writes today. +# +# There is no `sqlite3` binary in the container (node:20-alpine), so the +# snapshot is taken by the `better-sqlite3` the application itself already +# depends on. Its `.backup()` is the online API. `sqlite3` IS present on nebula +# and on this machine, and that is what verifies the result afterwards. +# +# ## The rule this block must not break +# +# The script's contract is that nothing is renamed into place until it has been +# read back and counted. That is kept: `engine_dump` writes a temporary name, +# `engine_verify` runs `PRAGMA integrity_check` and lists the tables, and +# `engine_summarise` counts them. A snapshot that cannot be read is refused, +# and so is one with no tables in it. +# =========================================================================== + +DUMP_SUFFIX=".sqlite" + +# Overridable so this can be pointed at a staging container, or at a local file +# for a rehearsal, without editing the script. +BACKUP_SSH_HOST="${BACKUP_SSH_HOST:-nebula}" +BACKUP_CONTAINER="${BACKUP_CONTAINER:-qn-website-dev}" +BACKUP_DB_PATH="${BACKUP_DB_PATH:-/app/db/queuenorth.db}" + +# `docker` is needed on the REMOTE host, not here, so it cannot be checked by +# the tool test — which only looks at this machine's PATH. ssh and sqlite3 can +# be, and are: sqlite3 is what verifies, and its absence would otherwise arrive +# as "the snapshot holds 0 tables", a sentence about the database that would +# send somebody looking at entirely the wrong thing. +engine_tools() { printf 'ssh sqlite3'; } + +engine_dump() { + local out="$1" + + # Written to a temporary path INSIDE the container first, then copied out. + # `docker cp` of a live database file would have exactly the torn-page + # problem the online API exists to avoid, so the order matters: snapshot + # first, copy the snapshot second. + local remote_tmp="/tmp/qn-backup-$$.sqlite" + + # The node one-liner is passed through ssh and docker exec, so it is + # single-quoted here and double-quoted there. It opens the live database + # read-only and asks SQLite itself to produce the snapshot. + if ! ssh -o BatchMode=yes "$BACKUP_SSH_HOST" \ + "docker exec '$BACKUP_CONTAINER' node -e \" + const Database = require('better-sqlite3'); + const db = new Database('$BACKUP_DB_PATH', { readonly: true }); + db.backup('$remote_tmp') + .then(() => { db.close(); process.exit(0); }) + .catch(e => { console.error(e.message); process.exit(1); }); + \"" >&2; then + ssh -o BatchMode=yes "$BACKUP_SSH_HOST" "docker exec '$BACKUP_CONTAINER' rm -f '$remote_tmp'" >/dev/null 2>&1 + return 1 + fi + + # Out of the container, onto the host, then down to here. Two hops because + # `docker cp` cannot write to a remote path and `scp` cannot read from inside + # a container. + local host_tmp="/tmp/qn-backup-$$.sqlite" + if ! ssh -o BatchMode=yes "$BACKUP_SSH_HOST" \ + "docker cp '$BACKUP_CONTAINER:$remote_tmp' '$host_tmp'" >&2; then + ssh -o BatchMode=yes "$BACKUP_SSH_HOST" "docker exec '$BACKUP_CONTAINER' rm -f '$remote_tmp'" >/dev/null 2>&1 + return 1 + fi + + # `cat` over ssh rather than scp: the caller passes an exact output path and + # this keeps the writer and the exit status in one pipeline we control. The + # redirect is on this side, so a failed transfer leaves a short file — which + # is precisely what engine_verify below is for. + if ! ssh -o BatchMode=yes "$BACKUP_SSH_HOST" "cat '$host_tmp'" >"$out"; then + ssh -o BatchMode=yes "$BACKUP_SSH_HOST" \ + "rm -f '$host_tmp'; docker exec '$BACKUP_CONTAINER' rm -f '$remote_tmp'" >/dev/null 2>&1 + return 1 + fi + + # Both temporaries, on both sides, whatever happened above. A snapshot of the + # whole lead table left in /tmp on a shared host is the kind of tidy-up that + # only looks optional. + ssh -o BatchMode=yes "$BACKUP_SSH_HOST" \ + "rm -f '$host_tmp'; docker exec '$BACKUP_CONTAINER' rm -f '$remote_tmp'" >/dev/null 2>&1 + + return 0 +} + +# Reads the snapshot back and writes what it found to $2; anything sqlite3 +# complains about goes to $3, kept separate so it cannot be counted as content. +# +# integrity_check FIRST, and its result is what decides. A truncated SQLite +# file will happily answer some queries — the header and the first pages are +# intact — so "the table list came back" is not evidence. `integrity_check` +# walks every page and says `ok` or says why not. +engine_verify() { + local dump="$1" toc="$2" err="$3" + local integrity + + integrity=$(sqlite3 "file:$dump?mode=ro" 'PRAGMA integrity_check;' 2>"$err") || return 1 + if [ "$integrity" != "ok" ]; then + printf 'integrity_check did not return ok:\n%s\n' "$integrity" >>"$err" + return 1 + fi + + # One line per table, plus its row count, so the summariser can count tables + # and a human reading the log can see the lead count move. Internal + # sqlite_% tables are excluded — they are not this project's data. + sqlite3 "file:$dump?mode=ro" \ + "SELECT 'TABLE ' || name FROM sqlite_master + WHERE type='table' AND name NOT LIKE 'sqlite\_%' ESCAPE '\' + ORDER BY name;" >"$toc" 2>>"$err" || return 1 + + return 0 +} + +# Prints "". +# +# SQLite has no schema-only/data-only split, so a table and its data are the +# same object and both columns are the same number. Printing it twice rather +# than printing a zero is deliberate: the caller treats a zero in either column +# as "this dump has no data", which would be false here and would refuse every +# backup this script ever takes. +engine_summarise() { + awk '$1 == "TABLE" { t++ } END { printf "%d %d\n", t + 0, t + 0 }' "$1" +} + +# =========================================================================== +# End of ENGINE block. +# =========================================================================== + +DRY_RUN="" +NO_PRUNE="" + +for arg in "$@"; do + case "$arg" in + --dry-run) DRY_RUN="yes" ;; + --no-prune) NO_PRUNE="yes" ;; + # No positional arguments exist, so one is a mistyped flag or a path + # somebody expected this to accept. Either way, acting on the rest of the + # command line as though it were fine is how a backup goes somewhere else. + *) die "unknown argument '$arg'. Usage: bash scripts/backup.sh [--dry-run] [--no-prune]" ;; + esac +done + +# --------------------------------------------------------------------------- +# Refuse to run half-configured. Everything checkable without a side effect, +# checked before the first side effect, and named one at a time so the message +# says which value is wrong rather than "configuration error". +# --------------------------------------------------------------------------- + +[ -n "$BACKUP_DIR" ] || die "set BACKUP_DIR — the directory to write dumps into. See the CONFIGURATION block." +[ -n "$BACKUP_NAME" ] || die "set BACKUP_NAME — the filename prefix for this project's dumps. See the CONFIGURATION block." + +# A relative path resolves against the working directory, and cron's working +# directory is not yours. The dumps would land somewhere else, and retention — +# globbing that same relative path — would find that other directory empty, +# report "nothing to remove", and let the real series grow until the disk +# filled. Both halves look healthy in isolation. +case "$BACKUP_DIR" in + /*) : ;; + *) die "BACKUP_DIR ('$BACKUP_DIR') must be an absolute path — a relative one + points somewhere different under cron than it does in your shell." ;; +esac + +# The glob and the delete guard are built from this. A name containing `*`, `?` +# or `/` would widen the candidate list past this project's dumps, which is the +# one thing the guard cannot make safe by checking paths. +case "$BACKUP_NAME" in + *[!A-Za-z0-9._-]*) die "BACKUP_NAME ('$BACKUP_NAME') may contain only letters, digits, dot, + underscore and hyphen — it is used as a filename glob." ;; +esac + +case "$KEEP" in + ''|*[!0-9]*) die "BACKUP_KEEP must be a whole number, got '$KEEP'." ;; +esac + +# Refused rather than clamped: a caller who typed 1 meant something, and it was +# not "destroy the only copy before checking the new one is worth having". +[ "$KEEP" -ge 2 ] || die "BACKUP_KEEP must be at least 2 — see the note above KEEP in this script." + +case "$MIN_TABLES" in + ''|*[!0-9]*) die "BACKUP_MIN_TABLES must be a whole number, got '$MIN_TABLES'." ;; +esac + +case "$SHRINK_PCT" in + ''|*[!0-9]*) die "BACKUP_SHRINK_PCT must be a whole number, got '$SHRINK_PCT'." ;; +esac + +for tool in $(engine_tools); do + command -v "$tool" >/dev/null 2>&1 \ + || die "$tool is not on PATH. Install the client tools for this engine + (Debian/Ubuntu: postgresql-client) and run this again." +done + +# awk is checked separately because it is not part of the engine and not part of +# coreutils: it is what turns the table of contents into a number, and cron's +# PATH is not your shell's. Without this check its absence surfaces further down +# as "the archive holds 0 table(s)" — a measurement, about the database, that +# nothing measured. +command -v awk >/dev/null 2>&1 \ + || die "awk is not on PATH. It is what counts what is inside the archive, and + without it this script cannot tell a good dump from an empty one. + Under cron, PATH is not the PATH your shell has." + +[ -d "$BACKUP_DIR" ] || die "BACKUP_DIR ('$BACKUP_DIR') does not exist. + Create it deliberately: mkdir -p -m 700 '$BACKUP_DIR' + It is not created here on purpose — a typo in the path is otherwise + indistinguishable from a first run, and the typo'd directory would fill + with a complete, correct-looking series that nobody restores from." + +# Resolved once, here, and used for every path comparison afterwards. Comparing +# the configured string instead would let `/srv/backups/../../etc` pass a +# prefix test while being nowhere near the backup directory. +BACKUP_DIR_REAL=$(cd "$BACKUP_DIR" && pwd -P) || die "cannot resolve BACKUP_DIR ('$BACKUP_DIR')." + +# Ahead of the writability check on purpose: for an unprivileged user / is not +# writable, so leaving this until later answers "permission denied" to somebody +# who is one `sudo` away from asking the same question again and being told yes. +[ "$BACKUP_DIR_REAL" != "/" ] || die "BACKUP_DIR resolves to / — retention would treat the root of the + filesystem as a directory of expendable files." + +[ -w "$BACKUP_DIR" ] || die "BACKUP_DIR ('$BACKUP_DIR') is not writable by $(id -un)." + +# A dump inside the checkout is one `git add -A` from being committed, and one +# push from being public, with every row in the database in it. It is also +# copied by every clone and every deploy from then on. The engine that made it +# does not put it back once that happens. +repo_root=$(git rev-parse --show-toplevel 2>/dev/null) || repo_root="" + +if [ -n "$repo_root" ] && repo_real=$(cd "$repo_root" && pwd -P); then + case "$BACKUP_DIR_REAL/" in + "$repo_real"/*) + die "BACKUP_DIR ('$BACKUP_DIR_REAL') is inside the repository at $repo_real. + Put it outside the checkout — a dump in a working tree is one 'git add -A' + away from being committed and pushed." ;; + esac +fi + +# Permissions are reported, not corrected. Changing the mode of a directory the +# operator created — which may be a mount, or shared with a sync agent — is a +# decision this script does not get to make on their behalf; failing to mention +# that every row in the database is world-readable would be worse. +dir_mode=$(ls -ld "$BACKUP_DIR_REAL" 2>/dev/null | cut -c5-10) +case "$dir_mode" in + ------) : ;; + '') say "note: could not read the permissions of $BACKUP_DIR_REAL." ;; + *) say "WARNING: $BACKUP_DIR_REAL is accessible to group or other (mode bits" + say " '$dir_mode'). A dump is every row in the database in one file." + say " chmod 700 it unless something else is meant to read these." ;; +esac + +# --------------------------------------------------------------------------- +# Connection. +# +# ENGINE-SPECIFIC, and sitting outside the ENGINE block above because the +# template's own copy did. Its header says everything engine-specific lives in +# that one block; this precondition check is the exception, and it is noted here +# rather than quietly worked around so the next person changing engines knows +# there are two places, not one. +# +# The Postgres version of this refused to run when neither PGDATABASE nor +# DATABASE_URL was set, because pg_dump with no target connects to a database +# named after the current user and produces a valid, empty archive. The SQLite +# equivalent of that accident is different but not better: an `ssh` to the wrong +# host, or a container name that no longer exists, and this script's job is to +# tell those apart from an empty database before it writes anything. +# +# So all three coordinates are asserted, and the container is asked whether it +# is actually running. MIN_TABLES below is the second net; this is the first. +# --------------------------------------------------------------------------- + +[ -n "$BACKUP_SSH_HOST" ] || die "BACKUP_SSH_HOST is empty. It names the host the database lives on — nebula, unless you are rehearsing against something else." +[ -n "$BACKUP_CONTAINER" ] || die "BACKUP_CONTAINER is empty. It names the container holding the database." +[ -n "$BACKUP_DB_PATH" ] || die "BACKUP_DB_PATH is empty. It is the path to the database file INSIDE the container." + +# Asked, not assumed. A stopped container answers this differently from a +# missing one, and both answer differently from a host that will not accept the +# connection — three problems with three fixes, which a single "backup failed" +# would flatten into one. +if ! remote_state=$(ssh -o BatchMode=yes -o ConnectTimeout=10 "$BACKUP_SSH_HOST" \ + "docker inspect -f '{{.State.Status}}' '$BACKUP_CONTAINER' 2>/dev/null" 2>/dev/null); then + die "could not reach '$BACKUP_SSH_HOST' over ssh, or docker there would not answer. + Nothing was backed up, and nothing is known about the database — this is + not 'no changes to back up'." +fi + +case "$remote_state" in + running) ;; + "") die "no container named '$BACKUP_CONTAINER' on '$BACKUP_SSH_HOST'. Check the name + before assuming the worst — a renamed container and a deleted one look + identical from here." ;; + *) die "container '$BACKUP_CONTAINER' on '$BACKUP_SSH_HOST' is '$remote_state', not running. + The online backup API needs the process alive. Start it, or take the + snapshot from the volume directly while nothing is writing to it." ;; +esac + +TARGET_DESC="${BACKUP_DB_PATH} in ${BACKUP_CONTAINER} on ${BACKUP_SSH_HOST}" + +# --------------------------------------------------------------------------- +# Names. Built here so the dry run and the real run cannot describe different +# files. +# +# UTC, always: local time repeats an hour every autumn, which puts two dumps +# out of order by name in the one direction that matters — retention reads that +# order to decide what is oldest. +# +# Nothing derived from the connection goes into the filename. A URL in a name +# sits in the directory listing forever and is copied offsite with the dumps. +# --------------------------------------------------------------------------- + +STAMP_GLOB='[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]T[0-9][0-9][0-9][0-9][0-9][0-9]Z' +STAMP=$(date -u +%Y%m%dT%H%M%SZ) || die "cannot read the clock." + +FINAL="${BACKUP_DIR_REAL}/${BACKUP_NAME}-${STAMP}${DUMP_SUFFIX}" +PART="${FINAL}.part" + +# Only a second run inside the same second reaches either of these, and the +# alternative in both cases is destroying a file this script promised to keep: +# a dump that has already been verified, or the .part evidence of a run that +# failed a moment ago and is about to be looked at. +[ ! -e "$FINAL" ] || die "$FINAL already exists. Refusing to overwrite a verified dump." +[ ! -e "$PART" ] || die "$PART already exists — it is the unverified remains of a run that + failed within this same second. Inspect or remove it before running again." + +human_size() { + local b="${1:-}" + + # An unmeasurable file is reported as unmeasured. Printing "0 B" for a file + # whose size could not be read is the same lie as printing "nothing to + # delete" for a directory that could not be listed. + case "$b" in + ''|*[!0-9]*) printf 'unknown size'; return 0 ;; + esac + + # Integer arithmetic rather than `du -h`, whose output format and rounding + # differ between GNU and BSD and which reports blocks allocated rather than + # bytes written. + if [ "$b" -lt 1024 ]; then printf '%s B' "$b" + elif [ "$b" -lt 1048576 ]; then printf '%s.%s KiB' "$((b / 1024))" "$((b * 10 / 1024 % 10))" + elif [ "$b" -lt 1073741824 ]; then printf '%s.%s MiB' "$((b / 1048576))" "$((b * 10 / 1048576 % 10))" + else printf '%s.%s GiB' "$((b / 1073741824))" "$((b * 10 / 1073741824 % 10))" + fi +} + +file_bytes() { + # `wc -c` is portable where `stat` is not: GNU wants -c%s and BSD wants -f%z. + # BSD pads the number with spaces, hence the tr. + # + # 2>/dev/null comes BEFORE the input redirect on purpose. Redirections are + # applied left to right, so with the other order the shell's own "no such + # file" message for a failed open is written before stderr has been silenced — + # a raw `backup.sh: line N: …` in the middle of otherwise formatted output. + # Callers read the empty result, which is what "unmeasurable" means here. + wc -c 2>/dev/null <"$1" | tr -d ' \n' +} + +# Every existing dump of this series, oldest first. Pathname expansion sorts, +# and LC_COLLATE=C above makes that sort byte-order, so no external sort is +# involved and no filename has to survive a round trip through word splitting. +# +# The glob is the exact shape this script writes — prefix, UTC stamp, suffix. +# Files that merely live in the directory are not candidates for anything: a +# `.part` from a failed run, a dump somebody copied in by hand, and another +# project's series all fail to match and are never counted or deleted. +list_dumps() { + printf '%s\n' "${BACKUP_DIR_REAL}/${BACKUP_NAME}-"$STAMP_GLOB"$DUMP_SUFFIX" +} + +# --------------------------------------------------------------------------- +# The delete guard. +# +# Called immediately before every `rm`, on the path about to be removed, and +# every check has to pass. It exists so that no configuration value, no symlink +# planted in the backup directory and no later edit to the glob can produce a +# deletion outside BACKUP_DIR_REAL — it does not trust the list it was given. +# --------------------------------------------------------------------------- +deletable() { + local path="$1" parent base + + case "$path" in + */*) parent="${path%/*}"; base="${path##*/}" ;; + *) return 1 ;; + esac + + # Resolved, not string-compared: `.` and `..` components make a text prefix + # test agree with a path that is somewhere else entirely. + parent=$(cd "$parent" 2>/dev/null && pwd -P) || return 1 + [ "$parent" = "$BACKUP_DIR_REAL" ] || return 1 + + # -f follows symlinks, so a link named like a dump would pass every other + # check while pointing at a file the checks were never made about. + [ -f "$path" ] || return 1 + [ ! -L "$path" ] || return 1 + + case "$base" in + "$BACKUP_NAME"-$STAMP_GLOB"$DUMP_SUFFIX") return 0 ;; + *) return 1 ;; + esac +} + +# --------------------------------------------------------------------------- +# Retention. +# +# Last, and never fatal: by the time this runs the dump has been written and +# verified, and housekeeping must not be able to report that a backup which +# succeeded failed. Everything it declines to do, it says out loud instead. +# --------------------------------------------------------------------------- +prune() { + [ -z "$NO_PRUNE" ] || { say "retention: skipped (--no-prune)."; return 0; } + + if [ ! -r "$BACKUP_DIR_REAL" ] || [ ! -x "$BACKUP_DIR_REAL" ]; then + say "retention: SKIPPED — cannot list $BACKUP_DIR_REAL." + say " That is 'I could not look', which is not the same answer as" + say " 'there is nothing to delete'. Nothing was removed." + return 0 + fi + + local -a candidates=() + local line + while IFS= read -r line; do + [ -n "$line" ] && candidates+=("$line") + done </dev/null | awk 'NR == 2 { print $4 }') + +case "$free_kb" in + ''|*[!0-9]*) + say "note: free space on $BACKUP_DIR_REAL could not be read — not checked." ;; + *) + if [ -z "$PREV_MEASURED" ] || [ "$PREV_BYTES" -eq 0 ]; then + say "note: $(human_size $((free_kb * 1024))) free; no measured previous dump to compare" + say " against, so whether that is enough is unknown." + elif [ $((free_kb * 1024)) -lt $((PREV_BYTES * 2)) ]; then + say "WARNING: $(human_size $((free_kb * 1024))) free, and the last dump was" + say " $(human_size "$PREV_BYTES"). This run may not fit." + fi ;; +esac + +# --------------------------------------------------------------------------- +# --dry-run stops here, having touched nothing. +# --------------------------------------------------------------------------- + +if [ -n "$DRY_RUN" ]; then + say "--dry-run: nothing was written or deleted. It would have:" + say " dumped ${TARGET_DESC}" + say " written ${PART}" + say " verified it with: sqlite3 ${PART} 'PRAGMA integrity_check' + a table list" + say " renamed it to ${FINAL}" + say "" + say "and then applied retention, shown here for real because it deletes:" + prune + report_parts + exit 0 +fi + +# --------------------------------------------------------------------------- +# One run at a time. +# +# mkdir is atomic on every filesystem this will meet, which flock and lock +# files are not. Two overlapping runs — a nightly job on a database that now +# takes longer than a day to dump — double the load on the server and race each +# other's retention pass. +# +# A stale lock stops backups, so this exits NON-ZERO and names the fix rather +# than skipping quietly. A skipped backup that reports success is the exact +# failure this whole script exists to prevent, and a lock is not allowed to +# reintroduce it. +# --------------------------------------------------------------------------- + +LOCK_DIR="${BACKUP_DIR_REAL}/.${BACKUP_NAME}.lock" +LOCK_HELD="" + +# mkdir's error is captured rather than discarded, because "the lock exists" and +# "the lock could not be created" are different failures with different fixes, +# and only the first one is about concurrency. A read-only mount, a full +# filesystem, or a missing mkdir under cron's PATH all make mkdir fail too; the +# directory is what distinguishes them, so it is looked at instead of assumed. +LOCK_ERR="" + +if ! LOCK_ERR=$(mkdir "$LOCK_DIR" 2>&1); then + if [ -d "$LOCK_DIR" ]; then + say "another run holds the lock at $LOCK_DIR" + [ ! -r "$LOCK_DIR/owner" ] || say " owner: $(cat "$LOCK_DIR/owner" 2>/dev/null)" + die "if no backup is running, that lock is stale: rmdir '$LOCK_DIR'" + fi + + # No lock directory, so nothing is holding anything. Saying otherwise here + # would send somebody to rmdir a path that does not exist, watch that change + # nothing, and go looking for a second backup process — while the real cause + # sits unread in mkdir's own message and every night's backup stays dead. + say "could not create the lock directory $LOCK_DIR, and no lock is present —" + say " so this is NOT another run. The directory could not be made at all." + [ -z "$LOCK_ERR" ] || say " mkdir said: $LOCK_ERR" + die "check that $BACKUP_DIR_REAL is on a writable, non-full filesystem." +fi + +LOCK_HELD="yes" +WORK="" + +# Armed in the same breath as the lock, not after the next command. Anything +# that exits between acquiring the lock and installing this trap leaves a stale +# lock behind, and a stale lock stops every future run — the trap tolerating an +# empty WORK is much cheaper than that. $WORK is quoted-empty-safe because rm +# is guarded on it. +trap '[ -z "$LOCK_HELD" ] || rm -rf "$LOCK_DIR"; [ -z "$WORK" ] || rm -rf "$WORK"' EXIT + +printf 'pid %s on %s since %s\n' "$$" "$(hostname 2>/dev/null || echo unknown)" \ + "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >"$LOCK_DIR/owner" 2>/dev/null + +# The .part is deliberately NOT cleaned up by that trap. It is evidence when +# verification fails, and the failure paths below print where it is. +WORK=$(mktemp -d) || die "cannot create a temporary directory" + +# --------------------------------------------------------------------------- +# Dump. +# --------------------------------------------------------------------------- + +say "dumping ${TARGET_DESC} -> ${PART}" + +# umask in a subshell so the archive is created 0600 from its first byte. Doing +# it with a chmod afterwards leaves a window in which every row in the database +# is readable by every user on the host, and that window is the whole dump. +if ! ( umask 077; engine_dump "$PART" ); then + say "the snapshot failed (the error from ssh/docker/better-sqlite3 is above)." + say "Nothing was renamed into" + say " place and nothing was deleted." + if [ -n "$PREV" ]; then + say " The newest verified dump is still $PREV" + else + say " There is no previously verified dump in $BACKUP_DIR_REAL." + fi + say " The partial file is kept for inspection: $PART" + exit 1 +fi + +[ -f "$PART" ] || die "the snapshot reported success but $PART does not exist. Nothing was renamed." + +BYTES=$(file_bytes "$PART") +[ -n "$BYTES" ] || die "cannot measure $PART. It has NOT been renamed into place." + +# --------------------------------------------------------------------------- +# Verify, before the file is allowed to become the backup. +# --------------------------------------------------------------------------- + +say "verifying ${PART}…" + +LISTING="$WORK/toc" +VERR="$WORK/toc.err" + +if ! engine_verify "$PART" "$LISTING" "$VERR"; then + say "VERIFICATION FAILED — the file cannot be read back as an archive." + say " $(human_size "$BYTES") written. The reader said:" + sed -n '1,10p' "$VERR" >&2 2>/dev/null + say "" + say " It has NOT been renamed into place and NOTHING was deleted." + say " Kept for inspection: $PART" + if [ -n "$PREV" ]; then + say " The newest verified dump is still $PREV" + else + say " There is no previously verified dump in $BACKUP_DIR_REAL." + fi + exit 1 +fi + +SUMMARY=$(engine_summarise "$LISTING") || SUMMARY="" + +read -r TABLES DATA_SECTIONS </dev/null 2>&1; then + printf '\033[1mcheck-env:\033[0m this bash has no `compgen`, so the environment cannot be\n' >&2 + printf '\033[1mcheck-env:\033[0m read reliably and nothing was checked. That is not a pass.\n' >&2 + exit 2 +fi + +_CE_ENV_NAMES=() +_CE_ENV_VALS=() +while IFS= read -r _CE_N; do + [ -n "$_CE_N" ] || continue + # `export FOO` with no value is exported-but-unset: it is not in a child's + # environment either, so it is absence rather than an empty value. `+x` + # keeps set-to-empty, which `docker run -e FOO` produces and which is a + # different bug from unset. + [ -n "${!_CE_N+x}" ] || continue + _CE_ENV_NAMES+=("$_CE_N") + _CE_ENV_VALS+=("${!_CE_N}") +done <<<"$(compgen -e)" +unset _CE_N + +# --------------------------------------------------------------------------- +# SPEC — the variables this project needs. One line each, then delete this +# banner. Adding a variable later is a one-line edit and nothing else. +# +# "NAME|required|kind|what breaks when it is missing or wrong" +# +# The description is not documentation. It is the sentence printed next to the +# failure at 03:00, so write the CONSEQUENCE ("sessions cannot be signed, every +# login 500s") rather than a restatement of the name ("the JWT secret"). +# +# required | optional — spelled exactly. A typo is refused rather than read as +# "optional", because silently downgrading a required +# variable is the one mistake this script cannot survive. +# +# Kinds: +# nonempty any non-empty string; for values whose shape is +# genuinely unknowable. Not a lazy default — it is the +# honest one, and it still catches unset and empty. +# int a whole number +# int:MIN..MAX a whole number within bounds, inclusive +# port 1-65535 +# url scheme://host[...]; rejects a bare `host:5432` +# path a filesystem path; existence is reported only when the +# source is this machine's own environment +# one-of:a,b,c exactly one of these, compared case-sensitively +# secret-min-length:N at least N characters; the value is never printed, and +# it is checked against well-known placeholders +# +# Examples of the FORM. Delete them — they are shapes, not your variables: +# +# "DATABASE_URL|required|url|nothing can read or write; every request 500s" +# "PORT|optional|port|the server binds its own default, which the proxy is not pointed at" +# "NODE_ENV|required|one-of:development,production,test|the wrong branch of every environment check" +# "SESSION_SECRET|required|secret-min-length:32|sessions are forgeable" +# "UPLOAD_DIR|optional|path|uploads land somewhere nothing serves and nothing backs up" +# --------------------------------------------------------------------------- + +SPEC=( + # --- Always. Wrong here and the site is wrong for everybody. --- + "NODE_ENV|required|one-of:development,production,test|CSP relaxes in dev and the HTTP-to-HTTPS redirect only fires in production; the wrong value ships dev CSP to the public origin, or redirects a local dev server into a loop" + "SERVER_PORT|required|port|Express binds 3001 by default and the reverse proxy in front of nebula is pointed at whatever this says; a mismatch is a site that is running and unreachable" + "CORS_ORIGIN|required|url|defaults to https://queuenorth.com, which is NOT this deployment. Wrong here and every form submission from qn.isnull.dev is blocked by the browser while the server logs nothing wrong" + "LOG_LEVEL|optional|one-of:error,warn,info,debug|defaults to info. At error you lose the request log, which is the only record of traffic this project keeps" + "RATE_LIMIT_PER_MINUTE|optional|int:1..1000|defaults to 5 per IP across all of /api. Unset is fine; set to something huge and the two POST endpoints are open to a bot overnight" + + # --- Anti-abuse. The only thing between two open POST endpoints and a bot. --- + "RECAPTCHA_ENABLED|required|one-of:true,false|anything other than the literal string true disables verification silently. Declared required so that switching it off is a decision somebody typed, not a variable somebody forgot" + "RECAPTCHA_SECRET_KEY|optional|secret-min-length:32|required in practice whenever RECAPTCHA_ENABLED=true — without it every verification fails open and the forms have no protection at all. Optional here only because this script cannot express the conditional" + "RECAPTCHA_MIN_SCORE|optional|nonempty|defaults to 0.5. A float, so it is not range-checked here; set it to 0 and every bot passes" + "VITE_RECAPTCHA_SITE_KEY|optional|nonempty|BUILD TIME, not runtime — Vite inlines it into dist/. Absent at build and the widget never loads, and no restart fixes it. It must be the SITE key; the secret key here ships the secret to every visitor" + + # --- CRM forwarding. Which half matters depends on the mode. --- + "ZOHO_FORWARDING_MODE|required|one-of:webtolead,api|selects which of the two blocks below is actually read. Production is webtolead; the api path is a configured standby. Wrong value and leads are written to SQLite and forwarded nowhere, silently, because the forward is fire-and-forget by design" + + "ZOHO_WEBTOLEAD_ENABLED|optional|one-of:true,false|the on switch for the mode production actually uses. False means no lead reaches the CRM and the visitor still sees success" + "ZOHO_WEBTOLEAD_URL|optional|url|defaults to the Zoho form endpoint. A wrong host means every forward posts into the void" + "ZOHO_WEBTOLEAD_XNQSJSDP|optional|secret-min-length:32|the form identifier. Missing or wrong and Zoho rejects the post; leads survive in SQLite and never appear in the CRM" + "ZOHO_WEBTOLEAD_XMIWTLD|optional|secret-min-length:32|the second form identifier, same consequence" + "ZOHO_WEBTOLEAD_ACTION_TYPE|optional|nonempty|base64 for the target module. Defaults to Leads; wrong and records land in the wrong module" + "ZOHO_WEBTOLEAD_RETURN_URL|optional|nonempty|the literal string null is correct here — server-to-server, nothing redirects. Not url-checked for that reason" + "ZOHO_WEBTOLEAD_ZC_GAD|optional|nonempty|Zoho ad-tracking field, legitimately empty" + + "ZOHO_ENABLED|optional|one-of:true,false|the on switch for the OAuth/REST standby path. Irrelevant while the mode is webtolead" + "ZOHO_CASES_ENABLED|optional|one-of:true,false|forwards support requests as Zoho Cases. Off means support tickets exist only in SQLite, where nobody is looking at them" + "ZOHO_API_DOMAIN|optional|url|the API host for the standby path. Datacenter-specific" + "ZOHO_ACCOUNTS_DOMAIN|optional|url|the OAUTH TOKEN host, which is a DIFFERENT domain from the API host. Pointing this at the API domain is the bug that shipped once and failed every token refresh in production" + "ZOHO_CLIENT_ID|optional|nonempty|standby path only" + "ZOHO_CLIENT_SECRET|optional|secret-min-length:16|standby path only" + "ZOHO_REFRESH_TOKEN|optional|secret-min-length:16|standby path only. Grants write access to the CRM" +) + +# --------------------------------------------------------------------------- +# Redaction. See "Secrets are measured, never printed" above. +# +# Matched case-insensitively against the whole variable name, so KEY catches +# API_KEY, KEYCLOAK_SECRET and MONKEY_HOST alike. The last is a false positive +# and costs nothing: its value is described rather than shown. +# --------------------------------------------------------------------------- +SECRET_NAME_PATTERN='SECRET|TOKEN|PASSWORD|PASSWD|PWD|CREDENTIAL|PRIVATE|SALT|SIGNATURE|SIGNING|AUTH|KEY|DSN|COOKIE' + +# Values a `secret-min-length` variable must not be, compared in lower case and +# never echoed. These are the strings typed to make the dev server start, which +# then travel to production inside a copied .env and satisfy every length rule. +PLACEHOLDER_SECRETS='changeme change-me change_me changethis secret mysecret supersecret password passwd hunter2 test testing example placeholder todo tbd xxx xxxx admin dev devsecret development your-secret-here your_secret_here notasecret 123456 12345678 abc123' + +# Redaction applies to NAMES read out of a file too, not only to values. +# +# A multi-line secret pasted into a .env — a PEM body, a wrapped base64 blob — +# has continuation lines carrying '=' padding, so `KEY=VALUE` parsing splits one +# into a "name" and a "value" and the name is a slice of the credential. Every +# message that prints a name out of a file therefore prints its LENGTH once it +# is longer than any name could plausibly be, which is the same rule values +# already follow. 40 covers the longest real variable names (the framework +# ones reach the low forties); a 64-column PEM line does not come close. +NAME_PRINT_MAX=40 + +say() { printf '\033[1mcheck-env:\033[0m %s\n' "$*" >&2; } + +# die is 2, not the 1 its counterpart in release.sh uses: everything that calls +# it is a reason this script could not measure the environment, and a caller +# must be able to tell "your configuration is wrong" from "I never found out". +die() { printf '\033[1mcheck-env:\033[0m %s\n' "$*" >&2; exit 2; } + +usage() { + say "usage: check-env.sh [--file PATH] [--quiet] [--list] [--help]" + say " --file PATH check that file instead of the process environment" + say " --quiet, -q print only failures (exit code still says everything)" + say " --list print the declared spec and check nothing" + say " --help, -h this message" +} + +trim() { + local s="$1" + s="${s#"${s%%[![:space:]]*}"}" + s="${s%"${s##*[![:space:]]}"}" + printf '%s' "$s" +} + +SOURCE_FILE="" +QUIET="" +LIST_ONLY="" + +while [ $# -gt 0 ]; do + case "$1" in + --file) shift; [ $# -gt 0 ] || die "--file needs a path."; SOURCE_FILE="$1" ;; + --file=*) SOURCE_FILE="${1#--file=}"; [ -n "$SOURCE_FILE" ] || die "--file needs a path." ;; + --quiet|-q) QUIET="yes" ;; + --list) LIST_ONLY="yes" ;; + -h|--help) usage; exit 0 ;; + *) usage; die "unknown argument '$1'." ;; + esac + shift +done + +# --------------------------------------------------------------------------- +# Read the spec, and refuse anything ambiguous. +# +# A malformed spec entry is a configuration error in this script, not a finding +# about the environment, so it exits 2 and reports nothing about the variables: +# a partial report from a spec that is half-understood is the same lie as a +# green one from an empty spec. +# --------------------------------------------------------------------------- + +if [ "${#SPEC[@]}" -eq 0 ]; then + say "SPEC is empty, so nothing was checked. That is not a pass." + say "" + say "This is a fresh template copy: declare the variables this project reads" + say "in the SPEC block near the top of this file, one line each —" + say ' "DATABASE_URL|required|url|nothing can read or write; every request 500s"' + die "then run this again." +fi + +SPEC_NAME=() +SPEC_REQ=() +SPEC_KIND=() +SPEC_PARAM=() +SPEC_DESC=() +NAME_WIDTH=4 + +entry_no=0 +for entry in "${SPEC[@]}"; do + entry_no=$((entry_no + 1)) + + # A description may legitimately contain '|', so read takes only four fields + # and the fourth keeps the rest. + IFS='|' read -r f_name f_req f_kind f_desc <<<"$entry" + + f_name=$(trim "${f_name:-}") + f_req=$(trim "${f_req:-}") + f_kind=$(trim "${f_kind:-}") + f_desc=$(trim "${f_desc:-}") + + [ -n "$f_name" ] && [ -n "$f_req" ] && [ -n "$f_kind" ] && [ -n "$f_desc" ] \ + || die "SPEC entry ${entry_no} is not NAME|required|kind|description: '${entry}'" + + # The shell cannot export a name it cannot parse, so a name that fails this + # could never have been set in the first place and every run would report it + # missing forever. + [[ "$f_name" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] \ + || die "SPEC entry ${entry_no}: '${f_name}' is not a usable variable name." + + # The environment snapshot has to live somewhere, and a variable it could + # collide with is one this script would measure against itself. Refused + # rather than measured wrong. + case "$f_name" in + _CE_*) die "SPEC entry ${entry_no}: '${f_name}' uses the _CE_ prefix, which this script reserves for the environment snapshot it takes before it defines anything. Rename the variable, or rename the snapshot at the top of this file." ;; + esac + + case "$f_req" in + required|optional) : ;; + *) die "SPEC entry ${entry_no} (${f_name}): expected 'required' or 'optional', got '${f_req}'. Spelling matters here — see the SPEC block." ;; + esac + + kind_name="${f_kind%%:*}" + kind_param="" + case "$f_kind" in *:*) kind_param="${f_kind#*:}" ;; esac + + case "$kind_name" in + nonempty|url|port|path) + [ -z "$kind_param" ] || die "SPEC entry ${entry_no} (${f_name}): kind '${kind_name}' takes no ':' argument." ;; + int) + if [ -n "$kind_param" ]; then + [[ "$kind_param" =~ ^[+-]?[0-9]+\.\.[+-]?[0-9]+$ ]] \ + || die "SPEC entry ${entry_no} (${f_name}): int bounds must look like 1..100, got '${kind_param}'." + fi ;; + one-of) + [ -n "$kind_param" ] || die "SPEC entry ${entry_no} (${f_name}): one-of needs a comma-separated list, e.g. one-of:development,production." ;; + secret-min-length) + [[ "$kind_param" =~ ^[0-9]+$ ]] && [ "$kind_param" -gt 0 ] \ + || die "SPEC entry ${entry_no} (${f_name}): secret-min-length needs a positive number, e.g. secret-min-length:32." ;; + *) + die "SPEC entry ${entry_no} (${f_name}): unknown kind '${kind_name}'. Valid: nonempty, int, port, url, path, one-of:a,b, secret-min-length:N." ;; + esac + + # A name declared twice gets two descriptions, and the report would print + # both — one of which is now out of date and neither of which is marked. + for existing in ${SPEC_NAME[@]+"${SPEC_NAME[@]}"}; do + [ "$existing" = "$f_name" ] && die "SPEC declares ${f_name} twice. Keep one line." + done + + SPEC_NAME+=("$f_name") + SPEC_REQ+=("$f_req") + SPEC_KIND+=("$kind_name") + SPEC_PARAM+=("$kind_param") + SPEC_DESC+=("$f_desc") + + [ "${#f_name}" -gt "$NAME_WIDTH" ] && NAME_WIDTH="${#f_name}" +done + +if [ -n "$LIST_ONLY" ]; then + say "declared in SPEC — nothing was checked:" + for (( i = 0; i < ${#SPEC_NAME[@]}; i++ )); do + kind="${SPEC_KIND[$i]}" + [ -n "${SPEC_PARAM[$i]}" ] && kind="${kind}:${SPEC_PARAM[$i]}" + printf ' %-*s %-8s %-22s %s\n' \ + "$NAME_WIDTH" "${SPEC_NAME[$i]}" "${SPEC_REQ[$i]}" "$kind" "${SPEC_DESC[$i]}" >&2 + done + exit 0 +fi + +# --------------------------------------------------------------------------- +# Display. Every message that mentions a value goes through show(), and that is +# the only place the redaction rule is enforced — so nothing else in this +# script may interpolate a value into a message. If you add a check, use it. +# --------------------------------------------------------------------------- + +is_secret() { + case "$2" in secret-min-length) return 0 ;; esac + local upper + upper=$(printf '%s' "$1" | tr '[:lower:]' '[:upper:]') + [[ "$upper" =~ $SECRET_NAME_PATTERN ]] +} + +show() { + local name="$1" kind="$2" value="$3" + + if is_secret "$name" "$kind"; then + printf 'set, %d characters (value withheld)' "${#value}" + return 0 + fi + + case "$kind" in + url) + # Scheme and host are what you need to spot the wrong environment; the + # path and query are where webhook and signed-URL secrets live, so they + # are summarised rather than shown. + local rest authority tail="" + case "$value" in + *://*) + rest="${value#*://}" + authority="${rest%%/*}" + authority="${authority%%\?*}" + case "$authority" in *@*) authority="***@${authority##*@}" ;; esac + case "$rest" in */*|*\?*) tail="/…" ;; esac + printf '%s://%s%s' "${value%%://*}" "$authority" "$tail" + return 0 ;; + esac + ;; + # Bounded, enumerable and not credential-shaped by construction. `literal` + # is the mode a failure message asks for when the value IS the finding — + # a port of '3000/tcp' cannot be explained without quoting it. + int|port|one-of|path|literal) : ;; + *) + # Unknown shape, so nothing is known about what might be inside it. + printf 'set, %d characters' "${#value}" + return 0 ;; + esac + + if [ "${#value}" -gt 60 ]; then + printf '%s…' "${value:0:59}" + else + printf '%s' "$value" + fi +} + +# --------------------------------------------------------------------------- +# Load the source. +# --------------------------------------------------------------------------- + +FILE_KEYS=() +FILE_VALS=() +FILE_LINES=() +FILE_BAD=() +FILE_NOTES=() + +load_file() { + local path="$1" lineno=0 line trimmed key val + + # Redirection, not `cat "$path" |`: a piped while-loop runs in a subshell and + # every array appended inside it is discarded at the `done`. + while IFS= read -r line || [ -n "$line" ]; do + lineno=$((lineno + 1)) + + case "$line" in + *$'\r') + line="${line%$'\r'}" + FILE_NOTES+=("line ${lineno}|has a CRLF line ending; most loaders keep the carriage return, making the value one invisible character longer than it looks") ;; + esac + + trimmed=$(trim "$line") + case "$trimmed" in ''|'#'*) continue ;; esac + + case "$trimmed" in + export\ *|export$'\t'*) + trimmed=$(trim "${trimmed#export}") + FILE_NOTES+=("line ${lineno}|uses 'export'; a shell sourcing this file is fine, but docker --env-file reads the name as 'export NAME'") ;; + esac + + case "$trimmed" in + *=*) key="${trimmed%%=*}"; val="${trimmed#*=}" ;; + *) FILE_BAD+=("line ${lineno}|no '=', so it sets nothing at all"); continue ;; + esac + + if [[ ! "$key" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then + if [[ "$(trim "$key")" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then + FILE_BAD+=("line ${lineno}|has spaces around '='; outside a shell those become part of the name and of the value") + elif [ "${#key}" -gt "$NAME_PRINT_MAX" ]; then + FILE_BAD+=("line ${lineno}|has ${#key} characters before its '=', so that is not a name and nothing can read it; the text is withheld in case this is one line of a wrapped multi-line secret") + else + FILE_BAD+=("line ${lineno}|'${key}' is not a usable variable name, so nothing can read it") + fi + continue + fi + + # Quotes are stripped here because that is what dotenv-style loaders do — + # but docker --env-file does not, so the two see different values. Said out + # loud rather than resolved, because only the operator knows which loader + # will read this file. + case "$val" in + \"*\"|\'*\') + val="${val:1:${#val}-2}" + FILE_NOTES+=("${key}|is quoted; dotenv-style loaders strip the quotes and docker --env-file keeps them, so the two disagree about this value") ;; + esac + + case "$val" in + *'${'*|*'$('*) + FILE_NOTES+=("${key}|contains a \$ expansion; dotenv-style loaders interpolate it and docker --env-file passes it through literally") ;; + esac + + local j + for (( j = 0; j < ${#FILE_KEYS[@]}; j++ )); do + if [ "${FILE_KEYS[$j]}" = "$key" ]; then + FILE_NOTES+=("${key}|is set twice (lines ${FILE_LINES[$j]} and ${lineno}); loaders disagree about which one wins") + break + fi + done + + FILE_KEYS+=("$key") + FILE_VALS+=("$val") + FILE_LINES+=("$lineno") + done < "$path" +} + +if [ -n "$SOURCE_FILE" ]; then + # Refused rather than fallen back on. You asked about a file; answering about + # the process environment instead would be a confident pass from a source you + # did not name — see status.sh on why a tool that answers about the wrong + # thing is worse than one that answers nothing. + [ -e "$SOURCE_FILE" ] || die "--file '${SOURCE_FILE}' does not exist. Refusing to check the process environment instead." + [ -f "$SOURCE_FILE" ] || die "--file '${SOURCE_FILE}' is not a regular file." + [ -r "$SOURCE_FILE" ] || die "--file '${SOURCE_FILE}' is not readable by this user." + load_file "$SOURCE_FILE" + SOURCE_LABEL="${SOURCE_FILE}" +else + SOURCE_LABEL="the process environment" +fi + +# The empty string means "not set here". Callers must not read it as "set to +# nothing" — FOUND says which. +FOUND="" +VALUE="" + +# The locals are prefixed for the same reason the snapshot is: `lookup`'s own +# `want` and `j` would shadow a SPEC entry of either name, and this is the one +# function that resolves a variable name at runtime. +lookup() { + local _ce_want="$1" _ce_j + FOUND="" + VALUE="" + + if [ -n "$SOURCE_FILE" ]; then + # No break: the last assignment wins, which is what a loader reading the + # file top to bottom into a map does. + for (( _ce_j = 0; _ce_j < ${#FILE_KEYS[@]}; _ce_j++ )); do + if [ "${FILE_KEYS[$_ce_j]}" = "$_ce_want" ]; then FOUND="yes"; VALUE="${FILE_VALS[$_ce_j]}"; fi + done + else + # The snapshot taken at the top, never the live variable: see the reasoning + # there. A name absent from the snapshot was absent from the environment, + # whatever this script may since have set in its own namespace. + for (( _ce_j = 0; _ce_j < ${#_CE_ENV_NAMES[@]}; _ce_j++ )); do + if [ "${_CE_ENV_NAMES[$_ce_j]}" = "$_ce_want" ]; then + FOUND="yes"; VALUE="${_CE_ENV_VALS[$_ce_j]}"; break + fi + done + fi +} + +# --------------------------------------------------------------------------- +# Shape checks. +# +# check_shape prints zero or more lines, each 'fail:' or 'note:' +# — a protocol rather than array appends because it is called in a command +# substitution, and an array appended inside a subshell is gone at the closing +# paren. +# --------------------------------------------------------------------------- + +check_shape() { + local name="$1" kind="$2" param="$3" value="$4" + + case "$value" in + ' '*|$'\t'*) printf 'fail:begins with whitespace, which is part of the value everywhere except a shell\n' ;; + esac + case "$value" in + *' '|*$'\t') printf 'fail:ends with whitespace, which is part of the value and breaks every comparison against it\n' ;; + esac + + case "$kind" in + url|port|int|one-of|path) + case "$value" in + *$'\n'*) printf 'fail:contains a newline, so nothing downstream will parse it\n'; return 0 ;; + esac ;; + esac + + case "$kind" in + nonempty) + # Presence and non-emptiness are the same test, already done by the + # caller. Reaching here means it passed; there is nothing else to know. + : ;; + + int) + if [[ ! "$value" =~ ^[+-]?[0-9]+$ ]]; then + printf 'fail:is not a whole number: %s\n' "$(show "$name" "$kind" "$value")" + return 0 + fi + local n="${value#+}" + # Bash arithmetic is 64-bit and errors past it, so an absurd number is + # rejected by length before anything tries to compare it. + if [ "${#n}" -gt 18 ]; then + printf 'fail:has %d digits; nothing downstream will read that as a number\n' "${#n}" + return 0 + fi + if [ -n "$param" ]; then + local lo="${param%%..*}" hi="${param##*..}" + if [ "$n" -lt "${lo#+}" ] || [ "$n" -gt "${hi#+}" ]; then + # Through show(), not the bare value: an out-of-range number under a + # credential-shaped name — TOTP_KEY, AUTH_TOKEN_TTL — is still a + # credential, and this is a failure message, which is precisely where + # the header says a careless validator leaks. The bounds come from the + # spec, so they are always safe to print. + printf 'fail:is %s, outside the allowed %s..%s\n' \ + "$(show "$name" "$kind" "$value")" "${lo#+}" "${hi#+}" + fi + fi ;; + + port) + case "$value" in + *://*) printf 'fail:is a URL, not a port number: %s\n' "$(show "$name" url "$value")"; return 0 ;; + */*) printf 'fail:looks like a compose port mapping (%s); a port field takes the number alone\n' "$(show "$name" literal "$value")"; return 0 ;; + esac + if [[ ! "$value" =~ ^[0-9]{1,5}$ ]] || [ "$value" -lt 1 ] || [ "$value" -gt 65535 ]; then + printf 'fail:is not a port; ports are 1-65535\n' + return 0 + fi ;; + + url) + case "$value" in + *://*) : ;; + *) + printf 'fail:has no scheme — %s is a host, not a URL; write it as scheme://host\n' "$(show "$name" literal "$value")" + return 0 ;; + esac + + local scheme="${value%%://*}" rest="${value#*://}" authority + if [[ ! "$scheme" =~ ^[A-Za-z][A-Za-z0-9+.-]*$ ]]; then + printf 'fail:has a scheme that is not a scheme\n' + return 0 + fi + + authority="${rest%%/*}" + authority="${authority%%\?*}" + case "$authority" in *@*) authority="${authority##*@}" ;; esac + + if [ -z "$authority" ]; then + printf 'fail:has no host between // and the path\n' + return 0 + fi + case "$authority" in + *' '*) printf 'fail:has a space in the host\n'; return 0 ;; + esac + + case "$authority" in + localhost|localhost:*|127.0.0.1|127.0.0.1:*|0.0.0.0|0.0.0.0:*|'[::1]'|'[::1]':*) + printf 'note:points at %s — correct on a laptop; inside a container that is the container itself, not the host\n' "$authority" ;; + host.docker.internal*) + printf 'note:points at host.docker.internal, which resolves on Docker Desktop and not on a Linux daemon without an extra_hosts entry\n' ;; + esac + + case "$value" in + */) printf 'note:ends with a slash, so anything joining a path onto it produces a double slash\n' ;; + esac ;; + + path) + case "$value" in + '~'*) printf 'fail:starts with ~, and nothing expands a tilde inside an environment variable — a process reading this opens a directory literally named ~\n' ;; + esac + case "$value" in + /*) : ;; + *) printf 'note:is relative, so it resolves against the working directory of whatever starts the process rather than the one you typed it in\n' ;; + esac + # Existence is a fact about THIS machine. Checking it against a file that + # describes another machine would report a fault that is not one, so in + # --file mode it is not checked and the report says so rather than + # leaving a silent gap. + if [ -z "$SOURCE_FILE" ] && [ ! -e "$value" ]; then + printf 'note:does not exist on this machine (this may be correct if the volume is mounted later)\n' + fi ;; + + one-of) + local rest="$param" opt matched="" ci_match="" + local lower_value lower_opt + lower_value=$(printf '%s' "$value" | tr '[:upper:]' '[:lower:]') + while [ -n "$rest" ]; do + opt="${rest%%,*}" + [ "$value" = "$opt" ] && matched="yes" + lower_opt=$(printf '%s' "$opt" | tr '[:upper:]' '[:lower:]') + [ "$lower_value" = "$lower_opt" ] && ci_match="$opt" + case "$rest" in *,*) rest="${rest#*,}" ;; *) rest="" ;; esac + done + if [ -z "$matched" ]; then + if [ -n "$ci_match" ]; then + printf 'fail:is %s, and the comparison is case-sensitive: write %s\n' "$(show "$name" one-of "$value")" "$ci_match" + else + printf 'fail:is %s; allowed: %s\n' "$(show "$name" one-of "$value")" "${param//,/, }" + fi + fi ;; + + secret-min-length) + # Length is the only property reported. Nothing below prints the value, + # and the comparison below pipes it to `tr` rather than passing it as an + # argument, so it never appears in this process's command line either. + if [ "${#value}" -lt "$param" ]; then + printf 'fail:is %d characters and the spec requires %d (value withheld)\n' "${#value}" "$param" + fi + local lower placeholder + lower=$(printf '%s' "$value" | tr '[:upper:]' '[:lower:]') + for placeholder in $PLACEHOLDER_SECRETS; do + if [ "$lower" = "$placeholder" ]; then + printf 'fail:is a well-known placeholder, not a secret — anyone can guess it (value withheld)\n' + break + fi + done ;; + esac +} + +# --------------------------------------------------------------------------- +# Measure everything, then report. Nothing exits early: one run must name every +# problem, or fixing them costs one restart each. +# --------------------------------------------------------------------------- + +MISSING=() +MALFORMED=() +NOTES=() +OPT_ABSENT=() +PRESENT_OK=() + +for (( i = 0; i < ${#SPEC_NAME[@]}; i++ )); do + name="${SPEC_NAME[$i]}" + req="${SPEC_REQ[$i]}" + kind="${SPEC_KIND[$i]}" + param="${SPEC_PARAM[$i]}" + desc="${SPEC_DESC[$i]}" + + lookup "$name" + + if [ -z "$FOUND" ]; then + if [ "$req" = "required" ]; then + MISSING+=("${name}|${desc}") + else + OPT_ABSENT+=("${name}|${desc}") + fi + continue + fi + + if [ -z "$VALUE" ]; then + if [ "$req" = "required" ]; then + MISSING+=("${name}|set, but empty — ${desc}") + else + # Set-to-empty is not unset, and code tests it both ways: a truthiness + # check sees nothing, a key-presence check sees it. Reported so the + # ambiguity is the operator's to resolve rather than this script's. + OPT_ABSENT+=("${name}|${desc}") + NOTES+=("${name}|is set to the empty string, which is not the same as unset to code that checks whether the key is present") + fi + continue + fi + + findings=$(check_shape "$name" "$kind" "$param" "$VALUE") + had_fail="" + + while IFS= read -r rline; do + [ -n "$rline" ] || continue + case "$rline" in + fail:*) MALFORMED+=("${name}|${rline#fail:}"); had_fail="yes" ;; + note:*) NOTES+=("${name}|${rline#note:}") ;; + esac + done <<<"$findings" + + [ -n "$had_fail" ] || PRESENT_OK+=("${name}|$(show "$name" "$kind" "$VALUE")") +done + +if [ -n "$SOURCE_FILE" ]; then + # Only for a file. The process environment carries hundreds of variables from + # the shell, the init system and every tool that ever exported one, so + # "undeclared" there would be noise; in a file every line was written on + # purpose, and one that matches nothing in the spec is usually a typo in a + # name — the thing that makes people swear the value is right there. + for (( j = 0; j < ${#FILE_KEYS[@]}; j++ )); do + fkey="${FILE_KEYS[$j]}" + known="" + for name in ${SPEC_NAME[@]+"${SPEC_NAME[@]}"}; do + [ "$name" = "$fkey" ] && known="yes" && break + done + if [ -z "$known" ]; then + if [ "${#fkey}" -gt "$NAME_PRINT_MAX" ]; then + FILE_NOTES+=("line ${FILE_LINES[$j]}|sets a ${#fkey}-character name that is not declared in SPEC and is too long to be a name at all; the text is withheld in case this is one line of a wrapped multi-line secret") + else + FILE_NOTES+=("${fkey}|is set on line ${FILE_LINES[$j]} but is not declared in SPEC — a misspelled name, or a spec that has fallen behind") + fi + fi + done +fi + +group() { + local heading="$1"; shift + [ $# -gt 0 ] || return 0 + say "" + say "$heading" + local e + for e in "$@"; do + printf ' %-*s %s\n' "$NAME_WIDTH" "${e%%|*}" "${e#*|}" >&2 + done +} + +n_missing="${#MISSING[@]}" +n_malformed="${#MALFORMED[@]}" +n_badlines="${#FILE_BAD[@]}" +n_problems=$(( n_missing + n_malformed + n_badlines )) + +# `${arr[@]+"${arr[@]}"}` throughout: under `set -u` a plain "${arr[@]}" on an +# empty array is an error in bash before 4.4, and macOS still ships 3.2. +if [ -z "$QUIET" ] || [ "$n_problems" -gt 0 ]; then + say "source: ${SOURCE_LABEL}" + if [ -n "$SOURCE_FILE" ]; then + say " the process environment was NOT consulted, on purpose." + fi +fi + +if [ -z "$QUIET" ]; then + group "ok — present and the right shape:" ${PRESENT_OK[@]+"${PRESENT_OK[@]}"} + group "optional, not set — this is fine:" ${OPT_ABSENT[@]+"${OPT_ABSENT[@]}"} + group "worth knowing — not failures:" ${NOTES[@]+"${NOTES[@]}"} + + # Kept apart from the value notes above: these are facts about how the FILE + # is written, and they apply to whichever loader reads it rather than to any + # one value being wrong. + if [ -n "$SOURCE_FILE" ]; then + group "about ${SOURCE_FILE} itself:" ${FILE_NOTES[@]+"${FILE_NOTES[@]}"} + fi + + if [ -n "$SOURCE_FILE" ]; then + for (( i = 0; i < ${#SPEC_KIND[@]}; i++ )); do + if [ "${SPEC_KIND[$i]}" = "path" ]; then + say "" + say "note: path existence was not checked — a file can describe a machine" + say " that is not this one. Run without --file to check paths here." + break + fi + done + fi +fi + +group "MALFORMED — set, but the wrong shape:" ${MALFORMED[@]+"${MALFORMED[@]}"} +group "UNREADABLE LINES — these set nothing:" ${FILE_BAD[@]+"${FILE_BAD[@]}"} +group "MISSING — required, not set:" ${MISSING[@]+"${MISSING[@]}"} + +if [ "$n_problems" -gt 0 ]; then + say "" + say "${n_missing} missing, ${n_malformed} malformed, ${n_badlines} unreadable line(s) in ${SOURCE_LABEL}." + # Malformed counts as a failure whether the variable was required or + # optional: not setting an optional variable is a choice, and setting it to + # something unusable is a mistake that reads as a working configuration. + say "fix the entries above and run this again — nothing that reads this" + say "environment can be trusted until then." + # Exit 1, not die's 2. The difference is the whole contract: 1 means this + # measured the environment and it is wrong, 2 means it never got far enough + # to find out. A caller that cannot tell those apart will retry the wrong one. + exit 1 +fi + +if [ -z "$QUIET" ]; then + say "" + say "ok — ${#PRESENT_OK[@]} set and well-formed, ${#OPT_ABSENT[@]} optional and absent, ${#SPEC_NAME[@]} declared." + say "shapes only: nothing was connected to, so this says the values are" + say "plausible, not that the services behind them are up." +fi + +exit 0 diff --git a/scripts/doc-triggers.py b/scripts/doc-triggers.py new file mode 100755 index 0000000..29443d1 --- /dev/null +++ b/scripts/doc-triggers.py @@ -0,0 +1,470 @@ +#!/usr/bin/env python3 +"""Which documents does a change fire, while there is still time to update them? + + python3 scripts/doc-triggers.py # everything dirty right now + python3 scripts/doc-triggers.py --staged # what is staged + python3 scripts/doc-triggers.py [path…] # specific paths + python3 scripts/doc-triggers.py --range HEAD~3..HEAD + +## The failure this catches + +Every document in this tree carries a `Review trigger:` — the change that should +send somebody back to it — and `WORK_CYCLE.md` requires the triggered documents +to be updated *in the same commit as the code*. Deciding which fired means +reading every `Governs:` line and matching globs in your head, once per commit. +It is a check with no output of its own, so it is the one that gets skipped when +the code is already green, and the cost is invisible until a reader trusts a +document that stopped being true: a reference manual six migrations behind, and +every reader in between believing it. + +**This is not the doc-review check.** That one asks which baselined documents are +*overdue*, from committed history — a governed path with a commit newer than the +document's `Last reviewed` date. It is a different question and it can only be +asked after the fact. By the time a change is committed without its document, the +thing this catches has already happened. + +Exit status is always 0. This is a prompt, not a gate: a trigger asks a human +whether the prose is still true, and a check that failed the build for that would +be bumped past rather than read. + +## Matching the trigger's verb, not only its glob + +A document is fired when a changed path matches its `Governs:` **and** the kind +of change matches its `Fires on:`. That second field is optional and almost +never needed; it exists for the case where `Governs:` is far broader than the +trigger. `DOC_TRUST_MAP.md` is the extreme — it governs `docs/**` while its +trigger is *any doc added, deleted or moved* — so on the glob alone it fired on +every edit to every document, forever, and correctly by the only rule there was. +A prompt that always fires is one people stop reading, and this one exits 0 by +design, so nothing forces the reading. + +The kinds are `added`, `deleted`, `moved` and `changed`, read from git's own +status letter. Absent, empty or unparseable means every kind, which is the +behaviour every document without the line still has. + +## Two things it deliberately does not do + +**It does not read `Exempt:` declarations.** Those mark a *required* document as +deliberately absent from a repository, and a document that does not exist cannot +govern a path. There is nothing here for them to change. + +**It cannot fire a document whose `Governs:` is prose.** Several in this template +govern a subject rather than a set of paths — `GUARDS.md` governs "structural +tests, source-grep assertions, probes, and any check whose passing is taken as +evidence", which is the honest description and matches no glob. Those documents +are listed separately at the end rather than silently ignored, because a reader +who sees only the matched list would reasonably conclude the others were checked +and cleared. +""" +from __future__ import annotations + +import fnmatch +import pathlib +import re +import subprocess +import sys + +def _find_root() -> pathlib.Path: + """The repository root, found rather than assumed. + + This was `parents[3]`, which is correct only while the script sits at + `docs/architecture/scripts/` — its home in the template. The moment a + project copies it to `scripts/`, as the template's own adoption + instructions say to, `parents[3]` climbs out of the repository entirely: in + a checkout at `~/Projects/thing/scripts/`, it resolves to `~/`, and the + script reports "no docs/ directory" about a directory two levels above the + project it was run in. + + So: walk up from the script looking for a directory that has both `docs/` + and `.git`, then fall back to either alone, then to git's own answer. + """ + here = pathlib.Path(__file__).resolve() + + for parent in here.parents: + if (parent / "docs").is_dir() and (parent / ".git").exists(): + return parent + + for parent in here.parents: + if (parent / "docs").is_dir(): + return parent + + result = subprocess.run( + ["git", "rev-parse", "--show-toplevel"], + cwd=here.parent, + capture_output=True, + text=True, + check=False, + ) + + if result.returncode == 0 and result.stdout.strip(): + return pathlib.Path(result.stdout.strip()) + + return here.parents[1] + + +ROOT = _find_root() +DOCS = ROOT / "docs" + +# The status header is a fenced block immediately after the H1, and a long value +# wraps onto continuation lines indented by two spaces: +# +# Governs: structural tests, source-grep assertions, probes, and any check whose +# passing is taken as evidence +# +# A regex that reads one line per field — the obvious implementation — truncates +# at the wrap and silently under-reports, which for this tool means quietly +# failing to name a document that should have been updated. So fields are +# assembled line by line instead. +FIELD_START = re.compile( + r"^(Status|Owner|Last reviewed|Governs|Review trigger|Fires on):\s*(.*)$" +) +HEADER_LINES = 16 + + +def header_of(doc: pathlib.Path) -> dict[str, str]: + """The status header, with wrapped values joined.""" + try: + lines = doc.read_text(encoding="utf-8").splitlines()[:HEADER_LINES] + except (OSError, UnicodeDecodeError): + return {} + + fields: dict[str, str] = {} + current: str | None = None + for line in lines: + match = FIELD_START.match(line) + if match: + current = match.group(1) + fields[current] = match.group(2).strip() + elif current and line.startswith((" ", "\t")) and line.strip(): + fields[current] = f"{fields[current]} {line.strip()}".strip() + elif line.strip().startswith("```") and fields: + break + return fields + + +# `Status` must be one of these four. `DOC_TRUST_MAP.md` states it as a rule with +# a checker behind it, and it is the one field that distinguishes a document from +# a template for one: `project-readme-template.md` carries +# `Status: `, and its `Governs` describes +# the README of the project that copies it, not anything in this repository. +STATUS_WORDS = {"Current", "Draft", "Superseded", "Archived"} + + +def governing_documents() -> list[pathlib.Path]: + """Every document that can fire, root ones included. + + The walk used to start at `docs/`, so the documents at the repository root + were not read at all — `README.md` and the two `START-HERE-*.md` carry a full + status header, govern real paths, and fired nothing ever, while the output + said "No document's Governs matched these paths". True of the tool and false + of the repository, which is the same shape as the gloss bug one level out. + + The root is read **non-recursively**: `ROOT.glob`, not `rglob`. A vendored + copy of this template, a scratch checkout, or somebody's directory of notes + would otherwise enrol its documents as governing this repository. + """ + docs = sorted(ROOT.glob("*.md")) + if DOCS.is_dir(): + docs += sorted(DOCS.rglob("*.md")) + return docs + + +def looks_like_path(glob: str) -> bool: + """Whether a `Governs:` entry is a path pattern rather than a subject. + + The same test `doc-claims.sh` uses: a slash, a wildcard, or a file + extension. Prose about what a document is authoritative for will have none + of them, and must not be treated as a glob that simply never matches. + """ + glob = glob.strip() + if not glob or " " in glob and "/" not in glob and "*" not in glob: + return False + return "/" in glob or "*" in glob or re.search(r"\.\w{1,5}$", glob) is not None + + +# A `Governs:` entry is split on commas, so an entry that explains itself after +# the glob arrives whole: `docs/data/** — the assets privacyllc.dev renders for +# this project`. Used as a glob that matches nothing, ever, and because it +# contains a slash `looks_like_path` calls it a path — so the document was +# neither fired nor listed among the ones no change can fire mechanically. It was +# simply absent, which is the one outcome a reader cannot notice. +# +# Three of the seven path-governing documents here were in that state from the +# first commit, `docs/data/img/README.md` among them: editing the branding assets +# had never once prompted the document that specifies their names and sizes. +GLOSS = re.compile(r"\s+(?:—|–|--)\s+") + + +def globs_in(entry: str) -> list[str]: + """The globs inside one `Governs:` entry, with any trailing gloss removed. + + The cut requires whitespace on both sides of the dash: `source-grep` and + `doc-claims` appear in these headers and a bare `-` would halve them. Tokens + are taken from the left of the gloss rather than from the whole entry, + because prose on the right can itself look like a path — `privacyllc.dev` + passes the extension test and would become a glob that fires on a file + nobody has. + + An entry yielding no token falls back to itself, so a shape not foreseen here + behaves exactly as it did before. + """ + head = GLOSS.split(entry, 1)[0] + return [tok for tok in head.split() if looks_like_path(tok)] or [entry] + + +def matches(path: str, glob: str) -> bool: + """Whether `path` is governed by `glob`. + + `**` means "and everything below", which `fnmatch` does not implement: its + `*` already crosses separators, so `a/**` never matches `a/b/c`. The two + forms these headers actually use are reduced to prefix tests. + """ + glob = glob.strip() + if not glob: + return False + if glob.endswith("/**"): + return path.startswith(glob[:-2]) or path == glob[:-3] + if glob.endswith("/"): + # A bare directory, as `githooks/README.md` governs ".githooks/". fnmatch + # would not match a file inside it. + return path.startswith(glob) + if "/**/" in glob: + head, tail = glob.split("/**/", 1) + return path.startswith(head + "/") and fnmatch.fnmatch(path, "*" + tail) + return fnmatch.fnmatch(path, glob) + + +# `Governs:` says *where* a document is authoritative; `Fires on:` says which +# kinds of change to that place its `Review trigger` actually names. The two come +# apart badly at the extreme: `DOC_TRUST_MAP.md` governs `docs/**`, the broadest +# glob in the tree, while its trigger is one of the narrowest — *any doc added, +# deleted or moved*. Matching on the glob alone fires it on every edit to every +# document forever, and a prompt that always fires is one people stop reading, +# which takes the true positives with it. +# +# Why a declared field rather than reading the trigger prose. The obvious first +# cut — look for `added`/`deleted`/`moved` and no `changed`/`change to` — was +# tried against the seven path-governing documents here and misclassified the one +# it exists to fix. `DOC_TRUST_MAP.md`'s trigger ends "any change to which doc +# owns a subject", so it reads as a change-verb; the clause is about which +# document owns a subject, not about a file being edited. Nothing lexical +# separates it from `architecture/README.md`'s "any change to a module boundary +# or a data shape", which genuinely does mean modification. Guessing at English +# and getting it wrong here is silent in the expensive direction: the document +# stops being prompted for and goes quietly stale. +# +# So the narrowing is declared or it does not happen. Absent, unparseable, or +# empty means fire on everything, which is the old behaviour — a document is only +# ever quietened by someone writing the line deliberately. +KIND_LETTERS = { + "added": {"A", "C"}, + "deleted": {"D"}, + "moved": {"R"}, + "changed": {"M", "T"}, +} +ALL_KINDS = {letter for letters in KIND_LETTERS.values() for letter in letters} +KIND_OF = {letter: word for word, letters in KIND_LETTERS.items() for letter in letters} + + +def fires_on(header: dict[str, str]) -> tuple[set[str], list[str]]: + """The status letters a document accepts, and any words not understood. + + Returns every letter when nothing is declared or the declaration cannot be + read, so the failure mode of a typo is a document that is prompted for too + often rather than one that is silently dropped. + """ + raw = header.get("Fires on", "").strip() + if not raw: + return ALL_KINDS, [] + + words = [w.strip().lower().rstrip(".") for w in re.split(r"[,;]|\band\b", raw)] + words = [w for w in words if w] + + letters: set[str] = set() + unknown: list[str] = [] + for word in words: + if word in KIND_LETTERS: + letters |= KIND_LETTERS[word] + else: + unknown.append(word) + + if unknown or not letters: + return ALL_KINDS, unknown or ["(empty)"] + return letters, [] + + +def _git(*args: str) -> str: + result = subprocess.run( + ["git", *args], cwd=ROOT, capture_output=True, text=True, check=False + ) + return result.stdout + + +def _name_status(*args: str) -> list[tuple[str, str]]: + """(letter, path) from a `--name-status` listing. + + A rename arrives as `R100oldnew`, so the path is taken from the + last field: the new name governs, as it did when only names were read. + """ + pairs = [] + for line in _git(*args, "--name-status").splitlines(): + if not line.strip(): + continue + fields = line.split("\t") + if len(fields) < 2 or not fields[0].strip(): + continue + pairs.append((fields[0].strip()[0].upper(), fields[-1].strip())) + return pairs + + +def _status_of_named(path: str) -> str: + """The kind of change a path named on the command line represents. + + There is no diff to read here, so it is inferred: gone from disk is a + deletion, present but untracked is an addition, and anything else is a + modification — the usual reason to ask about a path by name. + """ + if not (ROOT / path).exists(): + return "D" + return "M" if _git("ls-files", "--", path).strip() else "A" + + +def changed_paths(argv: list[str]) -> tuple[list[tuple[str, str]], str]: + if argv and argv[0] == "--staged": + return _name_status("diff", "--cached"), "staged" + if argv and argv[0] == "--range": + if len(argv) < 2: + sys.exit("doc-triggers: --range needs a revision range") + return _name_status("diff", argv[1]), f"range {argv[1]}" + if argv: + return [(_status_of_named(a), a) for a in argv], "named paths" + + # Untracked files are included on purpose: a brand-new module is the case + # most likely to need a document and least likely to be remembered, and it + # is invisible to `git diff`. + pairs = [] + for line in _git("status", "--porcelain").splitlines(): + if not line.strip(): + continue + index, worktree = line[0], line[1] + path = line[3:].strip() + if " -> " in path: # a rename; the new name governs + path = path.split(" -> ", 1)[1] + if "?" in (index, worktree): + letter = "A" # untracked: a file that is new + else: + letter = (index if index != " " else worktree).upper() + pairs.append((letter, path.strip('"'))) + return pairs, "working tree" + + +def main() -> int: + if not DOCS.is_dir(): + print(f"doc-triggers: no docs/ directory at {DOCS}") + return 0 + + paths, source = changed_paths(sys.argv[1:]) + if not paths: + print(f"doc-triggers: nothing changed in the {source}.") + return 0 + + print(f"doc-triggers: {len(paths)} path(s) from the {source}\n") + + fired: list[tuple[str, list[tuple[str, str]], str]] = [] + subject_only: list[str] = [] + wrong_kind: list[tuple[str, str]] = [] + unfilled: list[tuple[str, str]] = [] + + for doc in governing_documents(): + rel = str(doc.relative_to(ROOT)) + header = header_of(doc) + governs = header.get("Governs", "") + if not governs: + continue + + filled = header.get("Status", "") in STATUS_WORDS + + entries = [g.strip() for g in governs.split(",") if g.strip()] + # Classification reads the whole entry and extraction reads inside it: + # deciding "path or subject?" on a token would move documents between the + # two lists as a side effect of this fix. + path_globs = [g for e in entries if looks_like_path(e) for g in globs_in(e)] + if not path_globs: + if filled: + subject_only.append(rel) + continue + + letters, unknown = fires_on(header) + if unknown: + print( + f"doc-triggers: {rel} declares 'Fires on: " + f"{header.get('Fires on', '')}' — {', '.join(unknown)} not " + f"understood, so it fires on everything.\n" + ) + + matched = {(s, p) for s, p in paths for g in path_globs if matches(p, g)} + + if not filled: + # A template for a document rather than a document. Named only when it + # would otherwise have fired: a line on every run, about a file that is + # supposed to look like this, is the noise this tool keeps being fixed + # for. + if matched: + unfilled.append((rel, header.get("Status", "") or "(none)")) + continue + + hits = sorted({(s, p) for s, p in matched if s in letters}, key=lambda x: x[1]) + if hits: + fired.append((rel, hits, header.get("Review trigger", "(none stated)"))) + elif matched: + # Governed, and deliberately not prompted for: the paths changed in a + # way this document's trigger does not name. Said out loud, because a + # reader who saw nothing would have to guess whether it was checked. + wrong_kind.append((rel, header.get("Fires on", "").strip())) + + for rel, hits, trigger in fired: + print(f"\033[1m{rel}\033[0m") + for letter, hit in hits[:6]: + print(f" {KIND_OF.get(letter, letter.lower()):>7} {hit}") + if len(hits) > 6: + print(f" … and {len(hits) - 6} more") + print(f" trigger: {trigger}\n") + + if fired: + print(f"{len(fired)} document(s) govern something in this change.") + print("Read each trigger and decide — the rule is to update them in the") + print("SAME commit as the code, not afterwards.") + elif wrong_kind or unfilled: + # Distinct from matching nothing, and worth separating: a path here *is* + # governed, and the reason nothing fired is a declaration somebody wrote + # or a header nobody filled in, not an area no document claims. + print("Nothing fired, but these paths are governed — see below for which") + print("documents matched them and why each was not raised.") + else: + print("No document's Governs matched these paths. Worth a second look if") + print("this change added a module, a migration, or a new boundary — an") + print("unmatched path can also mean no document claims that area yet.") + + if wrong_kind: + print("\nGovern a path in this change but do not fire on this kind of") + print("change, by their own Fires on declaration:") + for rel, declared in wrong_kind: + print(f" {rel} — fires on {declared}") + + if unfilled: + print("\nGovern a path in this change but their status header is still a") + print("template, so they are not treated as documents of this repository:") + for rel, status in unfilled: + print(f" {rel} — Status: {status}") + + if subject_only: + print("\nNot checked here — these govern a subject rather than paths, so") + print("no change can fire them mechanically. Judge them yourself:") + for rel in subject_only: + print(f" {rel}") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/forgejo-issue.py b/scripts/forgejo-issue.py new file mode 100755 index 0000000..d3d8dfc --- /dev/null +++ b/scripts/forgejo-issue.py @@ -0,0 +1,586 @@ +#!/usr/bin/env python3 +"""Post, list and close Forgejo issues in the convention these projects use. + +Exists because every one of the rules below was learned by getting it wrong +once. The script is the enforcement; the skill is the explanation. + + create file an issue, refusing one that has no `Verify:` line or that + duplicates an existing title + batch file several from a JSON file, skipping ones already there + list open issues, grouped by milestone + close close with an evidence comment — "Done" is not a close + labels / milestones — what exists, with the ids the API wants + +Run any subcommand with --dry-run to see the payload and change nothing. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +import urllib.error +import urllib.parse +import urllib.request + +# Where the credentials live, when they are not already in the environment. +# +# Keep this file OUTSIDE the repository. A token in a file the repo can see is +# a token one `git add -A` away from being published — the same argument +# `release.sh` makes about its registry env. +ENV_FILE = os.environ.get("FORGEJO_ENV_FILE", os.path.expanduser("~/.forgejo.env")) + +# Cloudflare fronts the Forgejo instance and 1010-blocks Python's default +# urllib User-Agent (browser_signature_banned). Every call fails with a +# Cloudflare HTML body that looks nothing like a Forgejo error. Do not remove. +USER_AGENT = "curl/8.5.0" + +SEVERITY = ("P0", "P1", "P2", "release-blocker") + + +# ── plumbing ───────────────────────────────────────────────────────────────── + + +def die(msg: str, code: int = 1): + print(f"error: {msg}", file=sys.stderr) + sys.exit(code) + + +def load_env() -> tuple[str, str]: + """Read host + token. Never print the token; it is not registry-scoped — + it carries admin/push/pull over the whole API.""" + host = os.environ.get("FORGEJO_REGISTRY") + token = os.environ.get("FORGEJO_REGISTRY_TOKEN") + if not (host and token): + try: + with open(ENV_FILE, encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, v = line.split("=", 1) + v = v.strip().strip("'\"") + if k.strip() == "FORGEJO_REGISTRY" and not host: + host = v + elif k.strip() == "FORGEJO_REGISTRY_TOKEN" and not token: + token = v + except FileNotFoundError: + pass + if not (host and token): + die( + f"FORGEJO_REGISTRY / FORGEJO_REGISTRY_TOKEN not in the environment " + f"or {ENV_FILE}.\n" + f"Set FORGEJO_ENV_FILE to point somewhere else, or export both." + ) + return host, token + + +def detect_repo() -> str | None: + """owner/name from the git remote of the current directory.""" + try: + url = subprocess.run( + ["git", "remote", "get-url", "origin"], + capture_output=True, text=True, check=True, + ).stdout.strip() + except (subprocess.CalledProcessError, FileNotFoundError): + return None + m = re.search(r"[:/]([^/:]+)/([^/]+?)(?:\.git)?$", url) + return f"{m.group(1)}/{m.group(2)}" if m else None + + +class Api: + def __init__(self, host: str, token: str, repo: str, dry_run: bool = False): + self.base = f"https://{host}/api/v1" + self.token = token + self.repo = repo + self.dry_run = dry_run + + def _call(self, method: str, path: str, body=None, params=None): + url = f"{self.base}{path}" + if params: + url += "?" + urllib.parse.urlencode(params) + data = json.dumps(body).encode() if body is not None else None + req = urllib.request.Request(url, data=data, method=method) + req.add_header("Authorization", f"token {self.token}") + req.add_header("User-Agent", USER_AGENT) + if data: + req.add_header("Content-Type", "application/json") + try: + with urllib.request.urlopen(req, timeout=30) as resp: + raw = resp.read().decode() + return json.loads(raw) if raw.strip() else None + except urllib.error.HTTPError as e: + detail = e.read().decode()[:400] + if " dict[str, int]: + return {l["name"]: l["id"] + for l in (self.get(f"/repos/{self.repo}/labels", + {"limit": 100}) or [])} + + def milestones(self) -> dict[str, int]: + out = {} + for state in ("open", "closed"): + for m in (self.get(f"/repos/{self.repo}/milestones", + {"state": state, "limit": 100}) or []): + out[m["title"]] = m["id"] + return out + + def milestones_full(self, state="all") -> list[dict]: + states = ("open", "closed") if state == "all" else (state,) + out = [] + for s in states: + out.extend(self.get(f"/repos/{self.repo}/milestones", + {"state": s, "limit": 100}) or []) + return out + + def current_milestone(self) -> dict | None: + """The first OPEN milestone that still has open issues. + + 'First' is the order Forgejo returns, which is creation order — NOT a + numeric sort by title. A milestone created last therefore cannot become + current while an earlier one still has open issues, which is the lever + for filing work that must not disturb the card. + """ + openms = self.milestones_full("open") + for m in openms: + if m["open_issues"] > 0: + return m + return openms[0] if openms else None + + +# ── convention checks ──────────────────────────────────────────────────────── + + +def check_verify_line(body: str, title: str) -> str: + """Every issue ends with a Verify: line stating the acceptance check. + A deliverable nobody can re-test cannot be closed, so it must not be filed.""" + lines = [l for l in body.strip().splitlines() if l.strip()] + if not any(l.strip().startswith("Verify:") for l in lines): + die(f'"{title}" has no `Verify:` line. State the acceptance check — a\n' + " finding that cannot be re-tested cannot be closed.") + if not lines[-1].strip().startswith("Verify:"): + print(f'warning: "{title}" has a Verify: line but it is not last', + file=sys.stderr) + return body + + +def resolve_labels(names, available: dict[str, int]) -> list[int]: + """Names → ids, failing loudly. A typo'd severity label is reported by the + Command Center as *not adopted*, not as zero defects — silently dropping it + would hide the whole repo's defect count.""" + ids = [] + for n in names: + # A severity label that differs only in case is the dangerous one: it + # looks right in the UI and is invisible to a query by exact name. + if n not in available: + near = [a for a in available if a.lower() == n.lower()] + if near: + die(f"label {n!r} does not exist, but {near[0]!r} does. " + "Names are matched exactly — use that one.") + die(f"label {n!r} does not exist. " + f"Available: {', '.join(sorted(available)) or '(none)'}") + if n.upper() in {s.upper() for s in SEVERITY} and n not in SEVERITY: + die(f"severity label must be spelled exactly one of {SEVERITY}, " + f"got {n!r}") + ids.append(available[n]) + return ids + + +def find_duplicate(title: str, existing: list) -> dict | None: + t = title.strip().lower() + for i in existing: + if i["title"].strip().lower() == t: + return i + return None + + +def validate_milestone_title(title: str) -> None: + """Refuse a comma. `milestones=` takes a comma-separated list of names, so a + title containing one splits into names that do not exist, the filter DROPS, + and the query returns the newest open issue in the WHOLE repository — which + the card then presents as that milestone's next action. Percent-encoding does + not save it; Forgejo decodes before splitting. Measured on null/fruit-fall, + where `0.3.7 Logo, Icons & Branding` was renamed for exactly this reason.""" + if "," in title: + die(f"milestone title contains a comma: {title!r}\n" + " That silently breaks the `milestones=` filter and makes the project\n" + " card show the wrong next action. Rename it without the comma.") + first = title.strip().split()[0] if title.strip() else "" + if first and first.replace("v", "", 1).replace(".", "").isdigit(): + rest = title.strip()[len(first):].strip() + if rest: + print(f" note: the dashboard phase will show just {first!r} — a title " + f"starting with a\n version token has the rest dropped. Put a " + f"word first to keep it whole.") + + +_warned_current: set[str] = set() + + +def _warn_if_current(api: Api, milestone: str) -> None: + """The dashboard's next action is the NEWEST open issue in the current + milestone — not the highest priority; priority labels have no influence at + all. Filing a routine item into the milestone the team is working on + therefore replaces what the card shows.""" + if milestone in _warned_current: + return + _warned_current.add(milestone) + cur = api.current_milestone() + if cur and cur["title"] == milestone: + print(f" warning: {milestone!r} is the CURRENT milestone. The next action on " + "the project\n card is the NEWEST open issue in it, ignoring " + "priority — so this will\n replace whatever is shown there now. " + "To avoid that, file into a\n milestone created later; order is " + "creation order, not title order.") + + +# ── commands ───────────────────────────────────────────────────────────────── + + +def cmd_check(api: Api, args): + """The probes that prove the card is not lying. Run before and after filing. + + Everything here exists because Forgejo's filters FAIL OPEN: given a value + they cannot match they ignore the filter and return the unfiltered list, with + no error. A query returning plausible results is not evidence it filtered. + """ + print(f"tracker health — {api.repo}\n") + problems = 0 + + labels = api.labels() + missing = [n for n in SEVERITY if n not in labels] + if missing: + problems += 1 + print(f" FAIL severity labels missing: {', '.join(missing)}") + print(" a query for a label that does not exist matches EVERYTHING") + else: + print(" ok all four severity labels defined") + + openi = api.issues(state="open") + blockers = [i for i in openi + if any(l["name"] == "release-blocker" for l in i["labels"])] + if blockers: + print(f" WARN {len(blockers)} open release-blocker — takes over the whole card:") + for i in blockers[:5]: + print(f" #{i['number']} {i['title'][:58]}") + else: + print(" ok no release-blocker hijacking the card") + + orphans = [i for i in openi if not i.get("milestone")] + if orphans: + problems += 1 + print(f" FAIL {len(orphans)} open issue(s) with no milestone — invisible on the card:") + for i in orphans[:5]: + print(f" #{i['number']} {i['title'][:58]}") + else: + print(f" ok no orphan issues ({len(openi)} open)") + + openms = api.milestones_full("open") + commas = [m["title"] for m in openms if "," in m["title"]] + if commas: + problems += 1 + print(f" FAIL comma in milestone title — breaks the filter: {commas}") + else: + print(" ok no comma in any open milestone title") + + empty = [m["title"] for m in openms if m["open_issues"] == 0] + if empty: + print(f" WARN {len(empty)} open milestone(s) with nothing in them — the card") + print(f" will read 'Close milestone …': {empty[:3]}") + else: + print(" ok no empty open milestones") + + cur = api.current_milestone() + if cur: + first = cur["title"].strip().split()[0] + tok = first.replace("v", "", 1).replace(".", "") + print(f"\n current milestone : {cur['title']!r}") + print(f" phase shown : {(first if tok.isdigit() else cur['title'].strip())!r}") + nxt = api.get(f"/repos/{api.repo}/issues", + {"type": "issues", "state": "open", "limit": 1, + "milestones": cur["title"]}) or [] + if nxt: + i = nxt[0] + in_ms = (i.get("milestone") or {}).get("title") + if in_ms != cur["title"]: + problems += 1 + print(f" next action : {i['title']!r}") + print(" ^ FILTER DROPPED — that issue is in " + f"{in_ms!r}.\n The card is showing a " + "wrong next action.") + else: + print(f" next action : {i['title']!r}") + else: + print("\n no open milestones — the phase would be 'release'") + + print(f"\n{'PROBLEMS: ' + str(problems) if problems else 'All checks passed.'}") + return 1 if problems else 0 + + +def cmd_milestone(api: Api, args): + """Create a milestone. No due date is ever set — a due date means a + commitment, and an invented one is worse than none.""" + title = args.title.strip() + validate_milestone_title(title) + if title in api.milestones(): + print(f"exists {title!r} — not creating a second one") + return + desc = args.description or "" + if args.description_file: + desc = open(args.description_file, encoding="utf-8").read() + if not desc.strip(): + print(" note: no description. It should say what the batch is for and how " + "anybody\n will know it landed.") + m = api.post(f"/repos/{api.repo}/milestones", + {"title": title, "description": desc}) + print(f"created milestone {m.get('title', title)!r} (id {m.get('id', '?')})") + + +def create_one(api: Api, spec: dict, labels_map, ms_map, existing, + allow_dup=False) -> dict | None: + title = spec["title"].strip() + body = check_verify_line(spec.get("body", ""), title) + + dup = find_duplicate(title, existing) + if dup and not allow_dup: + print(f"skip #{dup['number']} already titled {title!r} " + f"({dup['state']}) — creates are not idempotent, so this is a skip " + f"not an error") + return None + + payload = {"title": title, "body": body} + if spec.get("labels"): + payload["labels"] = resolve_labels(spec["labels"], labels_map) + if "release-blocker" in spec["labels"]: + print(" warning: release-blocker does NOT filter by milestone, so one " + "stray label\n takes over the phase and next action for " + "the entire project. It means\n *nothing else can " + "proceed* — it is not a synonym for important; that is P1.") + if spec.get("milestone"): + m = spec["milestone"] + if m not in ms_map: + die(f"milestone {m!r} does not exist. Available: " + f"{', '.join(sorted(ms_map)) or '(none)'}") + payload["milestone"] = ms_map[m] + _warn_if_current(api, m) + else: + print(f" warning: {title[:48]!r} has no milestone — it can never become the " + "next action\n and never appears anywhere on the project card.") + + d = api.post(f"/repos/{api.repo}/issues", payload) + names = ",".join(l["name"] for l in d.get("labels", [])) + mile = (d.get("milestone") or {}).get("title", "—") + print(f"filed #{d['number']} [{names}] {d['title']}" + + (f" → {mile}" if mile != "—" else "")) + return d + + +def cmd_create(api: Api, args): + labels_map, ms_map = api.labels(), api.milestones() + existing = api.issues(state="all") + body = args.body + if args.body_file: + body = open(args.body_file, encoding="utf-8").read() + create_one(api, {"title": args.title, "body": body or "", + "labels": args.label, "milestone": args.milestone}, + labels_map, ms_map, existing, args.allow_duplicate) + + +def cmd_batch(api: Api, args): + specs = json.load(open(args.file, encoding="utf-8")) + if isinstance(specs, dict): + specs = specs.get("issues", []) + if not isinstance(specs, list): + die("batch file must be a JSON list, or an object with an 'issues' list") + labels_map, ms_map = api.labels(), api.milestones() + existing = api.issues(state="all") + filed = 0 + for spec in specs: + d = create_one(api, spec, labels_map, ms_map, existing, + args.allow_duplicate) + if d: + filed += 1 + existing.append({"number": d["number"], "title": d["title"], + "state": "open"}) + print(f"\n{filed} filed, {len(specs) - filed} skipped") + + +def cmd_list(api: Api, args): + issues = api.issues(state=args.state) + groups: dict[str, list] = {} + for i in issues: + groups.setdefault((i.get("milestone") or {}).get("title", + "(no milestone)"), + []).append(i) + for m in sorted(groups): + print(f"\n### {m}") + for i in sorted(groups[m], key=lambda x: x["number"]): + names = ",".join(l["name"] for l in i["labels"]) + print(f" #{i['number']:<4} [{names}] {i['title']}") + print(f"\n{len(issues)} {args.state} issue(s) — pull requests excluded") + + +def cmd_close(api: Api, args): + """Close with the evidence that proves it: a path, a symbol, a test name, + or the command that shows it. 'Done' is not a close.""" + ev = args.evidence.strip() + if len(ev) < 15: + die("evidence too thin. Give a path, a symbol, a test name, or the " + "command that proves it — 'Done' is not a close.") + api.post(f"/repos/{api.repo}/issues/{args.number}/comments", {"body": ev}) + api.patch(f"/repos/{api.repo}/issues/{args.number}", {"state": "closed"}) + print(f"closed #{args.number} with evidence") + print("note: prefer `closes #N` in the commit that does the work — the " + "tracker then records who and when from the thing that happened.") + + +def cmd_labels(api: Api, args): + labels = api.labels() + print("severity (exact names — queried by name by the Command Center):") + for s in SEVERITY: + print(f" {'✓' if s in labels else '✗ MISSING'} {s}" + + (f" id={labels[s]}" if s in labels else "")) + print("\nother:") + for n, i in sorted(labels.items()): + if n not in SEVERITY: + print(f" {n} id={i}") + + +def cmd_milestones(api: Api, args): + for state in ("open", "closed"): + ms = api.get(f"/repos/{api.repo}/milestones", + {"state": state, "limit": 100}) or [] + if ms: + print(f"\n{state}:") + for m in ms: + print(f" [{m['id']}] {m['title']} — open {m['open_issues']} / " + f"closed {m['closed_issues']}") + + +def main(): + # --repo and --dry-run are accepted on BOTH sides of the subcommand. Putting + # them only on the top-level parser means `… create "T" --dry-run` — the + # natural way to type it, and the position that matters most — dies with an + # argparse usage error instead of previewing. A safety flag that is easy to + # put in the wrong place is a safety flag that gets left off. + # default=SUPPRESS is load-bearing, not tidiness. With a normal default the + # subparser re-defines the same dest and argparse writes its default over + # whatever the top-level parser already parsed — so `--repo X create …` + # silently became repo=None and `--dry-run create …` silently became False. + # A --dry-run that quietly turns itself off is the worst possible bug in a + # tool whose job is writing to a live tracker. SUPPRESS leaves the attribute + # unset unless it was actually given, so neither position clobbers the other. + common = argparse.ArgumentParser(add_help=False) + common.add_argument("--repo", default=argparse.SUPPRESS, + help="owner/name (default: from git remote)") + common.add_argument("--dry-run", action="store_true", + default=argparse.SUPPRESS, + help="print payloads, change nothing") + + p = argparse.ArgumentParser( + parents=[common], + description="File Forgejo issues in the tracker convention.", + epilog="Every open issue is a denominator. Do not pad the tracker.") + sub = p.add_subparsers(dest="cmd", required=True) + + c = sub.add_parser("create", parents=[common], help="file one issue") + c.add_argument("title") + c.add_argument("--body", help="issue body; must end with a Verify: line") + c.add_argument("--body-file", help="read the body from a file") + c.add_argument("--label", action="append", default=[], + help="label name, repeatable") + c.add_argument("--milestone", help="milestone title") + c.add_argument("--allow-duplicate", action="store_true") + c.set_defaults(fn=cmd_create) + + b = sub.add_parser("batch", parents=[common], help="file several from a JSON file") + b.add_argument("file") + b.add_argument("--allow-duplicate", action="store_true") + b.set_defaults(fn=cmd_batch) + + l = sub.add_parser("list", parents=[common], help="open issues by milestone") + l.add_argument("--state", default="open", + choices=["open", "closed", "all"]) + l.set_defaults(fn=cmd_list) + + x = sub.add_parser("close", parents=[common], help="close with an evidence comment") + x.add_argument("number", type=int) + x.add_argument("evidence", help="what was checked and where") + x.set_defaults(fn=cmd_close) + + m = sub.add_parser("milestone", parents=[common], help="create a milestone (batch)") + m.add_argument("title") + m.add_argument("--description") + m.add_argument("--description-file") + m.set_defaults(fn=cmd_milestone) + + sub.add_parser("check", parents=[common], help="health probes — run before AND after filing" + ).set_defaults(fn=cmd_check) + sub.add_parser("labels", parents=[common], help="labels and their ids").set_defaults( + fn=cmd_labels) + sub.add_parser("milestones", parents=[common], help="milestones and their ids").set_defaults( + fn=cmd_milestones) + + args = p.parse_args() + # getattr, because SUPPRESS means the attribute may legitimately be absent. + repo = getattr(args, "repo", None) or detect_repo() + if not repo: + die("could not detect owner/name from the git remote — pass --repo") + host, token = load_env() + rc = args.fn(Api(host, token, repo, getattr(args, "dry_run", False)), args) + # `check` returns a count so it can gate a script; the rest return None. + sys.exit(rc or 0) + + +if __name__ == "__main__": + main() diff --git a/scripts/healthcheck.sh b/scripts/healthcheck.sh new file mode 100755 index 0000000..3cfe63a --- /dev/null +++ b/scripts/healthcheck.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash +# +# Liveness tick for a deployed service. +# +# ## Why this exists +# +# The five-minute healthcheck it replaced was prose handed to a model: "GET +# /healthz. Do NOT use /api/internal/v1/healthz." It duly reported a 404 +# on `/api/internal/v1/health` — a third path, neither the one it was told to +# use nor the one it was told to avoid, and one that had never existed. The site +# was healthy throughout. +# +# An explicit prohibition constrained one wrong URL and left every other wrong +# URL open, because the address was being re-derived on every run rather than +# read. So it is written here once, as a string in version control, and the +# whole class of failure goes with it. +# +# That failure is the expensive kind. The job's own state read `lastRunStatus: +# ok, consecutiveErrors: 0` while a red alert went to a DM — so the monitor was +# reporting itself healthy and crying wolf at the same time, and a monitor +# nobody believes is a monitor nobody has. +# +# ## It holds no credential, and that is the point +# +# `/healthz` is unauthenticated by design — `SECURITY_CHECKLIST.md` makes it a +# checklist item, and the container's own HEALTHCHECK uses it. So unlike +# `reconcile.sh` and `analyze.sh`, this script reads no token, sources no env +# file, and has nothing to leak. At 288 runs a day that is worth more than the +# extra assurance an authenticated probe would buy. +# +# The authenticated sibling, `/api/internal/v1/agent/health`, reports more and +# needs a token. It is deliberately not used here: this asks "is the site up", +# which is a question with a public answer. +# +# ## Both halves are checked +# +# 503 is a real answer, not an outage — `/api/health` returns it when the +# `SELECT 1` against SQLite fails, so the container is marked unhealthy while +# the marketing pages keep serving perfectly well. Status alone is therefore +# not a verdict; the check asserts HTTP 200 *and* `"status":"ok"` *and* +# `"db":"ok"`. +# +# ## The path, written down rather than re-derived +# +# This project's endpoint is `/api/health`. It is NOT `/healthz`, which is what +# the template's copy of this script probed and what a plausible guess produces. +# The whole argument above is about exactly this line, so it is stated once, +# here, and nowhere else. +# +# ## Installing it +# +# install -m 0755 healthcheck.sh ~/bin/healthcheck.sh +# # then, in the crontab — every five minutes +# */5 * * * * $HOME/bin/healthcheck.sh \ +# >> $HOME/.healthcheck.log 2>&1 +# +# On this deployment it is run by an OpenClaw cron job instead, which is the +# same thing with a scheduler that can also deliver the alert. + +set -euo pipefail + +# `-`, not `:-`. Unset means "no opinion, use production". Set-and-empty means +# a config is wrong, and substituting production for it would report the health +# of a site nobody asked about — quietly, and only where somebody was trying to +# point this somewhere else. +BASE_URL="${HEALTHCHECK_BASE_URL:-https://qn.isnull.dev}" +TIMEOUT="${PRIVACY_TIMEOUT:-15}" + +stamp() { date -Is; } +say() { printf '%s healthcheck: %s\n' "$(stamp)" "$*"; } + +if [ -z "$BASE_URL" ]; then + say "FAIL no base URL. HEALTHCHECK_BASE_URL is set but empty; unset it for the default." + exit 78 # EX_CONFIG +fi + +body=$(mktemp) +trap 'rm -f "$body"' EXIT + +# Assigned in the `if`, not with `|| echo "000"` appended. On a connection +# failure curl *already* prints "000" via --write-out and then exits non-zero, +# so appending a fallback produces "000000", which matches no branch below and +# reports "unexpected HTTP" for the one failure this script names explicitly. +# +# No -v and no --trace, ever. This request carries no credential, but the habit +# is the rule SECURITY.md states: a request URL or header must not reach a log. +if ! status=$( + curl --silent --show-error --output "$body" --write-out '%{http_code}' \ + --max-time "$TIMEOUT" \ + "$BASE_URL/api/health" +); then + status="000" +fi + +case "$status" in + 200) + # The status got us here; the body decides. Both fields are asserted rather + # than assumed because a 200 from Cloudflare, a cached page or an error page + # is still a 200, and none of them are this application answering. + # + # Both, not either: the app can answer `"status":"ok"` while its database + # handle is gone, and that is the state where the site looks fine and every + # form submission is being lost. + if grep -q '"status":"ok"' "$body" && grep -q '"db":"ok"' "$body"; then + say "ok" + elif grep -q '"status":"ok"' "$body"; then + say "FAIL 200 and the app is up, but the database is not answering." + say " Every form submission is failing. See docs/OPERATIONS.md." + exit 70 # EX_SOFTWARE + else + say "FAIL 200 but not this application. The response carried neither" + say " \"status\":\"ok\" nor a recognisable health body — check whether" + say " something in front of the origin answered instead." + exit 70 # EX_SOFTWARE + fi + ;; + 503) + # Named rather than left to the catch-all, because it is the one unhealthy + # answer this route is designed to give: migrations failed, the container is + # marked unhealthy, and the public pages are still being served. That is a + # different thing from the site being down and reads differently at 3am. + say "FAIL the database is not reachable (HTTP 503). The site is serving" + say " static pages; every lead and support submission is being lost." + exit 70 # EX_SOFTWARE + ;; + 000) + say "FAIL could not reach $BASE_URL" + exit 69 # EX_UNAVAILABLE + ;; + *) + # Includes 404. If this ever fires on a path this script wrote itself, the + # route moved — which is a thing to fix in one place rather than a thing for + # a caller to guess around. + say "FAIL unexpected HTTP $status from $BASE_URL/api/health" + exit 1 + ;; +esac diff --git a/scripts/preflight.sh b/scripts/preflight.sh new file mode 100755 index 0000000..0db04b1 --- /dev/null +++ b/scripts/preflight.sh @@ -0,0 +1,278 @@ +#!/usr/bin/env bash +# +# The checks you can run against a live URL in two minutes, before strangers do. +# +# bash scripts/preflight.sh # passive: headers and TLS +# bash scripts/preflight.sh --auth # also the two that generate load +# bash scripts/preflight.sh --dry-run # print the plan, contact nothing +# +# Exit codes: +# +# 0 every check that ran found nothing +# 1 findings, each one named +# 2 NOTHING WAS CHECKED — unreachable, unconfigured, or curl missing. Not a +# pass: a site that could not be contacted and a site with no problems must +# never exit the same way. +# +# =========================================================================== +# TEMPLATE COPY — configure this before the first run +# =========================================================================== +# +# Set PREFLIGHT_ORIGIN to the one origin this copy is allowed to test. +# +# Assumes: bash, coreutils, `curl`. +# +# ## Why this exists +# +# These are the findings that people who audit applications of this kind report +# seeing over and over, and every one of them is mechanical: a header that is +# absent, a scheme that is plain, a login that answers a thousand guesses, a +# reset form that confirms which addresses have accounts. None needs to be +# understood to be checked, which is exactly why it belongs in a script rather +# than in a page somebody re-reads before each release and then does not. +# +# *(precautionary)*: none of this has bitten a project here yet. It is included +# because the checks are cheap and the evidence for them is somebody else's. +# +# ## It refuses to run against anything but its own origin +# +# Two of these checks — repeated bad logins, and asking whether an account +# exists — are indistinguishable from an attack in somebody's log, and one of +# them deliberately generates failed authentications. So there is no URL +# argument that can point this anywhere: the target comes from PREFLIGHT_ORIGIN +# and a URL passed on the command line must match it, or the run is refused. +# +# That is `status.sh`'s argument for having no --host flag, with more at stake: +# there, a mistake reads the wrong machine, and here it hammers somebody else's +# login form from your address. +# +# The active checks are further gated behind --auth, so the default run sends +# exactly two GETs and could not be mistaken for anything. +# +# ## Present is not in force +# +# It also counts each header rather than only looking for one. More than one +# copy means the later ones are discarded — RFC 6797 requires exactly that for +# HSTS and browsers do the same elsewhere — so a directive can sit in a response, +# be read by a person as applying, and have never once applied. Two layers each +# adding their own is all it takes. +# +# ## What it cannot tell you +# +# A header being present is not a header being correct — a CSP of +# `default-src *` is a CSP. This reports presence, which is the part that is +# mechanically checkable, and a present-but-useless policy is a job for a person +# or for securityheaders.com's grade. Absence is the common case and the one +# this catches. + +set -uo pipefail + +say() { printf '\033[1mpreflight:\033[0m %s\n' "$*" >&2; } +die() { printf '\033[1mpreflight:\033[0m %s\n' "$*" >&2; exit 2; } + +# --------------------------------------------------------------------------- +# CONFIGURATION — set this one, then delete this banner. +# +# The single origin this copy may contact, e.g. https://app.example.com. +# Empty on purpose: a default here is a script that attacks whatever origin the +# project it was copied from happened to use. +# --------------------------------------------------------------------------- +ORIGIN="${PREFLIGHT_ORIGIN:-https://qn.isnull.dev}" + +# Paths used only by --auth. Unset means that check reports "not configured" +# rather than guessing at /login, which on the wrong app is a POST to something +# else entirely. +LOGIN_PATH="${PREFLIGHT_LOGIN_PATH:-}" +RESET_PATH="${PREFLIGHT_RESET_PATH:-}" +ATTEMPTS="${PREFLIGHT_LOGIN_ATTEMPTS:-12}" + +AUTH="" +DRY_RUN="" +TARGET="" + +while [ $# -gt 0 ]; do + case "$1" in + --auth) AUTH="yes"; shift ;; + --dry-run) DRY_RUN="yes"; shift ;; + -h|--help) + say "usage: bash scripts/preflight.sh [--auth] [--dry-run] []" + say " must match PREFLIGHT_ORIGIN. Set PREFLIGHT_LOGIN_PATH" + say " and PREFLIGHT_RESET_PATH for --auth." + exit 0 ;; + -*) die "unknown argument '$1'." ;; + *) TARGET="$1"; shift ;; + esac +done + +command -v curl >/dev/null 2>&1 || die "curl is not on PATH, so nothing could be checked." +[ -n "$ORIGIN" ] || die "set PREFLIGHT_ORIGIN — the one origin this copy may contact. See the CONFIGURATION block." + +host_of() { printf '%s' "$1" | sed -E 's#^[a-zA-Z]+://##; s#/.*$##; s#:.*$##'; } + +ORIGIN_HOST="$(host_of "$ORIGIN")" +[ -n "$ORIGIN_HOST" ] || die "PREFLIGHT_ORIGIN '$ORIGIN' does not look like a URL." + +if [ -n "$TARGET" ]; then + TARGET_HOST="$(host_of "$TARGET")" + if [ "$TARGET_HOST" != "$ORIGIN_HOST" ]; then + say "refusing: '$TARGET' is host '$TARGET_HOST', and this copy is configured" + say " for '$ORIGIN_HOST'." + say "Two of these checks generate failed logins and look like an attack in" + say "somebody's log. Pointing them at a host this copy was not configured" + say "for is the mistake the refusal exists to prevent — change" + say "PREFLIGHT_ORIGIN deliberately if you mean it." + exit 2 + fi +else + TARGET="$ORIGIN" +fi + +case "$ATTEMPTS" in ''|*[!0-9]*) die "PREFLIGHT_LOGIN_ATTEMPTS must be a whole number, got '$ATTEMPTS'." ;; esac + +if [ -n "$DRY_RUN" ]; then + say "--dry-run: nothing was contacted. Target: $TARGET" + say "would run:" + printf ' curl -sSI %s # headers, TLS\n' "$TARGET" >&2 + printf ' curl -sSI http://%s/ # is plaintext served or redirected\n' "$ORIGIN_HOST" >&2 + if [ -n "$AUTH" ]; then + printf ' %s POSTs of bad credentials to %s\n' "$ATTEMPTS" "${LOGIN_PATH:-}" >&2 + printf ' 1 reset request for an address that does not exist to %s\n' "${RESET_PATH:-}" >&2 + else + printf ' (--auth not given: the two active checks are skipped)\n' >&2 + fi + exit 0 +fi + +findings=0 +finding() { printf ' FINDING %s\n' "$*" >&2; findings=$((findings + 1)); } +ok() { printf ' ok %s\n' "$*" >&2; } +skip() { printf ' skipped %s\n' "$*" >&2; } + +# --------------------------------------------------------------------------- +# Passive: one request, several answers. +# --------------------------------------------------------------------------- + +HEADERS=$(curl -sS -I -L --max-time 20 "$TARGET" 2>/dev/null) \ + || die "could not reach $TARGET. Nothing was checked — this is not a report that the site is fine." +[ -n "$HEADERS" ] || die "$TARGET returned no headers. Nothing was checked." + +lower_headers=$(printf '%s' "$HEADERS" | tr '[:upper:]' '[:lower:]') + +# `curl -L` concatenates the headers of every response in the chain, so a header +# counted across all of them would report a redirect's copy as a duplicate of +# the final page's. Only the last response block is counted. +final_block() { awk 'tolower($0) ~ /^http\// { buf = "" } { buf = buf $0 "\n" } END { printf "%s", buf }'; } +final_headers=$(printf '%s' "$lower_headers" | final_block) +final_raw=$(printf '%s' "$HEADERS" | tr -d '\r' | final_block) + +# Present is not the same as in force. RFC 6797 section 8.1 is explicit for HSTS +# -- more than one and the agent MUST process only the first -- and browsers +# behave the same way for the others: the second copy is discarded in silence. +# Two layers each adding their own is the ordinary cause, and the result reads +# to a person as though both applied. +# +# Found on this script's first real target: an origin serving +# strict-transport-security: max-age=63072000; includeSubDomains +# strict-transport-security: max-age=63072000; preload +# where preload had never once been in force. +check_duplicate() { # + local name="$1" count first + count=$(printf '%s' "$final_headers" | grep -cE "^${name}:" || true) + [ "${count:-0}" -gt 1 ] || return 0 + # Counted against the lowercased copy, but quoted from the original: echoing + # `includesubdomains` back at somebody who wrote `includeSubDomains` reports a + # value they did not send. Directive names are case-insensitive; the report + # should still show what is actually on the wire. + # The FIRST occurrence, within the FINAL response block. Both halves matter: + # RFC 6797 processes the first and discards the rest, and `curl -L` hands us + # the headers of every hop, so searching the whole buffer would quote a + # redirect's copy. Quoted from the original rather than the lowercased copy — + # echoing `includesubdomains` at somebody who wrote `includeSubDomains` + # reports a value they never sent. + first=$(printf '%s' "$final_raw" | grep -iE "^${name}:" | head -n 1 | sed -E "s/^[^:]*:[[:space:]]*//") + finding "${count} ${name} headers — only the first is processed, so what is in force is '${first}' and every later copy is discarded silently" + return 1 +} + +case "$TARGET" in + https://*) ok "the target is https" ;; + *) finding "the target is not https — everything in transit is readable, including the session cookie" ;; +esac + +if printf '%s' "$lower_headers" | grep -q '^content-security-policy:'; then + check_duplicate content-security-policy && ok "content-security-policy present" +else + finding "no content-security-policy header — injected script has nothing to stop it" +fi + +# Either header answers the framing question; frame-ancestors is the modern one +# and x-frame-options the one older browsers read, so one of the two is enough. +if printf '%s' "$lower_headers" | grep -q '^x-frame-options:' \ + || printf '%s' "$lower_headers" | grep -q 'frame-ancestors'; then + check_duplicate x-frame-options && ok "framing policy present" +else + finding "neither x-frame-options nor a csp frame-ancestors — the page can be framed and clickjacked" +fi + +if printf '%s' "$lower_headers" | grep -q '^strict-transport-security:'; then + check_duplicate strict-transport-security && ok "strict-transport-security present" +else + finding "no strict-transport-security — the first request of each visit can still be plaintext" +fi + +# Plain HTTP: a redirect is the right answer; a 200 is a site served in the clear. +PLAIN=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 15 "http://$ORIGIN_HOST/" 2>/dev/null || printf 'none') +case "$PLAIN" in + 30[0-9]) ok "plain http redirects ($PLAIN)" ;; + none) skip "plain http did not answer at all, which is also fine" ;; + 200) finding "plain http answered 200 — the site is served unencrypted as well as encrypted" ;; + *) skip "plain http answered $PLAIN, which is neither a redirect nor a page" ;; +esac + +# --------------------------------------------------------------------------- +# Active, and only with --auth. These generate failed authentications. +# --------------------------------------------------------------------------- + +if [ -z "$AUTH" ]; then + skip "rate limiting and account enumeration (pass --auth, and read what it does first)" +else + if [ -z "$LOGIN_PATH" ]; then + skip "rate limiting — PREFLIGHT_LOGIN_PATH unset, and guessing at /login POSTs to whatever is there" + else + say "sending $ATTEMPTS failed logins to ${TARGET%/}$LOGIN_PATH — this will appear in the logs" + limited="" + for i in $(seq 1 "$ATTEMPTS"); do + code=$(curl -sS -o /dev/null -w '%{http_code}' --max-time 10 \ + -X POST -H 'Content-Type: application/json' \ + --data '{"email":"preflight-probe@example.invalid","password":"not-a-real-password"}' \ + "${TARGET%/}$LOGIN_PATH" 2>/dev/null || printf 'none') + case "$code" in 429|423) limited="$code at attempt $i"; break ;; esac + done + if [ -n "$limited" ]; then + ok "authentication is rate limited ($limited)" + else + finding "$ATTEMPTS failed logins in a row, no 429 and no lockout — a password list can be run against this overnight" + fi + fi + + if [ -z "$RESET_PATH" ]; then + skip "account enumeration — PREFLIGHT_RESET_PATH unset" + else + BODY=$(curl -sS --max-time 10 -X POST -H 'Content-Type: application/json' \ + --data '{"email":"definitely-not-registered-preflight@example.invalid"}' \ + "${TARGET%/}$RESET_PATH" 2>/dev/null || printf '') + if printf '%s' "$BODY" | grep -qiE "no (account|user)|not (found|registered)|does not exist|unknown email"; then + finding "the reset endpoint says an unregistered address is unknown — that confirms which addresses DO have accounts, which is the input to a phishing or credential-stuffing list" + else + ok "the reset endpoint does not reveal whether the address is registered" + fi + fi +fi + +say "" +if [ "$findings" -gt 0 ]; then + say "$findings finding(s) above." + exit 1 +fi + +say "no findings from the checks that ran." diff --git a/scripts/restore-check.sh b/scripts/restore-check.sh new file mode 100755 index 0000000..32505f8 --- /dev/null +++ b/scripts/restore-check.sh @@ -0,0 +1,249 @@ +#!/usr/bin/env bash +# +# Replay the newest dump into a scratch database, count what arrived, and time +# it. Then throw the scratch database away. +# +# bash scripts/restore-check.sh # the newest dump +# bash scripts/restore-check.sh --dry-run # print the plan, touch nothing +# bash scripts/restore-check.sh --file X # a specific dump, not the newest +# +# Exit codes, because a caller must be able to act on the answer: +# +# 0 restored, and what came back is plausible +# 1 the restore failed, or the result holds fewer tables than the minimum +# 2 NOTHING WAS CHECKED — unconfigured, no dump found, or a missing tool. +# Not a pass. A restore check that did not run and one that succeeded must +# never exit the same way. +# +# =========================================================================== +# THIS IS A REWRITE, NOT A CONFIGURED COPY +# =========================================================================== +# +# The template ships this as PostgreSQL — `pg_restore --clean` into a scratch +# database created and dropped over `psql`, with RESTORE_ADMIN_URL naming the +# server. Unlike `backup.sh`, which isolates its engine in one block precisely +# so it can be swapped, this script is engine-specific end to end: there was no +# seam to configure. So the argument was kept and the mechanism replaced. +# +# Everything below that reads like the original is deliberate. The exit codes, +# the refusal to accept a target, the timing, and the rule that a dump which +# merely *reads* is not a backup are all the template's and all still true. +# +# Assumes: bash, coreutils, `sqlite3`. +# +# ## Why this exists +# +# `backup.sh` says it plainly: it verifies the artefact, and only a restore +# verifies the backup. Its header describes this script as "the other half". +# +# A SQLite file that answers `PRAGMA integrity_check` is a well-formed +# database. It is not yet *your* database. Between those two facts sit the +# reasons a restore disappoints on the day it is needed: a snapshot of an empty +# database taken after the volume was recreated, a snapshot of the wrong +# container that reads perfectly and holds somebody else's rows, a schema from +# before a migration, or a table that exists with nothing in it. +# +# So this does not repeat the integrity check. It does what a recovery actually +# does: serialises the snapshot to SQL and replays it into an empty database. +# That exercises every CREATE and every INSERT, which is the part that fails. +# +# And there is a number nobody has that they will want badly: **how long it +# takes**. During an incident that number decides whether you restore or start +# apologising, and it is unknowable from the file size. This prints it every run. +# +# ## The dangerous part, and what is done about it +# +# A restore writes. Pointed at the live database it would overwrite every lead +# the site has ever taken, immediately and irreversibly. That is the entire risk +# surface of this script, and it is handled the way the template handles it — +# by never accepting a target at all: +# +# - There is no --into flag and no RESTORE_TARGET path. You cannot name the +# database to restore into, because naming it is the mistake. +# - The scratch database is created inside a fresh `mktemp -d`, under a name +# this script generates, and the whole directory is removed by a trap on +# every exit path including the failures. +# - The source dump is opened READ-ONLY, over a `file:…?mode=ro` URI, so a +# mistyped path cannot damage the thing it was pointed at either. +# +# ## What it does not prove +# +# That the application runs against the restored file. It proves the schema and +# the rows come back, which is the half that is mechanically checkable. Standing +# the container up against a restored volume is a person's job, and belongs in +# `docs/OPERATIONS.md` the day somebody does it. + +set -uo pipefail + +say() { printf '\033[1mrestore-check:\033[0m %s\n' "$*" >&2; } +die() { printf '\033[1mrestore-check:\033[0m %s\n' "$*" >&2; exit 2; } +fail() { printf '\033[1mrestore-check:\033[0m %s\n' "$*" >&2; exit 1; } + +# --------------------------------------------------------------------------- +# Configuration — shared with backup.sh, and read from the environment for that +# reason. A restore check pointed at another series answers confidently about +# the wrong database, which is worse than not running. +# --------------------------------------------------------------------------- +BACKUP_DIR="${BACKUP_DIR:-$HOME/backups/queue-north-website}" +BACKUP_NAME="${BACKUP_NAME:-queuenorth-leads}" +MIN_TABLES="${BACKUP_MIN_TABLES:-1}" + +# Refuse a restore that comes back with fewer rows than this ACROSS ALL TABLES. +# Zero is allowed and is the default, because a brand-new deployment legitimately +# has no leads yet — but set it once there are rows, and the check starts +# catching the snapshot-of-an-empty-volume case that no structural test can see. +MIN_ROWS="${RESTORE_MIN_ROWS:-0}" + +DUMP_SUFFIX=".sqlite" + +DRY_RUN="" +ONE_FILE="" + +while [ $# -gt 0 ]; do + case "$1" in + --dry-run) DRY_RUN="yes"; shift ;; + --file) + ONE_FILE="${2:-}" + [ -n "$ONE_FILE" ] || die "--file needs a path." + shift 2 ;; + -h|--help) + say "usage: bash scripts/restore-check.sh [--dry-run] [--file ]" + say " BACKUP_DIR and BACKUP_NAME are shared with backup.sh." + exit 0 ;; + *) die "unknown argument '$1'. There is deliberately no flag naming a restore target." ;; + esac +done + +# --------------------------------------------------------------------------- +# Refuse to run half-configured, before anything is touched, naming the missing +# value one at a time so the message says which. +# --------------------------------------------------------------------------- + +[ -n "$BACKUP_DIR" ] || die "set BACKUP_DIR — the same directory backup.sh writes to." +[ -n "$BACKUP_NAME" ] || die "set BACKUP_NAME — the same filename prefix backup.sh uses." +[ -d "$BACKUP_DIR" ] || die "BACKUP_DIR '$BACKUP_DIR' is not a directory. Nothing was checked." + +command -v sqlite3 >/dev/null 2>&1 \ + || die "sqlite3 is not on PATH, so nothing could be restored. Nothing was checked." + +# --------------------------------------------------------------------------- +# Choose the dump. Newest by modification time, within this series only. +# --------------------------------------------------------------------------- + +if [ -n "$ONE_FILE" ]; then + DUMP="$ONE_FILE" + [ -f "$DUMP" ] || die "'$DUMP' is not a file. Nothing was checked." +else + # `.part` files are half-written by definition and must never be selected; + # backup.sh only renames into place after it has verified, so the glob below + # matching the final suffix exactly is what keeps those out. + DUMP=$(find "$BACKUP_DIR" -maxdepth 1 -type f \ + -name "${BACKUP_NAME}-*${DUMP_SUFFIX}" -printf '%T@ %p\n' 2>/dev/null \ + | sort -rn | head -1 | cut -d' ' -f2-) + + [ -n "$DUMP" ] || die "no dump matching '${BACKUP_NAME}-*${DUMP_SUFFIX}' in $BACKUP_DIR. + Nothing was checked — which is not the same as a backup that failed, and + not the same as one that passed." +fi + +DUMP_SIZE=$(du -h "$DUMP" 2>/dev/null | cut -f1) + +if [ -n "$DRY_RUN" ]; then + say "--dry-run: nothing was created, written or removed. It would have:" + say " read $DUMP (${DUMP_SIZE:-size unknown}), read-only" + say " created a scratch database in a fresh mktemp -d" + say " restored sqlite3 '' < (sqlite3 'file:?mode=ro' .dump)" + say " counted tables and rows, requiring at least $MIN_TABLES table(s) and $MIN_ROWS row(s)" + say " removed the scratch directory, on every exit path" + exit 0 +fi + +# --------------------------------------------------------------------------- +# The scratch database. Created inside a temporary directory that the trap +# removes, so there is no path here that outlives the run. +# --------------------------------------------------------------------------- + +SCRATCH_DIR=$(mktemp -d) || die "could not create a temporary directory. Nothing was checked." +trap 'rm -rf "$SCRATCH_DIR"' EXIT + +SCRATCH="$SCRATCH_DIR/restorecheck_$$.db" +ERRFILE="$SCRATCH_DIR/err" + +say "restoring $DUMP (${DUMP_SIZE:-size unknown})" +say " -> $SCRATCH" + +# --------------------------------------------------------------------------- +# The restore, timed. +# +# `.dump` serialises schema and data to SQL; replaying it builds the database +# from nothing. That is the point — a file copy would prove only that `cp` +# works, which is the check this script exists to be better than. +# +# SECONDS is bash's own counter and needs no external date arithmetic. +# --------------------------------------------------------------------------- + +START=$SECONDS + +if ! sqlite3 "file:${DUMP}?mode=ro" .dump 2>"$ERRFILE" | sqlite3 "$SCRATCH" 2>>"$ERRFILE"; then + say "the restore failed. sqlite3 said:" + sed 's/^/ /' "$ERRFILE" >&2 + fail "$DUMP did not restore. This dump is not a backup." +fi + +ELAPSED=$(( SECONDS - START )) + +# A non-empty stderr with a zero exit is the case worth catching: sqlite3 will +# report a constraint or a duplicate and carry on, leaving a database that is +# missing exactly the rows it complained about. +if [ -s "$ERRFILE" ]; then + say "WARNING: the restore reported problems while still exiting zero:" + sed 's/^/ /' "$ERRFILE" >&2 + say " What follows counted whatever survived that." +fi + +# --------------------------------------------------------------------------- +# What came back. Counted, not assumed. +# --------------------------------------------------------------------------- + +TABLES=$(sqlite3 "$SCRATCH" \ + "SELECT count(*) FROM sqlite_master + WHERE type='table' AND name NOT LIKE 'sqlite\_%' ESCAPE '\';" 2>/dev/null) + +[ -n "$TABLES" ] || fail "the restored database could not be counted, so nothing about it is known." + +if [ "$TABLES" -lt "$MIN_TABLES" ]; then + fail "the restore produced $TABLES table(s), fewer than the $MIN_TABLES required. + A dump of an empty or wrong database restores perfectly and looks like this." +fi + +# Per-table row counts, built as SQL and then run — sqlite3 has no built-in for +# it. Printed per table rather than only as a total, because "leads: 0" beside +# "support_requests: 40" is a different incident from both being zero. +ROW_SQL=$(sqlite3 "$SCRATCH" \ + "SELECT 'SELECT ''' || name || ''', count(*) FROM \"' || name || '\";' + FROM sqlite_master + WHERE type='table' AND name NOT LIKE 'sqlite\_%' ESCAPE '\' + ORDER BY name;" 2>/dev/null) + +TOTAL_ROWS=0 +if [ -n "$ROW_SQL" ]; then + while IFS='|' read -r tname tcount; do + [ -n "$tname" ] || continue + printf ' %-24s %s row(s)\n' "$tname" "$tcount" >&2 + TOTAL_ROWS=$(( TOTAL_ROWS + tcount )) + done < <(printf '%s\n' "$ROW_SQL" | sqlite3 "$SCRATCH" 2>/dev/null) +fi + +if [ "$TOTAL_ROWS" -lt "$MIN_ROWS" ]; then + fail "the restore produced $TOTAL_ROWS row(s) across $TABLES table(s), fewer than the + $MIN_ROWS required by RESTORE_MIN_ROWS. Either the snapshot is of an empty + volume, or the threshold is stale." +fi + +say "" +say "restored in ${ELAPSED}s — $TABLES table(s), $TOTAL_ROWS row(s) total." +say "" +say "That number is the one an incident needs: it is how long this takes with" +say "the data as it is today, measured rather than guessed. Record it in" +say "docs/OPERATIONS.md beside the date, and update the 'Last verified restore'" +say "row — that row is the only thing separating a backup from a file." diff --git a/scripts/secrets.sh b/scripts/secrets.sh new file mode 100755 index 0000000..22cf322 --- /dev/null +++ b/scripts/secrets.sh @@ -0,0 +1,311 @@ +#!/usr/bin/env bash +# +# Credentials, before they are committed. +# +# ## Why this and not a generic scanner +# +# A general-purpose secret scanner knows about AWS keys and GitHub tokens. It +# does not know that *this* deployment issues `pllc_agent_`, or that its +# encrypted envelopes start `v2::`, or which of its environment variables +# hold a password. The project does know, and usually writes it down twice: once +# in whatever redacts its logs, and once in whatever redacts its outbound +# messages. +# +# So this reads the project's own patterns where they exist — point +# `SECRETS_PATTERN_FILE` at the module holding them — and falls back to a +# conservative built-in set. A scanner tuned to the shapes a project actually +# issues catches the leak a generic one misses, and stays quiet the rest of the +# time. +# +# ## What it scans +# +# By default the **staged diff**, which is the only moment a commit can still be +# stopped cheaply. `--tracked` scans every tracked file instead, which is what +# you want once, on adoption, to find what is already in the history's tip. +# +# bash scripts/secrets.sh # staged changes (use in pre-commit) +# bash scripts/secrets.sh --tracked # everything tracked, for an audit +# bash scripts/secrets.sh --built dist/ # the artifact users receive +# SECRETS_PATTERN_FILE=src/lib/log.ts bash scripts/secrets.sh +# bash scripts/secrets.sh --allow docs/examples/ +# +# ## --built, and why the repository is the wrong place to stop +# +# The two modes above scan what is in git. Neither sees the bundle, which is the +# only artifact a user actually receives — and a key can reach it without ever +# being committed, from an environment variable inlined at build time. Somebody +# auditing applications of this kind reported finding hardcoded credentials in +# the frontend bundle of seven of eight in a single week. +# +# So --built walks a build directory instead, with two tiers of result: +# +# findings, which fail eyJ (a JWT header), service_role, apikey=, Bearer, +# plus every pattern the other modes use +# noted, which do not anon, VITE_, REACT_APP_, NEXT_PUBLIC_ +# +# The second tier is printed and changes nothing. Those prefixes mean +# "deliberately shipped to the browser", so failing on them would be a +# permanently red gate, and a gate that is always red is one everybody has +# learned to ignore. But they are worth *seeing* enumerated: a Supabase anon key +# is safe exactly as far as row-level security makes it safe, and knowing it is +# out there is the input to that judgement rather than a substitute for it. +# +# ## What it cannot do +# +# It reads the working tree and the index. **A secret already committed is still +# in the history** after you delete it, and this will not tell you that — the +# fix there is a rotation, not a scan. Rotate first, then clean up. +# +# Exit codes: 0 nothing found. 1 a candidate found. 2 nothing was scanned. +# --------------------------------------------------------------------------- + +set -uo pipefail + +cd "$(git rev-parse --show-toplevel 2>/dev/null)" || { + printf 'secrets: not a git repository.\n' >&2 + exit 2 +} + +say() { printf 'secrets: %s\n' "$*" >&2; } + +# The delimiter for the masking substitution below. A real control byte, because +# these patterns contain both `/` and `|` and either would end the expression +# early. It cannot occur in a pattern and it cannot occur in source text. +MASK_D=$'\001' + +MODE="staged" +BUILT_DIR="" +ALLOW=() + +while [ $# -gt 0 ]; do + case "$1" in + --tracked) MODE="tracked"; shift ;; + --staged) MODE="staged"; shift ;; + --built) + MODE="built" + BUILT_DIR="${2:-}" + [ -n "$BUILT_DIR" ] || { say "--built needs a directory (dist/, build/, .next/…)"; exit 2; } + case "$BUILT_DIR" in -*) say "--built needs a directory, got '$BUILT_DIR'"; exit 2 ;; esac + shift 2 ;; + --allow) ALLOW+=("${2:-}"); shift 2 ;; + *) say "unknown argument: $1"; exit 2 ;; + esac +done + +# Shapes that reach a bundle and should not. Added to PATTERNS below only in +# --built mode: `eyJ` is the base64 of `{"` that every JWT header starts with, +# and it is far too eager to run against source, where it matches ordinary +# base64. In a bundle it is worth the noise. +BUILT_PATTERNS=( + # All three segments, not just the header. Masking removes exactly what the + # pattern matched, so a pattern that stops at the first dot redacts `eyJ...` + # and prints the payload and signature beside it -- which is the token. + 'eyJ[A-Za-z0-9_-]{10,}(\.[A-Za-z0-9_-]+){0,2}' + '\bservice_role\b' + '\bapikey["'"'"'[:space:]]*[:=]' +) + +# Shapes that are *meant* to be public. Reported, never failed on — see the +# header. A finding you cannot act on is a finding that teaches people to skip +# the report. +NOTED_PATTERNS=( + '\bNEXT_PUBLIC_[A-Z0-9_]+' + '\bVITE_[A-Z0-9_]+' + '\bREACT_APP_[A-Z0-9_]+' + '\banon["'"'"'[:space:]]*[:=]' + + # A Google reCAPTCHA key. Noted and never failed on, because the SITE key and + # the SECRET key are the same shape and this script cannot tell them apart — + # failing would mean failing on the site key, which Vite inlines into the + # bundle correctly and by design. + # + # So it reports, and a person confirms. In --built mode that report is the + # whole point: `SECURITY_CHECKLIST.md` carries "the reCAPTCHA key in dist/ is + # the SITE key, not the secret key" as a release check, and this is the line + # that puts the key in front of whoever is ticking it. Compare what it prints + # against VITE_RECAPTCHA_SITE_KEY; if it matches RECAPTCHA_SECRET_KEY instead, + # the secret has shipped to every visitor and rotating it is the only fix. + '\b6L[A-Za-z0-9_-]{38}\b' +) + +# The built-in set. Deliberately shapes that are *structurally* credential-like +# rather than words that merely appear near credentials — `password` in a +# sentence is not a leak, and a scanner that says it is gets muted. +PATTERNS=( + '//[^/@[:space:]:]+:[^/@[:space:]]+@' # user:pass@host in a URL + '[?&](token|key|secret|password|access_token|api_key)=[^&[:space:]"]+' + '\b(Bearer|Basic)[[:space:]]+[A-Za-z0-9._~+/=-]{20,}' # an authorization header + # Anchored to the start of a line or an `export`, because unanchored it + # matched `access_token = $1` in SQL and `apiKey=` in a property list — three + # findings in src/ that were column names, not credentials. + '(^|export )[A-Z][A-Z0-9_]*(SECRET|TOKEN|PASSWORD|API_KEY|PASSWD)[A-Z0-9_]*=[^[:space:]"'"'"']{8,}' + '-----BEGIN [A-Z ]*PRIVATE KEY-----' + '\bghp_[A-Za-z0-9]{20,}' # GitHub + '\bxox[baprs]-[A-Za-z0-9-]{10,}' # Slack + '\bAKIA[0-9A-Z]{16}\b' # AWS access key id + + # --- This project's own shapes. --- + # + # Zoho WebToLead form identifiers are 64 and 96 lowercase hex characters, and + # they are the reason this hook exists here at all: `xnQsjsdp` and `xmIwtLD` + # were hardcoded in index.html at 0d3af33 and in src/pages/Contact.jsx at + # 033bdf6, and reached four commits on what was then a PUBLIC repository + # before being moved to environment variables at 05b27d2. + # + # Two patterns rather than one. The first is the shape as it actually leaked — + # a bare literal in markup, with no variable name anywhere near it. The second + # catches the tidier form somebody would write next time. A bare 60+ hex run + # matches nothing else in this tree: checked across every tracked text file on + # 2026-08-18 and it found zero, which is what makes it safe to fail on. + '\b[0-9a-f]{60,}\b' + 'ZOHO_[A-Z0-9_]*[[:space:]]*[:=][[:space:]]*["'"'"']?[0-9a-f]{32,}' +) + +# The project's own shapes, if it has written them down. A `pllc_agent_` +# token is invisible to every generic scanner and obvious to the module that +# redacts it. +if [ -n "${SECRETS_PATTERN_FILE:-}" ] && [ -f "$SECRETS_PATTERN_FILE" ]; then + loaded=0 + + # Anchored on the closing `/flags,` and greedy to it, rather than on "no + # commas". The first version used `[^,]+`, which cannot cross the comma inside + # a bounded quantifier — so `pllc_[a-z]+_[0-9a-f]{8,}` was silently dropped + # along with every other `{n,}` pattern: four of six on the file this was + # written against, while the script printed that it had loaded them. + while IFS= read -r found; do + [ -n "$found" ] || continue + + PATTERNS+=("$found") + loaded=$((loaded + 1)) + done < <( + sed -nE 's/.*\[\/(.+)\/[gimsuy]*,[[:space:]]*".*/\1/p' "$SECRETS_PATTERN_FILE" 2>/dev/null || true + ) + + # The count, never a bare reassurance. "Loaded project patterns" over an empty + # list is the same lie as a green test run that executed nothing. + if [ "$loaded" -gt 0 ]; then + say "loaded $loaded project pattern(s) from $SECRETS_PATTERN_FILE" + else + say "WARNING: $SECRETS_PATTERN_FILE yielded no patterns — scanning with the" + say " built-in set only. Check the file holds regex literals." + fi +fi + +if [ "$MODE" = "built" ]; then + [ -d "$BUILT_DIR" ] || { say "no such directory: $BUILT_DIR"; say "Nothing was scanned, which is not a pass."; exit 2; } + + CONTENT="" + WHAT="the built output in $BUILT_DIR" + PATTERNS+=("${BUILT_PATTERNS[@]}") + + while IFS= read -r file; do + skip="" + for allowed in ${ALLOW[@]+"${ALLOW[@]}"}; do + case "$file" in *"$allowed"*) skip="yes" ;; esac + done + [ -n "$skip" ] && continue + + # Source maps are the build's own copy of the source and would double every + # finding; they are worth scanning on purpose, not by accident. + case "$file" in *.map) continue ;; esac + + file "$file" 2>/dev/null | grep -q "text" || continue + + # Relative to the build directory, not the absolute path find produced. + # The report truncates each line to keep a secret off the terminal, and an + # absolute path in a temp directory can consume that budget entirely -- + # leaving a finding that names a file and shows nothing about the match. + rel="${file#"$BUILT_DIR"/}" + CONTENT+="$(sed "s|^|${rel}: |" "$file")"$'\n' + done < <(find "$BUILT_DIR" -type f -size -20M 2>/dev/null) + +elif [ "$MODE" = "staged" ]; then + # Added lines only. A removed line containing a token is somebody deleting + # one, which is the opposite of a leak. + CONTENT="$(git diff --cached --unified=0 --no-color | grep '^+' | grep -v '^+++' || true)" + WHAT="staged changes" +else + CONTENT="" + WHAT="tracked files" + + while IFS= read -r file; do + skip="" + + for allowed in ${ALLOW[@]+"${ALLOW[@]}"}; do + case "$file" in *"$allowed"*) skip="yes" ;; esac + done + + [ -n "$skip" ] && continue + [ -f "$file" ] || continue + + # Text only; a webp full of bytes will match anything. + file "$file" 2>/dev/null | grep -q "text" || continue + + CONTENT+="$(sed "s|^|${file}: |" "$file")"$'\n' + done < <(git ls-files) +fi + +if [ -z "$CONTENT" ]; then + say "nothing to scan in $WHAT." + exit 2 +fi + +found=0 + +for pattern in "${PATTERNS[@]}"; do + while IFS= read -r hit; do + [ -n "$hit" ] || continue + + skip="" + for allowed in ${ALLOW[@]+"${ALLOW[@]}"}; do + case "$hit" in *"$allowed"*) skip="yes" ;; esac + done + [ -n "$skip" ] && continue + + # The match is masked, then the line is truncated. Truncation alone was not + # enough and used to be all there was: it bounds how much of a LONG value + # reaches the terminal and prints a short one whole, so the scanner + # published the very thing it was built to find — to the scrollback, the CI + # log, and wherever that log is shipped. + # + # \001 as the delimiter, because these patterns contain both `/` and `|` + # and either would end the expression early. It cannot occur in a pattern + # and it cannot occur in the text of a source file. + masked="$(printf '%s' "$hit" | sed -E "s${MASK_D}${pattern}${MASK_D}[redacted]${MASK_D}g" 2>/dev/null)" + [ -n "$masked" ] || masked="[a line matching a credential pattern, unprintable]" + printf ' %.120s…\n' "$masked" + found=$((found + 1)) + done < <(printf '%s\n' "$CONTENT" | grep -nEI "$pattern" 2>/dev/null | head -20) +done + +# The public-by-design tier. Printed, counted, and deliberately not fatal. +if [ "$MODE" = "built" ]; then + noted=0 + + for pattern in "${NOTED_PATTERNS[@]}"; do + while IFS= read -r hit; do + [ -n "$hit" ] || continue + if [ "$noted" -eq 0 ]; then + say "shipped to the browser on purpose — check each is meant to be public:" + fi + printf ' %.120s…\n' "$hit" + noted=$((noted + 1)) + done < <(printf '%s\n' "$CONTENT" | grep -oEI "$pattern" 2>/dev/null | sort -u | head -20) + done + + if [ "$noted" -gt 0 ]; then + say "$noted public reference(s) above. Not a failure: those prefixes mean" + say "the value was compiled in deliberately. A Supabase anon key is safe" + say "exactly as far as row-level security makes it safe — this is the input" + say "to that judgement, not a substitute for it." + fi +fi + +if [ "$found" -gt 0 ]; then + say "$found candidate credential(s) in $WHAT." + say "If one is real: rotate it first. Deleting the line does not remove it" + say "from a commit that already exists, and the scan cannot see history." + exit 1 +fi + +say "no credential shapes in $WHAT." diff --git a/scripts/status.sh b/scripts/status.sh new file mode 100755 index 0000000..dbec4aa --- /dev/null +++ b/scripts/status.sh @@ -0,0 +1,678 @@ +#!/usr/bin/env bash +# +# What is actually running right now on the host this project deploys to, and +# its logs. Read-only: it inspects and it tails, and it does nothing else. +# +# bash scripts/status.sh # up? which version? how long? +# bash scripts/status.sh --logs # last 100 lines +# bash scripts/status.sh --logs 500 +# bash scripts/status.sh --logs all +# bash scripts/status.sh --follow # stream until Ctrl-C +# bash scripts/status.sh --follow 20 # stream, after 20 lines of context +# bash scripts/status.sh --deployed-version # just the version, for scripts +# bash scripts/status.sh --dry-run # print the commands, contact nothing +# +# STATUS_CONTAINER=other-app bash scripts/status.sh # one-off, another target +# +# Exit codes, because the whole point is that a caller can act on the answer: +# +# 0 running — healthy, still starting, or no healthcheck defined +# 1 configuration or usage error; nothing was contacted +# 2 COULD NOT DETERMINE — host unreachable, daemon silent, or an error this +# does not recognise. This is not an outage report. Nothing is known. +# 3 the container does not exist on that host +# 4 the container exists but is not running +# 5 running, and its own healthcheck says unhealthy +# +# stdout carries only what a caller should parse — the log lines, or the one +# version string. The human report goes to stderr, so `status.sh +# --deployed-version` can be read straight into a variable without a filter. +# +# =========================================================================== +# TEMPLATE COPY — configure this before the first run +# =========================================================================== +# +# Copy to `scripts/status.sh` and set the two values in the CONFIGURATION block +# below. The script refuses to run until they are set: it has no defaults, +# deliberately, for the reason `release.sh` gives at length — a copied script +# carrying another project's host and container name answers confidently about +# the wrong machine, and an answer is exactly what you came here for. A status +# tool that lies is worse than no status tool, because you act on it. +# +# Assumes: bash, coreutils, git-less (it reads nothing from the repository), and +# docker on the target. `ssh` only when the target is not this machine. +# +# ## Why this exists +# +# Without it, "is it up?" is answered by typing docker incantations from memory, +# which in practice means `docker ps | grep`. That shows a name and a status and +# none of the four things you actually wanted: which version is deployed, how +# long it has been up, how many times it has restarted, and what its healthcheck +# thinks. So the question gets half-answered, and the half that is missing — +# restart count — is the one that distinguishes "running" from "crash-looping +# and up for nine seconds". +# +# The other half is worse. When there is no command that answers "what version +# is deployed", every script that needs the answer grows its own copy of the +# lookup. `release.sh` has one: a single `ssh … docker inspect --format` whose +# empty return value is the ONLY thing standing between the prune and deleting +# the published image production is currently running. That fact is load-bearing +# enough to deserve a name, a documented exit code, and one place to fix — not a +# line pasted into whichever script needed it that week, each copy free to +# disagree about what an empty answer means. +# +# Hence `--deployed-version`: one fact, on stdout, non-zero when it could not be +# read. A caller that treats non-zero as "stop" cannot make the prune's mistake. +# +# ## Unreachable is not stopped +# +# These are different facts and only one of them is an outage you caused: +# +# - the container is not running → something happened to the deployment +# - I could not reach the host → something happened to the network, or to +# ssh, or to the daemon, and the container +# is very probably still serving traffic +# +# Rendering the second as the first is how a 2am investigation starts by +# restarting a healthy service. So a failed connection exits 2 and says so in +# those words; only an inspect that succeeded and reported a non-running state +# exits 4. Nothing here ever prints "stopped", "0" or "ok" for something it did +# not measure — an unknown is printed as an unknown, every time. +# +# ssh runs with BatchMode=yes and a connect timeout, so a host that would prompt +# for a password or a host key is read as unreachable in seconds instead of +# hanging forever on a prompt nobody is watching. +# +# ## There is no --host flag +# +# Host and container come from the CONFIGURATION block, or from the environment +# for a one-off. A flag would make it one keystroke to read host A and file the +# answer under host B, and the environment form is self-documenting in shell +# history — which is where you will be reading it back from. +# +# ## What it deliberately does not do +# +# It never starts, stops, restarts, recreates, kills or removes anything, and it +# never pulls or prunes an image. Every docker verb it runs is `inspect`, +# `version` or `logs`. This is not an oversight to be fixed later: a status tool +# is the one command people run half-awake and without reading, so it must be +# safe to run half-awake and without reading. Fixing what it reports is a +# separate, deliberate act with a separate command. +# +# It does not compare the deployed version against this repository, and does not +# read package.json or any other file. Whether the deployed version is the RIGHT +# version is a judgement, and a judgement needs a human or a release script that +# owns the decision; this only reports what is there. +# +# It reports one configured container, not every container on the host. A tool +# that enumerates is a tool that finds another project's container and reports it +# as yours. + +set -uo pipefail + +# --------------------------------------------------------------------------- +# CONFIGURATION — set these two, then delete this banner. +# +# Both are empty on purpose. See the note at the top. +# --------------------------------------------------------------------------- + +# An ssh destination — a Host in ~/.ssh/config, an IP, or user@host — or the +# literal word `local` to ask this machine's own docker daemon. +# +# `local` is a word you have to type rather than what an empty value means, +# because "you did not configure a host" and "the host is this machine" must not +# look the same. Defaulting to local would make an unconfigured copy of this +# script quietly report on your laptop and call it production. +HOST="${STATUS_HOST:-nebula}" + +# The container name (or id) to report on. Exactly one. +CONTAINER="${STATUS_CONTAINER:-qn-website-dev}" + +# Seconds to wait for the ssh connection. Short on purpose: this command exists +# to be run when something might be wrong, and a status tool that hangs is +# indistinguishable from the outage it was meant to describe. +SSH_TIMEOUT="${STATUS_SSH_TIMEOUT:-10}" + +# Lines tailed by `--logs` with no number given. +TAIL_DEFAULT="${STATUS_TAIL:-100}" + +# The image label carrying the deployed version. This one has a default because +# it is an open standard rather than a fact about your project; override it only +# if your build stamps a different label. +VERSION_LABEL="${STATUS_VERSION_LABEL:-org.opencontainers.image.version}" + +# --------------------------------------------------------------------------- + +E_CONFIG=1 +E_UNKNOWN=2 +E_MISSING=3 +E_STOPPED=4 +E_UNHEALTHY=5 + +say() { printf '\033[1mstatus:\033[0m %s\n' "$*" >&2; } +die() { printf '\033[1mstatus:\033[0m %s\n' "$*" >&2; exit "$E_CONFIG"; } + +# die() with a chosen code. Every caller of this is a distinct fact a script +# downstream is expected to branch on, so the code is passed explicitly at the +# point the fact is established rather than mapped from a message afterwards. +quit() { local code="$1"; shift; printf '\033[1mstatus:\033[0m %s\n' "$*" >&2; exit "$code"; } + +field() { printf '\033[1mstatus:\033[0m %-9s %s\n' "$1" "$2" >&2; } + +usage() { + say "usage: bash scripts/status.sh [--logs [N|all]] [--follow [N]] [--deployed-version] [--dry-run]" + say " set STATUS_HOST and STATUS_CONTAINER, or edit the CONFIGURATION block." +} + +# --------------------------------------------------------------------------- +# Arguments, before the configuration check, so that `--help` answers on a +# fresh unconfigured copy — which is the one moment somebody needs it. +# --------------------------------------------------------------------------- + +MODE="report" +TAIL="" +DRY_RUN="" + +# `("$@")` with no arguments is an unbound expansion under `set -u` on bash +# before 4.4, which is still what ships on macOS. Guarded rather than assumed. +if [ "$#" -gt 0 ]; then ARGS=("$@"); else ARGS=(); fi + +# Refused rather than resolved last-flag-wins. `--deployed-version --logs` from a +# script would otherwise return a log stream on stdout to a caller that is about +# to compare it to a version string. +set_mode() { + [ "$MODE" = "report" ] || [ "$MODE" = "$1" ] \ + || die "--logs, --follow and --deployed-version ask different questions; pick one." + MODE="$1" +} + +i=0 +while [ "$i" -lt "${#ARGS[@]}" ]; do + arg="${ARGS[$i]}" + + case "$arg" in + --logs|-l|--follow|-f) + case "$arg" in + --logs|-l) set_mode "logs" ;; + *) set_mode "follow" ;; + esac + + # An optional operand rather than a required one, so `--logs` alone is a + # whole command. Only consumed when it looks like a count — otherwise + # `--logs --dry-run` would swallow the flag and silently mean something + # else. + if [ $((i + 1)) -lt "${#ARGS[@]}" ]; then + case "${ARGS[$((i + 1))]}" in + all|[0-9]*) TAIL="${ARGS[$((i + 1))]}"; i=$((i + 1)) ;; + esac + fi + ;; + --logs=*) set_mode "logs"; TAIL="${arg#*=}" ;; + --follow=*) set_mode "follow"; TAIL="${arg#*=}" ;; + --deployed-version) set_mode "version" ;; + --dry-run) DRY_RUN="yes" ;; + -h|--help) usage; exit 0 ;; + *) usage; die "unknown argument '$arg'." ;; + esac + + i=$((i + 1)) +done + +[ -n "$TAIL" ] || TAIL="$TAIL_DEFAULT" + +case "$TAIL" in + all) : ;; + ''|*[!0-9]*) die "line count must be a whole number or 'all', got '$TAIL'." ;; +esac + +case "$SSH_TIMEOUT" in + ''|*[!0-9]*) die "STATUS_SSH_TIMEOUT must be a whole number of seconds, got '$SSH_TIMEOUT'." ;; +esac + +# --------------------------------------------------------------------------- +# Refuse to run half-configured, before anything is contacted, and name the +# missing value one at a time so the message says which one. +# --------------------------------------------------------------------------- + +[ -n "$HOST" ] || die "set HOST (or STATUS_HOST) — the ssh host running the container, or the word 'local'. See the CONFIGURATION block." +[ -n "$CONTAINER" ] || die "set CONTAINER (or STATUS_CONTAINER) — the container name to report on." + +# ssh takes its destination positionally, so a host beginning with a dash is +# parsed as an option and the failure that follows describes something else +# entirely. Refused here, where the message can still name the real cause. +case "$HOST" in + -*) die "HOST ('$HOST') begins with '-'; ssh would read it as an option. Use user@host, or a Host from ~/.ssh/config." ;; +esac + +case "$CONTAINER" in + -*) die "CONTAINER ('$CONTAINER') begins with '-'; docker would read it as a flag." ;; +esac + +# The label name is interpolated into a Go template as a quoted string below. A +# quote or backslash in it would end that string early, and the template error +# that followed would be reported as "could not inspect" — an unknown, blamed on +# the host, caused here. +case "$VERSION_LABEL" in + ''|*[\"\\]*) die "STATUS_VERSION_LABEL must be a plain label name without quotes or backslashes, got '$VERSION_LABEL'." ;; +esac + +# --------------------------------------------------------------------------- +# Running docker here or over there. +# +# ssh joins its command arguments with spaces and hands the result to a remote +# shell, which splits it again. Anything containing a space — every Go template +# below does — arrives as several arguments unless it is quoted for that second +# parse. Quoting is done here, once, rather than by hand at each call site. +# --------------------------------------------------------------------------- + +shquote() { + local out="" a q="'" esc="'\\''" + + for a in "$@"; do + a="${a//$q/$esc}" + out="${out}${out:+ }'${a}'" + done + + printf '%s' "$out" +} + +# ServerAlive is here for --follow: a connection that dies mid-stream otherwise +# leaves the terminal sitting quietly forever, which looks exactly like a service +# that has stopped logging. Absence of output must not be able to mean two +# things, so the connection is made to fail loudly instead. +ssh_opts() { + printf '%s' "-o BatchMode=yes -o ConnectTimeout=${SSH_TIMEOUT} -o ServerAliveInterval=15 -o ServerAliveCountMax=3" +} + +SSH_WANT_TTY="" + +dock() { + if [ "$HOST" = "local" ]; then + docker "$@" + return + fi + + # A tty makes the remote `docker logs -f` receive a hangup when you press + # Ctrl-C. Without one it keeps running on the far side, holding the daemon's + # log stream open, and enough abandoned follows are a real resource leak on a + # box you are already worried about. Requested only when we have a tty to + # give, because ssh -t without one is a warning and no tty anyway. + if [ -n "$SSH_WANT_TTY" ] && [ -t 0 ]; then + # shellcheck disable=SC2046 # deliberate word-splitting of the option list + ssh -t $(ssh_opts) "$HOST" "docker $(shquote "$@")" + else + # shellcheck disable=SC2046 + ssh $(ssh_opts) "$HOST" "docker $(shquote "$@")" + fi +} + +# The remote form is printed on two lines — the connection, then the command +# string the far shell receives — because the single-line form is the same text +# quoted twice and the host and container names disappear into the punctuation. +# Those two names are the entire reason to run --dry-run. +dock_show() { + if [ "$HOST" = "local" ]; then + printf ' docker %s\n' "$(shquote "$@")" >&2 + else + printf ' ssh %s %s\n' "$(ssh_opts)" "$HOST" >&2 + printf ' docker %s\n' "$(shquote "$@")" >&2 + fi +} + +# --------------------------------------------------------------------------- +# Timestamps. +# +# Docker reports RFC3339 with nanoseconds; `date -d` is GNU and `date -j -f` is +# BSD, so both are tried and NEITHER working is reported as not knowing the +# uptime rather than as an uptime of zero. +# --------------------------------------------------------------------------- + +epoch_of() { + local t="${1%Z}" + t="${t%%.*}" + + local e + if e=$(date -u -d "${t}Z" +%s 2>/dev/null) && [ -n "$e" ]; then + printf '%s' "$e" + return 0 + fi + + if e=$(date -u -j -f '%Y-%m-%dT%H:%M:%S' "$t" +%s 2>/dev/null) && [ -n "$e" ]; then + printf '%s' "$e" + return 0 + fi + + return 1 +} + +human_duration() { + local s="$1" d h m + + d=$((s / 86400)); s=$((s % 86400)) + h=$((s / 3600)); s=$((s % 3600)) + m=$((s / 60)); s=$((s % 60)) + + if [ "$d" -gt 0 ]; then printf '%dd %dh %dm' "$d" "$h" "$m" + elif [ "$h" -gt 0 ]; then printf '%dh %dm' "$h" "$m" + elif [ "$m" -gt 0 ]; then printf '%dm %ds' "$m" "$s" + else printf '%ds' "$s" + fi +} + +# Elapsed time since an RFC3339 stamp, or nothing at all. +# +# The zero timestamp means the event never happened — a container created but +# never started carries 0001-01-01 in StartedAt, and subtracting it yields two +# millennia of uptime presented with a straight face. +# +# A negative result is clock skew between this machine and that one, not time +# travel, and is likewise reported as not knowing rather than as a number. +elapsed_since() { + local stamp="$1" at now diff + + case "$stamp" in + ''|0001-01-01*) return 1 ;; + esac + + at=$(epoch_of "$stamp") || return 1 + now=$(date -u +%s) || return 1 + + diff=$((now - at)) + [ "$diff" -ge 0 ] || return 1 + + human_duration "$diff" +} + +# --------------------------------------------------------------------------- +# One inspect, one round trip. +# +# Fields are read together rather than one call each: two calls can straddle a +# restart and produce a report that never described a single moment — an uptime +# from before the restart beside a restart count from after it. +# +# The free-text fields are LAST. `read` gives the final variable everything that +# remains, delimiters included, so a '|' inside an image name or a version label +# lands harmlessly in the last field instead of shifting every field after it. +# --------------------------------------------------------------------------- + +FMT='{{.State.Status}}|{{.State.Running}}|{{.RestartCount}}|{{.State.ExitCode}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}{{end}}|{{.State.StartedAt}}|{{.State.FinishedAt}}|{{.Image}}|{{.Config.Image}}|{{index .Config.Labels "'"$VERSION_LABEL"'"}}' + +C_STATUS=""; C_RUNNING=""; C_RESTARTS=""; C_EXIT=""; C_HEALTH="" +C_STARTED=""; C_FINISHED=""; C_DIGEST=""; C_IMAGE=""; C_VERSION="" + +# Work out WHY the inspect failed, and exit with the code for that specific +# fact. Reached only after a failure, so the extra round trips cost nothing in +# the normal case and buy the one distinction this script exists to preserve. +diagnose_and_exit() { + local err="$1" + + if [ "$HOST" != "local" ]; then + # shellcheck disable=SC2046 + if ! ssh $(ssh_opts) "$HOST" true >/dev/null 2>&1; then + say "cannot reach ${HOST} over ssh (BatchMode, ${SSH_TIMEOUT}s timeout)." + say " This is NOT a report that the container is down — nothing was" + say " determined about it. Check the network, the ssh key, and" + say " whether the host would have prompted for a password." + exit "$E_UNKNOWN" + fi + fi + + if ! dock version --format '{{.Server.Version}}' >/dev/null 2>&1; then + if [ "$HOST" = "local" ]; then + say "this machine's docker daemon did not answer." + else + say "reached ${HOST} over ssh, but its docker daemon did not answer." + fi + + say " The container's state is unknown — the daemon being down does" + say " not tell you whether the container was running before it went." + exit "$E_UNKNOWN" + fi + + # Matched case-insensitively against a lowercased copy, because the wording + # is not stable across docker versions: "Error: No such object: x" on one + # daemon here, "error: no such object: x" on another, and "No such container" + # from older ones. A case-sensitive match reports a container that is simply + # absent as an unknown — which is the safe direction to be wrong in, and + # still the wrong answer. + local lower + lower=$(printf '%s' "$err" | tr '[:upper:]' '[:lower:]') + + case "$lower" in + *"no such object"*|*"no such container"*) + quit "$E_MISSING" "there is no container named '${CONTAINER}' on ${HOST}." ;; + esac + + # An error nobody anticipated is printed verbatim and treated as an unknown. + # Guessing at an unfamiliar message is how "I could not check" becomes "there + # is nothing there". + say "could not inspect '${CONTAINER}' on ${HOST}, and the error is not one" + say " this script recognises. Nothing is known about the container:" + printf ' %s\n' "${err:-(no error output)}" >&2 + exit "$E_UNKNOWN" +} + +# The one file this script creates, and the one thing it ever deletes. +# +# The path is read when the trap FIRES rather than pasted into the trap's source +# text. `trap "rm -f '$f'" EXIT` looks equivalent and is not: it builds a shell +# command out of mktemp's output, which is built out of $TMPDIR, so a directory +# whose name contains a single quote closes that string early and everything +# after it runs as shell. `TMPDIR="/tmp/x'; rm -rf ~; echo '"` is a working +# exploit against the double-quoted form. This is the only destructive verb in +# the script and it must not be constructible from the environment. +# +# The -n guard means rm is never handed an empty path, and `--` means a path +# beginning with a dash is a path rather than a bundle of options. +ERRFILE="" + +# shellcheck disable=SC2329 # invoked by the EXIT trap below, not by name +cleanup() { + [ -n "$ERRFILE" ] || return 0 + rm -f -- "$ERRFILE" + return 0 +} + +trap cleanup EXIT + +inspect_container() { + local out rc + + ERRFILE=$(mktemp) || die "cannot create a temporary file" + + out=$(dock inspect --format "$FMT" "$CONTAINER" 2>"$ERRFILE") + rc=$? + + if [ "$rc" -ne 0 ] || [ -z "$out" ]; then + diagnose_and_exit "$(tr -d '\r' < "$ERRFILE" | head -n 3)" + fi + + IFS='|' read -r C_STATUS C_RUNNING C_RESTARTS C_EXIT C_HEALTH \ + C_STARTED C_FINISHED C_DIGEST C_IMAGE C_VERSION \ + </dev/null 2>&1 \ + || quit "$E_UNKNOWN" "docker is not on PATH, so nothing could be determined. Install it, or set HOST to the machine that runs the container." +else + command -v ssh >/dev/null 2>&1 \ + || quit "$E_UNKNOWN" "ssh is not on PATH, so ${HOST} could not be contacted." +fi + +inspect_container + +# --------------------------------------------------------------------------- +# --deployed-version: one fact on stdout, non-zero when it is not known. +# +# The contract callers depend on: a zero exit means the string on stdout was +# read from the running container's image. Anything else means stop — never +# "assume none", which is the assumption that lets a prune delete production's +# image. +# --------------------------------------------------------------------------- + +if [ "$MODE" = "version" ]; then + [ -n "$C_VERSION" ] \ + || quit "$E_UNKNOWN" "'${CONTAINER}' on ${HOST} is ${C_STATUS}, but its image carries no ${VERSION_LABEL} label — the deployed version is unknown." + + [ "$C_RUNNING" = "true" ] \ + || quit "$E_STOPPED" "'${CONTAINER}' on ${HOST} is ${C_STATUS}, not running. Its image says ${C_VERSION}, but nothing is serving from it." + + printf '%s\n' "$C_VERSION" + exit 0 +fi + +# --------------------------------------------------------------------------- +# --logs / --follow. +# +# The inspect above already ran, which is what makes an empty tail meaningful: +# the container is known to exist, so no output means it has logged nothing. +# Tailing blind cannot tell that apart from a container that is not there. +# --------------------------------------------------------------------------- + +if [ "$MODE" = "logs" ] || [ "$MODE" = "follow" ]; then + field container "${CONTAINER} on ${HOST}" + field state "${C_STATUS}${C_VERSION:+ (${C_VERSION})}" + + # Said before the first line of output, because a wall of old log lines from a + # container that died an hour ago is indistinguishable from a live one. + if [ "$C_RUNNING" != "true" ]; then + say "note: the container is ${C_STATUS}. These are the logs it left behind," + + if [ "$MODE" = "follow" ]; then + say " not a live stream — nothing is writing, so this will return at once." + else + say " not a live stream." + fi + fi + + if [ "$MODE" = "follow" ]; then + SSH_WANT_TTY="yes" + say "following ${CONTAINER} — Ctrl-C to stop." + dock logs --tail "$TAIL" --follow "$CONTAINER" + else + dock logs --tail "$TAIL" "$CONTAINER" + fi + + rc=$? + + # A tail that failed after a successful inspect is a new fact, not a quiet + # zero: the container exists, so an empty screen would otherwise read as a + # silent service. + [ "$rc" -eq 0 ] || quit "$E_UNKNOWN" "the log stream ended with status ${rc}; some or all of the output may be missing." + + # The logs printed either way; only the code differs. Exiting 0 here would + # report the documented meaning of 0 — "running" — for a container that is + # not, which is the same mistake as printing "ok" for something unmeasured. + # The state is the fact the exit code carries, in every mode. + [ "$C_RUNNING" = "true" ] \ + || quit "$E_STOPPED" "'${CONTAINER}' exists on ${HOST} but is ${C_STATUS}; the logs above are what it left behind, not a live service." + + exit 0 +fi + +# --------------------------------------------------------------------------- +# The report. +# --------------------------------------------------------------------------- + +field container "${CONTAINER} on ${HOST}" + +if [ "$C_RUNNING" = "true" ]; then + field state "running" +else + field state "${C_STATUS:-unknown} (exit code ${C_EXIT:-unknown})" +fi + +field image "${C_IMAGE:-(unknown)}" + +# The digest is printed beside the tag because the tag is a claim and the digest +# is the fact. A tag can be moved after a container was created, at which point +# the container's image reference names something it was never built from. +if [ -n "$C_DIGEST" ]; then + case "$C_DIGEST" in + sha256:*) + short="${C_DIGEST#sha256:}" + field digest "sha256:${short:0:12}" + ;; + *) + # Whatever this is, it is not a sha256 digest, so it is not labelled as + # one. The stripping form prints "sha256:" in front of the first twelve + # characters of anything at all — podman and older daemons report a bare + # image id here, and it would have been rendered as a digest it is not. + field digest "$C_DIGEST" + ;; + esac +fi + +if [ -n "$C_VERSION" ]; then + field version "$C_VERSION" +else + field version "(not set — this image carries no ${VERSION_LABEL} label)" +fi + +if [ "$C_RUNNING" = "true" ]; then + if up=$(elapsed_since "$C_STARTED"); then + field uptime "${up} (started ${C_STARTED})" + else + # Reported as unknown, with the raw stamp, rather than as a plausible + # number computed from a timestamp that could not be read. + field uptime "unknown — could not read the start time '${C_STARTED}'" + fi +else + if down=$(elapsed_since "$C_FINISHED"); then + field stopped "${down} ago (${C_FINISHED})" + else + field stopped "at '${C_FINISHED:-unknown}'" + fi +fi + +field restarts "${C_RESTARTS:-unknown}" + +# The count alone is not the signal; the count beside a short uptime is. Said +# out loud because a crash loop reads as "running" in every tool that shows only +# a state, and that is the case this command was written to catch. +if [ -n "$C_RESTARTS" ] && [ "$C_RESTARTS" != "0" ] && [ "$C_RUNNING" = "true" ]; then + say " ^ restarted ${C_RESTARTS} time(s). A climbing count with a short" + say " uptime is a crash loop, not a healthy service." +fi + +if [ -n "$C_HEALTH" ]; then + field health "$C_HEALTH" +else + # Not "healthy", and not "ok". No healthcheck is defined, so nothing about + # this container's health has been measured by anyone. + field health "not configured — this image defines no HEALTHCHECK, so nothing is being checked" +fi + +if [ "$C_RUNNING" != "true" ]; then + quit "$E_STOPPED" "'${CONTAINER}' exists on ${HOST} but is ${C_STATUS}." +fi + +if [ "$C_HEALTH" = "unhealthy" ]; then + quit "$E_UNHEALTHY" "'${CONTAINER}' is running, but its healthcheck reports unhealthy. Try --logs." +fi + +exit 0 diff --git a/scripts/verify.d/10-build b/scripts/verify.d/10-build new file mode 100755 index 0000000..b6454d2 --- /dev/null +++ b/scripts/verify.d/10-build @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# +# The build, which is the only gate this project actually has. +# +# There is no test runner and no typecheck here — no vitest, no jest, no +# tsconfig, and every source file is plain .jsx. `verify.sh` therefore detects +# nothing at all from package.json, whose script names are dev/build/preview/ +# start/docker:*, and would exit 2 ("nothing was verified") on a project that +# does have something worth checking. +# +# So this is that something, named honestly. `npm run build` is three steps — +# the client bundle, the SSR bundle, then scripts/prerender.js across every +# route — and it fails on a broken import, a missing asset, a syntax error, or a +# route that cannot be rendered to static HTML. That last one is worth more than +# it sounds: prerendering exercises every page component in Node, which is the +# closest thing this repository has to running its own UI. +# +# What it does NOT do is check behaviour. A form that posts to the wrong URL +# builds perfectly. Do not read a green row here as "the site works". +# +# Exit 0 built, 1 did not. +set -uo pipefail +cd "$(git rev-parse --show-toplevel)" || exit 1 + +if ! command -v npm >/dev/null 2>&1; then + echo "build: npm is not on PATH — this guard did not run." >&2 + exit 1 +fi + +npm run build diff --git a/scripts/verify.d/20-secrets b/scripts/verify.d/20-secrets new file mode 100755 index 0000000..f7bc824 --- /dev/null +++ b/scripts/verify.d/20-secrets @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# +# Credentials in the tracked tree. +# +# `scripts/secrets.sh` runs on the staged diff from the pre-commit hook, which +# is the cheap moment. This is the whole-tree version, run as part of verify so +# that something looks at what is ALREADY committed rather than only at what is +# arriving. +# +# The distinction earned itself here: the Zoho WebToLead tokens sat in four +# commits of a then-public repository for a month, and a staged-diff scan +# installed afterwards would never have mentioned them. +# +# Exit 0 clean, 1 findings, 2 the scanner could not run. +set -uo pipefail +cd "$(git rev-parse --show-toplevel)" || exit 1 + +[ -f scripts/secrets.sh ] || { echo "secrets: scripts/secrets.sh is missing — nothing was scanned." >&2; exit 2; } +bash scripts/secrets.sh --tracked diff --git a/scripts/verify.d/30-doc-headers b/scripts/verify.d/30-doc-headers new file mode 100755 index 0000000..68adb07 --- /dev/null +++ b/scripts/verify.d/30-doc-headers @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# +# Every document carries a complete, valid status header. +# +# ## Why this is a guard and not a convention +# +# `DOC_TRUST_MAP.md` makes two claims that nothing else enforces: the status +# word is one of exactly four, and `Review trigger` is the line that stops a +# document going quietly stale. A header carrying `Status` without +# `Review trigger` is the specific failure worth catching — it looks finished +# and is not. +# +# This repository is the reason. Before 2026-08-18 it had six markdown documents +# at its root with no headers at all, two of which described the project as +# being in "Phase 5" while the code was at 0.9.3. Nothing said so. +# +# Checked in the first sixteen lines, which is where a header lives. +# +# Exit 0 all conformant, 1 at least one is not, 2 no documents were found — +# which is not a pass, because it is what a moved docs/ directory looks like. +set -uo pipefail +cd "$(git rev-parse --show-toplevel)" || exit 1 + +VALID="Current Draft Superseded Archived" +bad=0 +seen=0 + +# docs/** at any depth, plus the root one level deep — the same scope +# doc-triggers.py reads, so a document one tool checks the other fires on. +while IFS= read -r f; do + [ -n "$f" ] || continue + seen=$((seen + 1)) + head16=$(head -16 "$f") + + status=$(printf '%s\n' "$head16" | sed -nE 's/^Status:[[:space:]]*([A-Za-z]+).*/\1/p' | head -1) + trigger=$(printf '%s\n' "$head16" | grep -c '^Review trigger:' || true) + governs=$(printf '%s\n' "$head16" | grep -c '^Governs:' || true) + + if [ -z "$status" ]; then + echo "no Status in the first 16 lines $f" >&2; bad=$((bad + 1)); continue + fi + case " $VALID " in + *" $status "*) ;; + *) echo "Status: '$status' is not one of the four $f" >&2; bad=$((bad + 1)) ;; + esac + [ "$trigger" -ge 1 ] || { echo "Status but no Review trigger — looks done $f" >&2; bad=$((bad + 1)); } + [ "$governs" -ge 1 ] || { echo "no Governs: line $f" >&2; bad=$((bad + 1)); } +done < <(git ls-files 'docs/**/*.md' 'docs/*.md' '*.md' 2>/dev/null) + +if [ "$seen" -eq 0 ]; then + echo "doc-headers: no tracked markdown found. Nothing was checked — that is not a pass." >&2 + exit 2 +fi + +if [ "$bad" -gt 0 ]; then + echo "doc-headers: $bad problem(s) across $seen document(s)." >&2 + exit 1 +fi + +echo "doc-headers: $seen document(s), all with a valid Status, Governs and Review trigger." diff --git a/scripts/verify.sh b/scripts/verify.sh new file mode 100755 index 0000000..9972c65 --- /dev/null +++ b/scripts/verify.sh @@ -0,0 +1,694 @@ +#!/usr/bin/env bash +# +# Run every check this project has, in one command, and print one table saying +# which passed, which failed and which did not run at all. +# +# bash scripts/verify.sh # everything +# bash scripts/verify.sh --quick # skip the test step, name it as skipped +# bash scripts/verify.sh --list # show the plan, run nothing +# bash scripts/verify.sh --only npm # only steps whose id contains 'npm' +# VERIFY_GUARD_DIR=scripts/verify.d bash scripts/verify.sh +# +# Exit codes: 0 everything that ran passed. 1 something failed, or the run +# could not start at all — a usage error, a bad VERIFY_GUARD_DIR, not a git +# repository. 2 nothing was verified: either no checks were detected, or every +# detected step skipped. Two is not a pass and CI must not treat it as one. +# +# One and two are both non-zero on purpose, so every refusal fails closed. If +# you need to tell "a check failed" apart from "it never got to run", read the +# message — a failed check always prints a table first. +# +# =========================================================================== +# TEMPLATE COPY — configure this before the first run +# =========================================================================== +# +# Copy to `scripts/verify.sh`, `chmod +x` it, and wire it in: for a Node +# project add `"verify": "bash scripts/verify.sh"` to package.json; for anything +# else call it from `pre-commit`. Then read the CONFIGURATION block below — +# two environment variables (VERIFY_GUARD_DIR, VERIFY_SLOW_STEP) and six lists +# of candidate script and target names. Every one is optional except the one +# you need, which is GUARD_DIR. +# +# Unlike its sibling release.sh this script has no host, image or container to +# get wrong, so it defaults to DETECTING rather than to refusing. The rule it +# inherits unchanged is the important half: it never claims to have checked +# something it did not check. A detector that finds nothing says so and exits +# non-zero; it does not print a green table. +# +# Assumes: bash, coreutils, git. Each detector additionally needs the toolchain +# it detects, and says so by name when that toolchain is missing instead of +# quietly dropping the step. +# +# ## Why this exists +# +# "Did I break anything" should not be a judgement call, and in a project with +# three separate check commands it always is. The commands live in different +# places — a package.json script, a Makefile target, a lint you have to +# remember — so the honest answer to "did you run everything" is usually "I ran +# the one I remembered". The suite passes, the typecheck was never run, and the +# breakage is found by the deploy. +# +# One entry point removes the judgement. It also removes the excuse: there is +# no "I ran the important one" when running all of them is the same amount of +# typing. +# +# The second reason is the summary. A chain of `&&` stops at the first failure, +# so a run tells you about one broken thing at a time and you pay the whole +# cost again to find the next. This runs every step even after one fails, so a +# single run tells you everything that is broken. +# +# ## Guard tests are the point +# +# This project family keeps getting bitten by bugs that are invisible to +# ordinary tests, because the test and the bug agree with each other: +# +# - A Date serialised across a SQL boundary came back shifted by the server's +# timezone. Every test asserted against the same shifted value, so the +# suite was green and the dates were wrong. +# - A React prop was spread over a form field and silently overwrote its +# `name`. The component rendered, the test rendered it, and the field +# submitted under the wrong key. +# - A `redirect()` was called inside a `try` block. Next.js implements +# redirect by throwing, so the `catch` swallowed it and every successful +# action reported failure. +# +# None of those is a logic error a unit test would catch. All three are SHAPES +# in the source: a shape you can grep for. The fix is a guard test — a script +# that greps the source for the shape and exits non-zero when it reappears — +# and this script is where guard tests belong, because verify.sh is the thing +# that actually gets run. +# +# Write them as small executables in GUARD_DIR. One shape per file, named for +# the bug, exiting non-zero with a message naming the file and line. They cost +# milliseconds, they run first here for exactly that reason, and they are the +# only mechanism in the repository that catches a bug the tests cannot see. +# +# ## SKIPPED is not PASS +# +# A step that did not run gets its own state in the table and its own colour, +# and it is never folded into the pass count. This matters more than it looks: +# the failure this script exists to prevent is a green table produced by +# checking nothing, and every path to that failure runs through a skip that was +# reported as a success. So a run where nothing actually executed exits 2 even +# though nothing failed, and --quick names the step it dropped rather than +# quietly shortening the table. +# +# ## What it deliberately does not do +# +# It does not fix anything. No --fix, no formatter writing to your files: this +# runs immediately before a commit, and a verify that edits the tree changes +# what you were about to commit into something you have not read. +# +# It does not touch git — no staging, no committing, no stash, no branch check. +# It answers one question about the working tree as it stands. +# +# It does not walk into workspaces or sub-packages. Detection runs at the +# repository root, once. A monorepo wants a guard script per package, or a +# Makefile target that fans out, and either is a step this will find. +# +# It has no --dry-run because it writes nothing OF ITS OWN: no file is created, +# deleted, truncated or moved anywhere in this script. That is not the same as +# "changes nothing", and the difference matters. Every step it runs is somebody +# else's program — a package.json script, a Makefile target, an executable in +# GUARD_DIR — and those inherit no restraint from here. Dropping a test +# database is the usual one. +# +# --list is the dry-run equivalent: it shows the exact command each step would +# run, and runs none of them. Read it once, on a new project, before trusting +# this in a hook. That is the only place the full list is visible before it +# executes. + +set -uo pipefail + +say() { printf '\033[1mverify:\033[0m %s\n' "$*" >&2; } +die() { printf '\033[1mverify:\033[0m %s\n' "$*" >&2; exit 1; } + +ROOT=$(git rev-parse --show-toplevel 2>/dev/null) \ + || die "not inside a git repository. Detection runs from the repository root so every detector looks in one place; cd into the repo and run this again." + +# Empty is checked separately from failed. `cd ""` succeeds and stays put, so an +# empty ROOT would not fail here — it would silently detect in whatever +# directory you happened to be in, which is the wrong answer delivered +# confidently. A bare repository is the way it happens. +[ -n "$ROOT" ] \ + || die "git reported an empty repository root — this looks like a bare repository, which has no working tree to check. Run this from a normal checkout." + +cd "$ROOT" || die "cannot cd to $ROOT" + +# Resolved once, with symlinks collapsed, so the guard-directory containment +# check below compares two paths of the same kind. +ROOT_ABS=$(pwd -P) + +# --------------------------------------------------------------------------- +# CONFIGURATION +# --------------------------------------------------------------------------- + +# Where guard scripts live, relative to the repository root. Empty by default: +# an unset path is a path that cannot point at the wrong thing, and this is the +# one value worth setting by hand — see "Guard tests are the point" above. +# +# Set and missing is a hard failure, not a skip. A guard directory that got +# renamed is precisely the case where silently running zero guards looks +# identical to running them all. +# +# IT MUST BE A DEDICATED DIRECTORY. Every executable file in it is RUN, with no +# allowlist — unlike the npm and make candidate lists below, which name the +# handful of scripts they are willing to invoke. That asymmetry is the whole +# hazard: point this at scripts/ and verify.sh runs release.sh; point it at a +# bin directory and it runs every binary there. detect_guards refuses the cases +# it can prove wrong (outside the repository, the repository root itself, the +# directory holding this script), but it cannot tell a guard from a deploy +# script that happens to sit beside one. Give guards their own directory and +# put nothing else in it. +GUARD_DIR="${VERIFY_GUARD_DIR:-scripts/verify.d}" + +# Which step --quick drops, by exact id (the left column of the table). Empty +# means "every step classified as a test", which is the right guess almost +# everywhere and is stated out loud when it is used. +SLOW_STEP="${VERIFY_SLOW_STEP:-}" + +# Candidate script and target names, checked against what the project actually +# has. Nothing here is run unless it exists — these are search terms, not +# defaults, and the distinction is load-bearing: `npm run lint` on a project +# without a lint script fails, and a verify that invents work to do fails for +# reasons that have nothing to do with the code. +# +# It is also an allowlist, and that is the other half of why it exists. A +# package.json contains scripts named `deploy` and `db:reset`. Running +# everything found would be a verify that publishes. +NODE_TYPECHECK_SCRIPTS="typecheck type-check tsc types" +NODE_LINT_SCRIPTS="lint lint:ci eslint format:check fmt:check" +NODE_TEST_SCRIPTS="test test:ci" + +# `verify` and `all` are absent on purpose: a Makefile in a repository holding +# this script very likely has a `verify` target that calls this script, and +# that is an infinite loop rather than a failed check. The guard below catches +# it anyway, for the `check` target that turns out to do the same thing. +MAKE_LINT_TARGETS="lint fmt-check format-check" +MAKE_TYPECHECK_TARGETS="typecheck type-check types" +MAKE_TEST_TARGETS="test check" + +# --------------------------------------------------------------------------- +# Refuse to be re-entered. +# +# A detected step that calls this script back — `make check` running +# scripts/verify.sh is the way it happens — recurses until the machine gives +# up, and the symptom is a hang rather than an error. Dying on the second entry +# turns that into one legible failure naming the step that did it. +# --------------------------------------------------------------------------- +if [ -n "${VERIFY_RUNNING:-}" ]; then + die "verify.sh invoked itself — a detected step calls this script back, which would recurse forever. Remove that target from the candidate lists in this file, or stop it calling verify." +fi +export VERIFY_RUNNING=1 + +QUICK="" +LIST="" +ONLY="" + +while [ $# -gt 0 ]; do + case "$1" in + --quick) QUICK="yes" ;; + --list) LIST="yes" ;; + --only) + shift + [ $# -gt 0 ] || die "--only needs a pattern. Usage: --only " + ONLY="$1" ;; + --only=*) ONLY="${1#--only=}" ;; + *) die "unknown argument '$1'. Usage: verify.sh [--quick] [--list] [--only ]" ;; + esac + shift +done + +if [ -t 2 ]; then + C_PASS=$'\033[32m'; C_FAIL=$'\033[31;1m'; C_SKIP=$'\033[33m'; C_OFF=$'\033[0m' +else + C_PASS=""; C_FAIL=""; C_SKIP=""; C_OFF="" +fi + +# --------------------------------------------------------------------------- +# The plan. Built completely before anything runs, so that "nothing was +# detected" is discovered before a single command executes rather than after a +# five-minute suite. +# +# Parallel arrays indexed together, iterated with a counted loop: `${arr[@]}` on +# an empty array is an unbound-variable error under `set -u` in bash 3.2, which +# is the bash on every stock macOS, and the empty case is the one that matters +# most here. +# --------------------------------------------------------------------------- +STEP_ID=() +STEP_CLASS=() +STEP_CMD=() +STEP_SKIP=() +STEP_RESULT=() +STEP_SECS=() + +add_step() { STEP_ID+=("$1"); STEP_CLASS+=("$2"); STEP_CMD+=("$3"); STEP_SKIP+=(""); } + +# A step that was found but cannot run. Recorded rather than dropped, because +# "this project has a mypy config and mypy is not installed" is information, +# and a silently shorter table is not. +add_skip() { STEP_ID+=("$1"); STEP_CLASS+=("skip"); STEP_CMD+=(""); STEP_SKIP+=("$2"); } + +q() { printf '%q' "$1"; } + +have() { command -v "$1" >/dev/null 2>&1; } + +# --------------------------------------------------------------------------- +# Guards first. They are greps: they finish before the toolchain has finished +# starting, and they are the only steps that catch the bug class described in +# the header. Putting them last would mean the cheapest answer arrives after +# the most expensive one. +# --------------------------------------------------------------------------- +detect_guards() { + [ -n "$GUARD_DIR" ] || return 0 + + [ -d "$GUARD_DIR" ] \ + || die "VERIFY_GUARD_DIR is set to '$GUARD_DIR' but there is no such directory. Create it, or unset the variable — a missing guard directory would otherwise run zero guards and look exactly like running all of them." + + # ------------------------------------------------------------------------- + # Containment. Everything below this point EXECUTES every file it finds, so a + # mis-set GUARD_DIR is not a wrong answer — it is arbitrary code with the + # developer's credentials, launched by the one command they were told is safe + # to run before every commit. + # + # The three refusals below are the cases that can be proven wrong rather than + # guessed at. Resolved with `cd`+`pwd -P` so that symlinks, `..` and relative + # paths all collapse to one comparable form before being judged. + # ------------------------------------------------------------------------- + local guard_abs self_abs + guard_abs=$(cd "$GUARD_DIR" 2>/dev/null && pwd -P) \ + || die "VERIFY_GUARD_DIR is set to '$GUARD_DIR' but that directory could not be entered (permissions?)." + + # Outside the repository. `..`, an absolute path and a symlink pointing out of + # the tree all land here. A guard is a check on THIS repository's source; a + # directory outside it holds someone else's programs. + case "$guard_abs/" in + "$ROOT_ABS"/*) ;; + *) die "VERIFY_GUARD_DIR '$GUARD_DIR' resolves to '$guard_abs', which is outside this repository ('$ROOT_ABS'). Every executable file in it would be RUN. Guard scripts belong in a dedicated directory inside the repository." ;; + esac + + # The repository root itself. Running every executable at the root means + # running whatever release, deploy or reset script the project keeps there. + [ "$guard_abs" != "$ROOT_ABS" ] \ + || die "VERIFY_GUARD_DIR points at the repository root. Every executable file at the root would be RUN as a guard, including any release, deploy or database script. Put guards in a dedicated subdirectory — scripts/verify.d is the convention." + + # The directory holding this script. This is the likely mistake, because it is + # where scripts live and it reads as the obvious answer: this template ships + # verify.sh beside release.sh, backup.sh and migrate.sh, and pointing the + # guard directory here would run all three and call the result a passing + # check. Skipped when $0 cannot be resolved, which loses nothing — the two + # refusals above still apply. + self_abs=$(cd "$(dirname -- "$0")" 2>/dev/null && pwd -P) || self_abs="" + if [ -n "$self_abs" ] && [ "$guard_abs" = "$self_abs" ]; then + die "VERIFY_GUARD_DIR points at the directory holding verify.sh itself ('$guard_abs'). Every executable file beside this script — release.sh, backup.sh, migrate.sh — would be RUN as a guard. Put guards in a dedicated subdirectory of their own." + fi + + local f path found="" + while IFS= read -r f; do + [ -n "$f" ] || continue + found="yes" + + # Made explicitly relative so the guard runs whether or not its directory is + # on PATH — but only when it is not already absolute, since './' in front of + # an absolute path silently resolves somewhere else entirely. + case "$f" in /*) path="$f" ;; *) path="./$f" ;; esac + + # A guard that lost its executable bit never runs and nothing notices. That + # is the same failure the whole script is about, one directory down. So is a + # guard symlinked in from a shared directory whose target has since moved: + # both are recorded, neither is dropped. + if [ -x "$f" ]; then + add_step "guard:$(basename "$f")" guard "$(q "$path")" + elif [ ! -e "$f" ]; then + add_skip "guard:$(basename "$f")" "broken symlink — it points at nothing, so this guard has not run since the target moved" + else + add_skip "guard:$(basename "$f")" "not executable — chmod +x it, or it will never run again either" + fi + + # -L so a symlinked guard, and a symlinked guard DIRECTORY, are both seen. + # Sharing one guard across sibling repositories by symlink is the normal way + # to do it, and plain `-type f` does not match a symlink: those guards were + # not skipped, not listed and not run, which is the silent-zero failure this + # script exists to prevent. `! -type d` rather than `-type f` so a broken + # symlink still surfaces above instead of vanishing again. + done < <(find -L "$GUARD_DIR" -maxdepth 1 ! -type d 2>/dev/null | LC_ALL=C sort) + + [ -n "$found" ] || say "note: $GUARD_DIR is configured but empty. See 'Guard tests are the point' at the top of this file for what belongs there." +} + +# --------------------------------------------------------------------------- +# Node. The script names come from package.json and nowhere else. +# +# Read with node rather than grepped, because a grep for '"test"' matches keys +# in devDependencies, in a nested tool config, and in lint-staged — and +# `npm run` on a name that is not a script exits non-zero, so the guess would +# surface as a failing check with no failing code behind it. +# --------------------------------------------------------------------------- +detect_node() { + [ -f package.json ] || return 0 + + if ! have npm || ! have node; then + add_skip "npm" "package.json is here but node/npm is not on PATH — none of its checks could run" + return 0 + fi + + local scripts rc + scripts=$(node -e 'try{const s=require(process.cwd()+"/package.json").scripts||{};process.stdout.write(Object.keys(s).join("\n"))}catch(e){process.exit(1)}' 2>/dev/null); rc=$? + + # Kept separate from "there are no scripts" below. Both end in zero npm steps + # and they need different answers: one is a project that has no checks yet, + # the other is a file this script could not read, which is also about to break + # every npm command anyone else runs today. + if [ "$rc" -ne 0 ]; then + add_skip "npm" "package.json could not be parsed by node — fix the JSON; no npm check could be read from it" + return 0 + fi + + if [ -z "$scripts" ]; then + add_skip "npm" "package.json declares no scripts — this Node project was not checked at all" + return 0 + fi + + local class list name matched="" + for class in typecheck lint test; do + case "$class" in + typecheck) list="$NODE_TYPECHECK_SCRIPTS" ;; + lint) list="$NODE_LINT_SCRIPTS" ;; + test) list="$NODE_TEST_SCRIPTS" ;; + esac + + for name in $list; do + printf '%s\n' "$scripts" | grep -qx -- "$name" || continue + add_step "npm:$name" "$class" "npm run $(q "$name")" + matched="yes" + # One per class. Two matches usually means `test` and `test:ci` are the + # same suite twice, and paying for a suite twice is how people stop + # running verify. + break + done + done + + if [ -z "$matched" ]; then + say "note: package.json has scripts, but none named like a check. Looked for:" + say " $NODE_TYPECHECK_SCRIPTS $NODE_LINT_SCRIPTS $NODE_TEST_SCRIPTS" + say " Add the real names to the candidate lists near the top of this file." + fi +} + +detect_make() { + local mk="" f + for f in Makefile makefile GNUmakefile; do + if [ -f "$f" ]; then mk="$f"; break; fi + done + [ -n "$mk" ] || return 0 + + if ! have make; then + add_skip "make" "$mk is here but make is not on PATH" + return 0 + fi + + local class list target matched="" + for class in typecheck lint test; do + case "$class" in + typecheck) list="$MAKE_TYPECHECK_TARGETS" ;; + lint) list="$MAKE_LINT_TARGETS" ;; + test) list="$MAKE_TEST_TARGETS" ;; + esac + + for target in $list; do + # Anchored at the start of the line so this matches a rule and not a + # .PHONY declaration or a variable that happens to contain the word. + grep -qE "^${target}[[:space:]]*:" "$mk" || continue + add_step "make:$target" "$class" "make $(q "$target")" + matched="yes" + break + done + done + + # Said out loud, for the same reason the npm detector says it. A Makefile + # whose target is `tests` or `ci` contributes nothing here, and in a project + # that ALSO has a package.json the run still prints a full green table — one + # that silently excludes the Makefile's suite. Contributing zero steps is + # information; contributing zero steps quietly is the failure. + if [ -z "$matched" ]; then + say "note: $mk is here, but none of its targets are named like a check. Looked for:" + say " $MAKE_TYPECHECK_TARGETS $MAKE_LINT_TARGETS $MAKE_TEST_TARGETS" + say " Add the real names to the candidate lists near the top of this file." + fi +} + +detect_cargo() { + [ -f Cargo.toml ] || return 0 + + if ! have cargo; then + add_skip "cargo" "Cargo.toml is here but cargo is not on PATH" + return 0 + fi + + if cargo fmt --version >/dev/null 2>&1; then + add_step "cargo:fmt" lint "cargo fmt --all -- --check" + else + add_skip "cargo:fmt" "rustfmt is not installed (rustup component add rustfmt)" + fi + + if cargo clippy --version >/dev/null 2>&1; then + # `-D warnings` is not strictness for its own sake: without it clippy prints + # its findings and exits 0, so the step passes whatever it found — a check + # that cannot fail. Loosen it here if this project has warnings it has + # decided to keep, but loosen it visibly. + add_step "cargo:clippy" lint "cargo clippy --all-targets -- -D warnings" + else + add_skip "cargo:clippy" "clippy is not installed (rustup component add clippy)" + fi + + add_step "cargo:test" test "cargo test" +} + +detect_python() { + [ -f pyproject.toml ] || return 0 + + PY_MATCHED="" + + # Configured in pyproject means the project uses it. A tool that merely + # happens to be installed on this machine is not a check this project has, + # and running it would invent a standard the repository never agreed to. + py_tool() { # section-regex binary class command + grep -qE "$1" pyproject.toml || return 0 + PY_MATCHED="yes" + if have "$2"; then + add_step "py:$2" "$3" "$4" + else + add_skip "py:$2" "configured in pyproject.toml but '$2' is not on PATH — activate the virtualenv, or install it" + fi + } + + py_tool '^\[tool\.ruff' ruff lint "ruff check ." + py_tool '^\[tool\.black' black lint "black --check ." + py_tool '^\[tool\.mypy' mypy typecheck "mypy ." + # pytest is matched anywhere in the file, not only as a [tool.pytest] section: + # most projects configure nothing and simply depend on it. + py_tool 'pytest' pytest test "pytest" + + if [ -z "$PY_MATCHED" ]; then + say "note: pyproject.toml is here, but it configures none of the tools this" + say " looks for (ruff, black, mypy, pytest). No Python check was run." + fi +} + +detect_go() { + [ -f go.mod ] || return 0 + + if ! have go; then + add_skip "go" "go.mod is here but go is not on PATH" + return 0 + fi + + add_step "go:vet" lint "go vet ./..." + add_step "go:test" test "go test ./..." +} + +detect_guards +detect_node +detect_make +detect_cargo +detect_python +detect_go + +# --------------------------------------------------------------------------- +# Nothing detected is a failure, and the loudest one here. +# +# A verify that checks nothing and prints success is worse than no verify at +# all: it converts "I have not checked" into "it passed" for everyone +# downstream, including the hook that trusts the exit code. +# --------------------------------------------------------------------------- +if [ "${#STEP_ID[@]}" -eq 0 ]; then + say "NO CHECKS DETECTED. Nothing was verified and nothing passed." + say "" + say "Looked at the repository root for: package.json scripts, a Makefile" + say "target, Cargo.toml, pyproject.toml, go.mod, and VERIFY_GUARD_DIR" + say "(currently ${GUARD_DIR:-unset})." + say "" + say "Give it something to run: add a check script to package.json, add a" + say "target to the Makefile, or set VERIFY_GUARD_DIR to a directory of" + say "executable guard scripts. Then run this again." + exit 2 +fi + +if [ -n "$ONLY" ]; then + n=${#STEP_ID[@]} + kept="" + for ((i = 0; i < n; i++)); do + case "${STEP_ID[$i]}" in + *"$ONLY"*) kept="yes" ;; + # An existing reason wins. A step already skipped because its tool is not + # installed is not skipped BY --only, and overwriting the reason would + # report that it was filtered out when in fact it could not have run — + # turning a real gap in the checks into a deliberate-looking choice. + *) [ -n "${STEP_SKIP[$i]}" ] || STEP_SKIP[$i]="--only $ONLY" ;; + esac + done + [ -n "$kept" ] || die "--only '$ONLY' matched none of the detected steps: $(printf '%s ' "${STEP_ID[@]}")" +fi + +# --quick, resolved against the plan rather than assumed. A SLOW_STEP naming a +# step this project does not have is a stale config that would silently skip +# nothing while reporting that it skipped, so it is refused. +if [ -n "$QUICK" ]; then + n=${#STEP_ID[@]} + hit="" + for ((i = 0; i < n; i++)); do + if [ -n "$SLOW_STEP" ]; then + [ "${STEP_ID[$i]}" = "$SLOW_STEP" ] || continue + else + [ "${STEP_CLASS[$i]}" = "test" ] || continue + fi + [ -n "${STEP_SKIP[$i]}" ] || STEP_SKIP[$i]="--quick" + hit="yes" + done + + if [ -z "$hit" ]; then + if [ -n "$SLOW_STEP" ]; then + die "VERIFY_SLOW_STEP is '$SLOW_STEP' but no detected step has that id: $(printf '%s ' "${STEP_ID[@]}")" + fi + say "note: --quick had no effect — no detected step is a test step." + elif [ -z "$SLOW_STEP" ]; then + say "--quick: skipping the test steps (set VERIFY_SLOW_STEP to an id to pick a different one)." + fi +fi + +n=${#STEP_ID[@]} + +if [ -n "$LIST" ]; then + say "$n step(s) detected. Nothing was run:" + for ((i = 0; i < n; i++)); do + if [ -n "${STEP_SKIP[$i]}" ]; then + say " $(printf '%-18s' "${STEP_ID[$i]}") SKIPPED — ${STEP_SKIP[$i]}" + else + say " $(printf '%-18s' "${STEP_ID[$i]}") ${STEP_CMD[$i]}" + fi + done + exit 0 +fi + +# --------------------------------------------------------------------------- +# Run everything. No early exit on failure: the point of one entry point is one +# answer, and an answer that stops at the first broken thing makes you pay the +# whole runtime again for the second. +# +# Output is streamed rather than captured, so a step that hangs is visible +# while it hangs instead of after it is killed. +# --------------------------------------------------------------------------- +started=$SECONDS +failed=0 +passed=0 +skipped=0 + +for ((i = 0; i < n; i++)); do + if [ -n "${STEP_SKIP[$i]}" ]; then + STEP_RESULT[$i]="SKIPPED" + STEP_SECS[$i]="-" + skipped=$((skipped + 1)) + continue + fi + + say "── ${STEP_ID[$i]} — ${STEP_CMD[$i]}" + t0=$SECONDS + eval "${STEP_CMD[$i]}" + code=$? + STEP_SECS[$i]=$((SECONDS - t0)) + + if [ "$code" -eq 0 ]; then + STEP_RESULT[$i]="PASS" + passed=$((passed + 1)) + else + STEP_RESULT[$i]="FAIL" + failed=$((failed + 1)) + say "${STEP_ID[$i]} failed (exit $code) — continuing so this run reports everything." + fi +done + +# --------------------------------------------------------------------------- +# The table. Three states, three words, three colours, and the word alone is +# enough when the colour is gone — this gets piped into files and hook output +# where nothing is a terminal. +# --------------------------------------------------------------------------- +say "" +say "───────────────────────────────────────────────────────────" + +for ((i = 0; i < n; i++)); do + case "${STEP_RESULT[$i]}" in + PASS) colour="$C_PASS" ;; + FAIL) colour="$C_FAIL" ;; + *) colour="$C_SKIP" ;; + esac + + line="$(printf ' %s%-8s%s %-26s %5s' \ + "$colour" "${STEP_RESULT[$i]}" "$C_OFF" "${STEP_ID[$i]}" "${STEP_SECS[$i]}")" + + if [ "${STEP_RESULT[$i]}" = "SKIPPED" ]; then + line="${line} ${STEP_SKIP[$i]}" + else + line="${line}s" + fi + + say "$line" +done + +say "───────────────────────────────────────────────────────────" + +elapsed=$((SECONDS - started)) + +# Counts are stated only for things that were measured. "0 failed" alongside +# four skips is true and misleading, so the skip count is never omitted and the +# exit code below refuses to call an all-skipped run a pass. +summary="${passed} passed" +[ "$failed" -eq 0 ] || summary="${summary}, ${failed} FAILED" +[ "$skipped" -eq 0 ] || summary="${summary}, ${skipped} SKIPPED (not run — not verified)" + +say "${summary} [${elapsed}s]" + +if [ "$failed" -gt 0 ]; then + say "" + say "Fix the FAILED step(s) above. To re-run one on its own:" + for ((i = 0; i < n; i++)); do + # Both halves shell-quoted, because both routinely contain spaces: this + # family of projects lives under directories like "Privacy LLC", and a step + # id is a guard's filename. Printed unquoted, the command this suggests + # parses as extra arguments and dies on the copy-paste. + [ "${STEP_RESULT[$i]}" = "FAIL" ] \ + && say " bash $(q "$0") --only $(q "${STEP_ID[$i]}")" + done + exit 1 +fi + +# Nothing failed and nothing ran. That is not a pass, and the exit code is the +# only part of this output a hook or a CI job will read. +if [ "$passed" -eq 0 ]; then + say "" + say "NOTHING WAS VERIFIED — every detected step skipped. Exiting non-zero:" + say "a clean table over an empty run is the one outcome this script exists" + say "to prevent." + exit 2 +fi + +exit 0