#!/usr/bin/env bash # # What is actually running right now on the host this project deploys to, and # its logs. Read-only: it inspects and it tails, and it does nothing else. # # bash scripts/status.sh # up? which version? how long? # bash scripts/status.sh --logs # last 100 lines # bash scripts/status.sh --logs 500 # bash scripts/status.sh --logs all # bash scripts/status.sh --follow # stream until Ctrl-C # bash scripts/status.sh --follow 20 # stream, after 20 lines of context # bash scripts/status.sh --deployed-version # just the version, for scripts # bash scripts/status.sh --dry-run # print the commands, contact nothing # # STATUS_CONTAINER=other-app bash scripts/status.sh # one-off, another target # # Exit codes, because the whole point is that a caller can act on the answer: # # 0 running — healthy, still starting, or no healthcheck defined # 1 configuration or usage error; nothing was contacted # 2 COULD NOT DETERMINE — host unreachable, daemon silent, or an error this # does not recognise. This is not an outage report. Nothing is known. # 3 the container does not exist on that host # 4 the container exists but is not running # 5 running, and its own healthcheck says unhealthy # # stdout carries only what a caller should parse — the log lines, or the one # version string. The human report goes to stderr, so `status.sh # --deployed-version` can be read straight into a variable without a filter. # # =========================================================================== # TEMPLATE COPY — configure this before the first run # =========================================================================== # # Copy to `scripts/status.sh` and set the two values in the CONFIGURATION block # below. The script refuses to run until they are set: it has no defaults, # deliberately, for the reason `release.sh` gives at length — a copied script # carrying another project's host and container name answers confidently about # the wrong machine, and an answer is exactly what you came here for. A status # tool that lies is worse than no status tool, because you act on it. # # Assumes: bash, coreutils, git-less (it reads nothing from the repository), and # docker on the target. `ssh` only when the target is not this machine. # # ## Why this exists # # Without it, "is it up?" is answered by typing docker incantations from memory, # which in practice means `docker ps | grep`. That shows a name and a status and # none of the four things you actually wanted: which version is deployed, how # long it has been up, how many times it has restarted, and what its healthcheck # thinks. So the question gets half-answered, and the half that is missing — # restart count — is the one that distinguishes "running" from "crash-looping # and up for nine seconds". # # The other half is worse. When there is no command that answers "what version # is deployed", every script that needs the answer grows its own copy of the # lookup. `release.sh` has one: a single `ssh … docker inspect --format` whose # empty return value is the ONLY thing standing between the prune and deleting # the published image production is currently running. That fact is load-bearing # enough to deserve a name, a documented exit code, and one place to fix — not a # line pasted into whichever script needed it that week, each copy free to # disagree about what an empty answer means. # # Hence `--deployed-version`: one fact, on stdout, non-zero when it could not be # read. A caller that treats non-zero as "stop" cannot make the prune's mistake. # # ## Unreachable is not stopped # # These are different facts and only one of them is an outage you caused: # # - the container is not running → something happened to the deployment # - I could not reach the host → something happened to the network, or to # ssh, or to the daemon, and the container # is very probably still serving traffic # # Rendering the second as the first is how a 2am investigation starts by # restarting a healthy service. So a failed connection exits 2 and says so in # those words; only an inspect that succeeded and reported a non-running state # exits 4. Nothing here ever prints "stopped", "0" or "ok" for something it did # not measure — an unknown is printed as an unknown, every time. # # ssh runs with BatchMode=yes and a connect timeout, so a host that would prompt # for a password or a host key is read as unreachable in seconds instead of # hanging forever on a prompt nobody is watching. # # ## There is no --host flag # # Host and container come from the CONFIGURATION block, or from the environment # for a one-off. A flag would make it one keystroke to read host A and file the # answer under host B, and the environment form is self-documenting in shell # history — which is where you will be reading it back from. # # ## What it deliberately does not do # # It never starts, stops, restarts, recreates, kills or removes anything, and it # never pulls or prunes an image. Every docker verb it runs is `inspect`, # `version` or `logs`. This is not an oversight to be fixed later: a status tool # is the one command people run half-awake and without reading, so it must be # safe to run half-awake and without reading. Fixing what it reports is a # separate, deliberate act with a separate command. # # It does not compare the deployed version against this repository, and does not # read package.json or any other file. Whether the deployed version is the RIGHT # version is a judgement, and a judgement needs a human or a release script that # owns the decision; this only reports what is there. # # It reports one configured container, not every container on the host. A tool # that enumerates is a tool that finds another project's container and reports it # as yours. set -uo pipefail # --------------------------------------------------------------------------- # CONFIGURATION — set these two, then delete this banner. # # Both are empty on purpose. See the note at the top. # --------------------------------------------------------------------------- # An ssh destination — a Host in ~/.ssh/config, an IP, or user@host — or the # literal word `local` to ask this machine's own docker daemon. # # `local` is a word you have to type rather than what an empty value means, # because "you did not configure a host" and "the host is this machine" must not # look the same. Defaulting to local would make an unconfigured copy of this # script quietly report on your laptop and call it production. HOST="${STATUS_HOST:-nebula}" # The container name (or id) to report on. Exactly one. CONTAINER="${STATUS_CONTAINER:-qn-website-dev}" # Seconds to wait for the ssh connection. Short on purpose: this command exists # to be run when something might be wrong, and a status tool that hangs is # indistinguishable from the outage it was meant to describe. SSH_TIMEOUT="${STATUS_SSH_TIMEOUT:-10}" # Lines tailed by `--logs` with no number given. TAIL_DEFAULT="${STATUS_TAIL:-100}" # The image label carrying the deployed version. This one has a default because # it is an open standard rather than a fact about your project; override it only # if your build stamps a different label. VERSION_LABEL="${STATUS_VERSION_LABEL:-org.opencontainers.image.version}" # --------------------------------------------------------------------------- E_CONFIG=1 E_UNKNOWN=2 E_MISSING=3 E_STOPPED=4 E_UNHEALTHY=5 say() { printf '\033[1mstatus:\033[0m %s\n' "$*" >&2; } die() { printf '\033[1mstatus:\033[0m %s\n' "$*" >&2; exit "$E_CONFIG"; } # die() with a chosen code. Every caller of this is a distinct fact a script # downstream is expected to branch on, so the code is passed explicitly at the # point the fact is established rather than mapped from a message afterwards. quit() { local code="$1"; shift; printf '\033[1mstatus:\033[0m %s\n' "$*" >&2; exit "$code"; } field() { printf '\033[1mstatus:\033[0m %-9s %s\n' "$1" "$2" >&2; } usage() { say "usage: bash scripts/status.sh [--logs [N|all]] [--follow [N]] [--deployed-version] [--dry-run]" say " set STATUS_HOST and STATUS_CONTAINER, or edit the CONFIGURATION block." } # --------------------------------------------------------------------------- # Arguments, before the configuration check, so that `--help` answers on a # fresh unconfigured copy — which is the one moment somebody needs it. # --------------------------------------------------------------------------- MODE="report" TAIL="" DRY_RUN="" # `("$@")` with no arguments is an unbound expansion under `set -u` on bash # before 4.4, which is still what ships on macOS. Guarded rather than assumed. if [ "$#" -gt 0 ]; then ARGS=("$@"); else ARGS=(); fi # Refused rather than resolved last-flag-wins. `--deployed-version --logs` from a # script would otherwise return a log stream on stdout to a caller that is about # to compare it to a version string. set_mode() { [ "$MODE" = "report" ] || [ "$MODE" = "$1" ] \ || die "--logs, --follow and --deployed-version ask different questions; pick one." MODE="$1" } i=0 while [ "$i" -lt "${#ARGS[@]}" ]; do arg="${ARGS[$i]}" case "$arg" in --logs|-l|--follow|-f) case "$arg" in --logs|-l) set_mode "logs" ;; *) set_mode "follow" ;; esac # An optional operand rather than a required one, so `--logs` alone is a # whole command. Only consumed when it looks like a count — otherwise # `--logs --dry-run` would swallow the flag and silently mean something # else. if [ $((i + 1)) -lt "${#ARGS[@]}" ]; then case "${ARGS[$((i + 1))]}" in all|[0-9]*) TAIL="${ARGS[$((i + 1))]}"; i=$((i + 1)) ;; esac fi ;; --logs=*) set_mode "logs"; TAIL="${arg#*=}" ;; --follow=*) set_mode "follow"; TAIL="${arg#*=}" ;; --deployed-version) set_mode "version" ;; --dry-run) DRY_RUN="yes" ;; -h|--help) usage; exit 0 ;; *) usage; die "unknown argument '$arg'." ;; esac i=$((i + 1)) done [ -n "$TAIL" ] || TAIL="$TAIL_DEFAULT" case "$TAIL" in all) : ;; ''|*[!0-9]*) die "line count must be a whole number or 'all', got '$TAIL'." ;; esac case "$SSH_TIMEOUT" in ''|*[!0-9]*) die "STATUS_SSH_TIMEOUT must be a whole number of seconds, got '$SSH_TIMEOUT'." ;; esac # --------------------------------------------------------------------------- # Refuse to run half-configured, before anything is contacted, and name the # missing value one at a time so the message says which one. # --------------------------------------------------------------------------- [ -n "$HOST" ] || die "set HOST (or STATUS_HOST) — the ssh host running the container, or the word 'local'. See the CONFIGURATION block." [ -n "$CONTAINER" ] || die "set CONTAINER (or STATUS_CONTAINER) — the container name to report on." # ssh takes its destination positionally, so a host beginning with a dash is # parsed as an option and the failure that follows describes something else # entirely. Refused here, where the message can still name the real cause. case "$HOST" in -*) die "HOST ('$HOST') begins with '-'; ssh would read it as an option. Use user@host, or a Host from ~/.ssh/config." ;; esac case "$CONTAINER" in -*) die "CONTAINER ('$CONTAINER') begins with '-'; docker would read it as a flag." ;; esac # The label name is interpolated into a Go template as a quoted string below. A # quote or backslash in it would end that string early, and the template error # that followed would be reported as "could not inspect" — an unknown, blamed on # the host, caused here. case "$VERSION_LABEL" in ''|*[\"\\]*) die "STATUS_VERSION_LABEL must be a plain label name without quotes or backslashes, got '$VERSION_LABEL'." ;; esac # --------------------------------------------------------------------------- # Running docker here or over there. # # ssh joins its command arguments with spaces and hands the result to a remote # shell, which splits it again. Anything containing a space — every Go template # below does — arrives as several arguments unless it is quoted for that second # parse. Quoting is done here, once, rather than by hand at each call site. # --------------------------------------------------------------------------- shquote() { local out="" a q="'" esc="'\\''" for a in "$@"; do a="${a//$q/$esc}" out="${out}${out:+ }'${a}'" done printf '%s' "$out" } # ServerAlive is here for --follow: a connection that dies mid-stream otherwise # leaves the terminal sitting quietly forever, which looks exactly like a service # that has stopped logging. Absence of output must not be able to mean two # things, so the connection is made to fail loudly instead. ssh_opts() { printf '%s' "-o BatchMode=yes -o ConnectTimeout=${SSH_TIMEOUT} -o ServerAliveInterval=15 -o ServerAliveCountMax=3" } SSH_WANT_TTY="" dock() { if [ "$HOST" = "local" ]; then docker "$@" return fi # A tty makes the remote `docker logs -f` receive a hangup when you press # Ctrl-C. Without one it keeps running on the far side, holding the daemon's # log stream open, and enough abandoned follows are a real resource leak on a # box you are already worried about. Requested only when we have a tty to # give, because ssh -t without one is a warning and no tty anyway. if [ -n "$SSH_WANT_TTY" ] && [ -t 0 ]; then # shellcheck disable=SC2046 # deliberate word-splitting of the option list ssh -t $(ssh_opts) "$HOST" "docker $(shquote "$@")" else # shellcheck disable=SC2046 ssh $(ssh_opts) "$HOST" "docker $(shquote "$@")" fi } # The remote form is printed on two lines — the connection, then the command # string the far shell receives — because the single-line form is the same text # quoted twice and the host and container names disappear into the punctuation. # Those two names are the entire reason to run --dry-run. dock_show() { if [ "$HOST" = "local" ]; then printf ' docker %s\n' "$(shquote "$@")" >&2 else printf ' ssh %s %s\n' "$(ssh_opts)" "$HOST" >&2 printf ' docker %s\n' "$(shquote "$@")" >&2 fi } # --------------------------------------------------------------------------- # Timestamps. # # Docker reports RFC3339 with nanoseconds; `date -d` is GNU and `date -j -f` is # BSD, so both are tried and NEITHER working is reported as not knowing the # uptime rather than as an uptime of zero. # --------------------------------------------------------------------------- epoch_of() { local t="${1%Z}" t="${t%%.*}" local e if e=$(date -u -d "${t}Z" +%s 2>/dev/null) && [ -n "$e" ]; then printf '%s' "$e" return 0 fi if e=$(date -u -j -f '%Y-%m-%dT%H:%M:%S' "$t" +%s 2>/dev/null) && [ -n "$e" ]; then printf '%s' "$e" return 0 fi return 1 } human_duration() { local s="$1" d h m d=$((s / 86400)); s=$((s % 86400)) h=$((s / 3600)); s=$((s % 3600)) m=$((s / 60)); s=$((s % 60)) if [ "$d" -gt 0 ]; then printf '%dd %dh %dm' "$d" "$h" "$m" elif [ "$h" -gt 0 ]; then printf '%dh %dm' "$h" "$m" elif [ "$m" -gt 0 ]; then printf '%dm %ds' "$m" "$s" else printf '%ds' "$s" fi } # Elapsed time since an RFC3339 stamp, or nothing at all. # # The zero timestamp means the event never happened — a container created but # never started carries 0001-01-01 in StartedAt, and subtracting it yields two # millennia of uptime presented with a straight face. # # A negative result is clock skew between this machine and that one, not time # travel, and is likewise reported as not knowing rather than as a number. elapsed_since() { local stamp="$1" at now diff case "$stamp" in ''|0001-01-01*) return 1 ;; esac at=$(epoch_of "$stamp") || return 1 now=$(date -u +%s) || return 1 diff=$((now - at)) [ "$diff" -ge 0 ] || return 1 human_duration "$diff" } # --------------------------------------------------------------------------- # One inspect, one round trip. # # Fields are read together rather than one call each: two calls can straddle a # restart and produce a report that never described a single moment — an uptime # from before the restart beside a restart count from after it. # # The free-text fields are LAST. `read` gives the final variable everything that # remains, delimiters included, so a '|' inside an image name or a version label # lands harmlessly in the last field instead of shifting every field after it. # --------------------------------------------------------------------------- FMT='{{.State.Status}}|{{.State.Running}}|{{.RestartCount}}|{{.State.ExitCode}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}{{end}}|{{.State.StartedAt}}|{{.State.FinishedAt}}|{{.Image}}|{{.Config.Image}}|{{index .Config.Labels "'"$VERSION_LABEL"'"}}' C_STATUS=""; C_RUNNING=""; C_RESTARTS=""; C_EXIT=""; C_HEALTH="" C_STARTED=""; C_FINISHED=""; C_DIGEST=""; C_IMAGE=""; C_VERSION="" # Work out WHY the inspect failed, and exit with the code for that specific # fact. Reached only after a failure, so the extra round trips cost nothing in # the normal case and buy the one distinction this script exists to preserve. diagnose_and_exit() { local err="$1" if [ "$HOST" != "local" ]; then # shellcheck disable=SC2046 if ! ssh $(ssh_opts) "$HOST" true >/dev/null 2>&1; then say "cannot reach ${HOST} over ssh (BatchMode, ${SSH_TIMEOUT}s timeout)." say " This is NOT a report that the container is down — nothing was" say " determined about it. Check the network, the ssh key, and" say " whether the host would have prompted for a password." exit "$E_UNKNOWN" fi fi if ! dock version --format '{{.Server.Version}}' >/dev/null 2>&1; then if [ "$HOST" = "local" ]; then say "this machine's docker daemon did not answer." else say "reached ${HOST} over ssh, but its docker daemon did not answer." fi say " The container's state is unknown — the daemon being down does" say " not tell you whether the container was running before it went." exit "$E_UNKNOWN" fi # Matched case-insensitively against a lowercased copy, because the wording # is not stable across docker versions: "Error: No such object: x" on one # daemon here, "error: no such object: x" on another, and "No such container" # from older ones. A case-sensitive match reports a container that is simply # absent as an unknown — which is the safe direction to be wrong in, and # still the wrong answer. local lower lower=$(printf '%s' "$err" | tr '[:upper:]' '[:lower:]') case "$lower" in *"no such object"*|*"no such container"*) quit "$E_MISSING" "there is no container named '${CONTAINER}' on ${HOST}." ;; esac # An error nobody anticipated is printed verbatim and treated as an unknown. # Guessing at an unfamiliar message is how "I could not check" becomes "there # is nothing there". say "could not inspect '${CONTAINER}' on ${HOST}, and the error is not one" say " this script recognises. Nothing is known about the container:" printf ' %s\n' "${err:-(no error output)}" >&2 exit "$E_UNKNOWN" } # The one file this script creates, and the one thing it ever deletes. # # The path is read when the trap FIRES rather than pasted into the trap's source # text. `trap "rm -f '$f'" EXIT` looks equivalent and is not: it builds a shell # command out of mktemp's output, which is built out of $TMPDIR, so a directory # whose name contains a single quote closes that string early and everything # after it runs as shell. `TMPDIR="/tmp/x'; rm -rf ~; echo '"` is a working # exploit against the double-quoted form. This is the only destructive verb in # the script and it must not be constructible from the environment. # # The -n guard means rm is never handed an empty path, and `--` means a path # beginning with a dash is a path rather than a bundle of options. ERRFILE="" # shellcheck disable=SC2329 # invoked by the EXIT trap below, not by name cleanup() { [ -n "$ERRFILE" ] || return 0 rm -f -- "$ERRFILE" return 0 } trap cleanup EXIT inspect_container() { local out rc ERRFILE=$(mktemp) || die "cannot create a temporary file" out=$(dock inspect --format "$FMT" "$CONTAINER" 2>"$ERRFILE") rc=$? if [ "$rc" -ne 0 ] || [ -z "$out" ]; then diagnose_and_exit "$(tr -d '\r' < "$ERRFILE" | head -n 3)" fi IFS='|' read -r C_STATUS C_RUNNING C_RESTARTS C_EXIT C_HEALTH \ C_STARTED C_FINISHED C_DIGEST C_IMAGE C_VERSION \ </dev/null 2>&1 \ || quit "$E_UNKNOWN" "docker is not on PATH, so nothing could be determined. Install it, or set HOST to the machine that runs the container." else command -v ssh >/dev/null 2>&1 \ || quit "$E_UNKNOWN" "ssh is not on PATH, so ${HOST} could not be contacted." fi inspect_container # --------------------------------------------------------------------------- # --deployed-version: one fact on stdout, non-zero when it is not known. # # The contract callers depend on: a zero exit means the string on stdout was # read from the running container's image. Anything else means stop — never # "assume none", which is the assumption that lets a prune delete production's # image. # --------------------------------------------------------------------------- if [ "$MODE" = "version" ]; then [ -n "$C_VERSION" ] \ || quit "$E_UNKNOWN" "'${CONTAINER}' on ${HOST} is ${C_STATUS}, but its image carries no ${VERSION_LABEL} label — the deployed version is unknown." [ "$C_RUNNING" = "true" ] \ || quit "$E_STOPPED" "'${CONTAINER}' on ${HOST} is ${C_STATUS}, not running. Its image says ${C_VERSION}, but nothing is serving from it." printf '%s\n' "$C_VERSION" exit 0 fi # --------------------------------------------------------------------------- # --logs / --follow. # # The inspect above already ran, which is what makes an empty tail meaningful: # the container is known to exist, so no output means it has logged nothing. # Tailing blind cannot tell that apart from a container that is not there. # --------------------------------------------------------------------------- if [ "$MODE" = "logs" ] || [ "$MODE" = "follow" ]; then field container "${CONTAINER} on ${HOST}" field state "${C_STATUS}${C_VERSION:+ (${C_VERSION})}" # Said before the first line of output, because a wall of old log lines from a # container that died an hour ago is indistinguishable from a live one. if [ "$C_RUNNING" != "true" ]; then say "note: the container is ${C_STATUS}. These are the logs it left behind," if [ "$MODE" = "follow" ]; then say " not a live stream — nothing is writing, so this will return at once." else say " not a live stream." fi fi if [ "$MODE" = "follow" ]; then SSH_WANT_TTY="yes" say "following ${CONTAINER} — Ctrl-C to stop." dock logs --tail "$TAIL" --follow "$CONTAINER" else dock logs --tail "$TAIL" "$CONTAINER" fi rc=$? # A tail that failed after a successful inspect is a new fact, not a quiet # zero: the container exists, so an empty screen would otherwise read as a # silent service. [ "$rc" -eq 0 ] || quit "$E_UNKNOWN" "the log stream ended with status ${rc}; some or all of the output may be missing." # The logs printed either way; only the code differs. Exiting 0 here would # report the documented meaning of 0 — "running" — for a container that is # not, which is the same mistake as printing "ok" for something unmeasured. # The state is the fact the exit code carries, in every mode. [ "$C_RUNNING" = "true" ] \ || quit "$E_STOPPED" "'${CONTAINER}' exists on ${HOST} but is ${C_STATUS}; the logs above are what it left behind, not a live service." exit 0 fi # --------------------------------------------------------------------------- # The report. # --------------------------------------------------------------------------- field container "${CONTAINER} on ${HOST}" if [ "$C_RUNNING" = "true" ]; then field state "running" else field state "${C_STATUS:-unknown} (exit code ${C_EXIT:-unknown})" fi field image "${C_IMAGE:-(unknown)}" # The digest is printed beside the tag because the tag is a claim and the digest # is the fact. A tag can be moved after a container was created, at which point # the container's image reference names something it was never built from. if [ -n "$C_DIGEST" ]; then case "$C_DIGEST" in sha256:*) short="${C_DIGEST#sha256:}" field digest "sha256:${short:0:12}" ;; *) # Whatever this is, it is not a sha256 digest, so it is not labelled as # one. The stripping form prints "sha256:" in front of the first twelve # characters of anything at all — podman and older daemons report a bare # image id here, and it would have been rendered as a digest it is not. field digest "$C_DIGEST" ;; esac fi if [ -n "$C_VERSION" ]; then field version "$C_VERSION" else field version "(not set — this image carries no ${VERSION_LABEL} label)" fi if [ "$C_RUNNING" = "true" ]; then if up=$(elapsed_since "$C_STARTED"); then field uptime "${up} (started ${C_STARTED})" else # Reported as unknown, with the raw stamp, rather than as a plausible # number computed from a timestamp that could not be read. field uptime "unknown — could not read the start time '${C_STARTED}'" fi else if down=$(elapsed_since "$C_FINISHED"); then field stopped "${down} ago (${C_FINISHED})" else field stopped "at '${C_FINISHED:-unknown}'" fi fi field restarts "${C_RESTARTS:-unknown}" # The count alone is not the signal; the count beside a short uptime is. Said # out loud because a crash loop reads as "running" in every tool that shows only # a state, and that is the case this command was written to catch. if [ -n "$C_RESTARTS" ] && [ "$C_RESTARTS" != "0" ] && [ "$C_RUNNING" = "true" ]; then say " ^ restarted ${C_RESTARTS} time(s). A climbing count with a short" say " uptime is a crash loop, not a healthy service." fi if [ -n "$C_HEALTH" ]; then field health "$C_HEALTH" else # Not "healthy", and not "ok". No healthcheck is defined, so nothing about # this container's health has been measured by anyone. field health "not configured — this image defines no HEALTHCHECK, so nothing is being checked" fi if [ "$C_RUNNING" != "true" ]; then quit "$E_STOPPED" "'${CONTAINER}' exists on ${HOST} but is ${C_STATUS}." fi if [ "$C_HEALTH" = "unhealthy" ]; then quit "$E_UNHEALTHY" "'${CONTAINER}' is running, but its healthcheck reports unhealthy. Try --logs." fi exit 0