Queue-North-Website/scripts/audit-html.js

#!/usr/bin/env node
//
// Audits what the site actually serves, in two modes.
//
//   node scripts/audit-html.js                      # dist/, run as guard 15-built-html
//   node scripts/audit-html.js --url https://queuenorth.com
//   node scripts/audit-html.js --url http://localhost:3001 --agents Googlebot
//
// Build mode is a gate: it reads dist/ and refuses on a finding. URL mode
// fetches every page in the live sitemap once per crawler user agent and reports
// what those crawlers actually receive. URL mode is a check to run AFTER a
// deploy, deliberately not wired into deploy.sh: a check that runs after
// publication cannot stop it, and pretending otherwise is worse than not having
// it (GUARDS.md rule 6).
//
// Exit 0 clean, 1 findings, 2 nothing was audited.
import { existsSync, readFileSync, readdirSync, statSync } from 'fs'
import path from 'path'
import { fileURLToPath } from 'url'
import { services } from '../src/data/services.js'
import { industries } from '../src/data/industries.js'
import { auditLinks, auditPage, auditSitemap, parseSitemap } from './lib/html-audit.js'
import { ROUTES } from './lib/routes.js'

const root = path.join(path.dirname(fileURLToPath(import.meta.url)), '..')
const distDir = path.join(root, 'dist')

// The crawlers this site is written for. A spoofed agent string is not the real
// crawler, and a "verified bots only" rule at the edge would answer this with a
// 403 while serving the real one, so treat a pass as evidence the ORIGIN is not
// blocking, not as proof the crawler is happy.
const AGENTS = {
  'OAI-SearchBot': 'Mozilla/5.0 (compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot)',
  PerplexityBot: 'Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)',
  ClaudeBot: 'Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)',
  Googlebot: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
  bingbot: 'Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)',
}

const argv = process.argv.slice(2)
const flag = (name) => {
  const at = argv.indexOf(`--${name}`)
  return at === -1 ? null : argv[at + 1]
}
const origin = flag('url')
const agents = (flag('agents')?.split(',') ?? Object.keys(AGENTS)).filter((name) => AGENTS[name])

const descriptionSource = (routePath) => {
  const service = services.find((item) => routePath === `/services/${item.id}`)
  if (service) return { shortDesc: service.shortDesc, approvedDescription: Boolean(service.page?.seo?.description) }
  const industry = industries.find((item) => routePath === `/industries/${item.id}`)
  if (industry) return { shortDesc: industry.shortDesc, approvedDescription: false }
  return { shortDesc: null, approvedDescription: false }
}

const report = (findings, checked, what) => {
  if (findings.length) {
    console.error(`audit: ${findings.length} finding(s) in ${what}:`)
    for (const finding of findings) console.error(`  ${finding}`)
    process.exit(1)
  }
  console.log(`audit: ${checked} page(s) in ${what}, nothing wrong.`)
}

// --- build mode --------------------------------------------------------------

const newestSourceMtime = () => {
  let newest = 0
  const walk = (dir) => {
    for (const entry of readdirSync(dir, { withFileTypes: true })) {
      const full = path.join(dir, entry.name)
      if (entry.isDirectory()) walk(full)
      else newest = Math.max(newest, statSync(full).mtimeMs)
    }
  }
  for (const dir of ['src', 'public', 'scripts/lib']) walk(path.join(root, dir))
  for (const file of ['index.html', 'scripts/prerender.js']) newest = Math.max(newest, statSync(path.join(root, file)).mtimeMs)
  return newest
}

const auditBuild = () => {
  const sitemapPath = path.join(distDir, 'sitemap.xml')
  if (!existsSync(sitemapPath)) {
    console.error('audit: dist/sitemap.xml is missing, so the build never finished and NOTHING was audited. Run npm run build.')
    process.exit(2)
  }
  // The prerender writes the sitemap last. A source file newer than it means
  // dist/ is stale, and auditing stale output is auditing nothing.
  if (newestSourceMtime() > statSync(sitemapPath).mtimeMs) {
    console.error('audit: dist/ is older than the sources, so NOTHING was audited. Run npm run build.')
    process.exit(2)
  }

  const entries = parseSitemap(readFileSync(sitemapPath, 'utf8'))
  const findings = [...auditSitemap(entries)]
  const pages = []

  for (const routePath of ROUTES) {
    const file = routePath === '/' ? path.join(distDir, 'index.html') : path.join(distDir, routePath, 'index.html')
    if (!existsSync(file)) {
      findings.push(`${routePath}: the build produced no page, so the server would answer it with 404.html`)
      continue
    }
    const html = readFileSync(file, 'utf8')
    pages.push([routePath, html])
    findings.push(...auditPage(html, { path: routePath, ...descriptionSource(routePath) }))
  }

  const notFound = path.join(distDir, '404.html')
  if (!existsSync(notFound)) findings.push('404.html: missing from the build')
  else findings.push(...auditPage(readFileSync(notFound, 'utf8'), { path: '/404', notFound: true }))

  findings.push(...auditLinks(pages, { fileExists: (href) => existsSync(path.join(distDir, href.replace(/^\//, ''))) }))

  report(findings, pages.length, 'the build')
}

// --- url mode ----------------------------------------------------------------

const fetchAs = async (url, agent) => {
  const response = await fetch(url, {
    headers: { 'User-Agent': AGENTS[agent] },
    redirect: 'manual',
    signal: AbortSignal.timeout(20000),
  })
  return { status: response.status, body: await response.text() }
}

const auditOrigin = async () => {
  let entries
  try {
    const response = await fetch(`${origin}/sitemap.xml`, { signal: AbortSignal.timeout(20000) })
    if (!response.ok) throw new Error(`HTTP ${response.status}`)
    entries = parseSitemap(await response.text())
  } catch (error) {
    console.error(`audit: could not read ${origin}/sitemap.xml (${error.message}), so NOTHING was audited.`)
    process.exit(2)
  }
  if (!entries.length) {
    console.error(`audit: ${origin}/sitemap.xml lists no pages, so NOTHING was audited.`)
    process.exit(2)
  }

  const findings = [...auditSitemap(entries, { requireLastmod: true })]
  const pages = []

  for (const entry of entries) {
    const url = `${origin}${entry.path}`
    const bodies = new Map()
    for (const agent of agents) {
      let result
      try {
        result = await fetchAs(url, agent)
      } catch (error) {
        findings.push(`${entry.path}: ${agent} could not fetch it (${error.message})`)
        continue
      }
      if (result.status !== 200) findings.push(`${entry.path}: ${agent} got HTTP ${result.status}`)
      bodies.set(agent, result.body)
    }

    const distinct = new Set(bodies.values())
    if (distinct.size > 1) {
      // Two different bodies can mean two different things, and only one of them
      // is a problem worth chasing: the origin choosing what to serve by agent,
      // or something in front of it varying every response. Cloudflare's email
      // obfuscation does the second, with a token that changes each time. Ask
      // the same agent twice before blaming the agents.
      let varies = false
      try {
        const again = await fetchAs(url, agents[0])
        varies = again.body !== bodies.get(agents[0])
      } catch {
        varies = false
      }
      findings.push(
        varies
          ? `${entry.path}: the response body changes between identical requests, so something in front of the origin is rewriting it (Cloudflare email obfuscation does this). Crawlers do not all receive the same page.`
          : `${entry.path}: crawlers were served different bytes (${bodies.size} agents, ${distinct.size} versions)`,
      )
    }
    const body = bodies.values().next().value
    if (body) {
      pages.push([entry.path, body])
      findings.push(...auditPage(body, { path: entry.path, ...descriptionSource(entry.path) }))
    }
  }

  findings.push(...auditLinks(pages, { fileExists: () => true }))
  report(findings, pages.length, `${origin} as ${agents.length} crawler(s)`)
}

if (origin) await auditOrigin()
else auditBuild()