From 6a455ae7e8d2d0ddcb971bb103ce852008849198 Mon Sep 17 00:00:00 2001 From: null Date: Thu, 10 Sep 2026 05:18:01 -0500 Subject: [PATCH] fix(build): tell a rewritten response from a page served by agent The live audit reported "crawlers were served different bytes" for /privacy-policy on qn.isnull.dev. That reads as the origin choosing what to serve by user agent, which would be serious. It is not what is happening: Cloudflare's email obfuscation rewrites the privacy address on that front door with a token that changes on every response, so any two fetches differ. The audit now asks the same agent twice before blaming the agents, and says which of the two it found. A misleading finding is how a checker gets ignored. Refs #228. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/audit-html.js | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/scripts/audit-html.js b/scripts/audit-html.js index f7cbbe6..a31b665 100644 --- a/scripts/audit-html.js +++ b/scripts/audit-html.js @@ -161,7 +161,23 @@ const auditOrigin = async () => { const distinct = new Set(bodies.values()) if (distinct.size > 1) { - findings.push(`${entry.path}: crawlers were served different bytes (${bodies.size} agents, ${distinct.size} versions)`) + // Two different bodies can mean two different things, and only one of them + // is a problem worth chasing: the origin choosing what to serve by agent, + // or something in front of it varying every response. Cloudflare's email + // obfuscation does the second, with a token that changes each time. Ask + // the same agent twice before blaming the agents. + let varies = false + try { + const again = await fetchAs(url, agents[0]) + varies = again.body !== bodies.get(agents[0]) + } catch { + varies = false + } + findings.push( + varies + ? `${entry.path}: the response body changes between identical requests, so something in front of the origin is rewriting it (Cloudflare email obfuscation does this). Crawlers do not all receive the same page.` + : `${entry.path}: crawlers were served different bytes (${bodies.size} agents, ${distinct.size} versions)`, + ) } const body = bodies.values().next().value if (body) {