Project-Template/docs/architecture/scripts/duplication.py

#!/usr/bin/env python3
"""Find code that exists twice.

## Why this is a script and not a code review

Duplication is the thing a reader is worst at. It is invisible unless both
copies are on screen together, and by the time there are three copies nobody
remembers there were ever two. On the repository this was written against, five
separate instances were found by reading in a single afternoon — a four-part
renderer copied into two components, six inlined copies of one ternary, a path
convention written out in four places, a settings key as a literal in three, and
two functions answering the same question with different precedence.

Every one was found by accident. A twenty-line heuristic finds fifty.

## What it reports, and what it deliberately does not

A run of normalised lines that appears in more than one place. Comments, blank
lines and imports are stripped first, so a shared licence header or a block of
imports is not a finding.

It is tuned for **near-silence**, per `GUARDS.md` rule 5: a check that is wrong
six times in ten is one people learn to skip. The defaults require a long run
and a substantial amount of text, so what survives is worth looking at. Loosen
them deliberately when hunting, not by default.

**It cannot tell you whether duplication is wrong.** Two similar-looking blocks
sometimes answer different questions and should stay apart — `notice-sections`
makes that argument about a producer's shape and a wire's shape. This finds
candidates; a person decides.

    python3 scripts/duplication.py                     # default roots
    python3 scripts/duplication.py src lib             # specific roots
    python3 scripts/duplication.py --lines 6 --chars 250   # hunt harder
    python3 scripts/duplication.py --ignore vendor/    # skip a tree

Exit codes: 0 nothing above the threshold. 1 duplication found. 2 nothing was
scanned, which is not a pass — an empty run and a clean run must not look the
same.
"""

from __future__ import annotations

import argparse
import hashlib
import os
import re
import sys
from collections import defaultdict

DEFAULT_ROOTS = ["src", "lib", "app", "scripts", "notices"]
DEFAULT_EXTS = (".ts", ".tsx", ".js", ".jsx", ".mjs", ".py", ".go", ".rs")
SKIP_DIRS = {"node_modules", ".next", ".git", "dist", "build", "__pycache__", "vendor"}

# A line that carries no logic. Stripping these is what stops a shared import
# block or a licence header being reported as duplication.
NOISE = re.compile(r"^\s*(//|#|\*|/\*|\*/|import\s|from\s+['\"]|use\s|require\()")


def normalised(path: str) -> list[tuple[int, str]]:
    """Real lines, with their original numbers, whitespace flattened."""
    out: list[tuple[int, str]] = []

    try:
        with open(path, encoding="utf8", errors="ignore") as handle:
            for number, raw in enumerate(handle, 1):
                line = raw.strip()

                if not line or NOISE.match(line):
                    continue

                # Punctuation-only lines — `}`, `});`, `)` — are structure, not
                # logic. Left in they make every closing brace look duplicated.
                if len(line) < 4:
                    continue

                out.append((number, re.sub(r"\s+", " ", line)))
    except OSError:
        return []

    return out


def collect(roots: list[str], exts: tuple[str, ...], ignore: list[str]) -> list[str]:
    files: list[str] = []

    for root in roots:
        if not os.path.isdir(root):
            continue

        for base, dirs, names in os.walk(root):
            dirs[:] = [d for d in dirs if d not in SKIP_DIRS]

            for name in names:
                if not name.endswith(exts):
                    continue

                path = os.path.join(base, name)

                if any(pattern in path for pattern in ignore):
                    continue

                files.append(path)

    return files


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__.split("\n")[0])
    parser.add_argument("roots", nargs="*", default=None)
    parser.add_argument("--lines", type=int, default=8, help="run length (default 8)")
    parser.add_argument("--chars", type=int, default=320, help="minimum text (default 320)")
    parser.add_argument("--ignore", action="append", default=[], help="substring to skip")
    parser.add_argument("--same-file", action="store_true", help="also report within one file")
    args = parser.parse_args()

    roots = args.roots or DEFAULT_ROOTS
    files = collect(roots, DEFAULT_EXTS, args.ignore)

    if not files:
        print(f"duplication: nothing to scan in {', '.join(roots)}.", file=sys.stderr)
        return 2

    blocks: dict[str, list[tuple[str, int]]] = defaultdict(list)

    for path in files:
        lines = normalised(path)

        for index in range(len(lines) - args.lines + 1):
            window = lines[index : index + args.lines]
            text = " ".join(text for _, text in window)

            if len(text) < args.chars:
                continue

            digest = hashlib.sha1(text.encode("utf8")).hexdigest()
            blocks[digest].append((path, window[0][0]))

    findings = []

    for sites in blocks.values():
        distinct_files = {path for path, _ in sites}

        if len(distinct_files) > 1 or (args.same_file and len(sites) > 1):
            findings.append(sites)

    if not findings:
        print(
            f"duplication: none at {args.lines}+ lines / {args.chars}+ chars "
            f"across {len(files)} file(s).",
            file=sys.stderr,
        )
        return 0

    # Overlapping windows report the same duplication many times. Keep one
    # finding per set of files, at its first site — otherwise a 40-line copy
    # produces 33 findings and reads as a disaster.
    seen: set[tuple[str, ...]] = set()
    shown = 0

    for sites in sorted(findings, key=lambda s: -len(s)):
        key = tuple(sorted({path for path, _ in sites}))

        if key in seen:
            continue

        seen.add(key)
        shown += 1
        print("duplicated block:")

        for path, line in sorted(sites)[:6]:
            print(f"  {path}:{line}")

    print(
        f"\nduplication: {shown} duplicated region(s) across {len(files)} file(s), "
        f"at {args.lines}+ lines and {args.chars}+ characters.",
        file=sys.stderr,
    )
    print(
        "Not every one is wrong — two blocks can answer different questions and "
        "belong apart. These are candidates; you decide.",
        file=sys.stderr,
    )

    return 1


if __name__ == "__main__":
    raise SystemExit(main())