Nexus/scripts/check_text_integrity.py

#!/usr/bin/env python3
"""Check release-facing text files for encoding and hidden-control issues.

This gate is intentionally small and boring: it catches the class of mistakes
that are easy to miss in Windows terminals, especially Markdown snippets that
accidentally contain control characters such as backspace (0x08).
"""

from __future__ import annotations

import argparse
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]

TEXT_SUFFIXES = {
    ".cfg",
    ".css",
    ".env",
    ".html",
    ".ini",
    ".js",
    ".json",
    ".jsonl",
    ".md",
    ".mjs",
    ".py",
    ".sh",
    ".sql",
    ".toml",
    ".ts",
    ".tsx",
    ".txt",
    ".yaml",
    ".yml",
}

TEXT_NAMES = {
    ".dockerignore",
    ".editorconfig",
    ".gitattributes",
    ".gitignore",
    "Dockerfile",
    "Dockerfile.prod",
}

SKIP_PARTS = {
    ".git",
    ".mypy_cache",
    ".pytest_cache",
    ".ruff_cache",
    ".venv",
    ".venv-py312-codex",
    ".venv-py314",
    ".playwright-mcp",
    "node_modules",
    "__pycache__",
    "test-results",
    "venv",
}

SKIP_REL_PREFIXES = {
    # Generated/browser-vendor bundles can legitimately contain terminal or
    # editor control-code tables. Source and docs remain scanned.
    "web/app/assets/",
    "web/app/vendor/",
}

ALLOWED_CONTROL = {"\n", "\r", "\t"}


def is_text_candidate(path: Path) -> bool:
    return path.name in TEXT_NAMES or path.suffix.lower() in TEXT_SUFFIXES


def should_skip(path: Path, root: Path) -> bool:
    try:
        rel = path.relative_to(root)
    except ValueError:
        rel = path
    rel_posix = rel.as_posix()
    if any(rel_posix.startswith(prefix) for prefix in SKIP_REL_PREFIXES):
        return True
    return bool(set(rel.parts) & SKIP_PARTS)


def check_file(path: Path) -> list[str]:
    errors: list[str] = []
    raw = path.read_bytes()
    try:
        text = raw.decode("utf-8")
    except UnicodeDecodeError as exc:
        return [f"{path}: UTF-8 decode failed at byte {exc.start}: {exc.reason}"]

    for index, char in enumerate(text):
        if ord(char) < 32 and char not in ALLOWED_CONTROL:
            line = text.count("\n", 0, index) + 1
            col = index - text.rfind("\n", 0, index)
            errors.append(f"{path}: hidden control character U+{ord(char):04X} at {line}:{col}")
            break

    # Specific guard for the typo produced by a malformed fenced code block:
    # `\x08ash instead of ```bash.
    if "`\x08ash" in text or "\x08ash" in text:
        errors.append(f"{path}: malformed bash code fence contains backspace control character")

    return errors


def iter_files(root: Path) -> list[Path]:
    files: list[Path] = []
    for path in root.rglob("*"):
        if not path.is_file():
            continue
        if should_skip(path, root):
            continue
        if not is_text_candidate(path):
            continue
        files.append(path)
    return sorted(files)


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "--root",
        action="append",
        type=Path,
        default=[ROOT],
        help="Root directory to scan. Can be passed multiple times.",
    )
    args = parser.parse_args()

    errors: list[str] = []
    scanned = 0
    for root in args.root:
        root = root.resolve()
        if not root.exists():
            errors.append(f"{root}: root does not exist")
            continue
        for path in iter_files(root):
            scanned += 1
            errors.extend(check_file(path))

    if errors:
        print("[text-integrity] FAILED", file=sys.stderr)
        for error in errors[:100]:
            print(error, file=sys.stderr)
        if len(errors) > 100:
            print(f"... {len(errors) - 100} more error(s)", file=sys.stderr)
        return 1

    print(f"[text-integrity] OK: scanned {scanned} text file(s)")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())