#!/usr/bin/env python3 """Check release-facing text files for encoding and hidden-control issues. This gate is intentionally small and boring: it catches the class of mistakes that are easy to miss in Windows terminals, especially Markdown snippets that accidentally contain control characters such as backspace (0x08). """ from __future__ import annotations import argparse import sys from pathlib import Path ROOT = Path(__file__).resolve().parents[1] TEXT_SUFFIXES = { ".cfg", ".css", ".env", ".html", ".ini", ".js", ".json", ".jsonl", ".md", ".mjs", ".py", ".sh", ".sql", ".toml", ".ts", ".tsx", ".txt", ".yaml", ".yml", } TEXT_NAMES = { ".dockerignore", ".editorconfig", ".gitattributes", ".gitignore", "Dockerfile", "Dockerfile.prod", } SKIP_PARTS = { ".git", ".mypy_cache", ".pytest_cache", ".ruff_cache", ".venv", ".venv-py312-codex", ".venv-py314", ".playwright-mcp", "node_modules", "__pycache__", "test-results", "venv", } SKIP_REL_PREFIXES = { # Generated/browser-vendor bundles can legitimately contain terminal or # editor control-code tables. Source and docs remain scanned. "web/app/assets/", "web/app/vendor/", } ALLOWED_CONTROL = {"\n", "\r", "\t"} def is_text_candidate(path: Path) -> bool: return path.name in TEXT_NAMES or path.suffix.lower() in TEXT_SUFFIXES def should_skip(path: Path, root: Path) -> bool: try: rel = path.relative_to(root) except ValueError: rel = path rel_posix = rel.as_posix() if any(rel_posix.startswith(prefix) for prefix in SKIP_REL_PREFIXES): return True return bool(set(rel.parts) & SKIP_PARTS) def check_file(path: Path) -> list[str]: errors: list[str] = [] raw = path.read_bytes() try: text = raw.decode("utf-8") except UnicodeDecodeError as exc: return [f"{path}: UTF-8 decode failed at byte {exc.start}: {exc.reason}"] for index, char in enumerate(text): if ord(char) < 32 and char not in ALLOWED_CONTROL: line = text.count("\n", 0, index) + 1 col = index - text.rfind("\n", 0, index) errors.append(f"{path}: hidden control character U+{ord(char):04X} at {line}:{col}") break # Specific guard for the typo produced by a malformed fenced code block: # `\x08ash instead of ```bash. if "`\x08ash" in text or "\x08ash" in text: errors.append(f"{path}: malformed bash code fence contains backspace control character") return errors def iter_files(root: Path) -> list[Path]: files: list[Path] = [] for path in root.rglob("*"): if not path.is_file(): continue if should_skip(path, root): continue if not is_text_candidate(path): continue files.append(path) return sorted(files) def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--root", action="append", type=Path, default=[ROOT], help="Root directory to scan. Can be passed multiple times.", ) args = parser.parse_args() errors: list[str] = [] scanned = 0 for root in args.root: root = root.resolve() if not root.exists(): errors.append(f"{root}: root does not exist") continue for path in iter_files(root): scanned += 1 errors.extend(check_file(path)) if errors: print("[text-integrity] FAILED", file=sys.stderr) for error in errors[:100]: print(error, file=sys.stderr) if len(errors) > 100: print(f"... {len(errors) - 100} more error(s)", file=sys.stderr) return 1 print(f"[text-integrity] OK: scanned {scanned} text file(s)") return 0 if __name__ == "__main__": raise SystemExit(main())