#!/usr/bin/env python3 """Audit whether CTC labels can be represented by a model's time axis. For a label of length L, CTC needs at least L timesteps plus one additional timestep for every pair of identical adjacent tokens. This tool independently checks that lower bound and the recognizer's configured maximum text length. PaddleOCR's character dictionaries are one-token-per-line. This audit accepts only single-codepoint dictionary entries so Python string iteration matches the label encoder exactly; multi-codepoint token dictionaries require a tokenizer- aware audit and are rejected. """ from __future__ import annotations import argparse import hashlib import json from pathlib import Path from typing import Any def sha256_of(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(1 << 20), b""): digest.update(chunk) return digest.hexdigest() def parse_label_line(raw: str, *, path: Path, line_number: int) -> tuple[str, str]: row = raw.rstrip("\r\n") if not row: raise ValueError(f"{path}:{line_number}: blank label row") try: image, text = row.split("\t", 1) except ValueError as exc: raise ValueError(f"{path}:{line_number}: expected imagetext") from exc if not image: raise ValueError(f"{path}:{line_number}: empty image path") if not text: raise ValueError(f"{path}:{line_number}: empty label") return image, text def ctc_min_timesteps(text: str) -> int: """Return CTC's exact lower bound for a codepoint-tokenized label.""" return len(text) + sum(left == right for left, right in zip(text, text[1:])) def audit_dictionary(path: Path, *, use_space_char: bool) -> tuple[dict[str, Any], set[str]]: entries = path.read_text(encoding="utf-8").splitlines() empty = [index for index, entry in enumerate(entries, start=1) if not entry] multitoken = [ {"line": index, "entry": entry, "codepoint_length": len(entry)} for index, entry in enumerate(entries, start=1) if len(entry) != 1 ] if empty or multitoken: raise ValueError( f"dictionary must contain exactly one codepoint per non-empty line: {path}; " f"empty={len(empty)}, multi_codepoint={len(multitoken)}" ) allowed = set(entries) if use_space_char: allowed.add(" ") return { "path": str(path), "sha256": sha256_of(path), "entries": len(entries), "single_codepoint_entries": True, "use_space_char": use_space_char, }, allowed def audit_label_file( path: Path, *, split: str, pool_width: int, max_text_length: int, allowed_characters: set[str], ) -> dict[str, Any]: rows: list[dict[str, Any]] = [] seen_images: set[str] = set() duplicate_images: list[str] = [] for line_number, raw in enumerate( path.read_text(encoding="utf-8").splitlines(keepends=True), start=1 ): image, text = parse_label_line(raw, path=path, line_number=line_number) if image in seen_images: duplicate_images.append(image) seen_images.add(image) unknown = sorted({char for char in text if char not in allowed_characters}) encoded = "".join(char for char in text if char in allowed_characters) adjacent_repeats = sum( left == right for left, right in zip(encoded, encoded[1:]) ) rows.append( { "line_number": line_number, "image": image, "raw_label_codepoints": len(text), "encoded_tokens": len(encoded), "adjacent_equal_pairs": adjacent_repeats, "ctc_min_timesteps": len(encoded) + adjacent_repeats, "unknown_codepoints": [f"U+{ord(char):04X}" for char in unknown], } ) if not rows: raise ValueError(f"label file has no rows: {path}") max_label = max(row["raw_label_codepoints"] for row in rows) max_ctc = max(row["ctc_min_timesteps"] for row in rows) over_text = [row for row in rows if row["raw_label_codepoints"] > max_text_length] over_pool = [row for row in rows if row["ctc_min_timesteps"] > pool_width] unknown_rows = [row for row in rows if row["unknown_codepoints"]] return { "split": split, "path": str(path), "sha256": sha256_of(path), "rows": len(rows), "unique_image_paths": len(seen_images), "duplicate_image_paths": sorted(set(duplicate_images)), "max_label_codepoints": max_label, "max_encoded_tokens": max(row["encoded_tokens"] for row in rows), "max_adjacent_equal_pairs": max( row["adjacent_equal_pairs"] for row in rows ), "max_ctc_min_timesteps": max_ctc, "over_max_text_length": len(over_text), "over_pool_width": len(over_pool), "rows_with_unknown_codepoints": len(unknown_rows), "unknown_codepoints": sorted( {codepoint for row in unknown_rows for codepoint in row["unknown_codepoints"]} ), "over_legacy_t40": sum(row["ctc_min_timesteps"] > 40 for row in rows), "over_legacy_eval_t100": sum( row["ctc_min_timesteps"] > 100 for row in rows ), "first_over_max_text_length": over_text[:10], "first_over_pool_width": over_pool[:10], "first_rows_with_unknown_codepoints": unknown_rows[:10], } def parse_split_path(value: str) -> tuple[str, Path]: try: split, raw_path = value.split("=", 1) except ValueError as exc: raise argparse.ArgumentTypeError("expected SPLIT=PATH") from exc if not split or not raw_path: raise argparse.ArgumentTypeError("expected non-empty SPLIT=PATH") return split, Path(raw_path) def build_report( *, dictionary: Path, label_files: list[tuple[str, Path]], pool_width: int, max_text_length: int, use_space_char: bool, ) -> dict[str, Any]: if pool_width <= 0: raise ValueError("pool_width must be positive") if max_text_length <= 0: raise ValueError("max_text_length must be positive") split_names = [split for split, _ in label_files] if len(split_names) != len(set(split_names)): raise ValueError("split names must be unique") dictionary_report, allowed_characters = audit_dictionary( dictionary, use_space_char=use_space_char ) splits = [ audit_label_file( path, split=split, pool_width=pool_width, max_text_length=max_text_length, allowed_characters=allowed_characters, ) for split, path in label_files ] issues: list[str] = [] for split in splits: if split["duplicate_image_paths"]: issues.append(f"{split['split']}: duplicate image paths") if split["over_max_text_length"]: issues.append( f"{split['split']}: {split['over_max_text_length']} labels exceed " f"max_text_length={max_text_length}" ) if split["over_pool_width"]: issues.append( f"{split['split']}: {split['over_pool_width']} labels require more " f"than pool_width={pool_width} CTC timesteps" ) if split["rows_with_unknown_codepoints"]: issues.append( f"{split['split']}: {split['rows_with_unknown_codepoints']} labels " "contain codepoints the Paddle label encoder would silently drop" ) return { "status": "PASS" if not issues else "FAIL", "tokenization": "one Unicode codepoint per dictionary entry", "ctc_minimum_formula": "label_codepoints + adjacent_equal_pairs", "pool_width": pool_width, "max_text_length": max_text_length, "dictionary": dictionary_report, "splits": splits, "issues": issues, } def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--dictionary", type=Path, required=True) parser.add_argument( "--label-file", action="append", type=parse_split_path, required=True, metavar="SPLIT=PATH", ) parser.add_argument("--pool-width", type=int, required=True) parser.add_argument("--max-text-length", type=int, required=True) parser.add_argument("--use-space-char", action="store_true") parser.add_argument("--out", type=Path) args = parser.parse_args(argv) report = build_report( dictionary=args.dictionary, label_files=args.label_file, pool_width=args.pool_width, max_text_length=args.max_text_length, use_space_char=args.use_space_char, ) rendered = json.dumps(report, ensure_ascii=False, indent=2) + "\n" if args.out: args.out.parent.mkdir(parents=True, exist_ok=True) args.out.write_text(rendered, encoding="utf-8") print(rendered, end="") return 0 if report["status"] == "PASS" else 2 if __name__ == "__main__": raise SystemExit(main())