Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified Download ocr-tech/tools/audit_ctc_label_feasibility.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 9.22 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tools/audit_ctc_label_feasibility.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech/tools/audit_ctc_label_feasibility.py
-
curl -L -o audit_ctc_label_feasibility.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tools/audit_ctc_label_feasibility.py
9.22 kB
| #!/usr/bin/env python3 | |
| """Audit whether CTC labels can be represented by a model's time axis. | |
| For a label of length L, CTC needs at least L timesteps plus one additional | |
| timestep for every pair of identical adjacent tokens. This tool independently | |
| checks that lower bound and the recognizer's configured maximum text length. | |
| PaddleOCR's character dictionaries are one-token-per-line. This audit accepts | |
| only single-codepoint dictionary entries so Python string iteration matches the | |
| label encoder exactly; multi-codepoint token dictionaries require a tokenizer- | |
| aware audit and are rejected. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import hashlib | |
| import json | |
| from pathlib import Path | |
| from typing import Any | |
| def sha256_of(path: Path) -> str: | |
| digest = hashlib.sha256() | |
| with path.open("rb") as handle: | |
| for chunk in iter(lambda: handle.read(1 << 20), b""): | |
| digest.update(chunk) | |
| return digest.hexdigest() | |
| def parse_label_line(raw: str, *, path: Path, line_number: int) -> tuple[str, str]: | |
| row = raw.rstrip("\r\n") | |
| if not row: | |
| raise ValueError(f"{path}:{line_number}: blank label row") | |
| try: | |
| image, text = row.split("\t", 1) | |
| except ValueError as exc: | |
| raise ValueError(f"{path}:{line_number}: expected image<TAB>text") from exc | |
| if not image: | |
| raise ValueError(f"{path}:{line_number}: empty image path") | |
| if not text: | |
| raise ValueError(f"{path}:{line_number}: empty label") | |
| return image, text | |
| def ctc_min_timesteps(text: str) -> int: | |
| """Return CTC's exact lower bound for a codepoint-tokenized label.""" | |
| return len(text) + sum(left == right for left, right in zip(text, text[1:])) | |
| def audit_dictionary(path: Path, *, use_space_char: bool) -> tuple[dict[str, Any], set[str]]: | |
| entries = path.read_text(encoding="utf-8").splitlines() | |
| empty = [index for index, entry in enumerate(entries, start=1) if not entry] | |
| multitoken = [ | |
| {"line": index, "entry": entry, "codepoint_length": len(entry)} | |
| for index, entry in enumerate(entries, start=1) | |
| if len(entry) != 1 | |
| ] | |
| if empty or multitoken: | |
| raise ValueError( | |
| f"dictionary must contain exactly one codepoint per non-empty line: {path}; " | |
| f"empty={len(empty)}, multi_codepoint={len(multitoken)}" | |
| ) | |
| allowed = set(entries) | |
| if use_space_char: | |
| allowed.add(" ") | |
| return { | |
| "path": str(path), | |
| "sha256": sha256_of(path), | |
| "entries": len(entries), | |
| "single_codepoint_entries": True, | |
| "use_space_char": use_space_char, | |
| }, allowed | |
| def audit_label_file( | |
| path: Path, | |
| *, | |
| split: str, | |
| pool_width: int, | |
| max_text_length: int, | |
| allowed_characters: set[str], | |
| ) -> dict[str, Any]: | |
| rows: list[dict[str, Any]] = [] | |
| seen_images: set[str] = set() | |
| duplicate_images: list[str] = [] | |
| for line_number, raw in enumerate( | |
| path.read_text(encoding="utf-8").splitlines(keepends=True), start=1 | |
| ): | |
| image, text = parse_label_line(raw, path=path, line_number=line_number) | |
| if image in seen_images: | |
| duplicate_images.append(image) | |
| seen_images.add(image) | |
| unknown = sorted({char for char in text if char not in allowed_characters}) | |
| encoded = "".join(char for char in text if char in allowed_characters) | |
| adjacent_repeats = sum( | |
| left == right for left, right in zip(encoded, encoded[1:]) | |
| ) | |
| rows.append( | |
| { | |
| "line_number": line_number, | |
| "image": image, | |
| "raw_label_codepoints": len(text), | |
| "encoded_tokens": len(encoded), | |
| "adjacent_equal_pairs": adjacent_repeats, | |
| "ctc_min_timesteps": len(encoded) + adjacent_repeats, | |
| "unknown_codepoints": [f"U+{ord(char):04X}" for char in unknown], | |
| } | |
| ) | |
| if not rows: | |
| raise ValueError(f"label file has no rows: {path}") | |
| max_label = max(row["raw_label_codepoints"] for row in rows) | |
| max_ctc = max(row["ctc_min_timesteps"] for row in rows) | |
| over_text = [row for row in rows if row["raw_label_codepoints"] > max_text_length] | |
| over_pool = [row for row in rows if row["ctc_min_timesteps"] > pool_width] | |
| unknown_rows = [row for row in rows if row["unknown_codepoints"]] | |
| return { | |
| "split": split, | |
| "path": str(path), | |
| "sha256": sha256_of(path), | |
| "rows": len(rows), | |
| "unique_image_paths": len(seen_images), | |
| "duplicate_image_paths": sorted(set(duplicate_images)), | |
| "max_label_codepoints": max_label, | |
| "max_encoded_tokens": max(row["encoded_tokens"] for row in rows), | |
| "max_adjacent_equal_pairs": max( | |
| row["adjacent_equal_pairs"] for row in rows | |
| ), | |
| "max_ctc_min_timesteps": max_ctc, | |
| "over_max_text_length": len(over_text), | |
| "over_pool_width": len(over_pool), | |
| "rows_with_unknown_codepoints": len(unknown_rows), | |
| "unknown_codepoints": sorted( | |
| {codepoint for row in unknown_rows for codepoint in row["unknown_codepoints"]} | |
| ), | |
| "over_legacy_t40": sum(row["ctc_min_timesteps"] > 40 for row in rows), | |
| "over_legacy_eval_t100": sum( | |
| row["ctc_min_timesteps"] > 100 for row in rows | |
| ), | |
| "first_over_max_text_length": over_text[:10], | |
| "first_over_pool_width": over_pool[:10], | |
| "first_rows_with_unknown_codepoints": unknown_rows[:10], | |
| } | |
| def parse_split_path(value: str) -> tuple[str, Path]: | |
| try: | |
| split, raw_path = value.split("=", 1) | |
| except ValueError as exc: | |
| raise argparse.ArgumentTypeError("expected SPLIT=PATH") from exc | |
| if not split or not raw_path: | |
| raise argparse.ArgumentTypeError("expected non-empty SPLIT=PATH") | |
| return split, Path(raw_path) | |
| def build_report( | |
| *, | |
| dictionary: Path, | |
| label_files: list[tuple[str, Path]], | |
| pool_width: int, | |
| max_text_length: int, | |
| use_space_char: bool, | |
| ) -> dict[str, Any]: | |
| if pool_width <= 0: | |
| raise ValueError("pool_width must be positive") | |
| if max_text_length <= 0: | |
| raise ValueError("max_text_length must be positive") | |
| split_names = [split for split, _ in label_files] | |
| if len(split_names) != len(set(split_names)): | |
| raise ValueError("split names must be unique") | |
| dictionary_report, allowed_characters = audit_dictionary( | |
| dictionary, use_space_char=use_space_char | |
| ) | |
| splits = [ | |
| audit_label_file( | |
| path, | |
| split=split, | |
| pool_width=pool_width, | |
| max_text_length=max_text_length, | |
| allowed_characters=allowed_characters, | |
| ) | |
| for split, path in label_files | |
| ] | |
| issues: list[str] = [] | |
| for split in splits: | |
| if split["duplicate_image_paths"]: | |
| issues.append(f"{split['split']}: duplicate image paths") | |
| if split["over_max_text_length"]: | |
| issues.append( | |
| f"{split['split']}: {split['over_max_text_length']} labels exceed " | |
| f"max_text_length={max_text_length}" | |
| ) | |
| if split["over_pool_width"]: | |
| issues.append( | |
| f"{split['split']}: {split['over_pool_width']} labels require more " | |
| f"than pool_width={pool_width} CTC timesteps" | |
| ) | |
| if split["rows_with_unknown_codepoints"]: | |
| issues.append( | |
| f"{split['split']}: {split['rows_with_unknown_codepoints']} labels " | |
| "contain codepoints the Paddle label encoder would silently drop" | |
| ) | |
| return { | |
| "status": "PASS" if not issues else "FAIL", | |
| "tokenization": "one Unicode codepoint per dictionary entry", | |
| "ctc_minimum_formula": "label_codepoints + adjacent_equal_pairs", | |
| "pool_width": pool_width, | |
| "max_text_length": max_text_length, | |
| "dictionary": dictionary_report, | |
| "splits": splits, | |
| "issues": issues, | |
| } | |
| def main(argv: list[str] | None = None) -> int: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--dictionary", type=Path, required=True) | |
| parser.add_argument( | |
| "--label-file", | |
| action="append", | |
| type=parse_split_path, | |
| required=True, | |
| metavar="SPLIT=PATH", | |
| ) | |
| parser.add_argument("--pool-width", type=int, required=True) | |
| parser.add_argument("--max-text-length", type=int, required=True) | |
| parser.add_argument("--use-space-char", action="store_true") | |
| parser.add_argument("--out", type=Path) | |
| args = parser.parse_args(argv) | |
| report = build_report( | |
| dictionary=args.dictionary, | |
| label_files=args.label_file, | |
| pool_width=args.pool_width, | |
| max_text_length=args.max_text_length, | |
| use_space_char=args.use_space_char, | |
| ) | |
| rendered = json.dumps(report, ensure_ascii=False, indent=2) + "\n" | |
| if args.out: | |
| args.out.parent.mkdir(parents=True, exist_ok=True) | |
| args.out.write_text(rendered, encoding="utf-8") | |
| print(rendered, end="") | |
| return 0 if report["status"] == "PASS" else 2 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |