ocr-projects-archive / ocr-tech /tools /audit_ctc_label_feasibility.py
prosperious's picture
Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified
Raw History Blame Contribute Delete
9.22 kB
#!/usr/bin/env python3
"""Audit whether CTC labels can be represented by a model's time axis.
For a label of length L, CTC needs at least L timesteps plus one additional
timestep for every pair of identical adjacent tokens. This tool independently
checks that lower bound and the recognizer's configured maximum text length.
PaddleOCR's character dictionaries are one-token-per-line. This audit accepts
only single-codepoint dictionary entries so Python string iteration matches the
label encoder exactly; multi-codepoint token dictionaries require a tokenizer-
aware audit and are rejected.
"""
from __future__ import annotations
import argparse
import hashlib
import json
from pathlib import Path
from typing import Any
def sha256_of(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1 << 20), b""):
digest.update(chunk)
return digest.hexdigest()
def parse_label_line(raw: str, *, path: Path, line_number: int) -> tuple[str, str]:
row = raw.rstrip("\r\n")
if not row:
raise ValueError(f"{path}:{line_number}: blank label row")
try:
image, text = row.split("\t", 1)
except ValueError as exc:
raise ValueError(f"{path}:{line_number}: expected image<TAB>text") from exc
if not image:
raise ValueError(f"{path}:{line_number}: empty image path")
if not text:
raise ValueError(f"{path}:{line_number}: empty label")
return image, text
def ctc_min_timesteps(text: str) -> int:
"""Return CTC's exact lower bound for a codepoint-tokenized label."""
return len(text) + sum(left == right for left, right in zip(text, text[1:]))
def audit_dictionary(path: Path, *, use_space_char: bool) -> tuple[dict[str, Any], set[str]]:
entries = path.read_text(encoding="utf-8").splitlines()
empty = [index for index, entry in enumerate(entries, start=1) if not entry]
multitoken = [
{"line": index, "entry": entry, "codepoint_length": len(entry)}
for index, entry in enumerate(entries, start=1)
if len(entry) != 1
]
if empty or multitoken:
raise ValueError(
f"dictionary must contain exactly one codepoint per non-empty line: {path}; "
f"empty={len(empty)}, multi_codepoint={len(multitoken)}"
)
allowed = set(entries)
if use_space_char:
allowed.add(" ")
return {
"path": str(path),
"sha256": sha256_of(path),
"entries": len(entries),
"single_codepoint_entries": True,
"use_space_char": use_space_char,
}, allowed
def audit_label_file(
path: Path,
*,
split: str,
pool_width: int,
max_text_length: int,
allowed_characters: set[str],
) -> dict[str, Any]:
rows: list[dict[str, Any]] = []
seen_images: set[str] = set()
duplicate_images: list[str] = []
for line_number, raw in enumerate(
path.read_text(encoding="utf-8").splitlines(keepends=True), start=1
):
image, text = parse_label_line(raw, path=path, line_number=line_number)
if image in seen_images:
duplicate_images.append(image)
seen_images.add(image)
unknown = sorted({char for char in text if char not in allowed_characters})
encoded = "".join(char for char in text if char in allowed_characters)
adjacent_repeats = sum(
left == right for left, right in zip(encoded, encoded[1:])
)
rows.append(
{
"line_number": line_number,
"image": image,
"raw_label_codepoints": len(text),
"encoded_tokens": len(encoded),
"adjacent_equal_pairs": adjacent_repeats,
"ctc_min_timesteps": len(encoded) + adjacent_repeats,
"unknown_codepoints": [f"U+{ord(char):04X}" for char in unknown],
}
)
if not rows:
raise ValueError(f"label file has no rows: {path}")
max_label = max(row["raw_label_codepoints"] for row in rows)
max_ctc = max(row["ctc_min_timesteps"] for row in rows)
over_text = [row for row in rows if row["raw_label_codepoints"] > max_text_length]
over_pool = [row for row in rows if row["ctc_min_timesteps"] > pool_width]
unknown_rows = [row for row in rows if row["unknown_codepoints"]]
return {
"split": split,
"path": str(path),
"sha256": sha256_of(path),
"rows": len(rows),
"unique_image_paths": len(seen_images),
"duplicate_image_paths": sorted(set(duplicate_images)),
"max_label_codepoints": max_label,
"max_encoded_tokens": max(row["encoded_tokens"] for row in rows),
"max_adjacent_equal_pairs": max(
row["adjacent_equal_pairs"] for row in rows
),
"max_ctc_min_timesteps": max_ctc,
"over_max_text_length": len(over_text),
"over_pool_width": len(over_pool),
"rows_with_unknown_codepoints": len(unknown_rows),
"unknown_codepoints": sorted(
{codepoint for row in unknown_rows for codepoint in row["unknown_codepoints"]}
),
"over_legacy_t40": sum(row["ctc_min_timesteps"] > 40 for row in rows),
"over_legacy_eval_t100": sum(
row["ctc_min_timesteps"] > 100 for row in rows
),
"first_over_max_text_length": over_text[:10],
"first_over_pool_width": over_pool[:10],
"first_rows_with_unknown_codepoints": unknown_rows[:10],
}
def parse_split_path(value: str) -> tuple[str, Path]:
try:
split, raw_path = value.split("=", 1)
except ValueError as exc:
raise argparse.ArgumentTypeError("expected SPLIT=PATH") from exc
if not split or not raw_path:
raise argparse.ArgumentTypeError("expected non-empty SPLIT=PATH")
return split, Path(raw_path)
def build_report(
*,
dictionary: Path,
label_files: list[tuple[str, Path]],
pool_width: int,
max_text_length: int,
use_space_char: bool,
) -> dict[str, Any]:
if pool_width <= 0:
raise ValueError("pool_width must be positive")
if max_text_length <= 0:
raise ValueError("max_text_length must be positive")
split_names = [split for split, _ in label_files]
if len(split_names) != len(set(split_names)):
raise ValueError("split names must be unique")
dictionary_report, allowed_characters = audit_dictionary(
dictionary, use_space_char=use_space_char
)
splits = [
audit_label_file(
path,
split=split,
pool_width=pool_width,
max_text_length=max_text_length,
allowed_characters=allowed_characters,
)
for split, path in label_files
]
issues: list[str] = []
for split in splits:
if split["duplicate_image_paths"]:
issues.append(f"{split['split']}: duplicate image paths")
if split["over_max_text_length"]:
issues.append(
f"{split['split']}: {split['over_max_text_length']} labels exceed "
f"max_text_length={max_text_length}"
)
if split["over_pool_width"]:
issues.append(
f"{split['split']}: {split['over_pool_width']} labels require more "
f"than pool_width={pool_width} CTC timesteps"
)
if split["rows_with_unknown_codepoints"]:
issues.append(
f"{split['split']}: {split['rows_with_unknown_codepoints']} labels "
"contain codepoints the Paddle label encoder would silently drop"
)
return {
"status": "PASS" if not issues else "FAIL",
"tokenization": "one Unicode codepoint per dictionary entry",
"ctc_minimum_formula": "label_codepoints + adjacent_equal_pairs",
"pool_width": pool_width,
"max_text_length": max_text_length,
"dictionary": dictionary_report,
"splits": splits,
"issues": issues,
}
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--dictionary", type=Path, required=True)
parser.add_argument(
"--label-file",
action="append",
type=parse_split_path,
required=True,
metavar="SPLIT=PATH",
)
parser.add_argument("--pool-width", type=int, required=True)
parser.add_argument("--max-text-length", type=int, required=True)
parser.add_argument("--use-space-char", action="store_true")
parser.add_argument("--out", type=Path)
args = parser.parse_args(argv)
report = build_report(
dictionary=args.dictionary,
label_files=args.label_file,
pool_width=args.pool_width,
max_text_length=args.max_text_length,
use_space_char=args.use_space_char,
)
rendered = json.dumps(report, ensure_ascii=False, indent=2) + "\n"
if args.out:
args.out.parent.mkdir(parents=True, exist_ok=True)
args.out.write_text(rendered, encoding="utf-8")
print(rendered, end="")
return 0 if report["status"] == "PASS" else 2
if __name__ == "__main__":
raise SystemExit(main())