| """How many chunks flagged is_table are genuinely tabular? | |
| `text_as_html` presence is the current test, and it flags 114 of 200 chunks. | |
| That is implausible for a 10-K: iXBRL uses <table> for page layout, so narrative | |
| prose ends up wrapped in one. Digit density separates the two cleanly enough to | |
| pick a threshold from the distribution rather than by guessing. | |
| Reads the saved corpus rather than re-partitioning — partitioning this filing | |
| segfaults intermittently, and the saved records already carry text and HTML. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| from pathlib import Path | |
| DEFAULT_CHUNKS = Path(__file__).resolve().parents[1] / "data" / "chunks_v2.json" | |
| def digit_ratio(text: str) -> float: | |
| if not text: | |
| return 0.0 | |
| return sum(char.isdigit() for char in text) / len(text) | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--chunks", type=Path, default=DEFAULT_CHUNKS) | |
| args = parser.parse_args() | |
| rows = json.loads(args.chunks.read_text(encoding="utf-8")) | |
| flagged = [] | |
| for record in rows: | |
| if not record.get("is_table"): | |
| continue | |
| text = str(record.get("content") or "") | |
| html = str(record.get("content_html") or "") | |
| flagged.append( | |
| { | |
| "id": record["id"], | |
| "section": record.get("section"), | |
| "ratio": digit_ratio(text), | |
| "rows": html.lower().count("<tr"), | |
| "chars": len(text), | |
| "head": " ".join(text.split())[:66], | |
| } | |
| ) | |
| print(f"{len(flagged)} of {len(rows)} chunks flagged is_table\n") | |
| buckets = [(0.0, 0.02), (0.02, 0.05), (0.05, 0.10), (0.10, 0.20), (0.20, 1.01)] | |
| for low, high in buckets: | |
| group = [f for f in flagged if low <= f["ratio"] < high] | |
| print(f" digit ratio {low:.2f}-{high:.2f} {len(group):>4} chunks") | |
| print("\nlowest digit density (prose wrapped in a layout table)") | |
| for item in sorted(flagged, key=lambda f: f["ratio"])[:8]: | |
| print(f" id={item['id']:>3} r={item['ratio']:.3f} tr={item['rows']:>3} {item['head']}") | |
| print("\nhighest digit density (real financial tables)") | |
| for item in sorted(flagged, key=lambda f: -f["ratio"])[:8]: | |
| print(f" id={item['id']:>3} r={item['ratio']:.3f} tr={item['rows']:>3} {item['head']}") | |
| for threshold in (0.02, 0.03, 0.05): | |
| kept = sum(1 for f in flagged if f["ratio"] >= threshold) | |
| print(f"\nthreshold {threshold:.2f} -> {kept} tables, {len(rows) - kept} prose") | |
| if __name__ == "__main__": | |
| main() | |