File size: 38,558 Bytes
2523bef | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 | """
Copyright (C) 2023 Artifex Software, Inc.
This file is part of PyMuPDF.
PyMuPDF is free software: you can redistribute it and/or modify it under the
terms of the GNU Affero General Public License as published by the Free
Software Foundation, either version 3 of the License, or (at your option)
any later version.
PyMuPDF is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
details.
You should have received a copy of the GNU Affero General Public License
along with MuPDF. If not, see <https://www.gnu.org/licenses/agpl-3.0.en.html>
Alternative licensing terms are available from the licensor.
For commercial licensing, see <https://www.artifex.com/> or contact
Artifex Software, Inc., 39 Mesa Street, Suite 108A, San Francisco,
CA 94129, USA, for further information.
---------------------------------------------------------------------
PyMuPDF table header detection and HTML serialization (opt-in extension).
Pure text-grid module (no pymupdf import): the header-region rules operate on a
row-major ``[[cell text]]`` grid, and the serializer turns a tagged placement
grid into an HTML ``<table>``. Used only by find_tables(refine=True) (via
pymupdf.table) and Table.to_html(); never runs on the default detection path.
"""
from __future__ import annotations
from dataclasses import dataclass
from statistics import mean
from typing import Any
import re
_TOKEN_RE = re.compile(r"[A-Za-z]+|\d+(?:[.,:/-]\d+)*|[%$€£¥()–—-]+")
_MONTH_RE = re.compile(
r"\b(?:jan|feb|mar|apr|may|jun|jul|aug|sep|sept|oct|nov|dec)[a-z]*\.?\b",
re.IGNORECASE,
)
_MONEY_RE = re.compile(r"(?:[$€£¥]\s*[-(]?\d|(?:usd|eur|gbp|jpy)\b)", re.IGNORECASE)
_PERCENT_RE = re.compile(r"[-(]?\d+(?:\.\d+)?\s*%")
_YEAR_RE = re.compile(r"\b(?:19|20)\d{2}\b")
_CODE_RE = re.compile(r"^(?=.*[A-Za-z])(?=.*\d)[A-Za-z0-9][A-Za-z0-9._/# -]{0,24}$")
_RANGE_RE = re.compile(r"(?:>=|<=|<|>| thru | through | to |\b\d+\s*[-–]\s*\d+\b)", re.IGNORECASE)
_DOTTED_ENUM_RE = re.compile(r"^\s*\d+(?:\.\d+){2,}\b")
_REF_TOKEN_RE = re.compile(r"[\[(]\s*\d{1,2}\s*[\])]|\[\s*\d{1,2}\s*\)")
_REF_FORMULA_ALLOWED_RE = re.compile(r"^[\s\d\[\]\(\)=+\-*/xX.]+$")
_UNIT_ROW_TERMS = ("in million", "in thousand", "in billion", "$ in", "usd", "'000", "%")
VALUE_TYPES = {"number", "money", "percent"}
LABEL_TYPES = {"text_label", "long_text", "code_id", "date_period", "unit"}
def count_alpha(text: str) -> int:
return sum(1 for char in text if char.isalpha())
def count_digit(text: str) -> int:
return sum(1 for char in text if char.isdigit())
def tokens(text: str) -> list[str]:
return _TOKEN_RE.findall(text)
def numeric_like(text: str) -> bool:
stripped = text.strip()
if not stripped:
return False
alpha = count_alpha(stripped)
digit = count_digit(stripped)
if digit == 0:
return False
if _MONTH_RE.search(stripped):
return False
return alpha / max(1, alpha + digit) <= 0.30
def date_or_period_like(text: str) -> bool:
stripped = text.strip()
if not stripped:
return False
return bool(_MONTH_RE.search(stripped) or re.search(r"\b(?:q[1-4]|fy|year|month|period)\b", stripped, re.I))
def cell_type(text: str) -> str:
stripped = " ".join(text.strip().split())
if not stripped:
return "empty"
alpha = count_alpha(stripped)
digit = count_digit(stripped)
token_count = len(tokens(stripped))
lowered = stripped.lower()
if _MONEY_RE.search(stripped):
return "money"
if _PERCENT_RE.fullmatch(stripped) or (stripped.endswith("%") and digit > 0 and alpha == 0):
return "percent"
if date_or_period_like(stripped) or (_YEAR_RE.fullmatch(stripped) and token_count == 1):
return "date_period"
if _RANGE_RE.search(stripped) and digit > 0 and token_count <= 8:
return "range"
if numeric_like(stripped):
return "number"
if lowered in {"%", "$", "$000", "$m", "usd", "eur", "gbp", "amount", "rate", "ratio"}:
return "unit"
if _CODE_RE.match(stripped) and token_count <= 4:
return "code_id"
if alpha > 0 and digit > 0 and token_count <= 6:
return "code_id"
if token_count >= 8:
return "long_text"
return "text_label"
def cell_type_ratios(texts: list[str]) -> dict[str, float]:
nonempty_types = [cell_type(text) for text in texts if text.strip()]
if not nonempty_types:
return {
"value_type_ratio": 0.0,
"label_type_ratio": 0.0,
"long_text_type_ratio": 0.0,
"code_type_ratio": 0.0,
"date_period_type_ratio": 0.0,
"range_type_ratio": 0.0,
}
total = len(nonempty_types)
return {
"value_type_ratio": sum(1 for item in nonempty_types if item in VALUE_TYPES) / total,
"label_type_ratio": sum(1 for item in nonempty_types if item in LABEL_TYPES) / total,
"long_text_type_ratio": sum(1 for item in nonempty_types if item == "long_text") / total,
"code_type_ratio": sum(1 for item in nonempty_types if item == "code_id") / total,
"date_period_type_ratio": sum(1 for item in nonempty_types if item == "date_period") / total,
"range_type_ratio": sum(1 for item in nonempty_types if item == "range") / total,
}
def row_features(texts: list[str]) -> dict[str, float]:
nonempty = [text for text in texts if text.strip()]
if not texts:
return {
"cell_count": 0.0,
"nonempty_count": 0.0,
"nonempty_ratio": 0.0,
"empty_ratio": 0.0,
"numeric_ratio": 0.0,
"alpha_ratio": 0.0,
"date_period_ratio": 0.0,
"avg_tokens": 0.0,
**cell_type_ratios([]),
}
return {
"cell_count": float(len(texts)),
"nonempty_count": float(len(nonempty)),
"nonempty_ratio": len(nonempty) / len(texts),
"empty_ratio": (len(texts) - len(nonempty)) / len(texts),
"numeric_ratio": sum(1 for text in nonempty if numeric_like(text)) / max(1, len(nonempty)),
"alpha_ratio": sum(1 for text in nonempty if count_alpha(text) > 0) / max(1, len(nonempty)),
"date_period_ratio": sum(1 for text in nonempty if date_or_period_like(text)) / max(1, len(nonempty)),
"avg_tokens": mean([len(tokens(text)) for text in nonempty]) if nonempty else 0.0,
**cell_type_ratios(texts),
}
def column_text(row: list[str], col: int) -> str:
return row[col] if col < len(row) else ""
def row_vector(texts: list[str]) -> dict[str, float]:
features = row_features(texts)
first = column_text(texts, 0)
rest = [text for text in texts[1:] if text.strip()]
rest_types = [cell_type(text) for text in rest]
first_nonempty = bool(first.strip())
return {
**features,
"first_empty": 0.0 if first_nonempty else 1.0,
"first_alpha": 1.0 if count_alpha(first) > 0 else 0.0,
"first_numeric": 1.0 if numeric_like(first) else 0.0,
"first_value_type": 1.0 if cell_type(first) in VALUE_TYPES else 0.0,
"first_label_type": 1.0 if cell_type(first) in LABEL_TYPES else 0.0,
"rest_numeric_ratio": sum(1 for text in rest if numeric_like(text)) / max(1, len(rest)),
"rest_alpha_ratio": sum(1 for text in rest if count_alpha(text) > 0) / max(1, len(rest)),
"rest_value_type_ratio": sum(1 for item in rest_types if item in VALUE_TYPES) / max(1, len(rest_types)),
"rest_label_type_ratio": sum(1 for item in rest_types if item in LABEL_TYPES) / max(1, len(rest_types)),
"rest_long_text_type_ratio": sum(1 for item in rest_types if item == "long_text") / max(1, len(rest_types)),
"avg_tokens_norm": min(1.0, features["avg_tokens"] / 8.0),
}
def header_candidate_row(row: list[str]) -> bool:
features = row_features(row)
if features["nonempty_count"] == 0:
return False
short_or_sparse = features["avg_tokens"] <= 6.0 or features["empty_ratio"] >= 0.20
return bool(short_or_sparse)
def grouped_header_scaffold(row: list[str]) -> bool:
features = row_features(row)
if features["cell_count"] == 0:
return False
texts = [text.strip() for text in row]
nonempty = [text for text in texts if text]
unique_nonempty = {text.lower() for text in nonempty}
repeated_labels = len(nonempty) >= 3 and len(unique_nonempty) <= max(1, len(nonempty) // 2)
sparse_labels = features["empty_ratio"] >= 0.20 or features["nonempty_ratio"] <= 0.70
return bool(sparse_labels or repeated_labels)
def data_row_like(row: list[str]) -> bool:
vector = row_vector(row)
return bool(
vector["first_alpha"] > 0
and vector["first_numeric"] == 0
and vector["rest_numeric_ratio"] >= 0.35
and vector["numeric_ratio"] >= 0.25
)
def dense_numeric_row_like(row: list[str]) -> bool:
vector = row_vector(row)
return bool(vector["numeric_ratio"] >= 0.70 and vector["alpha_ratio"] <= 0.35)
def money_or_percent_heavy(row: list[str]) -> bool:
types = [cell_type(text) for text in row if text.strip()]
if not types:
return False
return sum(1 for item in types if item in {"money", "percent"}) / len(types) >= 0.25
def rest_date_period_ratio(row: list[str]) -> float:
rest = [cell for cell in row[1:] if cell.strip()]
if not rest:
return 0.0
return sum(1 for cell in rest if cell_type(cell) == "date_period") / len(rest)
def first_row_period_restore_guard(row: list[str]) -> bool:
features = row_features(row)
return bool(features["date_period_type_ratio"] >= 0.25 or rest_date_period_ratio(row) >= 0.35)
def first_row_date_or_unit_restore_guard(row: list[str]) -> bool:
rest = [cell for cell in row[1:] if cell.strip()]
if not rest:
return False
rest_types = [cell_type(cell) for cell in rest]
dp_unit_ratio = sum(1 for item in rest_types if item in {"date_period", "unit"}) / len(rest_types)
money_or_percent = any(item in {"money", "percent"} for item in rest_types)
row_text = " ".join(row).lower()
unit_row = any(term in row_text for term in _UNIT_ROW_TERMS)
return bool((dp_unit_ratio >= 0.50 or unit_row) and not money_or_percent)
def first_row_restore_guards(row: list[str]) -> list[str]:
guards = []
if first_row_period_restore_guard(row):
guards.append("first_row_period_restore")
if first_row_date_or_unit_restore_guard(row):
guards.append("first_row_date_or_unit_restore")
return guards
def headerless_dotted_enum_guard(rows: list[list[str]]) -> bool:
if len(rows) < 3:
return False
row = rows[0]
cells = [cell.strip() for cell in row if cell and cell.strip()]
if not cells or not _DOTTED_ENUM_RE.search(cells[0]):
return False
features = row_features(row)
return bool(features["long_text_type_ratio"] >= 0.25 or features["avg_tokens"] >= 5.0)
def simple_integer_enum_header_like(row: list[str]) -> bool:
vector = row_vector(row)
if vector["first_label_type"] <= 0 or vector["first_value_type"] > 0:
return False
values = []
for text in [text.strip() for text in row[1:] if text.strip()]:
if not re.fullmatch(r"\d{1,2}", text):
return False
values.append(int(text))
if len(values) < 2:
return False
ordered = values == sorted(values)
unique = len(set(values)) == len(values)
contiguous = max(values) - min(values) == len(values) - 1
return bool(ordered and unique and contiguous)
def reference_formula_header_row_like(candidate: list[str], following_rows: list[list[str]]) -> bool:
"""A bracket/formula reference row under a grouped header, e.g. [3] = [1] * [2]."""
nonempty = [text.strip() for text in candidate if text.strip()]
if len(nonempty) < 3:
return False
if any(any(char.isalpha() for char in text) for text in nonempty):
return False
if any(("$" in text or "%" in text or "," in text) for text in nonempty):
return False
if not all(_REF_FORMULA_ALLOWED_RE.match(text) for text in nonempty):
return False
ref_cells = [text for text in nonempty if _REF_TOKEN_RE.search(text)]
formula_cells = [text for text in nonempty if "=" in text and len(_REF_TOKEN_RE.findall(text)) >= 2]
if len(ref_cells) / len(nonempty) < 0.75:
return False
if not formula_cells and len(ref_cells) < 4:
return False
if len(following_rows) < 2:
return False
following_profiles = row_profiles(following_rows)
body_features = body_window_features_from_profiles(following_profiles, 0)
if body_window_is_stable(body_features):
return True
if following_profiles[0]["section_label_row"] and len(following_profiles) >= 3:
shifted = body_window_features_from_profiles(following_profiles, 1)
return body_window_is_stable(shifted)
return False
def dense_integer_axis_row_like(row: list[str]) -> bool:
values = []
alpha_cells = 0
nonempty = [text.strip() for text in row if text.strip()]
for text in nonempty:
if re.fullmatch(r"\d{1,2}", text):
values.append(int(text))
elif any(char.isalpha() for char in text):
alpha_cells += 1
else:
return False
if len(values) < 5:
return False
unique = sorted(set(values))
if unique != list(range(unique[0], unique[-1] + 1)):
return False
if unique[0] not in {0, 1}:
return False
if alpha_cells > max(4, len(nonempty) // 5):
return False
return True
def sparse_prefix_extension_prefix_like(row: list[str]) -> bool:
features = row_features(row)
if features["nonempty_count"] == 0:
return False
if features["nonempty_count"] <= 2:
return features["alpha_ratio"] >= 0.50 or features["date_period_ratio"] >= 0.20
if features["empty_ratio"] < 0.25:
return False
return bool(features["alpha_ratio"] >= 0.50 or features["date_period_ratio"] >= 0.20)
def next_header_row_value_guard(candidate: list[str], following_rows: list[list[str]]) -> bool:
features = row_features(candidate)
vector = row_vector(candidate)
if features["nonempty_count"] < 2:
return False
if features["avg_tokens"] > 7.0 or features["long_text_type_ratio"] >= 0.35:
return False
if money_or_percent_heavy(candidate):
return False
if reference_formula_header_row_like(candidate, following_rows):
return True
if features["value_type_ratio"] > 0.85:
return False
# date_period_type_ratio recognises a bare-year header row that the
# keyword-based date_period_ratio misses; combine them only in this guard.
date_period_signal = max(features["date_period_ratio"], features["date_period_type_ratio"])
has_label_signal = (
features["alpha_ratio"] >= 0.30
or date_period_signal >= 0.20
or vector["first_label_type"] > 0
)
if not has_label_signal:
return False
if len(following_rows) < 2:
return False
following_profiles = row_profiles(following_rows)
body_features = body_window_features_from_profiles(following_profiles, 0)
if not body_window_is_stable(body_features):
return False
if features["numeric_ratio"] >= 0.50 or features["value_type_ratio"] >= 0.50:
if vector["first_label_type"] <= 0 and date_period_signal < 0.20:
return False
if vector["first_value_type"] > 0:
return False
types = [cell_type(text) for text in candidate if text.strip()]
type_count = len(types)
if not type_count:
return False
date_type_ratio = sum(1 for item in types if item == "date_period") / type_count
code_range_number_ratio = sum(1 for item in types if item in {"code_id", "range", "number"}) / type_count
if features["value_type_ratio"] > 0.25:
return simple_integer_enum_header_like(candidate)
if code_range_number_ratio >= 0.50 and date_type_ratio < 0.30:
return simple_integer_enum_header_like(candidate)
return True
def sparse_prefix_conservative_extension_guard(rows: list[list[str]], top_header_rows: int) -> bool:
if top_header_rows != 1 or len(rows) < 3:
return False
prefix = rows[0]
candidate = rows[1]
prefix_features = row_features(prefix)
candidate_features = row_features(candidate)
if prefix_features["empty_ratio"] < 0.50:
return False
if candidate_features["nonempty_count"] < 3 and candidate_features["date_period_ratio"] < 0.50:
return False
if not sparse_prefix_extension_prefix_like(prefix):
return False
if candidate_features["label_type_ratio"] < 0.50:
return False
if candidate_features["value_type_ratio"] >= 0.50:
return False
if section_label_row_like(candidate):
return False
following_profiles = row_profiles(rows[2:])
body_features = body_window_features_from_profiles(following_profiles, 0)
return body_window_is_stable(body_features)
def sparse_prefix_next_header_guard(rows: list[list[str]], top_header_rows: int) -> bool:
"""Return True when the first body row should be included as a second header row."""
if top_header_rows < 1 or top_header_rows >= len(rows) - 1:
return False
return sparse_prefix_conservative_extension_guard(rows, top_header_rows)
def rowspan_leaf_header_guard(rows: list[list[str]], top_header_rows: int) -> bool:
"""Promote a leaf-label row under a sparse/grouped rowspan header scaffold.
An empty first cell in the candidate row is the textual shadow of the row-0
rowspan label; treat it as header only when later rows form a stable body.
"""
if top_header_rows != 1 or len(rows) < 3:
return False
max_cols = max((len(row) for row in rows), default=0)
if max_cols < 3:
return False
prefix = rows[0]
candidate = rows[1]
prefix_features = row_features(prefix)
candidate_features = row_features(candidate)
candidate_vector = row_vector(candidate)
shifted_by_rowspan = len(candidate) < max_cols
if column_text(candidate, 0).strip() and not shifted_by_rowspan:
return False
sparse_rowspan_prefix = len(prefix) < max_cols and len(prefix) <= max(2, int(max_cols * 0.50))
if not (grouped_header_scaffold(prefix) or prefix_features["empty_ratio"] >= 0.35 or sparse_rowspan_prefix):
return False
if candidate_features["nonempty_count"] < max(2, int(max_cols * 0.45)):
return False
if candidate_features["value_type_ratio"] > 0.25 or candidate_features["numeric_ratio"] > 0.35:
return False
if candidate_vector["rest_label_type_ratio"] < 0.50 and candidate_features["alpha_ratio"] < 0.50:
return False
if section_label_row_like(candidate) or dense_numeric_row_like(candidate):
return False
following_profiles = row_profiles(rows[2:])
body_features = body_window_features_from_profiles(following_profiles, 0)
return body_window_is_stable(body_features)
def leading_body_record_like(row: list[str]) -> bool:
vector = row_vector(row)
if vector["nonempty_count"] < 2:
return False
first_is_label = vector["first_label_type"] > 0 and vector["first_value_type"] == 0
rest_is_values = vector["rest_numeric_ratio"] >= 0.50 or vector["rest_value_type_ratio"] >= 0.50
value_heavy = vector["numeric_ratio"] >= 0.35 or vector["value_type_ratio"] >= 0.35
return bool(first_is_label and rest_is_values and value_heavy and not grouped_header_scaffold(row))
def leading_long_form_row_like(row: list[str]) -> bool:
vector = row_vector(row)
return bool(
vector["nonempty_count"] >= 4
and vector["long_text_type_ratio"] >= 0.30
and vector["value_type_ratio"] <= 0.20
and vector["empty_ratio"] <= 0.35
)
def section_label_row_like(row: list[str]) -> bool:
if len(row) < 2:
return False
features = row_features(row)
first = column_text(row, 0).strip()
nonempty = [text for text in row if text.strip()]
first_only = bool(first and not [text for text in row[1:] if text.strip()])
sparse_label = (
features["nonempty_count"] <= 3
and features["empty_ratio"] >= 0.60
and features["alpha_ratio"] >= 0.50
)
return bool(first_only or (nonempty and sparse_label))
def centered_section_label_row_like(row: list[str]) -> bool:
if not section_label_row_like(row):
return False
nonempty_indices = [idx for idx, text in enumerate(row) if text.strip()]
return bool(len(nonempty_indices) == 1 and nonempty_indices[0] > 0 and len(row) >= 3)
def section_header_rows(rows: list[list[str]], top_header_rows: int) -> list[int]:
"""Return sparse section-label rows inside the detected top header prefix."""
section_rows = []
for row_idx, row in enumerate(rows[:top_header_rows]):
if row_idx == 0 or not centered_section_label_row_like(row):
continue
nonempty = [text.strip() for text in row if text.strip()]
if len(nonempty) != 1:
continue
if len(row) < 2:
continue
section_rows.append(row_idx)
return section_rows
def row_profile(row: list[str], index: int) -> dict[str, Any]:
vector = row_vector(row)
coordinate_header_row = dense_integer_axis_row_like(row)
body_value_row = (
vector["first_alpha"] > 0
and vector["first_numeric"] == 0
and vector["rest_numeric_ratio"] >= 0.35
and vector["numeric_ratio"] >= 0.25
)
body_dense_row = vector["numeric_ratio"] >= 0.70 and vector["alpha_ratio"] <= 0.35
body_long_form_row = (
vector["first_label_type"] > 0
and vector["first_value_type"] == 0
and vector["rest_long_text_type_ratio"] >= 0.50
and vector["value_type_ratio"] <= 0.25
and vector["empty_ratio"] <= 0.35
)
if coordinate_header_row:
body_value_row = False
body_dense_row = False
body_long_form_row = False
return {
"index": index,
"vector": vector,
"nonempty": vector["nonempty_count"] > 0,
"body_value_row": body_value_row,
"body_dense_row": body_dense_row,
"body_long_form_row": body_long_form_row,
"body_like_row": body_value_row or body_dense_row or body_long_form_row,
"coordinate_header_row": coordinate_header_row,
"leading_body_record_row": leading_body_record_like(row),
"leading_long_form_row": leading_long_form_row_like(row),
"section_label_row": section_label_row_like(row),
"grouped_header_scaffold": grouped_header_scaffold(row),
"header_candidate": header_candidate_row(row),
}
def row_profiles(rows: list[list[str]]) -> list[dict[str, Any]]:
return [row_profile(row, index) for index, row in enumerate(rows)]
def combine_body_vectors(vectors: list[dict[str, float]]) -> dict[str, float]:
if not vectors:
return {
"count": 0.0,
"first_label_ratio": 0.0,
"rest_numeric_ratio": 0.0,
"numeric_ratio": 0.0,
"alpha_ratio": 0.0,
"avg_tokens_norm": 0.0,
"body_row_ratio": 0.0,
"dense_numeric_row_ratio": 0.0,
"long_form_row_ratio": 0.0,
"value_type_ratio": 0.0,
"rest_value_type_ratio": 0.0,
"rest_long_text_type_ratio": 0.0,
"long_text_type_ratio": 0.0,
}
return {
"count": float(len(vectors)),
"first_label_ratio": sum(1 for item in vectors if item["first_label_type"] > 0 and item["first_value_type"] == 0)
/ len(vectors),
"rest_numeric_ratio": sum(item["rest_numeric_ratio"] for item in vectors) / len(vectors),
"numeric_ratio": sum(item["numeric_ratio"] for item in vectors) / len(vectors),
"alpha_ratio": sum(item["alpha_ratio"] for item in vectors) / len(vectors),
"avg_tokens_norm": sum(item["avg_tokens_norm"] for item in vectors) / len(vectors),
"value_type_ratio": sum(item["value_type_ratio"] for item in vectors) / len(vectors),
"rest_value_type_ratio": sum(item["rest_value_type_ratio"] for item in vectors) / len(vectors),
"rest_long_text_type_ratio": sum(item["rest_long_text_type_ratio"] for item in vectors) / len(vectors),
"long_text_type_ratio": sum(item["long_text_type_ratio"] for item in vectors) / len(vectors),
"body_row_ratio": sum(
1
for item in vectors
if (
item["first_alpha"] > 0
and item["first_numeric"] == 0
and item["rest_numeric_ratio"] >= 0.35
and item["numeric_ratio"] >= 0.25
)
or (item["numeric_ratio"] >= 0.70 and item["alpha_ratio"] <= 0.35)
)
/ len(vectors),
"dense_numeric_row_ratio": sum(
1 for item in vectors if item["numeric_ratio"] >= 0.70 and item["alpha_ratio"] <= 0.35
)
/ len(vectors),
"long_form_row_ratio": sum(
1
for item in vectors
if (
item["first_label_type"] > 0
and item["first_value_type"] == 0
and item["rest_long_text_type_ratio"] >= 0.50
and item["value_type_ratio"] <= 0.25
and item["empty_ratio"] <= 0.35
)
)
/ len(vectors),
}
def body_window_features_from_profiles(
profiles: list[dict[str, Any]], start: int, size: int = 4
) -> dict[str, float]:
vectors = [profile["vector"] for profile in profiles[start : start + size] if profile["nonempty"]]
return combine_body_vectors(vectors)
def body_window_is_stable(features: dict[str, float]) -> bool:
if features["count"] < 2:
return False
value_body = (
features["body_row_ratio"] >= 0.50
and features["rest_numeric_ratio"] >= 0.45
and features["first_label_ratio"] >= 0.35
)
dense_numeric_body = (
features["dense_numeric_row_ratio"] >= 0.50
and features["numeric_ratio"] >= 0.60
and features["alpha_ratio"] <= 0.55
)
long_form_body = (
features["long_form_row_ratio"] >= 0.50
and features["first_label_ratio"] >= 0.50
and features["rest_long_text_type_ratio"] >= 0.50
and features["value_type_ratio"] <= 0.25
and features["numeric_ratio"] <= 0.25
)
return value_body or dense_numeric_body or long_form_body
def boundary_distance(header_row: list[str], body_features: dict[str, float]) -> float:
header = row_vector(header_row)
distance = 0.0
distance += abs(header["numeric_ratio"] - body_features["numeric_ratio"])
distance += abs(header["alpha_ratio"] - body_features["alpha_ratio"])
distance += abs(header["rest_numeric_ratio"] - body_features["rest_numeric_ratio"])
distance += abs(header["first_alpha"] - body_features["first_label_ratio"])
distance += 0.5 * abs(header["avg_tokens_norm"] - body_features["avg_tokens_norm"])
if header["empty_ratio"] >= 0.20:
distance += 0.25
return distance
def header_prefix_ok(
rows: list[list[str]],
profiles: list[dict[str, Any]],
body_start: int,
body_features: dict[str, float],
) -> bool:
reasons = []
if body_start <= 1:
return True
if body_start > 5:
return False
candidate_profiles = profiles[1:body_start]
if not candidate_profiles:
return True
allowed_section_prefix = all(centered_section_label_row_like(rows[profile["index"]]) for profile in candidate_profiles)
coordinate_prefix = any(profile.get("coordinate_header_row", False) for profile in candidate_profiles)
disallowed_section_profiles = []
for profile in candidate_profiles:
if not profile["section_label_row"]:
continue
idx = int(profile["index"])
sparse_coordinate_tail = (
coordinate_prefix
and idx == body_start - 1
and profile["header_candidate"]
and not profile["body_like_row"]
and profile["vector"]["nonempty_count"] <= 3
and idx > 1
and profiles[idx - 1]["header_candidate"]
)
if not centered_section_label_row_like(rows[idx]) and not sparse_coordinate_tail:
disallowed_section_profiles.append(profile)
if any(profile["section_label_row"] for profile in candidate_profiles) and not allowed_section_prefix:
if disallowed_section_profiles:
reasons.append("section_label_before_body")
elif disallowed_section_profiles:
reasons.append("section_label_before_body")
if any(profile["body_like_row"] for profile in candidate_profiles):
reasons.append("body_like_row_before_body")
if not all(profile["header_candidate"] for profile in candidate_profiles):
reasons.append("non_header_candidate_before_body")
previous = profiles[body_start - 2]
last_candidate = profiles[body_start - 1]
contrast = boundary_distance(rows[body_start - 1], body_features)
if (
body_start == 2
and last_candidate["vector"]["first_empty"] == 0
and last_candidate["vector"]["numeric_ratio"] > 0.0
and not previous["grouped_header_scaffold"]
):
reasons.append("numeric_first_row_without_group_scaffold")
if last_candidate["body_dense_row"] and not previous["grouped_header_scaffold"]:
reasons.append("dense_candidate_without_group_scaffold")
if last_candidate["vector"]["alpha_ratio"] < 0.20 and not previous["grouped_header_scaffold"]:
reasons.append("low_alpha_candidate_without_group_scaffold")
if contrast < 1.05:
reasons.append("weak_header_body_contrast")
return not reasons
def body_start_candidates(
profiles: list[dict[str, Any]],
) -> list[dict[str, Any]]:
candidates = []
max_start = min(5, len(profiles) - 2)
for start in range(1, max_start + 1):
body_features = body_window_features_from_profiles(profiles, start)
start_row_body_like = profiles[start]["body_like_row"]
stable = start_row_body_like and body_window_is_stable(body_features)
prior_body_seen = any(
profile["body_like_row"] and not profile.get("coordinate_header_row", False)
for profile in profiles[1:start]
)
candidates.append(
{
"body_start": start,
"body_like": stable,
"start_row_body_like": start_row_body_like,
"prior_body_seen": prior_body_seen,
"body": body_features,
}
)
return candidates
def _nonempty_col_indices(row: list[str], max_cols: int) -> set[int]:
return {idx for idx in range(min(max_cols, len(row))) if column_text(row, idx).strip()}
def _header_column_incomplete(rows: list[list[str]], top_header_rows: int) -> bool:
if top_header_rows <= 0:
return False
max_cols = max((len(row) for row in rows), default=0)
if max_cols < 3:
return False
filled: set[int] = set()
for row in rows[:top_header_rows]:
filled.update(_nonempty_col_indices(row, max_cols))
return len(filled) < max_cols and len(filled) <= max(1, int(max_cols * 0.75))
def _bare_year_value_row_like(row: list[str]) -> bool:
nonempty = [text.strip() for text in row if text.strip()]
if len(nonempty) < 2:
return False
return all(_YEAR_RE.fullmatch(text) for text in nonempty)
def _undertag_extension_candidate(row: list[str], following_rows: list[list[str]], max_cols: int) -> bool:
features = row_features(row)
profile = row_profile(row, 0)
filled_cols = sorted(_nonempty_col_indices(row, max_cols))
nonempty_after_stub = [column_text(row, col) for col in range(1, min(max_cols, len(row))) if column_text(row, col).strip()]
reasons: list[str] = []
if features["nonempty_count"] < 2 or len(filled_cols) < 2:
reasons.append("single_or_sparse_row")
if len(nonempty_after_stub) < 1:
reasons.append("no_column_header_cells_after_stub")
if profile["body_like_row"] or data_row_like(row) or dense_numeric_row_like(row):
reasons.append("body_like_row")
if money_or_percent_heavy(row) or features["value_type_ratio"] > 0.35 or features["numeric_ratio"] > 0.50:
reasons.append("strong_value_row")
if _bare_year_value_row_like(row):
reasons.append("bare_year_value_row")
if section_label_row_like(row):
reasons.append("section_label_row")
if features["avg_tokens"] > 8.0 or features["long_text_type_ratio"] >= 0.35:
reasons.append("long_text_row")
if len(following_rows) >= 2:
following_profiles = row_profiles(following_rows)
body_features = body_window_features_from_profiles(following_profiles, 0)
if not body_window_is_stable(body_features):
reasons.append("following_body_not_stable")
label_signal = features["alpha_ratio"] >= 0.25 or features["date_period_ratio"] >= 0.20
if not label_signal:
reasons.append("weak_label_signal")
return not reasons
def extend_header_undertag(
rows: list[list[str]],
top_header_rows: int,
*,
cap: int = 3,
) -> int:
"""Promote under-tagged column-header rows immediately below the base boundary."""
max_cols = max((len(row) for row in rows), default=0)
if top_header_rows <= 0:
return top_header_rows
if top_header_rows >= len(rows):
return top_header_rows
if not _header_column_incomplete(rows, top_header_rows):
return top_header_rows
new_top = top_header_rows
for row_idx in range(top_header_rows, min(len(rows), top_header_rows + cap)):
if row_idx >= len(rows) - 1:
break
ok = _undertag_extension_candidate(rows[row_idx], rows[row_idx + 1 :], max_cols)
if not ok:
break
new_top = row_idx + 1
return new_top
def find_top_header_rows_by_body_change(
rows: list[list[str]],
) -> int:
if not rows:
return 0
if len(rows) == 1:
return 1
profiles = row_profiles(rows)
candidates = body_start_candidates(profiles)
for candidate in candidates:
body_start = int(candidate["body_start"])
if not candidate["body_like"] or candidate["prior_body_seen"]:
continue
if body_start == 1:
if profiles[0]["leading_body_record_row"]:
restore_guards = first_row_restore_guards(rows[0])
if restore_guards:
return 1
return 0
if headerless_dotted_enum_guard(rows):
return 0
if sparse_prefix_next_header_guard(rows, 1):
return 2
if rowspan_leaf_header_guard(rows, 1):
return 2
# A dense period/label header (row 0) over a value-like sub-row (bare
# years, enumerated headers) is not caught by the sparse-prefix guards.
if len(rows) >= 3 and next_header_row_value_guard(rows[1], rows[2:]):
return 2
return 1
if header_prefix_ok(rows, profiles, body_start, candidate["body"]):
return body_start
if headerless_dotted_enum_guard(rows):
return 0
if sparse_prefix_next_header_guard(rows, 1):
return 2
if rowspan_leaf_header_guard(rows, 1):
return 2
return 1
@dataclass(frozen=True)
class HeaderRegion:
top_header_rows: int
section_header_rows: tuple[int, ...]
def find_header_region(rows: list[list[str]]) -> HeaderRegion:
top_header_rows = find_top_header_rows_by_body_change(rows)
# Promote under-tagged column-header rows immediately below the detected
# header/body boundary.
top_header_rows = extend_header_undertag(rows, top_header_rows)
section_rows = tuple(section_header_rows(rows, top_header_rows))
return HeaderRegion(
top_header_rows=top_header_rows,
section_header_rows=section_rows,
)
# --- HTML serialization: tagged placement grid -> <table> --------------------
def collapse_cell_ws(text: str) -> str:
"""Whitespace-collapse a cell's text (runs of whitespace/newlines -> one space)."""
return " ".join(text.split())
def escape_html_text(text: str) -> str:
"""Escape a cell's text for the serializer: only ``& < >`` (quotes left literal)."""
return text.replace("&", "&").replace("<", "<").replace(">", ">")
def _cell_inner(text: str) -> str:
"""A cell's inner HTML: escaped non-empty lines joined by ``<br/>``."""
return "<br/>".join(escape_html_text(part.strip()) for part in text.splitlines() if part.strip())
def render_table_html(rows, section_header_rows=()) -> str:
"""Serialize a tagged placement grid to its final ``<table>`` HTML in one pass.
``rows`` is a row-major grid of cells duck-typed with ``text`` / ``colspan``
/ ``rowspan`` / ``tag`` (e.g. :class:`pymupdf.table.SpanCell`): each cell
emits its tag, its ``colspan`` / ``rowspan`` attributes and its ``<br/>``-
joined escaped inner HTML. A row whose index is in ``section_header_rows``
and that carries a single non-empty label collapses to one
``<th colspan=N>`` spanning the row."""
section_rows = set(section_header_rows or ())
parts = ["<table>"]
for row_idx, cells in enumerate(rows):
if row_idx in section_rows:
nonempty = [collapse_cell_ws(cell.text) for cell in cells if collapse_cell_ws(cell.text)]
if len(nonempty) == 1 and len(cells) >= 2:
parts.append(
'<tr><th colspan="%d">%s</th></tr>'
% (len(cells), escape_html_text(nonempty[0]))
)
continue
parts.append("<tr>")
for cell in cells:
attrs = ""
if cell.colspan > 1:
attrs += ' colspan="%d"' % cell.colspan
if cell.rowspan > 1:
attrs += ' rowspan="%d"' % cell.rowspan
parts.append(
"<%s%s>%s</%s>" % (cell.tag, attrs, _cell_inner(cell.text), cell.tag)
)
parts.append("</tr>")
parts.append("</table>")
return "".join(parts)
|