Tinghong-Ye22's picture
Add files using upload-large-folder tool
2523bef verified
Raw History Blame Contribute Delete
38.6 kB
"""
Copyright (C) 2023 Artifex Software, Inc.
This file is part of PyMuPDF.
PyMuPDF is free software: you can redistribute it and/or modify it under the
terms of the GNU Affero General Public License as published by the Free
Software Foundation, either version 3 of the License, or (at your option)
any later version.
PyMuPDF is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
details.
You should have received a copy of the GNU Affero General Public License
along with MuPDF. If not, see <https://www.gnu.org/licenses/agpl-3.0.en.html>
Alternative licensing terms are available from the licensor.
For commercial licensing, see <https://www.artifex.com/> or contact
Artifex Software, Inc., 39 Mesa Street, Suite 108A, San Francisco,
CA 94129, USA, for further information.
---------------------------------------------------------------------
PyMuPDF table header detection and HTML serialization (opt-in extension).
Pure text-grid module (no pymupdf import): the header-region rules operate on a
row-major ``[[cell text]]`` grid, and the serializer turns a tagged placement
grid into an HTML ``<table>``. Used only by find_tables(refine=True) (via
pymupdf.table) and Table.to_html(); never runs on the default detection path.
"""
from __future__ import annotations
from dataclasses import dataclass
from statistics import mean
from typing import Any
import re
_TOKEN_RE = re.compile(r"[A-Za-z]+|\d+(?:[.,:/-]\d+)*|[%$€£¥()–—-]+")
_MONTH_RE = re.compile(
r"\b(?:jan|feb|mar|apr|may|jun|jul|aug|sep|sept|oct|nov|dec)[a-z]*\.?\b",
re.IGNORECASE,
)
_MONEY_RE = re.compile(r"(?:[$€£¥]\s*[-(]?\d|(?:usd|eur|gbp|jpy)\b)", re.IGNORECASE)
_PERCENT_RE = re.compile(r"[-(]?\d+(?:\.\d+)?\s*%")
_YEAR_RE = re.compile(r"\b(?:19|20)\d{2}\b")
_CODE_RE = re.compile(r"^(?=.*[A-Za-z])(?=.*\d)[A-Za-z0-9][A-Za-z0-9._/# -]{0,24}$")
_RANGE_RE = re.compile(r"(?:>=|<=|<|>| thru | through | to |\b\d+\s*[-–]\s*\d+\b)", re.IGNORECASE)
_DOTTED_ENUM_RE = re.compile(r"^\s*\d+(?:\.\d+){2,}\b")
_REF_TOKEN_RE = re.compile(r"[\[(]\s*\d{1,2}\s*[\])]|\[\s*\d{1,2}\s*\)")
_REF_FORMULA_ALLOWED_RE = re.compile(r"^[\s\d\[\]\(\)=+\-*/xX.]+$")
_UNIT_ROW_TERMS = ("in million", "in thousand", "in billion", "$ in", "usd", "'000", "%")
VALUE_TYPES = {"number", "money", "percent"}
LABEL_TYPES = {"text_label", "long_text", "code_id", "date_period", "unit"}
def count_alpha(text: str) -> int:
return sum(1 for char in text if char.isalpha())
def count_digit(text: str) -> int:
return sum(1 for char in text if char.isdigit())
def tokens(text: str) -> list[str]:
return _TOKEN_RE.findall(text)
def numeric_like(text: str) -> bool:
stripped = text.strip()
if not stripped:
return False
alpha = count_alpha(stripped)
digit = count_digit(stripped)
if digit == 0:
return False
if _MONTH_RE.search(stripped):
return False
return alpha / max(1, alpha + digit) <= 0.30
def date_or_period_like(text: str) -> bool:
stripped = text.strip()
if not stripped:
return False
return bool(_MONTH_RE.search(stripped) or re.search(r"\b(?:q[1-4]|fy|year|month|period)\b", stripped, re.I))
def cell_type(text: str) -> str:
stripped = " ".join(text.strip().split())
if not stripped:
return "empty"
alpha = count_alpha(stripped)
digit = count_digit(stripped)
token_count = len(tokens(stripped))
lowered = stripped.lower()
if _MONEY_RE.search(stripped):
return "money"
if _PERCENT_RE.fullmatch(stripped) or (stripped.endswith("%") and digit > 0 and alpha == 0):
return "percent"
if date_or_period_like(stripped) or (_YEAR_RE.fullmatch(stripped) and token_count == 1):
return "date_period"
if _RANGE_RE.search(stripped) and digit > 0 and token_count <= 8:
return "range"
if numeric_like(stripped):
return "number"
if lowered in {"%", "$", "$000", "$m", "usd", "eur", "gbp", "amount", "rate", "ratio"}:
return "unit"
if _CODE_RE.match(stripped) and token_count <= 4:
return "code_id"
if alpha > 0 and digit > 0 and token_count <= 6:
return "code_id"
if token_count >= 8:
return "long_text"
return "text_label"
def cell_type_ratios(texts: list[str]) -> dict[str, float]:
nonempty_types = [cell_type(text) for text in texts if text.strip()]
if not nonempty_types:
return {
"value_type_ratio": 0.0,
"label_type_ratio": 0.0,
"long_text_type_ratio": 0.0,
"code_type_ratio": 0.0,
"date_period_type_ratio": 0.0,
"range_type_ratio": 0.0,
}
total = len(nonempty_types)
return {
"value_type_ratio": sum(1 for item in nonempty_types if item in VALUE_TYPES) / total,
"label_type_ratio": sum(1 for item in nonempty_types if item in LABEL_TYPES) / total,
"long_text_type_ratio": sum(1 for item in nonempty_types if item == "long_text") / total,
"code_type_ratio": sum(1 for item in nonempty_types if item == "code_id") / total,
"date_period_type_ratio": sum(1 for item in nonempty_types if item == "date_period") / total,
"range_type_ratio": sum(1 for item in nonempty_types if item == "range") / total,
}
def row_features(texts: list[str]) -> dict[str, float]:
nonempty = [text for text in texts if text.strip()]
if not texts:
return {
"cell_count": 0.0,
"nonempty_count": 0.0,
"nonempty_ratio": 0.0,
"empty_ratio": 0.0,
"numeric_ratio": 0.0,
"alpha_ratio": 0.0,
"date_period_ratio": 0.0,
"avg_tokens": 0.0,
**cell_type_ratios([]),
}
return {
"cell_count": float(len(texts)),
"nonempty_count": float(len(nonempty)),
"nonempty_ratio": len(nonempty) / len(texts),
"empty_ratio": (len(texts) - len(nonempty)) / len(texts),
"numeric_ratio": sum(1 for text in nonempty if numeric_like(text)) / max(1, len(nonempty)),
"alpha_ratio": sum(1 for text in nonempty if count_alpha(text) > 0) / max(1, len(nonempty)),
"date_period_ratio": sum(1 for text in nonempty if date_or_period_like(text)) / max(1, len(nonempty)),
"avg_tokens": mean([len(tokens(text)) for text in nonempty]) if nonempty else 0.0,
**cell_type_ratios(texts),
}
def column_text(row: list[str], col: int) -> str:
return row[col] if col < len(row) else ""
def row_vector(texts: list[str]) -> dict[str, float]:
features = row_features(texts)
first = column_text(texts, 0)
rest = [text for text in texts[1:] if text.strip()]
rest_types = [cell_type(text) for text in rest]
first_nonempty = bool(first.strip())
return {
**features,
"first_empty": 0.0 if first_nonempty else 1.0,
"first_alpha": 1.0 if count_alpha(first) > 0 else 0.0,
"first_numeric": 1.0 if numeric_like(first) else 0.0,
"first_value_type": 1.0 if cell_type(first) in VALUE_TYPES else 0.0,
"first_label_type": 1.0 if cell_type(first) in LABEL_TYPES else 0.0,
"rest_numeric_ratio": sum(1 for text in rest if numeric_like(text)) / max(1, len(rest)),
"rest_alpha_ratio": sum(1 for text in rest if count_alpha(text) > 0) / max(1, len(rest)),
"rest_value_type_ratio": sum(1 for item in rest_types if item in VALUE_TYPES) / max(1, len(rest_types)),
"rest_label_type_ratio": sum(1 for item in rest_types if item in LABEL_TYPES) / max(1, len(rest_types)),
"rest_long_text_type_ratio": sum(1 for item in rest_types if item == "long_text") / max(1, len(rest_types)),
"avg_tokens_norm": min(1.0, features["avg_tokens"] / 8.0),
}
def header_candidate_row(row: list[str]) -> bool:
features = row_features(row)
if features["nonempty_count"] == 0:
return False
short_or_sparse = features["avg_tokens"] <= 6.0 or features["empty_ratio"] >= 0.20
return bool(short_or_sparse)
def grouped_header_scaffold(row: list[str]) -> bool:
features = row_features(row)
if features["cell_count"] == 0:
return False
texts = [text.strip() for text in row]
nonempty = [text for text in texts if text]
unique_nonempty = {text.lower() for text in nonempty}
repeated_labels = len(nonempty) >= 3 and len(unique_nonempty) <= max(1, len(nonempty) // 2)
sparse_labels = features["empty_ratio"] >= 0.20 or features["nonempty_ratio"] <= 0.70
return bool(sparse_labels or repeated_labels)
def data_row_like(row: list[str]) -> bool:
vector = row_vector(row)
return bool(
vector["first_alpha"] > 0
and vector["first_numeric"] == 0
and vector["rest_numeric_ratio"] >= 0.35
and vector["numeric_ratio"] >= 0.25
)
def dense_numeric_row_like(row: list[str]) -> bool:
vector = row_vector(row)
return bool(vector["numeric_ratio"] >= 0.70 and vector["alpha_ratio"] <= 0.35)
def money_or_percent_heavy(row: list[str]) -> bool:
types = [cell_type(text) for text in row if text.strip()]
if not types:
return False
return sum(1 for item in types if item in {"money", "percent"}) / len(types) >= 0.25
def rest_date_period_ratio(row: list[str]) -> float:
rest = [cell for cell in row[1:] if cell.strip()]
if not rest:
return 0.0
return sum(1 for cell in rest if cell_type(cell) == "date_period") / len(rest)
def first_row_period_restore_guard(row: list[str]) -> bool:
features = row_features(row)
return bool(features["date_period_type_ratio"] >= 0.25 or rest_date_period_ratio(row) >= 0.35)
def first_row_date_or_unit_restore_guard(row: list[str]) -> bool:
rest = [cell for cell in row[1:] if cell.strip()]
if not rest:
return False
rest_types = [cell_type(cell) for cell in rest]
dp_unit_ratio = sum(1 for item in rest_types if item in {"date_period", "unit"}) / len(rest_types)
money_or_percent = any(item in {"money", "percent"} for item in rest_types)
row_text = " ".join(row).lower()
unit_row = any(term in row_text for term in _UNIT_ROW_TERMS)
return bool((dp_unit_ratio >= 0.50 or unit_row) and not money_or_percent)
def first_row_restore_guards(row: list[str]) -> list[str]:
guards = []
if first_row_period_restore_guard(row):
guards.append("first_row_period_restore")
if first_row_date_or_unit_restore_guard(row):
guards.append("first_row_date_or_unit_restore")
return guards
def headerless_dotted_enum_guard(rows: list[list[str]]) -> bool:
if len(rows) < 3:
return False
row = rows[0]
cells = [cell.strip() for cell in row if cell and cell.strip()]
if not cells or not _DOTTED_ENUM_RE.search(cells[0]):
return False
features = row_features(row)
return bool(features["long_text_type_ratio"] >= 0.25 or features["avg_tokens"] >= 5.0)
def simple_integer_enum_header_like(row: list[str]) -> bool:
vector = row_vector(row)
if vector["first_label_type"] <= 0 or vector["first_value_type"] > 0:
return False
values = []
for text in [text.strip() for text in row[1:] if text.strip()]:
if not re.fullmatch(r"\d{1,2}", text):
return False
values.append(int(text))
if len(values) < 2:
return False
ordered = values == sorted(values)
unique = len(set(values)) == len(values)
contiguous = max(values) - min(values) == len(values) - 1
return bool(ordered and unique and contiguous)
def reference_formula_header_row_like(candidate: list[str], following_rows: list[list[str]]) -> bool:
"""A bracket/formula reference row under a grouped header, e.g. [3] = [1] * [2]."""
nonempty = [text.strip() for text in candidate if text.strip()]
if len(nonempty) < 3:
return False
if any(any(char.isalpha() for char in text) for text in nonempty):
return False
if any(("$" in text or "%" in text or "," in text) for text in nonempty):
return False
if not all(_REF_FORMULA_ALLOWED_RE.match(text) for text in nonempty):
return False
ref_cells = [text for text in nonempty if _REF_TOKEN_RE.search(text)]
formula_cells = [text for text in nonempty if "=" in text and len(_REF_TOKEN_RE.findall(text)) >= 2]
if len(ref_cells) / len(nonempty) < 0.75:
return False
if not formula_cells and len(ref_cells) < 4:
return False
if len(following_rows) < 2:
return False
following_profiles = row_profiles(following_rows)
body_features = body_window_features_from_profiles(following_profiles, 0)
if body_window_is_stable(body_features):
return True
if following_profiles[0]["section_label_row"] and len(following_profiles) >= 3:
shifted = body_window_features_from_profiles(following_profiles, 1)
return body_window_is_stable(shifted)
return False
def dense_integer_axis_row_like(row: list[str]) -> bool:
values = []
alpha_cells = 0
nonempty = [text.strip() for text in row if text.strip()]
for text in nonempty:
if re.fullmatch(r"\d{1,2}", text):
values.append(int(text))
elif any(char.isalpha() for char in text):
alpha_cells += 1
else:
return False
if len(values) < 5:
return False
unique = sorted(set(values))
if unique != list(range(unique[0], unique[-1] + 1)):
return False
if unique[0] not in {0, 1}:
return False
if alpha_cells > max(4, len(nonempty) // 5):
return False
return True
def sparse_prefix_extension_prefix_like(row: list[str]) -> bool:
features = row_features(row)
if features["nonempty_count"] == 0:
return False
if features["nonempty_count"] <= 2:
return features["alpha_ratio"] >= 0.50 or features["date_period_ratio"] >= 0.20
if features["empty_ratio"] < 0.25:
return False
return bool(features["alpha_ratio"] >= 0.50 or features["date_period_ratio"] >= 0.20)
def next_header_row_value_guard(candidate: list[str], following_rows: list[list[str]]) -> bool:
features = row_features(candidate)
vector = row_vector(candidate)
if features["nonempty_count"] < 2:
return False
if features["avg_tokens"] > 7.0 or features["long_text_type_ratio"] >= 0.35:
return False
if money_or_percent_heavy(candidate):
return False
if reference_formula_header_row_like(candidate, following_rows):
return True
if features["value_type_ratio"] > 0.85:
return False
# date_period_type_ratio recognises a bare-year header row that the
# keyword-based date_period_ratio misses; combine them only in this guard.
date_period_signal = max(features["date_period_ratio"], features["date_period_type_ratio"])
has_label_signal = (
features["alpha_ratio"] >= 0.30
or date_period_signal >= 0.20
or vector["first_label_type"] > 0
)
if not has_label_signal:
return False
if len(following_rows) < 2:
return False
following_profiles = row_profiles(following_rows)
body_features = body_window_features_from_profiles(following_profiles, 0)
if not body_window_is_stable(body_features):
return False
if features["numeric_ratio"] >= 0.50 or features["value_type_ratio"] >= 0.50:
if vector["first_label_type"] <= 0 and date_period_signal < 0.20:
return False
if vector["first_value_type"] > 0:
return False
types = [cell_type(text) for text in candidate if text.strip()]
type_count = len(types)
if not type_count:
return False
date_type_ratio = sum(1 for item in types if item == "date_period") / type_count
code_range_number_ratio = sum(1 for item in types if item in {"code_id", "range", "number"}) / type_count
if features["value_type_ratio"] > 0.25:
return simple_integer_enum_header_like(candidate)
if code_range_number_ratio >= 0.50 and date_type_ratio < 0.30:
return simple_integer_enum_header_like(candidate)
return True
def sparse_prefix_conservative_extension_guard(rows: list[list[str]], top_header_rows: int) -> bool:
if top_header_rows != 1 or len(rows) < 3:
return False
prefix = rows[0]
candidate = rows[1]
prefix_features = row_features(prefix)
candidate_features = row_features(candidate)
if prefix_features["empty_ratio"] < 0.50:
return False
if candidate_features["nonempty_count"] < 3 and candidate_features["date_period_ratio"] < 0.50:
return False
if not sparse_prefix_extension_prefix_like(prefix):
return False
if candidate_features["label_type_ratio"] < 0.50:
return False
if candidate_features["value_type_ratio"] >= 0.50:
return False
if section_label_row_like(candidate):
return False
following_profiles = row_profiles(rows[2:])
body_features = body_window_features_from_profiles(following_profiles, 0)
return body_window_is_stable(body_features)
def sparse_prefix_next_header_guard(rows: list[list[str]], top_header_rows: int) -> bool:
"""Return True when the first body row should be included as a second header row."""
if top_header_rows < 1 or top_header_rows >= len(rows) - 1:
return False
return sparse_prefix_conservative_extension_guard(rows, top_header_rows)
def rowspan_leaf_header_guard(rows: list[list[str]], top_header_rows: int) -> bool:
"""Promote a leaf-label row under a sparse/grouped rowspan header scaffold.
An empty first cell in the candidate row is the textual shadow of the row-0
rowspan label; treat it as header only when later rows form a stable body.
"""
if top_header_rows != 1 or len(rows) < 3:
return False
max_cols = max((len(row) for row in rows), default=0)
if max_cols < 3:
return False
prefix = rows[0]
candidate = rows[1]
prefix_features = row_features(prefix)
candidate_features = row_features(candidate)
candidate_vector = row_vector(candidate)
shifted_by_rowspan = len(candidate) < max_cols
if column_text(candidate, 0).strip() and not shifted_by_rowspan:
return False
sparse_rowspan_prefix = len(prefix) < max_cols and len(prefix) <= max(2, int(max_cols * 0.50))
if not (grouped_header_scaffold(prefix) or prefix_features["empty_ratio"] >= 0.35 or sparse_rowspan_prefix):
return False
if candidate_features["nonempty_count"] < max(2, int(max_cols * 0.45)):
return False
if candidate_features["value_type_ratio"] > 0.25 or candidate_features["numeric_ratio"] > 0.35:
return False
if candidate_vector["rest_label_type_ratio"] < 0.50 and candidate_features["alpha_ratio"] < 0.50:
return False
if section_label_row_like(candidate) or dense_numeric_row_like(candidate):
return False
following_profiles = row_profiles(rows[2:])
body_features = body_window_features_from_profiles(following_profiles, 0)
return body_window_is_stable(body_features)
def leading_body_record_like(row: list[str]) -> bool:
vector = row_vector(row)
if vector["nonempty_count"] < 2:
return False
first_is_label = vector["first_label_type"] > 0 and vector["first_value_type"] == 0
rest_is_values = vector["rest_numeric_ratio"] >= 0.50 or vector["rest_value_type_ratio"] >= 0.50
value_heavy = vector["numeric_ratio"] >= 0.35 or vector["value_type_ratio"] >= 0.35
return bool(first_is_label and rest_is_values and value_heavy and not grouped_header_scaffold(row))
def leading_long_form_row_like(row: list[str]) -> bool:
vector = row_vector(row)
return bool(
vector["nonempty_count"] >= 4
and vector["long_text_type_ratio"] >= 0.30
and vector["value_type_ratio"] <= 0.20
and vector["empty_ratio"] <= 0.35
)
def section_label_row_like(row: list[str]) -> bool:
if len(row) < 2:
return False
features = row_features(row)
first = column_text(row, 0).strip()
nonempty = [text for text in row if text.strip()]
first_only = bool(first and not [text for text in row[1:] if text.strip()])
sparse_label = (
features["nonempty_count"] <= 3
and features["empty_ratio"] >= 0.60
and features["alpha_ratio"] >= 0.50
)
return bool(first_only or (nonempty and sparse_label))
def centered_section_label_row_like(row: list[str]) -> bool:
if not section_label_row_like(row):
return False
nonempty_indices = [idx for idx, text in enumerate(row) if text.strip()]
return bool(len(nonempty_indices) == 1 and nonempty_indices[0] > 0 and len(row) >= 3)
def section_header_rows(rows: list[list[str]], top_header_rows: int) -> list[int]:
"""Return sparse section-label rows inside the detected top header prefix."""
section_rows = []
for row_idx, row in enumerate(rows[:top_header_rows]):
if row_idx == 0 or not centered_section_label_row_like(row):
continue
nonempty = [text.strip() for text in row if text.strip()]
if len(nonempty) != 1:
continue
if len(row) < 2:
continue
section_rows.append(row_idx)
return section_rows
def row_profile(row: list[str], index: int) -> dict[str, Any]:
vector = row_vector(row)
coordinate_header_row = dense_integer_axis_row_like(row)
body_value_row = (
vector["first_alpha"] > 0
and vector["first_numeric"] == 0
and vector["rest_numeric_ratio"] >= 0.35
and vector["numeric_ratio"] >= 0.25
)
body_dense_row = vector["numeric_ratio"] >= 0.70 and vector["alpha_ratio"] <= 0.35
body_long_form_row = (
vector["first_label_type"] > 0
and vector["first_value_type"] == 0
and vector["rest_long_text_type_ratio"] >= 0.50
and vector["value_type_ratio"] <= 0.25
and vector["empty_ratio"] <= 0.35
)
if coordinate_header_row:
body_value_row = False
body_dense_row = False
body_long_form_row = False
return {
"index": index,
"vector": vector,
"nonempty": vector["nonempty_count"] > 0,
"body_value_row": body_value_row,
"body_dense_row": body_dense_row,
"body_long_form_row": body_long_form_row,
"body_like_row": body_value_row or body_dense_row or body_long_form_row,
"coordinate_header_row": coordinate_header_row,
"leading_body_record_row": leading_body_record_like(row),
"leading_long_form_row": leading_long_form_row_like(row),
"section_label_row": section_label_row_like(row),
"grouped_header_scaffold": grouped_header_scaffold(row),
"header_candidate": header_candidate_row(row),
}
def row_profiles(rows: list[list[str]]) -> list[dict[str, Any]]:
return [row_profile(row, index) for index, row in enumerate(rows)]
def combine_body_vectors(vectors: list[dict[str, float]]) -> dict[str, float]:
if not vectors:
return {
"count": 0.0,
"first_label_ratio": 0.0,
"rest_numeric_ratio": 0.0,
"numeric_ratio": 0.0,
"alpha_ratio": 0.0,
"avg_tokens_norm": 0.0,
"body_row_ratio": 0.0,
"dense_numeric_row_ratio": 0.0,
"long_form_row_ratio": 0.0,
"value_type_ratio": 0.0,
"rest_value_type_ratio": 0.0,
"rest_long_text_type_ratio": 0.0,
"long_text_type_ratio": 0.0,
}
return {
"count": float(len(vectors)),
"first_label_ratio": sum(1 for item in vectors if item["first_label_type"] > 0 and item["first_value_type"] == 0)
/ len(vectors),
"rest_numeric_ratio": sum(item["rest_numeric_ratio"] for item in vectors) / len(vectors),
"numeric_ratio": sum(item["numeric_ratio"] for item in vectors) / len(vectors),
"alpha_ratio": sum(item["alpha_ratio"] for item in vectors) / len(vectors),
"avg_tokens_norm": sum(item["avg_tokens_norm"] for item in vectors) / len(vectors),
"value_type_ratio": sum(item["value_type_ratio"] for item in vectors) / len(vectors),
"rest_value_type_ratio": sum(item["rest_value_type_ratio"] for item in vectors) / len(vectors),
"rest_long_text_type_ratio": sum(item["rest_long_text_type_ratio"] for item in vectors) / len(vectors),
"long_text_type_ratio": sum(item["long_text_type_ratio"] for item in vectors) / len(vectors),
"body_row_ratio": sum(
1
for item in vectors
if (
item["first_alpha"] > 0
and item["first_numeric"] == 0
and item["rest_numeric_ratio"] >= 0.35
and item["numeric_ratio"] >= 0.25
)
or (item["numeric_ratio"] >= 0.70 and item["alpha_ratio"] <= 0.35)
)
/ len(vectors),
"dense_numeric_row_ratio": sum(
1 for item in vectors if item["numeric_ratio"] >= 0.70 and item["alpha_ratio"] <= 0.35
)
/ len(vectors),
"long_form_row_ratio": sum(
1
for item in vectors
if (
item["first_label_type"] > 0
and item["first_value_type"] == 0
and item["rest_long_text_type_ratio"] >= 0.50
and item["value_type_ratio"] <= 0.25
and item["empty_ratio"] <= 0.35
)
)
/ len(vectors),
}
def body_window_features_from_profiles(
profiles: list[dict[str, Any]], start: int, size: int = 4
) -> dict[str, float]:
vectors = [profile["vector"] for profile in profiles[start : start + size] if profile["nonempty"]]
return combine_body_vectors(vectors)
def body_window_is_stable(features: dict[str, float]) -> bool:
if features["count"] < 2:
return False
value_body = (
features["body_row_ratio"] >= 0.50
and features["rest_numeric_ratio"] >= 0.45
and features["first_label_ratio"] >= 0.35
)
dense_numeric_body = (
features["dense_numeric_row_ratio"] >= 0.50
and features["numeric_ratio"] >= 0.60
and features["alpha_ratio"] <= 0.55
)
long_form_body = (
features["long_form_row_ratio"] >= 0.50
and features["first_label_ratio"] >= 0.50
and features["rest_long_text_type_ratio"] >= 0.50
and features["value_type_ratio"] <= 0.25
and features["numeric_ratio"] <= 0.25
)
return value_body or dense_numeric_body or long_form_body
def boundary_distance(header_row: list[str], body_features: dict[str, float]) -> float:
header = row_vector(header_row)
distance = 0.0
distance += abs(header["numeric_ratio"] - body_features["numeric_ratio"])
distance += abs(header["alpha_ratio"] - body_features["alpha_ratio"])
distance += abs(header["rest_numeric_ratio"] - body_features["rest_numeric_ratio"])
distance += abs(header["first_alpha"] - body_features["first_label_ratio"])
distance += 0.5 * abs(header["avg_tokens_norm"] - body_features["avg_tokens_norm"])
if header["empty_ratio"] >= 0.20:
distance += 0.25
return distance
def header_prefix_ok(
rows: list[list[str]],
profiles: list[dict[str, Any]],
body_start: int,
body_features: dict[str, float],
) -> bool:
reasons = []
if body_start <= 1:
return True
if body_start > 5:
return False
candidate_profiles = profiles[1:body_start]
if not candidate_profiles:
return True
allowed_section_prefix = all(centered_section_label_row_like(rows[profile["index"]]) for profile in candidate_profiles)
coordinate_prefix = any(profile.get("coordinate_header_row", False) for profile in candidate_profiles)
disallowed_section_profiles = []
for profile in candidate_profiles:
if not profile["section_label_row"]:
continue
idx = int(profile["index"])
sparse_coordinate_tail = (
coordinate_prefix
and idx == body_start - 1
and profile["header_candidate"]
and not profile["body_like_row"]
and profile["vector"]["nonempty_count"] <= 3
and idx > 1
and profiles[idx - 1]["header_candidate"]
)
if not centered_section_label_row_like(rows[idx]) and not sparse_coordinate_tail:
disallowed_section_profiles.append(profile)
if any(profile["section_label_row"] for profile in candidate_profiles) and not allowed_section_prefix:
if disallowed_section_profiles:
reasons.append("section_label_before_body")
elif disallowed_section_profiles:
reasons.append("section_label_before_body")
if any(profile["body_like_row"] for profile in candidate_profiles):
reasons.append("body_like_row_before_body")
if not all(profile["header_candidate"] for profile in candidate_profiles):
reasons.append("non_header_candidate_before_body")
previous = profiles[body_start - 2]
last_candidate = profiles[body_start - 1]
contrast = boundary_distance(rows[body_start - 1], body_features)
if (
body_start == 2
and last_candidate["vector"]["first_empty"] == 0
and last_candidate["vector"]["numeric_ratio"] > 0.0
and not previous["grouped_header_scaffold"]
):
reasons.append("numeric_first_row_without_group_scaffold")
if last_candidate["body_dense_row"] and not previous["grouped_header_scaffold"]:
reasons.append("dense_candidate_without_group_scaffold")
if last_candidate["vector"]["alpha_ratio"] < 0.20 and not previous["grouped_header_scaffold"]:
reasons.append("low_alpha_candidate_without_group_scaffold")
if contrast < 1.05:
reasons.append("weak_header_body_contrast")
return not reasons
def body_start_candidates(
profiles: list[dict[str, Any]],
) -> list[dict[str, Any]]:
candidates = []
max_start = min(5, len(profiles) - 2)
for start in range(1, max_start + 1):
body_features = body_window_features_from_profiles(profiles, start)
start_row_body_like = profiles[start]["body_like_row"]
stable = start_row_body_like and body_window_is_stable(body_features)
prior_body_seen = any(
profile["body_like_row"] and not profile.get("coordinate_header_row", False)
for profile in profiles[1:start]
)
candidates.append(
{
"body_start": start,
"body_like": stable,
"start_row_body_like": start_row_body_like,
"prior_body_seen": prior_body_seen,
"body": body_features,
}
)
return candidates
def _nonempty_col_indices(row: list[str], max_cols: int) -> set[int]:
return {idx for idx in range(min(max_cols, len(row))) if column_text(row, idx).strip()}
def _header_column_incomplete(rows: list[list[str]], top_header_rows: int) -> bool:
if top_header_rows <= 0:
return False
max_cols = max((len(row) for row in rows), default=0)
if max_cols < 3:
return False
filled: set[int] = set()
for row in rows[:top_header_rows]:
filled.update(_nonempty_col_indices(row, max_cols))
return len(filled) < max_cols and len(filled) <= max(1, int(max_cols * 0.75))
def _bare_year_value_row_like(row: list[str]) -> bool:
nonempty = [text.strip() for text in row if text.strip()]
if len(nonempty) < 2:
return False
return all(_YEAR_RE.fullmatch(text) for text in nonempty)
def _undertag_extension_candidate(row: list[str], following_rows: list[list[str]], max_cols: int) -> bool:
features = row_features(row)
profile = row_profile(row, 0)
filled_cols = sorted(_nonempty_col_indices(row, max_cols))
nonempty_after_stub = [column_text(row, col) for col in range(1, min(max_cols, len(row))) if column_text(row, col).strip()]
reasons: list[str] = []
if features["nonempty_count"] < 2 or len(filled_cols) < 2:
reasons.append("single_or_sparse_row")
if len(nonempty_after_stub) < 1:
reasons.append("no_column_header_cells_after_stub")
if profile["body_like_row"] or data_row_like(row) or dense_numeric_row_like(row):
reasons.append("body_like_row")
if money_or_percent_heavy(row) or features["value_type_ratio"] > 0.35 or features["numeric_ratio"] > 0.50:
reasons.append("strong_value_row")
if _bare_year_value_row_like(row):
reasons.append("bare_year_value_row")
if section_label_row_like(row):
reasons.append("section_label_row")
if features["avg_tokens"] > 8.0 or features["long_text_type_ratio"] >= 0.35:
reasons.append("long_text_row")
if len(following_rows) >= 2:
following_profiles = row_profiles(following_rows)
body_features = body_window_features_from_profiles(following_profiles, 0)
if not body_window_is_stable(body_features):
reasons.append("following_body_not_stable")
label_signal = features["alpha_ratio"] >= 0.25 or features["date_period_ratio"] >= 0.20
if not label_signal:
reasons.append("weak_label_signal")
return not reasons
def extend_header_undertag(
rows: list[list[str]],
top_header_rows: int,
*,
cap: int = 3,
) -> int:
"""Promote under-tagged column-header rows immediately below the base boundary."""
max_cols = max((len(row) for row in rows), default=0)
if top_header_rows <= 0:
return top_header_rows
if top_header_rows >= len(rows):
return top_header_rows
if not _header_column_incomplete(rows, top_header_rows):
return top_header_rows
new_top = top_header_rows
for row_idx in range(top_header_rows, min(len(rows), top_header_rows + cap)):
if row_idx >= len(rows) - 1:
break
ok = _undertag_extension_candidate(rows[row_idx], rows[row_idx + 1 :], max_cols)
if not ok:
break
new_top = row_idx + 1
return new_top
def find_top_header_rows_by_body_change(
rows: list[list[str]],
) -> int:
if not rows:
return 0
if len(rows) == 1:
return 1
profiles = row_profiles(rows)
candidates = body_start_candidates(profiles)
for candidate in candidates:
body_start = int(candidate["body_start"])
if not candidate["body_like"] or candidate["prior_body_seen"]:
continue
if body_start == 1:
if profiles[0]["leading_body_record_row"]:
restore_guards = first_row_restore_guards(rows[0])
if restore_guards:
return 1
return 0
if headerless_dotted_enum_guard(rows):
return 0
if sparse_prefix_next_header_guard(rows, 1):
return 2
if rowspan_leaf_header_guard(rows, 1):
return 2
# A dense period/label header (row 0) over a value-like sub-row (bare
# years, enumerated headers) is not caught by the sparse-prefix guards.
if len(rows) >= 3 and next_header_row_value_guard(rows[1], rows[2:]):
return 2
return 1
if header_prefix_ok(rows, profiles, body_start, candidate["body"]):
return body_start
if headerless_dotted_enum_guard(rows):
return 0
if sparse_prefix_next_header_guard(rows, 1):
return 2
if rowspan_leaf_header_guard(rows, 1):
return 2
return 1
@dataclass(frozen=True)
class HeaderRegion:
top_header_rows: int
section_header_rows: tuple[int, ...]
def find_header_region(rows: list[list[str]]) -> HeaderRegion:
top_header_rows = find_top_header_rows_by_body_change(rows)
# Promote under-tagged column-header rows immediately below the detected
# header/body boundary.
top_header_rows = extend_header_undertag(rows, top_header_rows)
section_rows = tuple(section_header_rows(rows, top_header_rows))
return HeaderRegion(
top_header_rows=top_header_rows,
section_header_rows=section_rows,
)
# --- HTML serialization: tagged placement grid -> <table> --------------------
def collapse_cell_ws(text: str) -> str:
"""Whitespace-collapse a cell's text (runs of whitespace/newlines -> one space)."""
return " ".join(text.split())
def escape_html_text(text: str) -> str:
"""Escape a cell's text for the serializer: only ``& < >`` (quotes left literal)."""
return text.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
def _cell_inner(text: str) -> str:
"""A cell's inner HTML: escaped non-empty lines joined by ``<br/>``."""
return "<br/>".join(escape_html_text(part.strip()) for part in text.splitlines() if part.strip())
def render_table_html(rows, section_header_rows=()) -> str:
"""Serialize a tagged placement grid to its final ``<table>`` HTML in one pass.
``rows`` is a row-major grid of cells duck-typed with ``text`` / ``colspan``
/ ``rowspan`` / ``tag`` (e.g. :class:`pymupdf.table.SpanCell`): each cell
emits its tag, its ``colspan`` / ``rowspan`` attributes and its ``<br/>``-
joined escaped inner HTML. A row whose index is in ``section_header_rows``
and that carries a single non-empty label collapses to one
``<th colspan=N>`` spanning the row."""
section_rows = set(section_header_rows or ())
parts = ["<table>"]
for row_idx, cells in enumerate(rows):
if row_idx in section_rows:
nonempty = [collapse_cell_ws(cell.text) for cell in cells if collapse_cell_ws(cell.text)]
if len(nonempty) == 1 and len(cells) >= 2:
parts.append(
'<tr><th colspan="%d">%s</th></tr>'
% (len(cells), escape_html_text(nonempty[0]))
)
continue
parts.append("<tr>")
for cell in cells:
attrs = ""
if cell.colspan > 1:
attrs += ' colspan="%d"' % cell.colspan
if cell.rowspan > 1:
attrs += ' rowspan="%d"' % cell.rowspan
parts.append(
"<%s%s>%s</%s>" % (cell.tag, attrs, _cell_inner(cell.text), cell.tag)
)
parts.append("</tr>")
parts.append("</table>")
return "".join(parts)