Download hfd_iclr2027_draft/tools/python/pymupdf/_table_refine.py from Tinghong-Ye22/code-iclr: direct link, hf CLI and curl.
- Browser
- Download file 31.8 kB
-
https://huggingface.co/Tinghong-Ye22/code-iclr/resolve/main/hfd_iclr2027_draft/tools/python/pymupdf/_table_refine.py
- Command line
-
hf download hf://Tinghong-Ye22/code-iclr/hfd_iclr2027_draft/tools/python/pymupdf/_table_refine.py
-
curl -L -o _table_refine.py https://huggingface.co/Tinghong-Ye22/code-iclr/resolve/main/hfd_iclr2027_draft/tools/python/pymupdf/_table_refine.py
31.8 kB
| """ | |
| Copyright (C) 2023 Artifex Software, Inc. | |
| This file is part of PyMuPDF. | |
| PyMuPDF is free software: you can redistribute it and/or modify it under the | |
| terms of the GNU Affero General Public License as published by the Free | |
| Software Foundation, either version 3 of the License, or (at your option) | |
| any later version. | |
| PyMuPDF is distributed in the hope that it will be useful, but WITHOUT ANY | |
| WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS | |
| FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more | |
| details. | |
| You should have received a copy of the GNU Affero General Public License | |
| along with MuPDF. If not, see <https://www.gnu.org/licenses/agpl-3.0.en.html> | |
| Alternative licensing terms are available from the licensor. | |
| For commercial licensing, see <https://www.artifex.com/> or contact | |
| Artifex Software, Inc., 39 Mesa Street, Suite 108A, San Francisco, | |
| CA 94129, USA, for further information. | |
| --------------------------------------------------------------------- | |
| PyMuPDF table grid refinement (opt-in extension). | |
| Provides the refine_grid / refine_grid_structure / refine_grid_rows API that | |
| refines a detected table's cell grid using page text and vector-graphics | |
| geometry. Re-exported by pymupdf.table; never runs on the default | |
| find_tables() path. | |
| """ | |
| import itertools | |
| import re | |
| import pymupdf | |
| # Grid refinement runs three splitters, each taking (page, grid) and returning a | |
| # grid without mutating the page: | |
| # * _split_shaded_rows split rows the line grid merged but a cell | |
| # background-shading rectangle separates, | |
| # * _split_undersegmented_columns split a column that jams several values into | |
| # one cell, | |
| # * _split_overmerged_rows split body rows that collapsed several | |
| # records into a single grid row. | |
| # Word selection uses center-point cell membership with rotated/vertical span | |
| # substitution; it is independent of the CHARS/extract_words path in | |
| # pymupdf.table, so refinement needs no CHARS state. | |
| _REFINE_LINE_GAP = 3.0 # center-y gap (points) that groups body words into lines | |
| # --- word selection: center-point membership + rotated-span substitution ----- | |
| def _refine_rawdict_spans(page): | |
| """Flattened rawdict text spans carrying line-direction metadata. | |
| Only used to locate vertical/rotated text so those words can be replaced by | |
| span-level bboxes in _refine_page_words. Minimal fields (bbox/text/dir/wmode) | |
| -- the only ones the consumers read.""" | |
| spans = [] | |
| raw = page.get_text("rawdict") | |
| for block in raw.get("blocks", []): | |
| if block.get("type") != 0: | |
| continue | |
| for line in block.get("lines", []): | |
| direction = line.get("dir") | |
| wmode = line.get("wmode") | |
| for span in line.get("spans", []): | |
| text = "".join(str(char.get("c", "")) for char in span.get("chars", [])) | |
| if not text.strip(): | |
| continue | |
| spans.append( | |
| { | |
| "bbox": [float(v) for v in span.get("bbox", [])], | |
| "text": text, | |
| "dir": direction, | |
| "wmode": wmode, | |
| } | |
| ) | |
| return spans | |
| def _refine_is_vertical_or_rotated(span): | |
| """True when a rawdict span's direction indicates non-horizontal text flow. | |
| wmode != 0 is vertical; otherwise compare |dir_x| vs |dir_y| (a missing or | |
| unparsable direction counts as horizontal).""" | |
| if span.get("wmode") not in (None, 0): | |
| return True | |
| direction = span.get("dir") | |
| if not isinstance(direction, (list, tuple)) or len(direction) < 2: | |
| return False | |
| try: | |
| dx = abs(float(direction[0])) | |
| dy = abs(float(direction[1])) | |
| except (TypeError, ValueError): | |
| return False | |
| return dy > dx | |
| def _refine_page_words(page): | |
| """Center-point word list for grid refinement, cached on the page object. | |
| Extract page words once (page.get_text("words")), then substitute vertical/ | |
| rotated text: any horizontal word whose center falls inside a rotated span's | |
| bbox is dropped, and each rotated span is appended as a single word.""" | |
| cache = getattr(page, "_refine_words_cache", None) | |
| if cache is not None: | |
| return cache | |
| words = [ | |
| (float(w[0]), float(w[1]), float(w[2]), float(w[3]), str(w[4])) | |
| for w in page.get_text("words") | |
| if str(w[4]).strip() | |
| ] | |
| rotated_spans = [ | |
| span | |
| for span in _refine_rawdict_spans(page) | |
| if _refine_is_vertical_or_rotated(span) and str(span.get("text") or "").strip() | |
| ] | |
| if rotated_spans: | |
| rotated_rects = [pymupdf.Rect(span.get("bbox") or []) for span in rotated_spans] | |
| kept = [] | |
| for word in words: | |
| wx0, wy0, wx1, wy1, _ = word | |
| center = pymupdf.Point((wx0 + wx1) * 0.5, (wy0 + wy1) * 0.5) | |
| if any(rect.contains(center) for rect in rotated_rects if not rect.is_empty): | |
| continue | |
| kept.append(word) | |
| for span, rect in zip(rotated_spans, rotated_rects): | |
| if rect.is_empty: | |
| continue | |
| kept.append( | |
| (float(rect.x0), float(rect.y0), float(rect.x1), float(rect.y1), str(span["text"])) | |
| ) | |
| words = sorted(kept, key=lambda item: (item[1], item[0])) | |
| try: | |
| setattr(page, "_refine_words_cache", words) | |
| except Exception: | |
| pass | |
| return words | |
| def _refine_word_in_rect(wx0, wy0, wx1, wy1, x0, y0, x1, y1): | |
| """Word-to-cell membership by CENTER point (inclusive), not clip-overlap. | |
| A word straddling a column line is claimed by exactly one cell.""" | |
| cx = (wx0 + wx1) * 0.5 | |
| cy = (wy0 + wy1) * 0.5 | |
| return x0 <= cx <= x1 and y0 <= cy <= y1 | |
| def _refine_words_in_rect(page, rect): | |
| """Page words whose center lies in rect, as (x0, y0, x1, y1, text) tuples.""" | |
| x0, y0, x1, y1 = float(rect.x0), float(rect.y0), float(rect.x1), float(rect.y1) | |
| return [ | |
| (wx0, wy0, wx1, wy1, text) | |
| for wx0, wy0, wx1, wy1, text in _refine_page_words(page) | |
| if _refine_word_in_rect(wx0, wy0, wx1, wy1, x0, y0, x1, y1) | |
| ] | |
| def _refine_has_digit(text): | |
| """Loose value-like predicate: the text contains any digit.""" | |
| return bool(re.search(r"\d", text.strip())) | |
| def _refine_has_digit_no_alpha(text): | |
| """Strict value-like predicate: contains a digit and no letters.""" | |
| stripped = text.strip() | |
| return bool(re.search(r"\d", stripped)) and not any(char.isalpha() for char in stripped) | |
| # --- stage 1: split rows separated by cell background shading ---------------- | |
| def _refine_is_white(fill, *, threshold=0.95): | |
| if fill is None: | |
| return True | |
| try: | |
| r, g, b = fill[:3] | |
| except Exception: | |
| return True | |
| return float(r) > threshold and float(g) > threshold and float(b) > threshold | |
| def _refine_table_rect(cells, table_bbox): | |
| if table_bbox is not None: | |
| rect = pymupdf.Rect(table_bbox) | |
| if not rect.is_empty: | |
| return rect | |
| rects = [ | |
| pymupdf.Rect(cell) | |
| for row in cells | |
| for cell in row | |
| if cell is not None and not pymupdf.Rect(cell).is_empty | |
| ] | |
| if not rects: | |
| return None | |
| rect = pymupdf.Rect(rects[0]) | |
| for item in rects[1:]: | |
| rect.include_rect(item) | |
| return rect | |
| def _refine_raw_shaded_rects(page, table_rect, *, min_dim): | |
| out = [] | |
| page_width = float(page.rect.width) | |
| for drawing in page.get_drawings(): | |
| if _refine_is_white(drawing.get("fill")): | |
| continue | |
| for item in drawing.get("items", []): | |
| if item[0] != "re": | |
| continue | |
| rect = pymupdf.Rect(item[1]) | |
| rect.normalize() | |
| if rect.width >= 0.95 * page_width: | |
| continue | |
| if rect.width < min_dim or rect.height < min_dim: | |
| continue | |
| if (rect & table_rect).is_empty: | |
| continue | |
| out.append(rect) | |
| return out | |
| def _refine_cell_background_rects(raw_rects, *, contain_frac=0.9): | |
| keep = [] | |
| for index, rect in enumerate(raw_rects): | |
| area = rect.get_area() | |
| if area <= 0: | |
| continue | |
| nested = any( | |
| other_index != index | |
| and (rect & other).get_area() >= contain_frac * area | |
| and other.get_area() > area * 1.05 | |
| for other_index, other in enumerate(raw_rects) | |
| ) | |
| if not nested: | |
| keep.append(rect) | |
| return keep | |
| def _refine_cluster(values, *, tolerance): | |
| clusters = [] | |
| current = [] | |
| for value in sorted(values): | |
| if current and value - current[-1] > tolerance: | |
| clusters.append(sum(current) / len(current)) | |
| current = [] | |
| current.append(value) | |
| if current: | |
| clusters.append(sum(current) / len(current)) | |
| return clusters | |
| def _refine_border_lines(page, table_rect): | |
| xs = set() | |
| ys = set() | |
| for drawing in page.get_drawings(): | |
| stroked = drawing.get("type") in ("s", "fs") | |
| for item in drawing.get("items", []): | |
| kind = item[0] | |
| if kind == "l": | |
| p1, p2 = item[1], item[2] | |
| if ( | |
| abs(p1.y - p2.y) < 1.0 | |
| and abs(p1.x - p2.x) > 10.0 | |
| and table_rect.y0 - 2.0 < (p1.y + p2.y) * 0.5 < table_rect.y1 + 2.0 | |
| ): | |
| ys.add(round((p1.y + p2.y) * 0.5, 1)) | |
| elif ( | |
| abs(p1.x - p2.x) < 1.0 | |
| and abs(p1.y - p2.y) > 10.0 | |
| and table_rect.x0 - 2.0 < (p1.x + p2.x) * 0.5 < table_rect.x1 + 2.0 | |
| ): | |
| xs.add(round((p1.x + p2.x) * 0.5, 1)) | |
| continue | |
| if kind != "re": | |
| continue | |
| rect = pymupdf.Rect(item[1]) | |
| rect.normalize() | |
| if (rect & table_rect).is_empty: | |
| continue | |
| if rect.height < 2.0 and rect.width > 10.0: | |
| ys.add(round((rect.y0 + rect.y1) * 0.5, 1)) | |
| elif rect.width < 2.0 and rect.height > 10.0: | |
| xs.add(round((rect.x0 + rect.x1) * 0.5, 1)) | |
| elif stroked: | |
| ys.add(round(rect.y0, 1)) | |
| ys.add(round(rect.y1, 1)) | |
| xs.add(round(rect.x0, 1)) | |
| xs.add(round(rect.x1, 1)) | |
| return sorted(xs), sorted(ys) | |
| def _refine_drop_near(edges, borders, *, tolerance): | |
| return [edge for edge in edges if not any(abs(edge - border) <= tolerance for border in borders)] | |
| def _refine_has_text(words, x0, y0, x1, y1, *, margin): | |
| for wx0, wy0, wx1, wy1, text in words: | |
| if not str(text).strip(): | |
| continue | |
| cx = (wx0 + wx1) * 0.5 | |
| cy = (wy0 + wy1) * 0.5 | |
| if x0 + margin < cx < x1 - margin and y0 + margin < cy < y1 - margin: | |
| return True | |
| return False | |
| def _refine_content_filter_row_edges(words, y0, y1, candidates, x0, x1, *, margin): | |
| edges = sorted(edge for edge in candidates if y0 + margin < edge < y1 - margin) | |
| while edges: | |
| bounds = [y0, *edges, y1] | |
| empty_index = None | |
| for index in range(len(bounds) - 1): | |
| if not _refine_has_text(words, x0, bounds[index], x1, bounds[index + 1], margin=margin): | |
| empty_index = index | |
| break | |
| if empty_index is None: | |
| break | |
| if empty_index < len(edges): | |
| edges.pop(empty_index) | |
| elif empty_index - 1 >= 0: | |
| edges.pop(empty_index - 1) | |
| else: | |
| break | |
| return edges | |
| def _split_shaded_rows( | |
| page, | |
| cells, | |
| table_bbox=None, | |
| *, | |
| cluster_tolerance=3.0, | |
| margin=2.0, | |
| min_dim=6.0, | |
| border_tolerance=2.5, | |
| ): | |
| """Split rows the line grid merged but a cell background rectangle separates. | |
| Cell background-shading rectangles give row boundaries the border lines do | |
| not; candidate y-edges from those rectangles (minus edges that coincide with | |
| real borders, minus edges that would cut an empty band) subdivide each row.""" | |
| table_rect = _refine_table_rect(cells, table_bbox) | |
| if table_rect is None or not cells: | |
| return cells | |
| raw_rects = _refine_raw_shaded_rects(page, table_rect, min_dim=min_dim) | |
| background_rects = _refine_cell_background_rects(raw_rects) | |
| if not background_rects: | |
| return cells | |
| y_edges = _refine_cluster( | |
| [edge for rect in background_rects for edge in (float(rect.y0), float(rect.y1))], | |
| tolerance=cluster_tolerance, | |
| ) | |
| _border_xs, border_ys = _refine_border_lines(page, table_rect) | |
| y_edges = _refine_drop_near(y_edges, border_ys, tolerance=border_tolerance) | |
| if not y_edges: | |
| return cells | |
| words = _refine_page_words(page) | |
| new_cells = [] | |
| added_rows = 0 | |
| for row in cells: | |
| live = [pymupdf.Rect(cell) for cell in row if cell is not None] | |
| if not live: | |
| new_cells.append(row) | |
| continue | |
| ry0 = min(float(cell.y0) for cell in live) | |
| ry1 = max(float(cell.y1) for cell in live) | |
| rx0 = min(float(cell.x0) for cell in live) | |
| rx1 = max(float(cell.x1) for cell in live) | |
| cuts = _refine_content_filter_row_edges(words, ry0, ry1, y_edges, rx0, rx1, margin=margin) | |
| if not cuts: | |
| new_cells.append(row) | |
| continue | |
| bounds = [ry0, *cuts, ry1] | |
| for band_index in range(len(bounds) - 1): | |
| band = [] | |
| for cell in row: | |
| if cell is None: | |
| band.append(None) | |
| else: | |
| rect = pymupdf.Rect(cell) | |
| band.append( | |
| [float(rect.x0), bounds[band_index], float(rect.x1), bounds[band_index + 1]] | |
| ) | |
| new_cells.append(band) | |
| added_rows += len(bounds) - 2 | |
| if added_rows <= 0: | |
| return cells | |
| return new_cells | |
| # --- stage 2: split under-segmented columns ---------------------------------- | |
| def _refine_value_like(text): | |
| return _refine_has_digit_no_alpha(text) | |
| def _refine_span_value_like(text): | |
| return _refine_has_digit(text) | |
| def _refine_cell_span_signal(page, cell, *, gap, line_tolerance): | |
| """True if a cell holds two value-like groups separated by a wide gap on one | |
| text line -- the signal that a single grid column packs several columns.""" | |
| rect = pymupdf.Rect(cell) | |
| words = [] | |
| for x0, y0, x1, y1, text in _refine_words_in_rect(page, rect): | |
| stripped = str(text).strip() | |
| if not stripped: | |
| continue | |
| cx = (float(x0) + float(x1)) / 2.0 | |
| cy = (float(y0) + float(y1)) / 2.0 | |
| if rect.x0 <= cx <= rect.x1 and rect.y0 <= cy <= rect.y1: | |
| words.append((float(x0), float(x1), cy, stripped)) | |
| if len(words) < 2: | |
| return False | |
| words.sort(key=lambda item: item[2]) | |
| lines = [] | |
| current = [words[0]] | |
| for word in words[1:]: | |
| if word[2] - current[-1][2] > line_tolerance: | |
| lines.append(current) | |
| current = [word] | |
| else: | |
| current.append(word) | |
| lines.append(current) | |
| for line in lines: | |
| line.sort(key=lambda item: item[0]) | |
| for index in range(len(line) - 1): | |
| if line[index + 1][0] - line[index][1] <= gap: | |
| continue | |
| left = " ".join(word[3] for word in line[: index + 1]) | |
| right = " ".join(word[3] for word in line[index + 1 :]) | |
| if _refine_span_value_like(left) and _refine_span_value_like(right): | |
| return True | |
| return False | |
| def _refine_detect_span_columns( | |
| page, cells, *, gap=12.0, line_tolerance=4.0, support_ratio=0.3, min_support=2 | |
| ): | |
| """Return the column indices that fire the under-segmentation gate: columns | |
| where enough rows carry the two-value-group signal (a support threshold).""" | |
| rows = len(cells) | |
| support_threshold = max(min_support, int(support_ratio * rows)) | |
| col_hits = {} | |
| for row in cells: | |
| for col_index, cell in enumerate(row): | |
| if cell is None: | |
| continue | |
| if _refine_cell_span_signal(page, cell, gap=gap, line_tolerance=line_tolerance): | |
| col_hits[col_index] = col_hits.get(col_index, 0) + 1 | |
| return sorted(col for col, count in col_hits.items() if count >= support_threshold) | |
| def _refine_column_words(page, cells, col): | |
| rows = [] | |
| for row in cells: | |
| words = [] | |
| if col < len(row) and row[col] is not None: | |
| rect = pymupdf.Rect(row[col]) | |
| for x0, y0, x1, y1, text in _refine_words_in_rect(page, rect): | |
| if not str(text).strip(): | |
| continue | |
| cx = (float(x0) + float(x1)) / 2.0 | |
| cy = (float(y0) + float(y1)) / 2.0 | |
| if rect.x0 <= cx <= rect.x1 and rect.y0 <= cy <= rect.y1: | |
| words.append((float(x0), float(x1), str(text))) | |
| rows.append(sorted(words)) | |
| return rows | |
| def _refine_cut_xs(rows_words, *, bridge_tolerance=0): | |
| intervals = [(x0, x1, row_index) for row_index, words in enumerate(rows_words) for (x0, x1, _) in words] | |
| if len(intervals) < 2: | |
| return [] | |
| lo = min(x0 for x0, _, _ in intervals) | |
| hi = max(x1 for _, x1, _ in intervals) | |
| if hi - lo < 1: | |
| return [] | |
| channels = [] | |
| current = None | |
| x = lo | |
| while x <= hi: | |
| crossing_rows = {row_index for x0, x1, row_index in intervals if x0 < x < x1} | |
| if len(crossing_rows) > bridge_tolerance: | |
| if current is not None: | |
| channels.append((current[0], current[1])) | |
| current = None | |
| else: | |
| current = [x, x] if current is None else [current[0], x] | |
| x += 1.0 | |
| if current is not None: | |
| channels.append((current[0], current[1])) | |
| cuts = [] | |
| for left_edge, right_edge in channels: | |
| if left_edge <= lo + 1 or right_edge >= hi - 1: | |
| continue | |
| cut_x = (left_edge + right_edge) / 2.0 | |
| crossing_count = sum(1 for words in rows_words if any(x0 < cut_x < x1 for x0, x1, _ in words)) | |
| # Evaluate the symmetry/value guards on non-bridging rows only, so a | |
| # merged label spanning value subcolumns does not block column recovery. | |
| body_rows = [words for words in rows_words if not any(x0 < cut_x < x1 for x0, x1, _ in words)] | |
| left_rows = right_rows = 0 | |
| left_ok = right_ok = 0 | |
| left_n = right_n = 0 | |
| left_values = [] | |
| right_values = [] | |
| for words in body_rows: | |
| left = [text for x0, x1, text in words if x1 <= cut_x] | |
| right = [text for x0, x1, text in words if x0 >= cut_x] | |
| if left: | |
| left_rows += 1 | |
| left_n += 1 | |
| left_ok += _refine_value_like(" ".join(left)) | |
| left_values.extend(left) | |
| if right: | |
| right_rows += 1 | |
| right_n += 1 | |
| right_ok += _refine_value_like(" ".join(right)) | |
| right_values.extend(right) | |
| if left_rows < 2 or right_rows < 2: | |
| continue | |
| if crossing_count > 0: | |
| if not _refine_value_like(" ".join(left_values)) or not _refine_value_like(" ".join(right_values)): | |
| continue | |
| elif (left_n and left_ok / left_n < 0.5) or (right_n and right_ok / right_n < 0.5): | |
| continue | |
| cuts.append(cut_x) | |
| return sorted(cuts) | |
| def _refine_split_columns(page, cells, *, bridge_tolerance=0, allowed_cols=None): | |
| ncols = max((len(row) for row in cells), default=0) | |
| active_cols = None if allowed_cols is None else set(allowed_cols) | |
| col_cuts = {} | |
| for col in range(ncols): | |
| if active_cols is not None and col not in active_cols: | |
| continue | |
| cuts = _refine_cut_xs(_refine_column_words(page, cells, col), bridge_tolerance=bridge_tolerance) | |
| if cuts: | |
| col_cuts[col] = cuts | |
| if not col_cuts: | |
| return cells | |
| new_cells = [] | |
| for row in cells: | |
| new_row = [] | |
| for col, cell in enumerate(row): | |
| cuts = col_cuts.get(col) | |
| if not cuts: | |
| new_row.append(cell) | |
| elif cell is None: | |
| new_row.extend([None] * (len(cuts) + 1)) | |
| else: | |
| x0, y0, x1, y1 = pymupdf.Rect(cell) | |
| xs = [x0] + [cut_x for cut_x in cuts if x0 < cut_x < x1] + [x1] | |
| for index in range(len(xs) - 1): | |
| new_row.append([xs[index], y0, xs[index + 1], y1]) | |
| new_cells.append(new_row) | |
| return new_cells | |
| def _split_undersegmented_columns( | |
| page, | |
| cells, | |
| *, | |
| gap=12.0, | |
| line_tolerance=4.0, | |
| support_ratio=0.3, | |
| min_support=2, | |
| bridge_tolerance=0, | |
| ): | |
| """Split under-segmented columns, but only those that fire the value-group | |
| gate -- so ordinary single-value columns are never disturbed.""" | |
| fired_cols = _refine_detect_span_columns( | |
| page, | |
| cells, | |
| gap=gap, | |
| line_tolerance=line_tolerance, | |
| support_ratio=support_ratio, | |
| min_support=min_support, | |
| ) | |
| if not fired_cols: | |
| return cells | |
| return _refine_split_columns( | |
| page, cells, bridge_tolerance=bridge_tolerance, allowed_cols=set(fired_cols) | |
| ) | |
| # --- stage 3: split over-merged body rows ------------------------------------ | |
| def _refine_rects_from_cells(cells): | |
| return [pymupdf.Rect(cell) for row in cells for cell in row if cell is not None] | |
| def _refine_union_rect(rects): | |
| if not rects: | |
| return None | |
| rect = pymupdf.Rect(rects[0]) | |
| for item in rects[1:]: | |
| rect.include_rect(item) | |
| return rect | |
| def _refine_best_column_bounds(cells): | |
| best_row = max( | |
| cells, | |
| key=lambda row: (sum(1 for cell in row if cell is not None), len(row)), | |
| default=[], | |
| ) | |
| bounds = sorted( | |
| (float(rect.x0), float(rect.x1)) | |
| for cell in best_row | |
| if cell is not None | |
| for rect in [pymupdf.Rect(cell)] | |
| if rect.x1 > rect.x0 | |
| ) | |
| return bounds | |
| def _refine_cluster_words_by_y(words): | |
| """Group center-point words into body "lines" by a fixed center-y gap, | |
| returning each line's union bbox and joined text.""" | |
| if not words: | |
| return [] | |
| words = sorted(words, key=lambda item: ((item[1] + item[3]) / 2.0, item[0])) | |
| clusters = [[words[0]]] | |
| last_center = (words[0][1] + words[0][3]) / 2.0 | |
| for word in words[1:]: | |
| center = (word[1] + word[3]) / 2.0 | |
| if center - last_center > _REFINE_LINE_GAP: | |
| clusters.append([word]) | |
| else: | |
| clusters[-1].append(word) | |
| last_center = center | |
| lines = [] | |
| for cluster in clusters: | |
| lines.append( | |
| { | |
| "x0": min(word[0] for word in cluster), | |
| "y0": min(word[1] for word in cluster), | |
| "x1": max(word[2] for word in cluster), | |
| "y1": max(word[3] for word in cluster), | |
| "text": " ".join(word[4] for word in sorted(cluster, key=lambda item: item[0])), | |
| } | |
| ) | |
| return lines | |
| def _refine_line_columns(line, col_bounds): | |
| cols = set() | |
| line_x0 = float(line["x0"]) | |
| line_x1 = float(line["x1"]) | |
| center = (line_x0 + line_x1) / 2.0 | |
| for index, (x0, x1) in enumerate(col_bounds): | |
| overlap = min(line_x1, x1) - max(line_x0, x0) | |
| if overlap > 0 or x0 <= center <= x1: | |
| cols.add(index) | |
| return cols | |
| def _refine_page_words_in_rect(page, rect): | |
| words = [] | |
| rx0, ry0, rx1, ry1 = float(rect.x0), float(rect.y0), float(rect.x1), float(rect.y1) | |
| for x0, y0, x1, y1, text in _refine_page_words(page): | |
| cx = (x0 + x1) * 0.5 | |
| cy = (y0 + y1) * 0.5 | |
| if rx0 <= cx <= rx1 and ry0 <= cy <= ry1: | |
| text = str(text).strip() | |
| if text: | |
| words.append((float(x0), float(y0), float(x1), float(y1), text)) | |
| return words | |
| def _refine_merge_overlapping_lines(lines, *, frac): | |
| if not lines: | |
| return [] | |
| items = sorted(lines) | |
| merged = [[float(items[0][0]), float(items[0][1])]] | |
| for y0, y1 in items[1:]: | |
| current = merged[-1] | |
| overlap = current[1] - float(y0) | |
| min_height = min(current[1] - current[0], float(y1) - float(y0)) | |
| if min_height > 0 and overlap / min_height >= frac: | |
| current[0] = min(current[0], float(y0)) | |
| current[1] = max(current[1], float(y1)) | |
| else: | |
| merged.append([float(y0), float(y1)]) | |
| return [(y0, y1) for y0, y1 in merged] | |
| def _refine_detect_row_overmerge(page, cells, *, clean_threshold=0.85, header_row_count=1): | |
| """Decide whether the body rows are over-merged. Returns the geometry needed | |
| to re-cut them (body bbox, column bounds, per-record y-bounds) or None. | |
| Fires only when the body text clusters into clean multi-column "record" lines | |
| (clean_ratio >= clean_threshold) and there are more records than existing body | |
| rows -- i.e. several records collapsed into one grid row.""" | |
| col_bounds = _refine_best_column_bounds(cells) | |
| header_row_count = max(1, min(int(header_row_count), len(cells))) if cells else 0 | |
| existing_body_rows = max(0, len(cells) - header_row_count) | |
| if len(cells) <= header_row_count: | |
| return None | |
| if len(col_bounds) < 2: | |
| return None | |
| body_rect = _refine_union_rect(_refine_rects_from_cells(cells[header_row_count:])) | |
| if body_rect is None or body_rect.is_empty: | |
| return None | |
| words = _refine_page_words_in_rect(page, body_rect) | |
| lines = _refine_cluster_words_by_y(words) | |
| if not lines: | |
| return None | |
| min_cols = max(2, (len(col_bounds) + 1) // 2) # == max(2, ceil(len/2)) | |
| record_lines = [] | |
| for line in lines: | |
| cols = sorted(_refine_line_columns(line, col_bounds)) | |
| if len(cols) >= min_cols: | |
| record_lines.append(line) | |
| clean_ratio = len(record_lines) / len(lines) | |
| if clean_ratio < clean_threshold: | |
| return None | |
| if len(record_lines) <= existing_body_rows: | |
| return None | |
| return { | |
| "body_bbox": [body_rect.x0, body_rect.y0, body_rect.x1, body_rect.y1], | |
| "col_bounds": [[round(x0, 2), round(x1, 2)] for x0, x1 in col_bounds], | |
| "record_line_bounds": [[float(line["y0"]), float(line["y1"])] for line in record_lines], | |
| "existing_body_rows": existing_body_rows, | |
| "header_row_count": header_row_count, | |
| } | |
| def _split_overmerged_rows( | |
| page, cells, *, clean_threshold=0.85, merge_overlap_frac=0.35, header_row_count=1 | |
| ): | |
| """Re-cut over-merged body rows into one grid row per detected record. | |
| Keeps the header rows intact, then rebuilds the body as evenly-bounded rows | |
| (cut at the midpoints between consecutive record centers) across the detected | |
| column bounds. header_row_count is the number of leading header rows to keep.""" | |
| meta = _refine_detect_row_overmerge( | |
| page, cells, clean_threshold=clean_threshold, header_row_count=header_row_count | |
| ) | |
| if meta is None: | |
| return cells | |
| body_bbox = pymupdf.Rect(meta["body_bbox"]) | |
| col_bounds = [(float(x0), float(x1)) for x0, x1 in meta["col_bounds"]] | |
| raw_line_bounds = [(float(y0), float(y1)) for y0, y1 in meta["record_line_bounds"]] | |
| line_bounds = _refine_merge_overlapping_lines(raw_line_bounds, frac=merge_overlap_frac) | |
| if len(line_bounds) <= int(meta["existing_body_rows"]): | |
| return cells | |
| centers = [(y0 + y1) / 2.0 for y0, y1 in line_bounds] | |
| edges = [float(body_bbox.y0)] | |
| for prev, nxt in zip(centers, centers[1:]): | |
| edges.append((prev + nxt) / 2.0) | |
| edges.append(float(body_bbox.y1)) | |
| header_rows = cells[: int(meta["header_row_count"])] | |
| new_body = [] | |
| for row_index in range(len(line_bounds)): | |
| y0 = edges[row_index] | |
| y1 = edges[row_index + 1] | |
| new_row = [] | |
| for x0, x1 in col_bounds: | |
| rect = pymupdf.Rect(x0, y0, x1, y1) | |
| new_row.append([float(rect.x0), float(rect.y0), float(rect.x1), float(rect.y1)]) | |
| new_body.append(new_row) | |
| return [*header_rows, *new_body] | |
| # --- public seam ------------------------------------------------------------- | |
| def refine_grid_structure(page, cells, *, table_bbox=None): | |
| """Structural half of refine_grid: split shaded rows, then under-segmented | |
| columns. Exposed separately so a caller that must know the header/body | |
| boundary of the post-structure grid can insert that computation between the | |
| two phases. table_bbox, when given, bounds the shaded-rectangle search; | |
| otherwise the cells' union is used.""" | |
| cells = _split_shaded_rows(page, cells, table_bbox) | |
| cells = _split_undersegmented_columns(page, cells) | |
| return cells | |
| def refine_grid_rows(page, cells, *, header_row_count=1, clean_threshold=0.85, merge_overlap_frac=0.35): | |
| """Row half of refine_grid: split body rows that collapsed several records | |
| into one grid row. header_row_count is the number of leading header rows to | |
| keep intact; callers that resolve a header region pass it in, and the default | |
| of 1 is a conservative single-header assumption.""" | |
| return _split_overmerged_rows( | |
| page, | |
| cells, | |
| clean_threshold=clean_threshold, | |
| merge_overlap_frac=merge_overlap_frac, | |
| header_row_count=header_row_count, | |
| ) | |
| def refine_grid(page, cells, *, table_bbox=None, header_row_count=1): | |
| """Refine a detected table's cell grid and return the refined grid. | |
| ``cells`` is a row-major grid: a list of rows, each a list of ``[x0, y0, x1, | |
| y1]`` cell rectangles (``None`` for a gap). The grid is refined in three | |
| stages -- split rows that cell background shading separates, split columns | |
| that jam several values into one cell, and split body rows that merged | |
| several records -- using the page's words and vector graphics. The result is | |
| a new grid in the same format (rows may be added and rows may widen). | |
| ``table_bbox`` optionally bounds the table (else the cells' union is used). | |
| ``header_row_count`` is the number of leading header rows to preserve when | |
| re-cutting body rows (default 1). This is the all-in-one convenience wrapper; | |
| a caller needing the header boundary of the intermediate grid can instead | |
| call :func:`refine_grid_structure` then :func:`refine_grid_rows`. | |
| """ | |
| cells = refine_grid_structure(page, cells, table_bbox=table_bbox) | |
| cells = refine_grid_rows(page, cells, header_row_count=header_row_count) | |
| return cells | |
| def _refine_cells_to_grid(cells): | |
| """Convert a Table's flat cell list into the row-major grid refine_grid wants. | |
| Mirrors Table.rows: columns are the sorted distinct left edges, rows are the | |
| cells grouped by top edge, each padded with None where a column has no cell.""" | |
| if not cells: | |
| return [] | |
| xs = sorted({c[0] for c in cells}) | |
| col_index = {x: i for i, x in enumerate(xs)} | |
| grid = [] | |
| ordered = sorted(cells, key=lambda c: (c[1], c[0])) | |
| for _y, row_cells in itertools.groupby(ordered, key=lambda c: c[1]): | |
| row = [None] * len(xs) | |
| for cell in row_cells: | |
| row[col_index[cell[0]]] = [float(cell[0]), float(cell[1]), float(cell[2]), float(cell[3])] | |
| grid.append(row) | |
| return grid | |
| def _refine_grid_to_cells(grid): | |
| """Flatten a refined grid back to a Table's flat (x0, y0, x1, y1) cell list.""" | |
| return [ | |
| (float(cell[0]), float(cell[1]), float(cell[2]), float(cell[3])) | |
| for row in grid | |
| for cell in row | |
| if cell is not None | |
| ] | |