Spaces:
Running
Running
File size: 4,472 Bytes
71276a4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 | """
visual_explainer.py — Caption/table understanding for Paper2Lab.
This is caption-grounded visual understanding. It does not use image pixels yet.
For the hackathon MVP, this gives useful figure/table summaries without GPU cost.
"""
from __future__ import annotations
import re
from typing import Any, Dict, List
_METRIC_WORDS = [
"accuracy", "precision", "recall", "f1", "auc", "bleu", "rouge", "loss", "perplexity",
"score", "performance", "results", "comparison", "evaluation", "training cost", "p-value",
]
_METHOD_WORDS = [
"architecture", "pipeline", "framework", "workflow", "model", "method", "procedure",
"attention", "encoder", "decoder", "algorithm", "overview",
]
_DATA_WORDS = [
"dataset", "data", "samples", "patients", "images", "sentences", "articles", "studies",
"distribution", "statistics", "characteristics",
]
def _clean(text: str) -> str:
text = text or ""
text = re.sub(r"\s+", " ", text)
return text.strip(" .;:\n\t")
def _label_type(label: str) -> str:
low = (label or "").lower()
if "table" in low or "tbl" in low:
return "table"
if "figure" in low or "fig" in low:
return "figure"
if "algorithm" in low:
return "algorithm"
if "scheme" in low:
return "scheme"
return "visual"
def _purpose(caption: str, visual_type: str) -> str:
low = caption.lower()
if any(w in low for w in _METHOD_WORDS):
return "method_or_architecture"
if any(w in low for w in _METRIC_WORDS):
return "results_or_evaluation"
if any(w in low for w in _DATA_WORDS):
return "data_or_dataset_description"
if visual_type == "table":
return "structured_results_or_metadata"
return "illustrative_figure"
def _summary_from_caption(label: str, caption: str, visual_type: str) -> str:
caption = _clean(caption)
if not caption:
return f"{label} is a {visual_type}, but no caption text was extracted."
# Keep concise, but grounded in caption text.
if len(caption) <= 220:
return caption
first_sentence = re.split(r"(?<=[.!?])\s+", caption)[0]
return _clean(first_sentence[:260])
def _summarize_table_data(table: Dict[str, Any]) -> str | None:
data = table.get("data")
if not isinstance(data, list) or not data:
return None
rows = [r for r in data if isinstance(r, list)]
if not rows:
return None
n_rows = len(rows)
n_cols = max((len(r) for r in rows), default=0)
header = rows[0] if rows else []
header_text = ", ".join(str(x).strip() for x in header if str(x).strip())[:180]
if header_text:
return f"Extracted table with approximately {n_rows} rows and {n_cols} columns. Header fields include: {header_text}."
return f"Extracted table with approximately {n_rows} rows and {n_cols} columns."
def explain_figures_and_tables(extracted: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Return concise explanations for extracted captions and tables."""
outputs: List[Dict[str, Any]] = []
for cap in extracted.get("captions", []) or []:
label = cap.get("label", "")
caption = _clean(cap.get("caption", ""))
visual_type = _label_type(label)
outputs.append({
"label": label,
"type": visual_type,
"purpose": _purpose(caption, visual_type),
"summary": _summary_from_caption(label, caption, visual_type),
"evidence": caption,
"page_number": cap.get("page_number"),
})
# Add tables that have data but no caption match.
existing_table_pages = {(o.get("page_number"), o.get("label")) for o in outputs if o.get("type") == "table"}
for table in extracted.get("tables", []) or []:
page = table.get("page_number")
caption = _clean(table.get("caption") or "")
label = f"Table extracted on page {page}" if page is not None else "Extracted table"
if (page, label) in existing_table_pages:
continue
data_summary = _summarize_table_data(table)
outputs.append({
"label": label,
"type": "table",
"purpose": _purpose(caption or data_summary or "", "table"),
"summary": caption or data_summary or "A table was detected, but its content could not be summarized reliably.",
"evidence": caption or data_summary or "",
"page_number": page,
})
return outputs[:20]
|