Upload 7 files

backend/media_analysis/figure_analysis.py

@@ -158,6 +158,12 @@ def analyze_media_question(question: str, selected_papers: list[dict], *, enable
         records, normalization_warnings = normalize_records(records)
         result["structured_data"] = records
         result["normalization_warnings"] = normalization_warnings
+        from .record_checks import validate_extraction
+        records, validation_warnings, coverage = validate_extraction(records, items if evidence_mode != "vision" else [])
+        result["structured_data"] = records
+        result["validation_warnings"] = validation_warnings
+        result["extraction_coverage"] = coverage
+
         result["csv_attachment"] = csv_attachment(records, result.get("sources") or [])
         if warning:
             result["export_warning"] = warning

backend/media_analysis/record_checks.py

@@ -0,0 +1,80 @@
+"""Non-destructive checks for model-produced tabular rows; warnings are not corrections."""
+from __future__ import annotations
+import re
+from collections import Counter
+
+_CLASS = re.compile(r"\b(Class\s+(?:I{1,3}|IV|V|[1-9]\d*))\b", re.I)
+
+def validate_extraction(records: list[dict], table_evidence: list[dict] | None = None) -> tuple[list[dict], list[str], dict]:
+    """Return standardized copies plus specific warnings and observable coverage.
+
+    Does not invent missing values or signs and does not treat model notes as verification.
+    """
+    output=[]; warnings=[]
+    for i, row in enumerate(records, 1):
+        r=dict(row)
+        parameter=str(r.get('parameter') or '').strip()
+        if not parameter:
+            warnings.append(f'Record {i}: missing parameter')
+        std=r.get('standard_range')
+        label=r.get('class_label')
+        if isinstance(std,str):
+            matches=list(_CLASS.finditer(std))
+            if matches:
+                inferred=matches[0].group(1)
+                if not label:
+                    r['class_label']=inferred
+                elif re.sub(r'\s+',' ',str(label).strip()).casefold()!=inferred.casefold():
+                    warnings.append(f'Record {i} ({parameter}): class_label conflicts with standard_range')
+                remainder=_CLASS.sub('',std).strip(' ;,:')
+                if remainder:
+                    r['standard_range']=remainder
+                else:
+                    warnings.append(f'Record {i} ({parameter}): standard_range only contained a class label')
+        if isinstance(r.get('source_quality'),str) and len(r['source_quality'])>100:
+            # Never delete model-provided evidence notes; keep intact, downstream presentation may summarize.
+            pass
+        output.append(r)
+    # Detect omissions only if a source contains a word at the percentage x-position
+    # on the same parameter row; source row layout may be complex, so report as review.
+    known_source = {}
+    for ev in table_evidence or []:
+        data=ev.get('table_data') or {}
+        for row in data.get('text_rows') or []:
+            words=row.get('positioned_words') or row.get('words') or []
+            text=str(row.get('text') or '')
+            for w in words:
+                try: x=float(w[1] if isinstance(w,list) else w.get('x',w.get('bbox_points',[None])[0]))
+                except (TypeError,ValueError,IndexError):continue
+                token=str(w[0] if isinstance(w,list) else w.get('text') or '')
+                if 442<=x<=460 and re.fullmatch(r'(?:\d+(?:\.\d+)?%?|0)',token):
+                    known_source.setdefault('percent_rows',[]).append((text,token))
+    # Contextual null is valid for blank printed cells; never auto-fill null as zero.
+    counts=Counter(str(r.get('parameter') or '') for r in output)
+    for i,r in enumerate(output,1):
+        if r.get('classification') and r.get('percent_samples') is None:
+            warnings.append(f'Record {i} ({r.get("parameter")}, {r.get("classification")}): percentage is null; check whether source cell is blank or extraction omitted it')
+        if r.get('standard_range') is None and r.get('classification'):
+            warnings.append(f'Record {i} ({r.get("parameter")}): classification has no standard_range')
+    # Check parameter coverage from first-column starts in positional data.
+    source_names=[]
+    for ev in table_evidence or []:
+        data=ev.get('table_data') or {}
+        for row in data.get('text_rows') or []:
+            words=row.get('positioned_words') or row.get('words') or []
+            if not words: continue
+            first=words[0]
+            try:x=float(first[1] if isinstance(first,list) else first.get('x',first.get('bbox_points',[None])[0]))
+            except (TypeError,ValueError,IndexError):continue
+            token=str(first[0] if isinstance(first,list) else first.get('text') or '')
+            if 55<=x<=85 and token and token.casefold() not in ('parameters','sample') and token not in source_names:
+                source_names.append(token)
+    present={x.casefold().rstrip('%') for x in counts if x}
+    missing=[n for n in source_names if n.casefold().rstrip('%') not in present]
+    if missing:
+        warnings.append('Possible missing source parameters (positional heuristic): '+', '.join(missing))
+    summary={'record_count':len(output),'parameter_count':len(counts)-int('' in counts),
+             'source_parameter_candidates':source_names,'possibly_missing_parameters':missing,
+             'null_percentage_count':sum(r.get('classification') is not None and r.get('percent_samples') is None for r in output),
+             'warning_count':len(warnings)}
+    return output,warnings,summary

backend/media_analysis/vision_client.py

@@ -125,7 +125,10 @@ def analyze_images(question: str, evidence: list[dict], *, model: str | None = N
             "Use stable field names: parameter, unit, minimum, maximum, mean, std, "
             "standard_range, class_label, classification, percent_samples, references, "
             "estimated (boolean), extraction_method (printed_table|positioned_text|visual_estimate), "
-            "and source_quality. Preserve original printed parameter names, units, inequality "
+            "and source_quality. Keep source_quality brief (e.g., image_checked, ambiguous, unverified). "
+            "For class labels (Class I, II, III), populate class_label separately and "
+            "exclude the class name from standard_range. Never guess blank percentages as zero. "
+            "Preserve original printed parameter names, units, inequality "
             "signs, and numeric precision. Numeric values must be JSON numbers (no thousands "
             "separators); use *_printed fields if preservation of original formatting matters. "
             "For ambiguous OCR signs, compare against image; if still unclear use null and "