Upload 6 files

backend/media_analysis/data_export.py

@@ -12,6 +12,14 @@ import json
 import re
 
 
+# Only accepted tabular fields become output columns; other fields may be kept
+# for backward compatibility with earlier full-record responses.
+TABLE_COLUMNS = ["parameter", "unit", "minimum", "maximum", "mean", "std",
+                 "standard_range", "class_label", "classification",
+                 "percent_samples", "references"]
+CHART_COLUMNS = ["point_id", "x", "y", "unit_x", "unit_y", "estimated"]
+
+
 def decode_structured_answer(answer: str) -> tuple[str, list[dict], str | None]:
     s = (answer or "").strip()
     if s.startswith("```"):
@@ -20,13 +28,27 @@ def decode_structured_answer(answer: str) -> tuple[str, list[dict], str | None]:
     try:
         obj = json.loads(s)
     except (ValueError, TypeError):
-        return answer, [], "Model did not produce machine-readable JSON; CSV not generated"
-    if not isinstance(obj, dict) or not isinstance(obj.get("records"), list):
-        return answer, [], "Model returned no structured records; CSV not generated"
-    rows = obj["records"]
+        return answer, [], "Model output was not complete JSON; CSV not generated"
+    if not isinstance(obj, dict):
+        return answer, [], "Model output must be a JSON object"
+    readable = str(obj.get("answer") or "")
+    if isinstance(obj.get("columns"), list) and isinstance(obj.get("rows"), list):
+        columns = obj["columns"]
+        if columns not in (TABLE_COLUMNS, CHART_COLUMNS):
+            return readable, [], "Unsupported or incorrectly ordered data columns; CSV not generated"
+        data_rows = obj["rows"]
+        if len(data_rows) > 250 or any(not isinstance(r, list) or len(r) != len(columns) for r in data_rows):
+            return readable, [], "Invalid row length or record count; CSV not generated"
+        if any(any(not isinstance(v, (str, int, float, bool, type(None))) for v in row) for row in data_rows):
+            return readable, [], "Nested values in data rows; CSV not generated"
+        return readable, [dict(zip(columns, r)) for r in data_rows], None
+    # Legacy full-record schema remains supported for existing clients.
+    rows = obj.get("records")
+    if not isinstance(rows, list):
+        return readable, [], "No structured records or compact rows; CSV not generated"
     if len(rows) > 250 or any(not isinstance(r,dict) for r in rows):
-        return str(obj.get("answer") or ""), [], "Invalid record structure"
-    return str(obj.get("answer") or ""), rows, None
+        return readable, [], "Invalid legacy record structure"
+    return readable, rows, None
 
 
 NUMERIC_FIELDS = {"minimum", "maximum", "mean", "std", "value", "x", "y", "percent_value"}
@@ -81,7 +103,9 @@ def csv_attachment(records: list[dict], sources: list[dict]) -> dict | None:
                 allowed.add(key)
     if not allowed:
         return None
-    fields = sorted(allowed - reserved) + sorted(reserved)
+    preferred = TABLE_COLUMNS + CHART_COLUMNS
+    fields = [x for x in preferred if x in allowed] + sorted(allowed - reserved - set(preferred)) + [
+        "paper_id", "paper_title", "doi", "paper_key", "media_label", "page", "extraction_method", "estimated", "source_quality"]
     output = io.StringIO(newline="")
     writer = csv.DictWriter(output, fieldnames=fields, extrasaction="ignore")
     writer.writeheader()
@@ -92,7 +116,9 @@ def csv_attachment(records: list[dict], sources: list[dict]) -> dict | None:
             for k, source_key in [("paper_id","paper_id"),("paper_title","paper_title"),("doi","doi"),("paper_key","paper_key"),("media_label","label"),("page","page")]:
                 row[k] = src.get(source_key) or ""
         # Never silently claim exactness for unverified positioned text.
-        row.setdefault("extraction_method", "model_assisted")
+        row.setdefault("extraction_method", "printed_table_transcription" if "parameter" in rec else "model_assisted")
+        row.setdefault("estimated", False if "parameter" in rec else "")
+        row.setdefault("source_quality", "model_extracted_unverified")
         writer.writerow({k:("'"+v if isinstance(v,str) and v[:1] in ("=","+","-","@") else v)
                          for k,v in row.items()})
     data=output.getvalue().encode("utf-8-sig")

backend/media_analysis/figure_analysis.py

@@ -154,7 +154,10 @@ def analyze_media_question(question: str, selected_papers: list[dict], *, enable
     if export_csv:
         from .data_export import decode_structured_answer, csv_attachment, normalize_records
         answer, records, warning = decode_structured_answer(result.get("answer", ""))
-        result["answer"] = answer
+        result["answer"] = answer if not warning else (
+            "Structured extraction did not complete; no downloadable CSV was generated. "
+            "Please retry or narrow the request."
+        )
         records, normalization_warnings = normalize_records(records)
         result["structured_data"] = records
         result["normalization_warnings"] = normalization_warnings

backend/media_analysis/vision_client.py

@@ -116,22 +116,26 @@ def analyze_images(question: str, evidence: list[dict], *, model: str | None = N
     )
     if structured_output:
         instructions += (
-            " Return ONLY valid JSON with keys answer (brief readable summary) and records "
-            "(array containing exactly the requested extracted rows). For a request to extract "
-            "all data, include every parameter and every classification subrow, standard range, "
-            "percentage, reference, units and descriptive statistics, without silently omitting "
-            "rows. In repeated subrows, do not invent or repeat descriptive statistics that "
-            "occur only in the first source row: use null. "
-            "Use stable field names: parameter, unit, minimum, maximum, mean, std, "
-            "standard_range, class_label, classification, percent_samples, references, "
-            "estimated (boolean), extraction_method (printed_table|positioned_text|visual_estimate), "
-            "and source_quality. Preserve original printed parameter names, units, inequality "
-            "signs, and numeric precision. Numeric values must be JSON numbers (no thousands "
-            "separators); use *_printed fields if preservation of original formatting matters. "
-            "For ambiguous OCR signs, compare against image; if still unclear use null and "
-            "explain ambiguity. Source-quality is descriptive, never a proof of verification. "
-            "When extracting multiple sources include paper_id and media_label per record. "
-            "The answer must not claim a complete extraction if rows are omitted."
+            " For downloadable data extraction return ONLY one compact valid JSON object with "
+            "keys 'answer', 'columns', and 'rows'. 'answer' MUST be one short sentence, "
+            "NOT a reproduction of the table. 'columns' MUST be exactly "
+            "['parameter','unit','minimum','maximum','mean','std','standard_range',"
+            "'class_label','classification','percent_samples','references']. "
+            "'rows' MUST be an array of positional arrays following those columns, with "
+            "one array per ACTUAL classification/data subrow in source order. "
+            "Preserve ALL requested cells, parameters, subrows, units, percentage values, "
+            "inequalities and reference text. For stats/units/reference spanning multiple "
+            "subrows, give the value only on its printed first row and null thereafter. "
+            "Use null for genuinely blank/unreadable cells; never infer a missing percentage. "
+            "Numbers must be JSON numbers with no thousands separators, preserving signs. "
+            "Class I/II/III goes in class_label, not standard_range. "
+            "Print-table transcriptions are NOT visual estimates. Do not add repeated "
+            "paper IDs, sources, confidence descriptions, provenance, CSV text, or any "
+            "other fields to rows: the server adds metadata and generates the CSV. "
+            "Do NOT include Markdown fences or a duplicate CSV in the answer. "
+            "If source is an image plot rather than a table, use columns "
+            "['point_id','x','y','unit_x','unit_y','estimated'] and data rows; "
+            "numerical visual estimates must have estimated=true."
         )
     content: list[dict] = [{"type": "input_text", "text": f"Question: {question.strip()}"}]
     used = []