Spaces:
Running on Zero
Running on Zero
zhiyi commited on
Commit ·
a8a3c84
1
Parent(s): de12820
Make deep report validation semantically tolerant
Browse files- app.py +19 -17
- tests/test_app.py +9 -0
app.py
CHANGED
|
@@ -109,14 +109,10 @@ class ReportSection(BaseModel):
|
|
| 109 |
raise ValueError("comparison content requires columns and at least two rows")
|
| 110 |
if self.layout == "comparison" and not all(isinstance(row, dict) for row in content.get("rows", [])):
|
| 111 |
raise ValueError("comparison rows must be objects")
|
| 112 |
-
if self.layout
|
| 113 |
-
content.
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
if self.layout == "evidence-gap" and not all(
|
| 117 |
-
content.get(key) for key in ("interpretation", "hypotheses", "required_inputs")
|
| 118 |
-
):
|
| 119 |
-
raise ValueError("evidence-gap requires interpretation, hypotheses, and required_inputs")
|
| 120 |
return self
|
| 121 |
|
| 122 |
|
|
@@ -666,7 +662,8 @@ def build_report(
|
|
| 666 |
"{columns:[...],rows:[{name,values:[...]}]}; process uses {steps:[{name,description}]} or "
|
| 667 |
"{common_steps:[...],branches:[{name,steps:[...]}]}; equation uses {equation,symbols,interpretation}; "
|
| 668 |
"paper-analysis uses {method,evidence,limitations}; evidence-gap uses "
|
| 669 |
-
"{interpretation,related_topics,hypotheses,required_inputs}.
|
|
|
|
| 670 |
"section, concise points, explicit assumptions, comparable dimensions, and no repeated background. Distinguish "
|
| 671 |
"retrieved facts from analysis or hypotheses. Never invent sources, papers, metrics, or recency claims."
|
| 672 |
)
|
|
@@ -694,14 +691,17 @@ def build_report(
|
|
| 694 |
4200,
|
| 695 |
)
|
| 696 |
sections_result = align_sections_to_plan(sections_result, batch)
|
| 697 |
-
|
| 698 |
-
|
| 699 |
-
|
| 700 |
-
|
| 701 |
-
|
| 702 |
-
|
| 703 |
-
|
| 704 |
-
|
|
|
|
|
|
|
|
|
|
| 705 |
recovered_sections: list[ReportSection] = []
|
| 706 |
for plan in batch:
|
| 707 |
one_result = chat_json(
|
|
@@ -800,6 +800,8 @@ def field_label(key: Any) -> str:
|
|
| 800 |
"method": "方法", "mechanism": "机制", "problem": "问题", "innovation": "创新",
|
| 801 |
"tradeoffs": "权衡", "interpretation": "解释", "related_topics": "相关主题",
|
| 802 |
"hypotheses": "待验证假设", "required_inputs": "所需补充信息", "findings": "发现",
|
|
|
|
|
|
|
| 803 |
}
|
| 804 |
normalized = str(key).strip().lower().replace("-", "_")
|
| 805 |
return labels.get(normalized, str(key).replace("_", " ").replace("-", " ").strip().title())
|
|
|
|
| 109 |
raise ValueError("comparison content requires columns and at least two rows")
|
| 110 |
if self.layout == "comparison" and not all(isinstance(row, dict) for row in content.get("rows", [])):
|
| 111 |
raise ValueError("comparison rows must be objects")
|
| 112 |
+
if self.layout in {"paper-analysis", "evidence-gap"}:
|
| 113 |
+
meaningful_blocks = [value for value in content.values() if value not in (None, "", [], {})]
|
| 114 |
+
if len(meaningful_blocks) < 2:
|
| 115 |
+
raise ValueError(f"{self.layout} requires at least two substantive information blocks")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 116 |
return self
|
| 117 |
|
| 118 |
|
|
|
|
| 662 |
"{columns:[...],rows:[{name,values:[...]}]}; process uses {steps:[{name,description}]} or "
|
| 663 |
"{common_steps:[...],branches:[{name,steps:[...]}]}; equation uses {equation,symbols,interpretation}; "
|
| 664 |
"paper-analysis uses {method,evidence,limitations}; evidence-gap uses "
|
| 665 |
+
"{interpretation,related_topics,hypotheses,required_inputs}. Equivalent topic-appropriate field names are allowed; "
|
| 666 |
+
"never add empty placeholder fields merely to match an example. Write for slides, not an essay: one conclusion per "
|
| 667 |
"section, concise points, explicit assumptions, comparable dimensions, and no repeated background. Distinguish "
|
| 668 |
"retrieved facts from analysis or hypotheses. Never invent sources, papers, metrics, or recency claims."
|
| 669 |
)
|
|
|
|
| 691 |
4200,
|
| 692 |
)
|
| 693 |
sections_result = align_sections_to_plan(sections_result, batch)
|
| 694 |
+
try:
|
| 695 |
+
validated_batch = validate_stage_with_repairs(
|
| 696 |
+
sections_result,
|
| 697 |
+
SectionsEnvelope,
|
| 698 |
+
f"report sections batch {batch_index // 4 + 1}",
|
| 699 |
+
{**shared_context, "approved_section_plan": [item.model_dump() for item in batch]},
|
| 700 |
+
4200,
|
| 701 |
+
)
|
| 702 |
+
except RuntimeError:
|
| 703 |
+
validated_batch = None
|
| 704 |
+
if validated_batch is None or len(validated_batch.sections) != len(batch):
|
| 705 |
recovered_sections: list[ReportSection] = []
|
| 706 |
for plan in batch:
|
| 707 |
one_result = chat_json(
|
|
|
|
| 800 |
"method": "方法", "mechanism": "机制", "problem": "问题", "innovation": "创新",
|
| 801 |
"tradeoffs": "权衡", "interpretation": "解释", "related_topics": "相关主题",
|
| 802 |
"hypotheses": "待验证假设", "required_inputs": "所需补充信息", "findings": "发现",
|
| 803 |
+
"known_evidence": "已有证据", "unknowns": "证据缺口", "next_steps": "下一步验证",
|
| 804 |
+
"open_questions": "待回答问题", "method_summary": "方法概述", "experimental_results": "实验结果",
|
| 805 |
}
|
| 806 |
normalized = str(key).strip().lower().replace("-", "_")
|
| 807 |
return labels.get(normalized, str(key).replace("_", " ").replace("-", " ").strip().title())
|
tests/test_app.py
CHANGED
|
@@ -117,6 +117,15 @@ class AppTests(unittest.TestCase):
|
|
| 117 |
section = {"title": "Evidence gap", "purpose": "Separate known facts from hypotheses.", "takeaway": "More evidence is required before choosing a method.", "layout": "evidence-gap", "content": {"interpretation": "The term may be ambiguous.", "related_topics": ["Adjacent field"], "hypotheses": ["Possible interpretation"], "required_inputs": ["Definition or internal data"]}, "source_ids": []}
|
| 118 |
self.assertEqual(ResearchReport.model_validate(base_report(section)).sections[0].layout, "evidence-gap")
|
| 119 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 120 |
def test_invalid_empty_section(self):
|
| 121 |
section = {"title": "Empty", "purpose": "This section should fail validation.", "takeaway": "There is no substantive supporting content.", "layout": "text", "content": {}, "source_ids": []}
|
| 122 |
with self.assertRaises(ValidationError):
|
|
|
|
| 117 |
section = {"title": "Evidence gap", "purpose": "Separate known facts from hypotheses.", "takeaway": "More evidence is required before choosing a method.", "layout": "evidence-gap", "content": {"interpretation": "The term may be ambiguous.", "related_topics": ["Adjacent field"], "hypotheses": ["Possible interpretation"], "required_inputs": ["Definition or internal data"]}, "source_ids": []}
|
| 118 |
self.assertEqual(ResearchReport.model_validate(base_report(section)).sections[0].layout, "evidence-gap")
|
| 119 |
|
| 120 |
+
def test_evidence_gap_accepts_equivalent_semantic_fields(self):
|
| 121 |
+
section = {"title": "Evidence gap", "purpose": "Separate available evidence from unresolved questions.", "takeaway": "Deployment requires evidence that is not yet public.", "layout": "evidence-gap", "content": {"known_evidence": ["A related result exists"], "unknowns": ["No direct benchmark"], "next_steps": ["Run an internal evaluation"]}, "source_ids": []}
|
| 122 |
+
validated = ResearchReport.model_validate(base_report(section))
|
| 123 |
+
self.assertEqual(validated.sections[0].content["unknowns"], ["No direct benchmark"])
|
| 124 |
+
|
| 125 |
+
def test_paper_analysis_accepts_equivalent_semantic_fields(self):
|
| 126 |
+
section = {"title": "Paper analysis", "purpose": "Analyze a paper without enforcing field spelling.", "takeaway": "The result is useful under a bounded condition.", "layout": "paper-analysis", "content": {"method_summary": "A compact architecture", "experimental_results": ["A controlled comparison"], "caveats": ["A narrow dataset"]}, "source_ids": []}
|
| 127 |
+
self.assertEqual(ResearchReport.model_validate(base_report(section)).sections[0].layout, "paper-analysis")
|
| 128 |
+
|
| 129 |
def test_invalid_empty_section(self):
|
| 130 |
section = {"title": "Empty", "purpose": "This section should fail validation.", "takeaway": "There is no substantive supporting content.", "layout": "text", "content": {}, "source_ids": []}
|
| 131 |
with self.assertRaises(ValidationError):
|