zhiyi commited on
Commit
a8a3c84
·
1 Parent(s): de12820

Make deep report validation semantically tolerant

Browse files
Files changed (2) hide show
  1. app.py +19 -17
  2. tests/test_app.py +9 -0
app.py CHANGED
@@ -109,14 +109,10 @@ class ReportSection(BaseModel):
109
  raise ValueError("comparison content requires columns and at least two rows")
110
  if self.layout == "comparison" and not all(isinstance(row, dict) for row in content.get("rows", [])):
111
  raise ValueError("comparison rows must be objects")
112
- if self.layout == "paper-analysis" and not all(
113
- content.get(key) for key in ("method", "evidence", "limitations")
114
- ):
115
- raise ValueError("paper-analysis requires method, evidence, and limitations")
116
- if self.layout == "evidence-gap" and not all(
117
- content.get(key) for key in ("interpretation", "hypotheses", "required_inputs")
118
- ):
119
- raise ValueError("evidence-gap requires interpretation, hypotheses, and required_inputs")
120
  return self
121
 
122
 
@@ -666,7 +662,8 @@ def build_report(
666
  "{columns:[...],rows:[{name,values:[...]}]}; process uses {steps:[{name,description}]} or "
667
  "{common_steps:[...],branches:[{name,steps:[...]}]}; equation uses {equation,symbols,interpretation}; "
668
  "paper-analysis uses {method,evidence,limitations}; evidence-gap uses "
669
- "{interpretation,related_topics,hypotheses,required_inputs}. Write for slides, not an essay: one conclusion per "
 
670
  "section, concise points, explicit assumptions, comparable dimensions, and no repeated background. Distinguish "
671
  "retrieved facts from analysis or hypotheses. Never invent sources, papers, metrics, or recency claims."
672
  )
@@ -694,14 +691,17 @@ def build_report(
694
  4200,
695
  )
696
  sections_result = align_sections_to_plan(sections_result, batch)
697
- validated_batch = validate_stage_with_repairs(
698
- sections_result,
699
- SectionsEnvelope,
700
- f"report sections batch {batch_index // 4 + 1}",
701
- {**shared_context, "approved_section_plan": [item.model_dump() for item in batch]},
702
- 4200,
703
- )
704
- if len(validated_batch.sections) != len(batch):
 
 
 
705
  recovered_sections: list[ReportSection] = []
706
  for plan in batch:
707
  one_result = chat_json(
@@ -800,6 +800,8 @@ def field_label(key: Any) -> str:
800
  "method": "方法", "mechanism": "机制", "problem": "问题", "innovation": "创新",
801
  "tradeoffs": "权衡", "interpretation": "解释", "related_topics": "相关主题",
802
  "hypotheses": "待验证假设", "required_inputs": "所需补充信息", "findings": "发现",
 
 
803
  }
804
  normalized = str(key).strip().lower().replace("-", "_")
805
  return labels.get(normalized, str(key).replace("_", " ").replace("-", " ").strip().title())
 
109
  raise ValueError("comparison content requires columns and at least two rows")
110
  if self.layout == "comparison" and not all(isinstance(row, dict) for row in content.get("rows", [])):
111
  raise ValueError("comparison rows must be objects")
112
+ if self.layout in {"paper-analysis", "evidence-gap"}:
113
+ meaningful_blocks = [value for value in content.values() if value not in (None, "", [], {})]
114
+ if len(meaningful_blocks) < 2:
115
+ raise ValueError(f"{self.layout} requires at least two substantive information blocks")
 
 
 
 
116
  return self
117
 
118
 
 
662
  "{columns:[...],rows:[{name,values:[...]}]}; process uses {steps:[{name,description}]} or "
663
  "{common_steps:[...],branches:[{name,steps:[...]}]}; equation uses {equation,symbols,interpretation}; "
664
  "paper-analysis uses {method,evidence,limitations}; evidence-gap uses "
665
+ "{interpretation,related_topics,hypotheses,required_inputs}. Equivalent topic-appropriate field names are allowed; "
666
+ "never add empty placeholder fields merely to match an example. Write for slides, not an essay: one conclusion per "
667
  "section, concise points, explicit assumptions, comparable dimensions, and no repeated background. Distinguish "
668
  "retrieved facts from analysis or hypotheses. Never invent sources, papers, metrics, or recency claims."
669
  )
 
691
  4200,
692
  )
693
  sections_result = align_sections_to_plan(sections_result, batch)
694
+ try:
695
+ validated_batch = validate_stage_with_repairs(
696
+ sections_result,
697
+ SectionsEnvelope,
698
+ f"report sections batch {batch_index // 4 + 1}",
699
+ {**shared_context, "approved_section_plan": [item.model_dump() for item in batch]},
700
+ 4200,
701
+ )
702
+ except RuntimeError:
703
+ validated_batch = None
704
+ if validated_batch is None or len(validated_batch.sections) != len(batch):
705
  recovered_sections: list[ReportSection] = []
706
  for plan in batch:
707
  one_result = chat_json(
 
800
  "method": "方法", "mechanism": "机制", "problem": "问题", "innovation": "创新",
801
  "tradeoffs": "权衡", "interpretation": "解释", "related_topics": "相关主题",
802
  "hypotheses": "待验证假设", "required_inputs": "所需补充信息", "findings": "发现",
803
+ "known_evidence": "已有证据", "unknowns": "证据缺口", "next_steps": "下一步验证",
804
+ "open_questions": "待回答问题", "method_summary": "方法概述", "experimental_results": "实验结果",
805
  }
806
  normalized = str(key).strip().lower().replace("-", "_")
807
  return labels.get(normalized, str(key).replace("_", " ").replace("-", " ").strip().title())
tests/test_app.py CHANGED
@@ -117,6 +117,15 @@ class AppTests(unittest.TestCase):
117
  section = {"title": "Evidence gap", "purpose": "Separate known facts from hypotheses.", "takeaway": "More evidence is required before choosing a method.", "layout": "evidence-gap", "content": {"interpretation": "The term may be ambiguous.", "related_topics": ["Adjacent field"], "hypotheses": ["Possible interpretation"], "required_inputs": ["Definition or internal data"]}, "source_ids": []}
118
  self.assertEqual(ResearchReport.model_validate(base_report(section)).sections[0].layout, "evidence-gap")
119
 
 
 
 
 
 
 
 
 
 
120
  def test_invalid_empty_section(self):
121
  section = {"title": "Empty", "purpose": "This section should fail validation.", "takeaway": "There is no substantive supporting content.", "layout": "text", "content": {}, "source_ids": []}
122
  with self.assertRaises(ValidationError):
 
117
  section = {"title": "Evidence gap", "purpose": "Separate known facts from hypotheses.", "takeaway": "More evidence is required before choosing a method.", "layout": "evidence-gap", "content": {"interpretation": "The term may be ambiguous.", "related_topics": ["Adjacent field"], "hypotheses": ["Possible interpretation"], "required_inputs": ["Definition or internal data"]}, "source_ids": []}
118
  self.assertEqual(ResearchReport.model_validate(base_report(section)).sections[0].layout, "evidence-gap")
119
 
120
+ def test_evidence_gap_accepts_equivalent_semantic_fields(self):
121
+ section = {"title": "Evidence gap", "purpose": "Separate available evidence from unresolved questions.", "takeaway": "Deployment requires evidence that is not yet public.", "layout": "evidence-gap", "content": {"known_evidence": ["A related result exists"], "unknowns": ["No direct benchmark"], "next_steps": ["Run an internal evaluation"]}, "source_ids": []}
122
+ validated = ResearchReport.model_validate(base_report(section))
123
+ self.assertEqual(validated.sections[0].content["unknowns"], ["No direct benchmark"])
124
+
125
+ def test_paper_analysis_accepts_equivalent_semantic_fields(self):
126
+ section = {"title": "Paper analysis", "purpose": "Analyze a paper without enforcing field spelling.", "takeaway": "The result is useful under a bounded condition.", "layout": "paper-analysis", "content": {"method_summary": "A compact architecture", "experimental_results": ["A controlled comparison"], "caveats": ["A narrow dataset"]}, "source_ids": []}
127
+ self.assertEqual(ResearchReport.model_validate(base_report(section)).sections[0].layout, "paper-analysis")
128
+
129
  def test_invalid_empty_section(self):
130
  section = {"title": "Empty", "purpose": "This section should fail validation.", "takeaway": "There is no substantive supporting content.", "layout": "text", "content": {}, "source_ids": []}
131
  with self.assertRaises(ValidationError):