Generate and validate reports in structured stages

README.md

@@ -18,6 +18,8 @@ Turn an algorithm research topic into an interactive, paper-backed HTML presenta
 
 Users can disable online paper search. In that mode the deck relies on the model's existing knowledge and visibly discloses that papers and recency were not externally verified. Download controls sit in the presentation toolbar: HTML for the standalone interactive deck, JSON for structured research data, and ZIP for both.
 
+Report generation is staged: raw papers are compressed into evidence cards, then the model produces an outline, a causal technical lineage and method comparison, and section content. Every stage is validated with Pydantic and receives at most two targeted repair attempts before rendering, preventing valid-but-empty JSON from producing blank slides.
+
 ## Space configuration
 
 Secret:

app.py

@@ -18,7 +18,7 @@ import gradio as gr
 import httpx
 import spaces
 from huggingface_hub import InferenceClient
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, ValidationError
 from pypdf import PdfReader
 
 
@@ -47,6 +47,82 @@ class ResearchRequest(BaseModel):
     paper_urls: list[str] = Field(default_factory=list, max_length=12)
 
 
+class EvidenceCard(BaseModel):
+    paper_id: str = Field(min_length=1)
+    title: str = Field(min_length=2)
+    year: str = Field(min_length=1)
+    lineage_role: str = Field(min_length=3)
+    problem: str = Field(min_length=10)
+    core_idea: str = Field(min_length=10)
+    delta: str = Field(min_length=10)
+    evidence: list[str] = Field(min_length=1, max_length=4)
+    limitations: list[str] = Field(min_length=1, max_length=4)
+
+
+class EvidenceCardsEnvelope(BaseModel):
+    cards: list[EvidenceCard] = Field(min_length=1)
+
+
+class SectionPlan(BaseModel):
+    id: str = Field(min_length=2)
+    title: str = Field(min_length=2)
+    purpose: str = Field(min_length=10)
+    paper_ids: list[str] = Field(default_factory=list)
+
+
+class ReportOutline(BaseModel):
+    title: str = Field(min_length=2)
+    subtitle: str = Field(min_length=2)
+    executive_summary: list[str] = Field(min_length=3, max_length=5)
+    section_plan: list[SectionPlan] = Field(min_length=5, max_length=10)
+    open_questions: list[str] = Field(min_length=2, max_length=5)
+
+
+class LineageNode(BaseModel):
+    year: str = Field(min_length=1)
+    method: str = Field(min_length=2)
+    paper_id: str = ""
+    bottleneck: str = Field(min_length=10)
+    innovation: str = Field(min_length=10)
+    limitation: str = Field(min_length=10)
+
+
+class MethodComparison(BaseModel):
+    method: str = Field(min_length=2)
+    mechanism: str = Field(min_length=10)
+    strengths: str = Field(min_length=10)
+    limitations: str = Field(min_length=10)
+    engineering: str = Field(min_length=10)
+
+
+class TechnicalSynthesis(BaseModel):
+    lineage: list[LineageNode] = Field(min_length=4, max_length=9)
+    comparisons: list[MethodComparison] = Field(min_length=3, max_length=8)
+
+
+class ReportSection(BaseModel):
+    title: str = Field(min_length=2)
+    takeaway: str = Field(min_length=10)
+    body: list[str] = Field(min_length=2, max_length=5)
+    paper_ids: list[str] = Field(default_factory=list)
+    details: str = Field(min_length=20)
+
+
+class SectionsEnvelope(BaseModel):
+    sections: list[ReportSection] = Field(min_length=5, max_length=10)
+
+
+class ResearchReport(BaseModel):
+    title: str = Field(min_length=2)
+    subtitle: str = Field(min_length=2)
+    executive_summary: list[str] = Field(min_length=3, max_length=5)
+    lineage: list[LineageNode] = Field(min_length=4, max_length=9)
+    sections: list[ReportSection] = Field(min_length=5, max_length=10)
+    comparisons: list[MethodComparison] = Field(min_length=3, max_length=8)
+    open_questions: list[str] = Field(min_length=2, max_length=5)
+    english_understanding: dict[str, Any] = Field(default_factory=dict)
+
+
 DEPTHS = {
     "quick": {"discover": 30, "screen": 10, "deep": 2},
     "standard": {"discover": 60, "screen": 20, "deep": 4},
@@ -329,42 +405,201 @@ def extract_pdf(paper: dict[str, Any], max_chars: int = 14000) -> str:
     return "\n".join(pages)[:max_chars]
 
 
+def validate_stage_with_repairs(
+    result: dict[str, Any],
+    model_type: type[BaseModel],
+    stage_name: str,
+    source_context: dict[str, Any],
+    max_tokens: int,
+) -> BaseModel:
+    candidate = result
+    for attempt in range(3):
+        try:
+            return model_type.model_validate(candidate)
+        except ValidationError as exc:
+            if attempt == 2:
+                raise RuntimeError(f"{stage_name} remained invalid after two repairs: {exc}") from exc
+            candidate = chat_json(
+                f"Repair the {stage_name} JSON. Preserve valid content, fill missing fields, and return only the complete corrected object.",
+                json.dumps({
+                    "validation_errors": exc.errors(include_url=False),
+                    "object_to_repair": candidate,
+                    "source_context": source_context,
+                    "schema": model_type.model_json_schema(),
+                }, ensure_ascii=False),
+                max_tokens,
+            )
+    raise RuntimeError(f"{stage_name} validation failed")
+
+
 def build_report(
     request: ResearchRequest,
     papers: list[dict[str, Any]],
     deep_texts: dict[str, str],
     english_understanding: dict[str, Any],
 ) -> dict[str, Any]:
-    evidence = []
+    raw_evidence = []
     for paper in papers:
         item = {k: paper.get(k) for k in ("id", "title", "summary", "authors", "published", "updated", "url")}
         if paper["id"] in deep_texts:
-            item["selected_full_text_extract"] = deep_texts[paper["id"]]
-        evidence.append(item)
-    payload = {
+            item["selected_text_extract"] = deep_texts[paper["id"]]
+        raw_evidence.append(item)
+
+    if raw_evidence:
+        card_payload = {
+            "request": request.model_dump(),
+            "papers": raw_evidence,
+            "requirements": {
+                "one_card_per_useful_paper": True,
+                "paraphrase_instead_of_copy": True,
+                "cite_only_supplied_paper_ids": True,
+                "schema": EvidenceCardsEnvelope.model_json_schema(),
+            },
+        }
+        cards_result = chat_json(
+            "Create compact evidence cards for an algorithm literature review. Keep experimental conditions and limitations. "
+            "Do not invent results, citations, or paper IDs. Return only the required JSON object and no empty fields.",
+            json.dumps(card_payload, ensure_ascii=False),
+            3600,
+        )
+        cards = validate_stage_with_repairs(
+            cards_result,
+            EvidenceCardsEnvelope,
+            "evidence cards",
+            {"request": request.model_dump(), "papers": raw_evidence},
+            3600,
+        ).model_dump()["cards"]
+    else:
+        cards = []
+
+    shared_context = {
         "request": request.model_dump(),
         "english_understanding": english_understanding,
         "research_cutoff": datetime.now(timezone.utc).date().isoformat(),
-        "evidence": evidence,
-        "schema": {
-            "title": "string",
-            "subtitle": "string",
-            "executive_summary": ["string"],
-            "lineage": [{"year": "string", "method": "string", "paper_id": "string", "bottleneck": "string", "innovation": "string", "limitation": "string"}],
-            "sections": [{"title": "string", "takeaway": "string", "body": ["string"], "paper_ids": ["string"], "details": "string"}],
-            "comparisons": [{"method": "string", "mechanism": "string", "strengths": "string", "limitations": "string", "engineering": "string"}],
-            "open_questions": ["string"],
-        },
+        "evidence_cards": cards,
+        "evidence_rule": (
+            "Use only supplied paper IDs for citations. State evidence gaps instead of inventing support."
+            if cards else
+            "Online evidence is disabled. Do not invent paper IDs or claim that recent literature was verified."
+        ),
     }
-    report = chat_json(
-        "You are a rigorous algorithm research lead. Return only JSON matching the supplied schema. "
-        "Trace classical ideas to current methods, distinguish preprints, avoid incomparable benchmark claims, "
-        "and cite only supplied paper IDs. Write concise presentation-ready content.",
-        json.dumps(payload, ensure_ascii=False),
-        3800,
+
+    outline_result = chat_json(
+        "Design a concise technical research presentation for algorithm engineers. Return only JSON matching the schema. "
+        "The executive summary must contain substantive conclusions. Every section needs a distinct purpose.",
+        json.dumps({
+            **shared_context,
+            "requirements": {
+                "executive_summary_items": "3 to 5",
+                "section_plan_items": "5 to 10",
+                "open_question_items": "2 to 5",
+                "schema": ReportOutline.model_json_schema(),
+            },
+        }, ensure_ascii=False),
+        3000,
+    )
+    outline = validate_stage_with_repairs(
+        outline_result,
+        ReportOutline,
+        "report outline",
+        shared_context,
+        3200,
+    )
+
+    synthesis_result = chat_json(
+        "Build a causal technical lineage and a consistent method comparison. Return only JSON matching the schema. "
+        "Each lineage node must connect a predecessor bottleneck to an innovation and remaining limitation. "
+        "Compare every method on the same engineering-relevant dimensions and avoid unsupported numeric rankings.",
+        json.dumps({
+            **shared_context,
+            "outline": outline.model_dump(),
+            "requirements": {
+                "lineage_nodes": "4 to 9",
+                "comparison_methods": "3 to 8",
+                "schema": TechnicalSynthesis.model_json_schema(),
+            },
+        }, ensure_ascii=False),
+        3200,
+    )
+    synthesis = validate_stage_with_repairs(
+        synthesis_result,
+        TechnicalSynthesis,
+        "technical synthesis",
+        {**shared_context, "outline": outline.model_dump()},
+        3600,
+    )
+
+    sections_result = chat_json(
+        "Write presentation-ready technical sections from the approved outline. Return only JSON matching the schema. "
+        "Explain mechanisms before benchmark claims. Each takeaway must be supported by its body. "
+        "Use only supplied paper IDs; use an empty paper_ids list when external evidence is disabled.",
+        json.dumps({
+            **shared_context,
+            "outline": outline.model_dump(),
+            "lineage": [item.model_dump() for item in synthesis.lineage],
+            "requirements": {
+                "one_section_per_outline_item": True,
+                "body_items_per_section": "2 to 5",
+                "schema": SectionsEnvelope.model_json_schema(),
+            },
+        }, ensure_ascii=False),
+        5200,
+    )
+    sections = validate_stage_with_repairs(
+        sections_result,
+        SectionsEnvelope,
+        "report sections",
+        {**shared_context, "outline": outline.model_dump()},
+        5200,
     )
-    report["english_understanding"] = english_understanding
-    return report
+
+    assembled = {
+        "title": outline.title,
+        "subtitle": outline.subtitle,
+        "executive_summary": outline.executive_summary,
+        "lineage": [item.model_dump() for item in synthesis.lineage],
+        "sections": [item.model_dump() for item in sections.sections],
+        "comparisons": [item.model_dump() for item in synthesis.comparisons],
+        "open_questions": outline.open_questions,
+        "english_understanding": english_understanding,
+    }
+    allowed_ids = {paper["id"] for paper in papers}
+    for node in assembled["lineage"]:
+        if node.get("paper_id") not in allowed_ids:
+            node["paper_id"] = ""
+    for section in assembled["sections"]:
+        section["paper_ids"] = [paper_id for paper_id in section["paper_ids"] if paper_id in allowed_ids]
+    return validate_report_with_repairs(assembled, shared_context)
+
+
+def validate_report_with_repairs(report: dict[str, Any], context: dict[str, Any]) -> dict[str, Any]:
+    candidate = report
+    for attempt in range(3):
+        try:
+            validated = ResearchReport.model_validate(candidate).model_dump()
+            allowed_ids = {card.get("paper_id") for card in context.get("evidence_cards", [])}
+            for node in validated["lineage"]:
+                if node.get("paper_id") not in allowed_ids:
+                    node["paper_id"] = ""
+            for section in validated["sections"]:
+                section["paper_ids"] = [paper_id for paper_id in section["paper_ids"] if paper_id in allowed_ids]
+            return validated
+        except ValidationError as exc:
+            if attempt == 2:
+                raise RuntimeError(f"Report remained incomplete after two repairs: {exc}") from exc
+            candidate = chat_json(
+                "Repair an incomplete algorithm research report. Preserve valid content and fill only missing or invalid fields. "
+                "Return the complete report as one JSON object matching the schema. Do not invent paper IDs.",
+                json.dumps({
+                    "validation_errors": exc.errors(include_url=False),
+                    "report_to_repair": candidate,
+                    "evidence_cards": context.get("evidence_cards", []),
+                    "schema": ResearchReport.model_json_schema(),
+                }, ensure_ascii=False),
+                5200,
+            )
+            candidate["english_understanding"] = context.get("english_understanding", {})
+    raise RuntimeError("Report validation failed")
 
 
 def esc(value: Any) -> str:
@@ -494,7 +729,10 @@ def run_research_ui(
         english_understanding["evidence_mode"] = "Model knowledge only; online paper search was disabled by the user."
         progress(0.45, desc="已关闭联网检索，使用模型已有知识生成")
     progress(0.68, desc="DeepSeek-V4-Flash 正在生成技术路线")
-    report = build_report(request, screen, deep_texts, english_understanding)
+    try:
+        report = build_report(request, screen, deep_texts, english_understanding)
+    except (ValidationError, RuntimeError) as exc:
+        raise gr.Error(f"报告结构生成失败，模型在两次定向修复后仍未满足完整性要求：{exc}") from exc
     cutoff = datetime.now(timezone.utc).date().isoformat()
     content = render_presentation(report, screen, cutoff)
     job_dir = DATA_DIR / uuid.uuid4().hex

tests/test_app.py

@@ -1,6 +1,8 @@
 import unittest
 
-from app import deduplicate, render_presentation, safe_slug
+from pydantic import ValidationError
+
+from app import ResearchReport, deduplicate, render_presentation, safe_slug
 
 
 class AppTests(unittest.TestCase):
@@ -17,6 +19,36 @@ class AppTests(unittest.TestCase):
         self.assertIn("<!doctype html>", output)
         self.assertIn("保存 PDF", output)
 
+    def test_empty_report_is_rejected(self):
+        with self.assertRaises(ValidationError):
+            ResearchReport.model_validate({})
+
+    def test_complete_report_is_accepted(self):
+        section = {
+            "title": "Method evolution",
+            "takeaway": "Each transition addresses a concrete bottleneck.",
+            "body": ["First supported point.", "Second supported point."],
+            "paper_ids": [],
+            "details": "Sufficient implementation and evidence details for validation.",
+        }
+        report = {
+            "title": "Test report",
+            "subtitle": "Validated structured output",
+            "executive_summary": ["Conclusion one is substantive.", "Conclusion two is substantive.", "Conclusion three is substantive."],
+            "lineage": [
+                {"year": str(2020 + i), "method": f"Method {i}", "paper_id": "", "bottleneck": "A concrete predecessor bottleneck.", "innovation": "A concrete technical innovation.", "limitation": "A remaining technical limitation."}
+                for i in range(4)
+            ],
+            "sections": [section for _ in range(5)],
+            "comparisons": [
+                {"method": f"Method {i}", "mechanism": "A sufficiently detailed mechanism.", "strengths": "A sufficiently detailed advantage.", "limitations": "A sufficiently detailed limitation.", "engineering": "A sufficiently detailed engineering note."}
+                for i in range(3)
+            ],
+            "open_questions": ["What evidence would change the choice?", "Which constraint dominates deployment?"],
+            "english_understanding": {},
+        }
+        self.assertEqual(ResearchReport.model_validate(report).title, "Test report")
+
 
 if __name__ == "__main__":
     unittest.main()