R14-CIT-A1: adaptive citation sufficiency gate

core/generation_runner.py

@@ -34,9 +34,15 @@ from utils.file_parser import extract_text_from_files
 
 ProgressCallback = Callable[..., None]
 
-MIN_FINAL_REFERENCES = 50
-LITERATURE_EVIDENCE_TARGET = 75
-CITATION_PLAN_TARGET = 70
+CITATION_SEMANTIC_SURVIVAL_RATIO = 0.78
+CITATION_ROLE_RULES = {
+    "introduction": (3, 180),
+    "theory": (3, 180),
+    "theoretical_framing": (3, 180),
+    "methodology": (2, 280),
+    "methods": (2, 280),
+    "discussion": (2, 220),
+}
 TRANSLATION_CONTRACT_VERSION = 'hierarchical-multilingual-v5'
 
 
@@ -1493,6 +1499,66 @@ class GenerationRunner:
             "ratio": round(ratio, 4), "status": status, "recommendation": recommendation, "nonfatal": True,
         }
 
+    @staticmethod
+    def _citation_eligible_section_ids(mode: str) -> set[str]:
+        return (
+            {"introduction", "theoretical_framing", "methods", "discussion"}
+            if str(mode).lower() == "asq"
+            else {"introduction", "theory", "methodology", "discussion"}
+        )
+
+    @classmethod
+    def _adaptive_citation_requirements(cls, tasks, mode: str):
+        """Derive evidence demand and semantic-survival reserve from manuscript units."""
+        allowed = cls._citation_eligible_section_ids(mode)
+        minimums = {}
+        reserve_targets = {}
+        for task in tasks or []:
+            section_id = str(task.get("section_id") or "")
+            display_name = str(task.get("display_name") or "").strip()
+            if section_id not in allowed or not display_name:
+                continue
+            base, words_per_source = CITATION_ROLE_RULES.get(section_id, (2, 240))
+            target_words = max(1, int(task.get("target_words") or 1))
+            minimum = max(base, int(math.ceil(target_words / max(1, words_per_source))))
+            reserve = max(
+                minimum,
+                int(math.ceil(minimum / max(0.01, CITATION_SEMANTIC_SURVIVAL_RATIO))),
+            )
+            minimums[display_name] = minimum
+            reserve_targets[display_name] = reserve
+        return minimums, reserve_targets
+
+    @staticmethod
+    def _adaptive_literature_candidate_target(target_words: int, agent) -> int:
+        """Adaptive retrieval aspiration; never a final reference-count quota."""
+        dynamic = max(24, int(math.ceil(max(1, int(target_words or 1)) / 90.0)))
+        configured_cap = max(1, int(getattr(agent, "lit_target_papers", dynamic) or dynamic))
+        return min(configured_cap, dynamic)
+
+    @staticmethod
+    def _citation_task_context(task, research_logic_contract="", synthesis_result="") -> str:
+        parts = [
+            str(task.get("display_name") or ""),
+            str(task.get("required_content") or ""),
+            " ".join(str(value) for value in (task.get("concepts") or []) if str(value).strip()),
+            str(research_logic_contract or "")[:1600],
+            str(synthesis_result or "")[:1600],
+        ]
+        return re.sub(r"\s+", " ", " ".join(part for part in parts if part)).strip()
+
+    @staticmethod
+    def _literature_insufficient_error(shortfalls) -> RuntimeError:
+        details = "; ".join(
+            f"{name}={actual}/{required}"
+            for name, actual, required in (shortfalls or [])[:8]
+        )
+        return RuntimeError(
+            "E_LITERATURE_INSUFFICIENT: high-relevance claim-specific literature is insufficient "
+            f"for the planned manuscript units ({details}). Please revise the research topic or "
+            "keywords and try again; unrelated references will not be used to satisfy a quota."
+        )
+
     def _resolve_paths(self, items) -> list[str]:
         output = []
         for item in items or []:
@@ -1615,16 +1681,16 @@ class GenerationRunner:
         refs_text = str(state.get("refs_text") or "")
         if not refs_text:
             self._progress(8, "S2 + OpenAlex")
+            literature_candidate_target = self._adaptive_literature_candidate_target(target_words, agent)
             refs_text = agent.fetch_references_parallel(
                 topic,
                 keywords,
-                min_required=LITERATURE_EVIDENCE_TARGET,
+                min_required=literature_candidate_target,
                 progress_callback=lambda: self._progress(9, "S2 + OpenAlex"),
             )
-            if len(agent.citation_map) < MIN_FINAL_REFERENCES:
-                raise RuntimeError(
-                    f"Insufficient eligible references: {len(agent.citation_map)}/{MIN_FINAL_REFERENCES}"
-                )
+            if not agent.citation_map:
+                raise self._literature_insufficient_error([("retrieved literature pool", 0, 1)])
+            state["literature_candidate_target"] = literature_candidate_target
             if user_lit_context:
                 core_refs = agent.parse_user_core_literature(user_lit_context)
                 if core_refs:
@@ -1699,29 +1765,86 @@ class GenerationRunner:
             if jit_texts:
                 rag_engine._initialize_chunks([jit_texts])
             self._progress(15, "citation_planning")
-            citation_plan = agent.build_claim_evidence_plan(
-                adaptive_structure, topic, synthesis_result, mode=mode, target_unique=CITATION_PLAN_TARGET
+            citation_minimums, citation_reserve_targets = self._adaptive_citation_requirements(
+                adaptive_structure, mode
             )
-            canonical_concept_ledger = self._build_canonical_concept_ledger(
-                mode, data_metadata, keywords=keywords, adaptive_structure=adaptive_structure,
-                research_logic_contract=research_logic_contract,
+            citation_plan_target = sum(citation_reserve_targets.values())
+            citation_plan = agent.build_claim_evidence_plan(
+                adaptive_structure, topic, synthesis_result, mode=mode,
+                target_unique=max(1, citation_plan_target),
             )
-            planned_unique = {
-                item.get("tag")
+
+            # Fill unit-specific reserve gaps before expensive drafting. Existing
+            # retrieved evidence is preferred; bounded contextual JIT is last resort.
+            planned_tags = {
+                str(item.get("tag") or "")
                 for items in citation_plan.values()
                 for item in items
                 if item.get("tag")
             }
-            if len(planned_unique) < MIN_FINAL_REFERENCES:
-                raise RuntimeError(
-                    "Insufficient claim-evidence plan before drafting: "
-                    f"{len(planned_unique)}/{MIN_FINAL_REFERENCES}."
+            task_by_name = {
+                str(task.get("display_name") or ""): task
+                for task in adaptive_structure
+                if str(task.get("display_name") or "")
+            }
+            for display_name, reserve_target in citation_reserve_targets.items():
+                planned_items = citation_plan.setdefault(display_name, [])
+                shortfall = max(0, reserve_target - len(planned_items))
+                if shortfall <= 0:
+                    continue
+                section_context = self._citation_task_context(
+                    task_by_name.get(display_name, {}), research_logic_contract, synthesis_result
+                )
+                recovery_items = agent.select_contextual_citation_recovery_items(
+                    [], section_name=display_name, section_context=section_context,
+                    needed=shortfall, excluded_tags=planned_tags,
                 )
+                for item in recovery_items:
+                    tag = str(item.get("tag") or "")
+                    if tag and tag not in planned_tags:
+                        planned_items.append(item)
+                        planned_tags.add(tag)
+                # External JIT is not used merely to hit the reserve target.
+                # It is triggered only when the unit remains below its actual
+                # evidence-demand minimum after exhausting already retrieved
+                # relevant literature.
+                minimum_shortfall = max(
+                    0, int(citation_minimums.get(display_name, 0)) - len(planned_items)
+                )
+                if minimum_shortfall > 0:
+                    jit_items = agent.fetch_contextual_jit_recovery_items(
+                        topic=topic, section_name=display_name, section_context=section_context,
+                        needed=minimum_shortfall, excluded_tags=planned_tags,
+                    )
+                    for item in jit_items:
+                        tag = str(item.get("tag") or "")
+                        if tag and tag not in planned_tags:
+                            planned_items.append(item)
+                            planned_tags.add(tag)
+
+            shortfalls = [
+                (name, len(citation_plan.get(name) or []), required)
+                for name, required in citation_minimums.items()
+                if len(citation_plan.get(name) or []) < required
+            ]
+            if shortfalls:
+                raise self._literature_insufficient_error(shortfalls)
+            logging.info(
+                "Adaptive citation feasibility passed: minimum_total=%s reserve_target=%s planned_unique=%s units=%s",
+                sum(citation_minimums.values()), citation_plan_target, len(planned_tags), len(citation_minimums),
+            )
+            canonical_concept_ledger = self._build_canonical_concept_ledger(
+                mode, data_metadata, keywords=keywords, adaptive_structure=adaptive_structure,
+                research_logic_contract=research_logic_contract,
+            )
             state.update(
                 {
                     "adaptive_structure": adaptive_structure,
                     "quote_invocation_plan": quote_invocation_plan,
                     "citation_plan": citation_plan,
+                    "citation_minimums": citation_minimums,
+                    "citation_reserve_targets": citation_reserve_targets,
+                    "citation_validated_tags_by_unit": {},
                     "research_logic_contract": research_logic_contract,
                     "working_abstract": working_abstract,
                     "working_keywords": working_keywords,
@@ -1753,6 +1876,16 @@ class GenerationRunner:
             state.get("research_logic_contract") or agent.global_memory.get("research_logic_contract") or ""
         )
         citation_plan = state.get("citation_plan") if isinstance(state.get("citation_plan"), dict) else {}
+        citation_minimums = dict(state.get("citation_minimums") or {})
+        citation_reserve_targets = dict(state.get("citation_reserve_targets") or {})
+        citation_validated_tags_by_unit = {
+            str(name): list(tags or [])
+            for name, tags in dict(state.get("citation_validated_tags_by_unit") or {}).items()
+        }
+        if not citation_minimums:
+            citation_minimums, citation_reserve_targets = self._adaptive_citation_requirements(
+                adaptive_structure, mode
+            )
         translation_sections = dict(state.get('translation_sections') or {})
         translation_glossary = dict(state.get('translation_glossary') or {})
         canonical_concept_ledger = state.get('canonical_concept_ledger')
@@ -1866,9 +1999,20 @@ class GenerationRunner:
         if not citation_plan:
             self._progress(15, "citation_planning")
             citation_plan = agent.build_claim_evidence_plan(
-                adaptive_structure, topic, synthesis_result, mode=mode, target_unique=CITATION_PLAN_TARGET
+                adaptive_structure, topic, synthesis_result, mode=mode,
+                target_unique=max(1, sum(citation_reserve_targets.values())),
             )
+            shortfalls = [
+                (name, len(citation_plan.get(name) or []), required)
+                for name, required in citation_minimums.items()
+                if len(citation_plan.get(name) or []) < required
+            ]
+            if shortfalls:
+                raise self._literature_insufficient_error(shortfalls)
             state["citation_plan"] = citation_plan
+            state["citation_minimums"] = citation_minimums
+            state["citation_reserve_targets"] = citation_reserve_targets
+            state["citation_validated_tags_by_unit"] = citation_validated_tags_by_unit
 
         quote_invocation_plan = (
             dict(state.get("quote_invocation_plan") or {}) if mode == "asq" else {}
@@ -2105,9 +2249,13 @@ Output only final academic prose. Do not report the word count."""
             used_after = set(agent.used_citations)
             survived = used_after - used_before
             if assigned_tag_list and not is_quarantined:
+                adaptive_minimum = max(1, int(citation_minimums.get(display_name, 1)))
                 required_survival = min(
                     len(assigned_tag_list),
-                    max(1, int(math.ceil(len(assigned_tag_list) * 0.78))),
+                    max(
+                        adaptive_minimum,
+                        int(math.ceil(len(assigned_tag_list) * CITATION_SEMANTIC_SURVIVAL_RATIO)),
+                    ),
                 )
                 if len(survived) < required_survival:
                     missing_items = [
@@ -2127,6 +2275,13 @@ Output only final academic prose. Do not report the word count."""
                                 excluded_tags=used_after | {str(item.get("tag") or "") for item in recovery_items},
                             )
                         )
+                    planned_tags_for_unit = {str(item.get("tag") or "") for item in planned_items}
+                    for recovery_item in recovery_items[:needed]:
+                        tag = str(recovery_item.get("tag") or "")
+                        if tag and tag not in planned_tags_for_unit:
+                            planned_items.append(recovery_item)
+                            planned_tags_for_unit.add(tag)
+                    citation_plan[display_name] = planned_items
                     supplemental_raw = agent.build_citation_recovery_paragraph(
                         recovery_items[:needed], section_name=display_name, section_context=en_resolved
                     )
@@ -2140,6 +2295,15 @@ Output only final academic prose. Do not report the word count."""
                             en_resolved = en_resolved.rstrip() + "\n\n" + supplemental_resolved.strip()
                         used_after = set(agent.used_citations)
                         survived = used_after - used_before
+            if assigned_tag_list and not is_quarantined:
+                adaptive_minimum = max(1, int(citation_minimums.get(display_name, 1)))
+                if len(survived) < adaptive_minimum:
+                    raise self._literature_insufficient_error(
+                        [(display_name, len(survived), adaptive_minimum)]
+                    )
+                citation_validated_tags_by_unit[display_name] = sorted(
+                    survived, key=agent._citation_tag_number
+                )
             if mode == "asq" and section_id == "findings":
                 en_resolved = self._render_planned_qualitative_quotes(
                     en_resolved,
@@ -2188,6 +2352,10 @@ Output only final academic prose. Do not report the word count."""
                     "context_tail": context_tail,
                     "accumulated_deviation": accumulated_deviation,
                     "quality_diagnostics": quality_diagnostics,
+                    "citation_plan": citation_plan,
+                    "citation_minimums": citation_minimums,
+                    "citation_reserve_targets": citation_reserve_targets,
+                    "citation_validated_tags_by_unit": citation_validated_tags_by_unit,
                     "agent_state": agent.export_state(),
                     "data_metadata": data_metadata,
                 }
@@ -2215,18 +2383,33 @@ Output only final academic prose. Do not report the word count."""
                 if ref and ref not in seen_refs:
                     references.append(f"- {ref}")
                     seen_refs.add(ref)
-            if len(references) < MIN_FINAL_REFERENCES:
-                raise RuntimeError(
-                    "Insufficient final cited references after semantic validation: "
-                    f"{len(references)}/{MIN_FINAL_REFERENCES}."
-                )
+            if not references:
+                raise self._literature_insufficient_error([("final validated references", 0, 1)])
+            final_shortfalls = []
+            for name, minimum in citation_minimums.items():
+                validated_tags = set(citation_validated_tags_by_unit.get(name) or [])
+                if not validated_tags:
+                    validated_tags = {
+                        str(item.get("tag") or "")
+                        for item in (citation_plan.get(name) or [])
+                        if str(item.get("tag") or "") in agent.used_citations
+                    }
+                if len(validated_tags) < int(minimum):
+                    final_shortfalls.append((name, len(validated_tags), int(minimum)))
+            if final_shortfalls:
+                raise self._literature_insufficient_error(final_shortfalls)
+            final_reference_count = len(references)
+            logging.info(
+                "Adaptive citation final Gate passed: validated_references=%s units=%s",
+                final_reference_count, len(citation_minimums),
+            )
             full_draft_en = full_draft_en.rstrip() + "\n\n# References\n" + "\n\n".join(references)
             actual_words = self._english_count_for_target(full_draft_en)
             self._upsert_quality_diagnostic(
                 quality_diagnostics, self._validate_target_length(full_draft_en, target_words)
             )
             body_only, _reference_tail = agent.validate_reference_tail(
-                full_draft_en, minimum_references=MIN_FINAL_REFERENCES
+                full_draft_en, minimum_references=max(1, final_reference_count)
             )
             agent.validate_manuscript_structure(body_only, adaptive_structure, translated=False)
             agent.enforce_resolved_claim_specific_citations(body_only)
@@ -2245,7 +2428,7 @@ Output only final academic prose. Do not report the word count."""
             full_draft_en = replace_abstract(full_draft_en, final_abstract)
             full_draft_en = self._replace_keywords(full_draft_en, final_keywords)
             body_only, _reference_tail = agent.validate_reference_tail(
-                full_draft_en, minimum_references=MIN_FINAL_REFERENCES
+                full_draft_en, minimum_references=max(1, final_reference_count)
             )
             agent.validate_manuscript_structure(body_only, adaptive_structure, translated=False)
             agent.enforce_resolved_claim_specific_citations(body_only)
@@ -2282,6 +2465,11 @@ Output only final academic prose. Do not report the word count."""
                     "final_abstract": final_abstract,
                     "final_keywords": final_keywords,
                     "front_matter_phase": "final",
+                    "final_reference_count": final_reference_count,
+                    "citation_plan": citation_plan,
+                    "citation_minimums": citation_minimums,
+                    "citation_reserve_targets": citation_reserve_targets,
+                    "citation_validated_tags_by_unit": citation_validated_tags_by_unit,
                     "quality_diagnostics": quality_diagnostics,
                     "english_finalized": True,
                     "translation_complete": False,
@@ -2305,7 +2493,7 @@ Output only final academic prose. Do not report the word count."""
             )
             self._shutdown_translation_worker(wait=True)
             body_only, reference_tail = agent.validate_reference_tail(
-                full_draft_en, minimum_references=MIN_FINAL_REFERENCES
+                full_draft_en, minimum_references=max(1, int(state.get("final_reference_count") or 1))
             )
             section_order = self._unique_top_level_sections(adaptive_structure)
             if not section_order:

core/pipeline_agent.py

@@ -2698,9 +2698,9 @@ INPUT:
 
         At most one query is issued to each literature backend. New references
         are admitted only when title/abstract evidence has distinctive lexical
-        overlap with the current manuscript unit; otherwise the final >=50 gate
-        is allowed to fail rather than padding the manuscript with irrelevant
-        literature.
+        overlap with the current manuscript unit; otherwise the adaptive
+        evidence-sufficiency Gate is allowed to fail rather than padding the
+        manuscript with irrelevant literature.
         """
         needed = max(0, int(needed or 0))
         if needed <= 0: