R14-TQ2-PQ1-R4-R3 Final Citation Ledger English References Preservation

core/generation_runner.py

@@ -44,6 +44,7 @@ CITATION_ROLE_RULES = {
     "discussion": (2, 220),
 }
 TRANSLATION_CONTRACT_VERSION = 'hierarchical-multilingual-v8'
+FINAL_CITATION_LEDGER_VERSION = 'resolved-final-ledger-v1'
 
 
 def _clean_trailing_references(text, keywords=("reference", "references", "reference list", "参考文献")):
@@ -78,6 +79,13 @@ def _normalize_target_manuscript_typography(text: str, target_lang: str) -> str:
     if "chinese" not in language.casefold() and "中文" not in language:
         return value
     value = value.replace("\r\n", "\n").replace("\r", "\n")
+    # The English References tail is immutable data, not target-language prose.
+    # Chinese punctuation normalization therefore stops before References.
+    reference_match = re.search(r"(?m)^#\s+参考文献\s*$", value)
+    frozen_reference_tail = ""
+    if reference_match:
+        frozen_reference_tail = value[reference_match.start():]
+        value = value[:reference_match.start()]
     lines = value.split("\n")
     normalized = []
     in_fence = False
@@ -93,7 +101,8 @@ def _normalize_target_manuscript_typography(text: str, target_lang: str) -> str:
         current = re.sub(r"[ \t]+([，。！？；：])", r"\1", line)
         current = re.sub(r"([，。！？；：])[ \t]+(?=\S)", r"\1", current)
         normalized.append(current)
-    return "\n".join(normalized)
+    normalized_body = "\n".join(normalized)
+    return normalized_body + frozen_reference_tail
 
 
 class _OrderedTranslationWorker:
@@ -1859,6 +1868,15 @@ class GenerationRunner:
         # Restore agent state before continuing any dependent step.
         if isinstance(state.get("agent_state"), dict):
             agent.load_state(state["agent_state"])
+        if state.get("english_finalized") and state.get("final_citation_ledger_version") != FINAL_CITATION_LEDGER_VERSION:
+            # R4-R3: an R4-R2 checkpoint may contain a correct resolved body but
+            # an empty References tail caused by rescanning already-resolved CITE
+            # tags. Re-enter only finalization/translation/package gates; completed
+            # English units and the same task/reservation remain authoritative.
+            logging.warning("Final citation ledger contract advanced; re-running finalization from frozen completed units.")
+            state["english_finalized"] = False
+            state["translation_complete"] = False
+            state["package_validated"] = False
 
         user_lit_context = str(state.get("user_lit_context") or "")
         user_data_md = str(state.get("user_data_md") or "")
@@ -2725,15 +2743,10 @@ Output only final academic prose. Do not report the word count."""
                 repeated_quotes = agent.find_repeated_qualitative_quotes(full_draft_en, data_metadata)
                 if repeated_quotes:
                     raise RuntimeError(f"Repeated qualitative quotations: {repeated_quotes[:5]}")
-            actual_used_tags = {f"[{tag.upper()}]" for tag in re.findall(r"CITE-\d+", full_draft_en, flags=re.I)}
-            agent.used_citations = set(tag for tag in actual_used_tags if tag in agent.citation_map)
-            references = []
-            seen_refs = set()
-            for tag in sorted(agent.used_citations, key=agent._citation_tag_number):
-                ref = str((agent.citation_map.get(tag) or {}).get("ref") or "").strip()
-                if ref and ref not in seen_refs:
-                    references.append(f"- {ref}")
-                    seen_refs.add(ref)
+            # R4-R3: the body is already citation-resolved at this point. Never
+            # rescan it for raw CITE tags; that destroys a valid citation ledger.
+            final_citation_ledger = agent.finalize_resolved_citation_ledger(full_draft_en)
+            references = agent.build_final_reference_lines(final_citation_ledger)
             if not references:
                 self._record_evidence_coverage_event(
                     state, [("final validated references", 0, 1)],
@@ -2754,7 +2767,7 @@ Output only final academic prose. Do not report the word count."""
                     citation_required_claim_counts_by_unit.get(unit_name, 0) or 0
                 )
             method_section_id = 'methods' if mode == 'asq' else 'methodology'
-            method_validated_tags = set(validated_by_top_level.get(method_section_id) or []) & set(agent.used_citations)
+            method_validated_tags = set(validated_by_top_level.get(method_section_id) or []) & set(final_citation_ledger)
             method_authority_tags = {
                 tag for tag in method_validated_tags
                 if 'method_authority' in set((agent.citation_map.get(tag) or {}).get('evidence_lanes') or [])
@@ -2813,6 +2826,7 @@ Output only final academic prose. Do not report the word count."""
             )
             agent.validate_manuscript_structure(body_only, adaptive_structure, translated=False)
             agent.enforce_resolved_claim_specific_citations(body_only)
+            agent.validate_final_citation_bijection(body_only, _reference_tail, final_citation_ledger)
 
             # Body is now frozen. Generate both Final Abstract and Final Keywords
             # from the completed manuscript/evidence, then replace the provisional
@@ -2832,6 +2846,10 @@ Output only final academic prose. Do not report the word count."""
             )
             agent.validate_manuscript_structure(body_only, adaptive_structure, translated=False)
             agent.enforce_resolved_claim_specific_citations(body_only)
+            post_front_ledger = agent.finalize_resolved_citation_ledger(body_only)
+            if set(post_front_ledger) != set(final_citation_ledger):
+                raise RuntimeError("final_front_matter_changed_citation_ledger")
+            agent.validate_final_citation_bijection(body_only, _reference_tail, final_citation_ledger)
             self._upsert_quality_diagnostic(
                 quality_diagnostics, self._validate_target_length(full_draft_en, target_words)
             )
@@ -2866,6 +2884,8 @@ Output only final academic prose. Do not report the word count."""
                     "final_keywords": final_keywords,
                     "front_matter_phase": "final",
                     "final_reference_count": final_reference_count,
+                    "final_citation_ledger": list(final_citation_ledger),
+                    "final_citation_ledger_version": FINAL_CITATION_LEDGER_VERSION,
                     "evidence_status": str(state.get("evidence_status") or "adequate"),
                     "evidence_risk_detected": bool(state.get("evidence_risk_detected")),
                     "evidence_coverage_events": list(state.get("evidence_coverage_events") or []),
@@ -2951,6 +2971,7 @@ Output only final academic prose. Do not report the word count."""
                 full_draft_translated, target_lang
             )
             agent.validate_translation_alignment(full_draft_en, full_draft_translated)
+            agent.validate_reference_preservation(full_draft_en, full_draft_translated, target_lang)
             self._publish_target_preview(
                 full_draft_translated, 'translation_complete', force=True, complete=True
             )

core/pipeline_agent.py

@@ -1794,6 +1794,151 @@ USER KEYWORD SEEDS: {keywords}
             )
         return body, tail
 
+    @staticmethod
+    def _resolved_parenthetical_citations(text):
+        """Return resolved parenthetical author-year citation markers in prose.
+
+        This deliberately excludes bare years such as ``(2026)``.  SmartPaper's
+        claim-specific citation compiler emits parenthetical author-year markers,
+        so any such marker in the frozen body must be traceable to citation_map.
+        """
+        pattern = re.compile(
+            r"\((?=[^()\n]*[A-Za-zÀ-ÖØ-öø-ÿ'’.-]+(?:\s+(?:&\s+[A-Za-zÀ-ÖØ-öø-ÿ'’.-]+|et\s+al\.))?\s*,\s*(?:19|20)\d{2}[a-z]?\b)[^()\n]+\)",
+            re.IGNORECASE,
+        )
+        return [match.group(0).strip() for match in pattern.finditer(str(text or ''))]
+
+    def finalize_resolved_citation_ledger(self, resolved_body):
+        """Freeze the final citation ledger from the *resolved* manuscript body.
+
+        R4-R3 authority rule: once ``[CITE-XX]`` has been compiled to
+        ``(Author, year)``, raw CITE tags are no longer a valid source of truth.
+        The final ledger is rebuilt from the resolved markers that are actually
+        present in the frozen body and the immutable citation_map metadata.
+        Stale pre-deletion citations are dropped; orphan/ambiguous citations fail
+        closed rather than producing a misleading References list.
+        """
+        body = str(resolved_body or '').strip()
+        if re.search(r'CITE-\d+', body, re.I):
+            raise DeepSeekRequestError('Final citation ledger received unresolved CITE tags.')
+
+        marker_index = {}
+        for tag, metadata in (self.citation_map or {}).items():
+            marker = str((metadata or {}).get('in_text') or '').strip()
+            ref = str((metadata or {}).get('ref') or '').strip()
+            if not marker or not ref:
+                continue
+            marker_index.setdefault(marker, []).append(str(tag))
+
+        observed = self._resolved_parenthetical_citations(body)
+        orphan = sorted({marker for marker in observed if marker not in marker_index})
+        if orphan:
+            raise DeepSeekRequestError(
+                'Final citation ledger found resolved citation(s) without authoritative metadata: '
+                + '; '.join(orphan[:5])
+            )
+
+        ledger = set()
+        ambiguous = []
+        for marker in sorted(set(observed)):
+            tags = marker_index.get(marker) or []
+            if len(tags) == 1:
+                ledger.add(tags[0])
+                continue
+            distinct_refs = {
+                str((self.citation_map.get(tag) or {}).get('ref') or '').strip()
+                for tag in tags
+                if str((self.citation_map.get(tag) or {}).get('ref') or '').strip()
+            }
+            previously_used = [tag for tag in tags if tag in set(self.used_citations or set())]
+            if len(distinct_refs) == 1:
+                ledger.add(sorted(tags, key=self._citation_tag_number)[0])
+            elif len(previously_used) == 1:
+                ledger.add(previously_used[0])
+            else:
+                ambiguous.append((marker, sorted(tags, key=self._citation_tag_number)))
+        if ambiguous:
+            raise DeepSeekRequestError(
+                'Final citation ledger found ambiguous author-year marker(s): '
+                + '; '.join(f"{marker} -> {tags}" for marker, tags in ambiguous[:5])
+            )
+
+        self.used_citations = set(ledger)
+        logging.info(
+            'Final citation ledger frozen: resolved_markers=%s unique_references=%s',
+            len(observed), len(ledger),
+        )
+        return sorted(ledger, key=self._citation_tag_number)
+
+    def build_final_reference_lines(self, ledger_tags):
+        """Build one English reference entry per authoritative ledger source."""
+        references = {}
+        for tag in ledger_tags or []:
+            metadata = self.citation_map.get(str(tag)) or {}
+            ref = str(metadata.get('ref') or '').strip()
+            if not ref:
+                raise DeepSeekRequestError(f'Final citation ledger metadata missing reference text: {tag}')
+            references.setdefault(re.sub(r'\s+', ' ', ref).casefold(), ref)
+        # Reader-facing References follow the conventional alphabetical order.
+        return [f"- {ref}" for ref in sorted(references.values(), key=lambda value: value.casefold())]
+
+    def validate_final_citation_bijection(self, resolved_body, reference_tail, ledger_tags):
+        """Require exact bidirectional consistency between prose and References."""
+        body = str(resolved_body or '').strip()
+        tail = str(reference_tail or '').strip()
+        ledger = list(ledger_tags or [])
+        expected = []
+        for tag in ledger:
+            metadata = self.citation_map.get(str(tag)) or {}
+            marker = str(metadata.get('in_text') or '').strip()
+            ref = str(metadata.get('ref') or '').strip()
+            if not marker or not ref:
+                raise DeepSeekRequestError(f'Final citation ledger is incomplete: {tag}')
+            if marker not in body:
+                raise DeepSeekRequestError(f'Final citation ledger contains a citation absent from body: {tag}')
+            expected.append(re.sub(r'\s+', ' ', ref).strip())
+
+        actual = [
+            re.sub(r'\s+', ' ', line[2:]).strip()
+            for line in tail.splitlines()
+            if line.strip().startswith('- ')
+        ]
+        if len(actual) != len(set(actual)):
+            raise DeepSeekRequestError('Final References contains duplicate entries.')
+        if set(actual) != set(expected):
+            missing = sorted(set(expected) - set(actual))[:5]
+            extra = sorted(set(actual) - set(expected))[:5]
+            raise DeepSeekRequestError(
+                f'Final citation-reference bijection failed: missing={missing}, extra={extra}.'
+            )
+
+        frozen = self.finalize_resolved_citation_ledger(body)
+        if set(frozen) != set(ledger):
+            raise DeepSeekRequestError(
+                'Final citation-reference bijection failed: resolved-body ledger changed after References assembly.'
+            )
+        return True
+
+    @classmethod
+    def validate_reference_preservation(cls, english_text, translated_text, target_lang):
+        """References are immutable English data in ZH/RU/KK deliverables."""
+        source = str(english_text or '').strip()
+        target = str(translated_text or '').strip()
+        source_parts = re.split(r'(?m)^#\s+References\s*$', source, maxsplit=1)
+        if len(source_parts) != 2:
+            raise DeepSeekRequestError('Reference preservation requires one English References tail.')
+        source_tail = source_parts[1].strip()
+        heading = re.escape(cls._localized_reference_heading(target_lang))
+        target_parts = re.split(rf'(?m)^#\s+{heading}\s*$', target, maxsplit=1)
+        if len(target_parts) != 2:
+            raise DeepSeekRequestError('Target manuscript is missing the localized References heading.')
+        target_tail = target_parts[1].strip()
+        if source_tail != target_tail:
+            raise DeepSeekRequestError(
+                'Target-language References changed from the frozen English References tail.'
+            )
+        return True
+
     @staticmethod
     def find_repeated_qualitative_quotes(text, data_metadata):
         violations = []
@@ -7400,6 +7545,12 @@ Decision must be CONFIRMED_BLOCKING or NOT_CONFIRMED. evidence_code must be one
                     'Russian whole-unit language signal diagnostic | kazakh_specific=%s cyrillic=%s | nonfatal=true',
                     len(kazakh_specific), len(cyrillic),
                 )
+        if re.search(r'(?m)^#\s+References\s*$', source):
+            cls.validate_reference_preservation(source, target, target_lang=
+                'Russian (Русский)' if re.search(r'(?m)^#\s+Литература\s*$', target) else
+                'Kazakh (Қазақша)' if re.search(r'(?m)^#\s+Пайдаланылған әдебиеттер\s*$', target) else
+                'Simplified Chinese (简体中文)'
+            )
         return True
 
     def enforce_claim_specific_citations(self, text):