Blame for core/generation_runner.py

# -*- coding: utf-8 -*-
"""R3-Free headless SmartPaper generation runner.

The runner contains no Streamlit calls. It is safe to execute in a subprocess
whose lifetime is independent from Streamlit ScriptRunner reruns and browser
connections. Checkpoints are task-scoped, immutable-by-sequence JSON snapshots.
"""
from __future__ import annotations

import hashlib
import json
import logging
import math
import os
import re
import time
from datetime import datetime, timezone, timedelta
from pathlib import Path
from typing import Callable, Any

from config import APP_CONFIG, AMJ_PROMPTS, ASQ_PROMPTS
from core.checkpoint_store import TaskCheckpointStore
from core.meta_extractor import extract_golden_metadata
from core.pipeline_agent import SSciPipelineAgent
from core.production_guard import build_validated_package, replace_abstract
from core.qualitative_engine import get_qualitative_engine
from core.quantitative_engine import get_quantitative_engine
from core.rag_engine import initialize_rag
from utils.file_parser import extract_text_from_files

ProgressCallback = Callable[..., None]

MIN_FINAL_REFERENCES = 50
LITERATURE_EVIDENCE_TARGET = 75
CITATION_PLAN_TARGET = 70


def _clean_trailing_references(text, keywords=("reference", "references", "reference list", "参考文献")):
    lines = str(text or "").split("\n")
    for i in range(len(lines) - 1, max(-1, len(lines) - 100), -1):
        if any(re.match(rf"^##?\s*{kw}", lines[i], re.IGNORECASE) for kw in keywords):
            return "\n".join(lines[:i]).strip()
    return str(text or "").strip()


def _clean_markdown_headings(text):
    return re.sub(r"(?m)^#+\s+(.*)$", r"**\1**", str(text or ""))


def _canonical_request_hash(request_data: dict[str, Any]) -> str:
    material = dict(request_data or {})
    material.pop("request_hash", None)
    encoded = json.dumps(material, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
    return hashlib.sha256(encoded).hexdigest()


class GenerationRunner:
    def __init__(
        self,
        *,
        task_dir: str | os.PathLike,
        request_data: dict[str, Any],
        execution_epoch: int,
        progress_callback: ProgressCallback | None = None,
        cancel_checker: Callable[[], bool] | None = None,
    ):
        self.task_dir = Path(task_dir)
        self.request = dict(request_data)
        self.execution_epoch = int(execution_epoch)
        self.progress_callback = progress_callback or (lambda **_: None)
        self.cancel_checker = cancel_checker or (lambda: False)
        expected_hash = str(self.request.get("request_hash") or "")
        actual_hash = _canonical_request_hash(self.request)
        if not expected_hash or expected_hash != actual_hash:
            raise RuntimeError("request_hash_mismatch")
        self.request_hash = expected_hash
        self.checkpoints = TaskCheckpointStore(self.task_dir, self.request_hash, self.execution_epoch)
        self.output_dir = self.task_dir / "outputs"
        self.output_dir.mkdir(parents=True, exist_ok=True)
        self.agent = None
        preview_token = str(self.request.get("preview_token") or "").strip()
        self.preview_token = preview_token if re.fullmatch(r"[A-Za-z0-9_-]{32,128}", preview_token) else ""
        self.preview_path = None
        self._last_preview_write = 0.0
        self._preview_english_text = ""
        self._preview_english_stage = ""
        self._preview_english_complete = False
        self._preview_target_text = ""
        self._preview_target_stage = "translation_waiting"
        self._preview_target_complete = False
        if self.preview_token:
            preview_dir = Path(__file__).resolve().parents[1] / "static" / "previews"
            preview_dir.mkdir(parents=True, exist_ok=True)
            self.preview_path = preview_dir / f"{self.preview_token}.json"
            # Best-effort cleanup of stale display-only previews; task/checkpoint
            # authority is unaffected by this cache.
            cutoff = time.time() - 24 * 3600
            for old_preview in preview_dir.glob("*.json"):
                try:
                    if old_preview.stat().st_mtime < cutoff:
                        old_preview.unlink()
                except OSError:
                    pass

    def _compute_request_counts(self) -> tuple[int, int, int]:
        """Return UI metrics: DeepSeek requests and eligible literature results."""
        agent = self.agent
        if agent is None:
            return 0, 0, 0
        metrics = getattr(agent, "source_metrics", {}) or {}
        s2_metric = metrics.get("s2", {}) if isinstance(metrics, dict) else {}
        oa_metric = metrics.get("oa", {}) if isinstance(metrics, dict) else {}
        return (
            int(getattr(agent, "ds_calls", 0) or 0),
            int(s2_metric.get("eligible_result_count", 0) or 0),
            int(oa_metric.get("eligible_result_count", 0) or 0),
        )

    def _progress(self, pct: int, node: str, *, checkpoint_seq=None, checkpoint_stage=None) -> None:
        ds_calls, s2_calls, oa_calls = self._compute_request_counts()
        self.progress_callback(
            progress_pct=max(0, min(100, int(pct))),
            current_node=str(node or "")[:240],
            checkpoint_seq=checkpoint_seq,
            checkpoint_stage=checkpoint_stage,
            ds_calls=ds_calls,
            s2_calls=s2_calls,
            oa_calls=oa_calls,
        )

    def _checkpoint(self, stage: str, state: dict[str, Any], pct: int) -> None:
        seq, normalized_stage = self.checkpoints.save(stage, state)
        self._progress(pct, normalized_stage, checkpoint_seq=seq, checkpoint_stage=normalized_stage)

    @staticmethod
    def _preview_visible_text(value: str) -> str:
        text = str(value or "")
        # Never expose hidden reasoning. Raw CITE placeholders are also hidden
        # from the user because they have not passed semantic citation QA yet;
        # the authoritative manuscript buffer remains untouched.
        text = re.sub(r"(?is)<think>.*?(?:</think>|$)", "", text)
        text = re.sub(r"\s*\[?CITE-\d+\]?\s*", " ", text, flags=re.I)
        text = re.sub(r"[ \t]{2,}", " ", text)
        return text.strip()

    def _write_preview_payload(self, *, force: bool = False) -> None:
        if self.preview_path is None:
            return
        now = time.monotonic()
        if not force and now - self._last_preview_write < 0.45:
            return
        payload = {
            "task_id": str(self.request.get("task_id") or ""),
            "request_hash": self.request_hash,
            "execution_epoch": self.execution_epoch,
            "stage": str(self._preview_english_stage or "")[:240],
            "text": self._preview_visible_text(self._preview_english_text),
            "complete": bool(self._preview_english_complete),
            "target_stage": str(self._preview_target_stage or "")[:240],
            "target_text": self._preview_visible_text(self._preview_target_text),
            "target_complete": bool(self._preview_target_complete),
            "updated_at": datetime.now(timezone.utc).isoformat(),
        }
        temporary = self.preview_path.with_suffix(".json.tmp")
        temporary.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
        os.replace(temporary, self.preview_path)
        self._last_preview_write = now

    def _publish_preview(self, text: str, stage: str, *, force: bool = False, complete: bool = False) -> None:
        self._preview_english_text = str(text or "")
        self._preview_english_stage = str(stage or "")
        self._preview_english_complete = bool(complete)
        self._write_preview_payload(force=force)

    def _publish_target_preview(self, text: str, stage: str, *, force: bool = True, complete: bool = False) -> None:
        self._preview_target_text = str(text or "")
        self._preview_target_stage = str(stage or "")
        self._preview_target_complete = bool(complete)
        self._write_preview_payload(force=force)

    def _stream_text(self, generator, *, preview_prefix: str | None = None, preview_stage: str = "") -> str:
        chunks = []
        for chunk in generator:
            value = str(chunk or "")
            if not value:
                continue
            chunks.append(value)
            if preview_prefix is not None:
                self._publish_preview(preview_prefix + "".join(chunks), preview_stage, force=False)
        text = "".join(chunks)
        if preview_prefix is not None:
            self._publish_preview(preview_prefix + text, preview_stage, force=True)
        return text

    @staticmethod
    def _text_hash(value: str) -> str:
        normalized = re.sub(r'\s+', ' ', str(value or '')).strip()
        return hashlib.sha256(normalized.encode('utf-8')).hexdigest()

    @staticmethod
    def _unique_top_level_sections(tasks):
        output = []
        seen = set()
        for task in tasks or []:
            key = str(task.get('section_id') or task.get('section') or '').strip()
            title = str(task.get('section') or '').strip()
            if key and title and key not in seen:
                seen.add(key); output.append((key, title))
        return output

    @staticmethod
    def _extract_top_level_section(manuscript: str, title: str) -> str:
        value = str(manuscript or '')
        match = re.search(
            rf'(?ms)^#\s+{re.escape(str(title))}\s*$\n(.*?)(?=^#\s+[^#].*?$|\Z)', value
        )
        if not match:
            return ''
        return f"# {title}\n" + match.group(1).strip()

    @staticmethod
    def _extract_front_matter(manuscript: str, first_section_title: str) -> str:
        value = str(manuscript or '')
        match = re.search(rf'(?m)^#\s+{re.escape(str(first_section_title))}\s*$', value)
        return value[:match.start()].strip() if match else ''

    @staticmethod
    def _replace_keywords(manuscript: str, keywords) -> str:
        value = str(manuscript or '')
        if isinstance(keywords, (list, tuple)):
            rendered = ', '.join(str(item).strip() for item in keywords if str(item).strip())
        else:
            rendered = str(keywords or '').strip()
        pattern = re.compile(r'(?ms)(^##\s+Keywords\s*$\n)(.*?)(?=^#\s+[^#].*?$|\Z)')
        matches = list(pattern.finditer(value))
        if len(matches) != 1:
            raise RuntimeError(f'keywords_heading_count_invalid:{len(matches)}')
        match = matches[0]
        return value[:match.start()] + match.group(1) + rendered + '\n\n' + value[match.end():].lstrip('\n')

    @staticmethod
    def _section_is_complete(tasks, completed_display_names, section_key: str) -> bool:
        relevant = [
            str(task.get('display_name') or '') for task in (tasks or [])
            if str(task.get('section_id') or task.get('section') or '') == str(section_key)
        ]
        done = set(str(item) for item in (completed_display_names or []))
        return bool(relevant) and all(name in done for name in relevant)

    @staticmethod
    def _glossary_version(glossary) -> str:
        encoded = json.dumps(dict(glossary or {}), ensure_ascii=False, sort_keys=True).encode('utf-8')
        return hashlib.sha256(encoded).hexdigest()[:16]

    @staticmethod
    def _assemble_partial_translation(translation_sections, adaptive_structure, *, include_front=False) -> str:
        mapping = translation_sections if isinstance(translation_sections, dict) else {}
        parts = []
        if include_front:
            record = mapping.get('__front_matter__') or {}
            if record.get('qa_status') == 'passed' and record.get('translated_text'):
                parts.append(str(record['translated_text']).strip())
        for key, _title in GenerationRunner._unique_top_level_sections(adaptive_structure):
            record = mapping.get(key) or {}
            if record.get('qa_status') == 'passed' and record.get('translated_text'):
                parts.append(str(record['translated_text']).strip())
        return '\n\n'.join(part for part in parts if part).strip()

    def _translate_frozen_unit(
        self, agent, *, unit_key: str, unit_name: str, source_text: str, target_lang: str,
        translation_sections: dict, translation_glossary: dict, adaptive_structure, include_front=False,
    ):
        source = str(source_text or '').strip()
        if not source:
            return translation_sections, translation_glossary
        source_hash = self._text_hash(source)
        existing = dict((translation_sections or {}).get(unit_key) or {})
        if (
            existing.get('qa_status') == 'passed'
            and existing.get('source_hash') == source_hash
            and existing.get('target_lang') == target_lang
            and existing.get('translated_text')
        ):
            return translation_sections, translation_glossary
        partial = self._assemble_partial_translation(translation_sections, adaptive_structure, include_front=include_front)
        self._publish_target_preview(partial, f'translating:{unit_name}', force=True, complete=False)
        result = agent.translate_frozen_section(
            source, target_lang, glossary=translation_glossary, section_name=unit_name, max_attempts=2
        )
        updated_glossary = dict(result.get('glossary') or translation_glossary or {})
        updated_sections = dict(translation_sections or {})
        updated_sections[unit_key] = {
            'source_hash': source_hash,
            'target_lang': target_lang,
            'glossary_version': self._glossary_version(updated_glossary),
            'translated_text': str(result.get('text') or '').strip(),
            'qa_status': 'passed',
            'attempts': int(result.get('attempts') or 1),
            'completed_at': datetime.now(timezone.utc).isoformat(),
        }
        partial = self._assemble_partial_translation(updated_sections, adaptive_structure, include_front=include_front)
        self._publish_target_preview(partial, f'translation_ready:{unit_name}', force=True, complete=False)
        return updated_sections, updated_glossary

    @staticmethod
    def _bounded_target(base_target: int, accumulated_deviation: float, remaining_including_current: int, safe_chunk: int) -> int:
        base = max(80, int(base_target))
        correction = -float(accumulated_deviation) / max(1, int(remaining_including_current))
        lower = max(80, int(round(base * 0.75)))
        upper = min(int(safe_chunk * 1.5), max(lower, int(round(base * 1.25))))
        return max(lower, min(upper, int(round(base + correction))))

    @staticmethod
    def _english_count_for_target(manuscript: str) -> int:
        body = re.split(r"(?m)^#\s+References\s*$", str(manuscript or ""), maxsplit=1)[0]
        body = re.split(r"(?m)^#\s+Appendix: Data Structure and Evidence\s*$", body, maxsplit=1)[0]
        # Markdown punctuation is ignored; contractions/hyphenated terms stay one token.
        return len(re.findall(r"\b[A-Za-z0-9]+(?:[-'][A-Za-z0-9]+)*\b", body))

    def _validate_target_length(self, manuscript: str, target_words: int) -> None:
        quality = APP_CONFIG.get("manuscript_validation", {})
        minimum = int(target_words * float(quality.get("minimum_total_ratio", 0.97)))
        maximum = int(target_words * float(quality.get("maximum_total_ratio", 1.03)))
        actual = self._english_count_for_target(manuscript)
        if actual < minimum or actual > maximum:
            raise RuntimeError(
                f"English manuscript length outside validated range: actual={actual}, "
                f"required={minimum}-{maximum}, target={target_words}."
            )

    def _resolve_paths(self, items) -> list[str]:
        output = []
        for item in items or []:
            path = (self.task_dir / str(item)).resolve()
            if self.task_dir.resolve() not in path.parents:
                raise RuntimeError("input_path_escape")
            if path.is_file():
                output.append(str(path))
        return output

    def run(self) -> dict[str, Any]:
        req = self.request
        mode = str(req.get("mode") or "").lower()
        if mode not in {"amj", "asq"}:
            raise RuntimeError("invalid_generation_mode")
        target_words = int(req.get("target_words") or 0)
        if not APP_CONFIG["target_words_min"] <= target_words <= APP_CONFIG["target_words_max"]:
            raise RuntimeError("target_words_out_of_range")

        ds_key = str(os.environ.get("DEEPSEEK_API_KEY") or "").strip()
        s2_key = str(os.environ.get("S2_API_KEY") or "").strip()
        if not ds_key:
            raise RuntimeError("DEEPSEEK_API_KEY_missing")

        if mode == "amj":
            APP_CONFIG.update(AMJ_PROMPTS)
        else:
            APP_CONFIG.update(ASQ_PROMPTS)

        agent = SSciPipelineAgent(ds_key, s2_key)
        self.agent = agent
        agent.set_cancel_checker(self.cancel_checker)
        # DeepSeek/search activity marks the executor as genuinely alive; the
        # heartbeat thread decides separately when to renew D1's lease.
        agent.set_execution_progress_hook(lambda: self._progress_activity())
        safe_chunk_words = int(agent.chunk_words_default)
        quant_engine = get_quantitative_engine()
        qual_engine = get_qualitative_engine(agent)

        state = self.checkpoints.load_latest() or {}
        if state:
            self._preview_english_text = str(state.get('full_draft_en') or '')
            self._preview_target_text = str(state.get('full_draft_translated') or '')
            logging.info(
                "[SP_EXECUTOR] resume checkpoint seq=%s stage=%s",
                state.get("checkpoint_seq"), state.get("checkpoint_stage"),
            )

        topic = str(req.get("topic") or "").strip()
        keywords = str(req.get("keywords") or "").strip()
        target_lang = str(req.get("target_lang") or "").strip()
        output_suffix = str(req.get("output_suffix") or "Target").strip()
        interface_language = str(req.get("interface_language") or "zh").strip()
        reference_paths = self._resolve_paths(req.get("reference_files") or [])
        data_paths = self._resolve_paths(req.get("data_files") or [])
        interview_paths = self._resolve_paths(req.get("interview_files") or [])

        base_evidence_contract = str(APP_CONFIG.get("evidence_integrity_contract") or "")

        # Restore agent state before continuing any dependent step.
        if isinstance(state.get("agent_state"), dict):
            agent.load_state(state["agent_state"])

        user_lit_context = str(state.get("user_lit_context") or "")
        user_data_md = str(state.get("user_data_md") or "")
        data_metadata = dict(state.get("data_metadata") or {})

        if not state.get("evidence_ready"):
            self._progress(3, "inputs_and_evidence")
            if reference_paths:
                user_lit_context = extract_text_from_files(reference_paths[:20])
                all_paper_texts = [extract_text_from_files([path]) for path in reference_paths[:20]]
                agent.global_memory["core_golden_metadata"] = extract_golden_metadata(agent, all_paper_texts)
            if mode == "amj":
                data_file = data_paths[0] if data_paths else None
                _, user_data_md, _, data_metadata = quant_engine.resolve_empirical_data(
                    data_file, interface_language, topic=topic, keywords=keywords, agent=agent
                )
            else:
                raw_interviews = extract_text_from_files(interview_paths[:1]) if interview_paths else ""
                user_data_md, _, data_metadata = qual_engine.resolve_qualitative_data(topic, keywords, raw_interviews)
            state.update(
                {
                    "topic": topic,
                    "keywords": keywords,
                    "target_lang": target_lang,
                    "target_words": target_words,
                    "mode": mode,
                    "interface_language": interface_language,
                    "user_lit_context": user_lit_context,
                    "user_data_md": user_data_md,
                    "data_metadata": data_metadata,
                    "agent_state": agent.export_state(),
                    "evidence_ready": True,
                }
            )
            self._checkpoint("evidence_ready", state, 6)

        refs_text = str(state.get("refs_text") or "")
        if not refs_text:
            self._progress(8, "S2 + OpenAlex")
            refs_text = agent.fetch_references_parallel(
                topic,
                keywords,
                min_required=LITERATURE_EVIDENCE_TARGET,
                progress_callback=lambda: self._progress(9, "S2 + OpenAlex"),
            )
            if len(agent.citation_map) < MIN_FINAL_REFERENCES:
                raise RuntimeError(
                    f"Insufficient eligible references: {len(agent.citation_map)}/{MIN_FINAL_REFERENCES}"
                )
            if user_lit_context:
                core_refs = agent.parse_user_core_literature(user_lit_context)
                if core_refs:
                    refs_text = (
                        "【USER PROVIDED CORE LITERATURE】\n" + core_refs
                        + "\n\n【RETRIEVED LITERATURE】\n" + refs_text
                    )
            state.update({"refs_text": refs_text, "agent_state": agent.export_state()})
            self._checkpoint("literature_ready", state, 11)

        rag_engine = initialize_rag([value for value in (user_lit_context, refs_text) if value])

        synthesis_result = str(state.get("synthesis_result") or "")
        if not synthesis_result:
            self._progress(12, "literature_synthesis")
            synthesis_result = self._stream_text(
                agent.stream_synthesize_literature(topic, refs_text, user_core_literature=user_lit_context)
            )
            synthesis_result = re.sub(r"<think>.*?</think>", "", synthesis_result, flags=re.DOTALL).strip()
            state.update({"synthesis_result": synthesis_result, "agent_state": agent.export_state()})
            self._checkpoint("synthesis_ready", state, 13)

        adaptive_structure = state.get("adaptive_structure")
        if not isinstance(adaptive_structure, list) or not adaptive_structure:
            self._progress(14, "research_logic")
            research_logic_response = self._stream_text(
                agent.stream_generate_research_logic(
                    topic, keywords, synthesis_result, refs_text,
                    data_evidence=user_data_md, mode=mode,
                )
            )
            research_logic_contract = re.sub(
                r"<think>.*?</think>", "", research_logic_response, flags=re.DOTALL
            ).strip()
            if not research_logic_contract:
                raise RuntimeError("research_logic_contract_missing")
            agent.global_memory["research_logic_contract"] = research_logic_contract

            self._progress(14, "working_front_matter")
            working_front = agent.generate_working_front_matter(
                topic, keywords, research_logic_contract, synthesis_result,
                data_evidence=user_data_md, mode=mode,
            )
            working_abstract = str(working_front.get("abstract") or "").strip()
            working_keywords = list(working_front.get("keywords") or [])
            if not working_abstract or not working_keywords:
                raise RuntimeError("working_front_matter_missing")
            agent.global_memory["working_abstract"] = working_abstract

            self._progress(14, "outline_planning")
            plan = agent.generate_outline(topic, keywords, mode=mode, data_metadata=data_metadata)
            en_topic = str(plan.get("en_topic", topic))
            structure = plan.get("structure", [])
            full_draft_en = (
                f"# {en_topic}\n\n## Abstract\n{working_abstract}\n\n"
                f"## Keywords\n{', '.join(working_keywords)}\n\n"
            )
            # Publish the Working Abstract/Keywords immediately, before JIT and
            # citation planning, so the user sees the manuscript anchor before
            # Introduction while those longer planning stages continue.
            self._publish_preview(full_draft_en, "working_front_matter", force=True)
            front_budget = max(180, int(APP_CONFIG.get("word_budget", {}).get("front_matter_words", 220)))
            body_target_words = max(1000, target_words - front_budget)
            adaptive_structure = agent.build_generation_tasks(structure, body_target_words, safe_chunk_words)
            for task in adaptive_structure:
                task["is_first"] = bool(task.get("section_first"))
            jit_texts = agent.prefetch_jit_literature(topic, keywords, adaptive_structure)
            if jit_texts:
                rag_engine._initialize_chunks([jit_texts])
            self._progress(15, "citation_planning")
            citation_plan = agent.build_claim_evidence_plan(
                adaptive_structure, topic, synthesis_result, mode=mode, target_unique=CITATION_PLAN_TARGET
            )
            planned_unique = {
                item.get("tag")
                for items in citation_plan.values()
                for item in items
                if item.get("tag")
            }
            if len(planned_unique) < MIN_FINAL_REFERENCES:
                raise RuntimeError(
                    "Insufficient claim-evidence plan before drafting: "
                    f"{len(planned_unique)}/{MIN_FINAL_REFERENCES}."
                )
            state.update(
                {
                    "adaptive_structure": adaptive_structure,
                    "citation_plan": citation_plan,
                    "research_logic_contract": research_logic_contract,
                    "working_abstract": working_abstract,
                    "working_keywords": working_keywords,
                    "front_matter_phase": "working",
                    "body_target_words": body_target_words,
                    "full_draft_en": full_draft_en,
                    "full_draft_translated": "",
                    "translation_sections": {},
                    "translation_glossary": {},
                    "completed_sections": [],
                    "context_tail": "",
                    "accumulated_deviation": 0.0,
                    "agent_state": agent.export_state(),
                    "outline_ready": True,
                }
            )
            self._publish_preview(full_draft_en, "outline_ready", force=True)
            self._checkpoint("outline_ready", state, 15)
        else:
            full_draft_en = str(state.get("full_draft_en") or "")
            # JIT text itself is already represented in agent citation state;
            # refs + user literature rebuild a safe RAG baseline on resume.

        completed_sections = list(state.get("completed_sections") or [])
        context_tail = str(state.get("context_tail") or "")
        accumulated_deviation = float(state.get("accumulated_deviation") or 0.0)
        research_logic_contract = str(
            state.get("research_logic_contract") or agent.global_memory.get("research_logic_contract") or ""
        )
        citation_plan = state.get("citation_plan") if isinstance(state.get("citation_plan"), dict) else {}
        translation_sections = dict(state.get('translation_sections') or {})
        translation_glossary = dict(state.get('translation_glossary') or {})
        working_abstract = str(state.get("working_abstract") or "").strip()
        working_keywords = list(state.get("working_keywords") or [])
        if working_abstract:
            agent.global_memory["working_abstract"] = working_abstract
        if self._preview_target_text == '':
            self._preview_target_text = str(state.get('full_draft_translated') or '')

        # Backward-compatible resume path for a pre-R3.2.8.3 checkpoint that has
        # an outline but no Working Abstract yet. Generate it once, checkpoint it,
        # and reuse it thereafter; never regenerate an unchanged anchor on resume.
        if not working_abstract and not state.get("english_finalized"):
            working_front = agent.generate_working_front_matter(
                topic, keywords, research_logic_contract, synthesis_result,
                data_evidence=user_data_md, mode=mode,
            )
            working_abstract = str(working_front.get("abstract") or "").strip()
            working_keywords = list(working_front.get("keywords") or [])
            if not working_abstract or not working_keywords:
                raise RuntimeError("working_front_matter_missing")
            agent.global_memory["working_abstract"] = working_abstract
            full_draft_en = replace_abstract(full_draft_en, working_abstract)
            full_draft_en = self._replace_keywords(full_draft_en, working_keywords)
            state.update({
                "working_abstract": working_abstract,
                "working_keywords": working_keywords,
                "front_matter_phase": "working",
                "full_draft_en": full_draft_en,
                "agent_state": agent.export_state(),
            })
            self._publish_preview(full_draft_en, "working_front_matter", force=True)
            self._checkpoint("working_front_matter_ready", state, 15)

        # Translate the QA-frozen Working front matter before Introduction. The
        # same __front_matter__ record is hash-reused on resume and invalidated
        # later when Final Abstract/Keywords replace the working front matter.
        if working_abstract and not state.get("english_finalized"):
            section_order_for_front = self._unique_top_level_sections(adaptive_structure)
            if section_order_for_front:
                working_front_source = self._extract_front_matter(full_draft_en, section_order_for_front[0][1])
                translation_sections, translation_glossary = self._translate_frozen_unit(
                    agent, unit_key='__front_matter__', unit_name='Working Front Matter',
                    source_text=working_front_source, target_lang=target_lang,
                    translation_sections=translation_sections, translation_glossary=translation_glossary,
                    adaptive_structure=adaptive_structure, include_front=True,
                )
                working_target = self._assemble_partial_translation(
                    translation_sections, adaptive_structure, include_front=True
                )
                state.update({
                    "full_draft_en": full_draft_en,
                    "full_draft_translated": working_target,
                    "translation_sections": translation_sections,
                    "translation_glossary": translation_glossary,
                    "agent_state": agent.export_state(),
                })
                self._checkpoint("working_front_matter_translated", state, 16)
        if not citation_plan:
            self._progress(15, "citation_planning")
            citation_plan = agent.build_claim_evidence_plan(
                adaptive_structure, topic, synthesis_result, mode=mode, target_unique=CITATION_PLAN_TARGET
            )
            state["citation_plan"] = citation_plan

        total_steps = len(adaptive_structure)
        for sec in adaptive_structure:
            display_name = sec["display_name"]
            if display_name in completed_sections:
                continue
            if self.cancel_checker():
                raise RuntimeError("task_cancel_requested")
            remaining = sum(1 for item in adaptive_structure if item["display_name"] not in completed_sections)
            adjusted_target = self._bounded_target(
                int(sec["target_words"]), accumulated_deviation, remaining, safe_chunk_words
            )
            sec_title = sec["section"]
            subsection = sec.get("subsection")
            is_quarantined = any(
                word in sec_title.lower()
                for word in ["abstract", "keyword", "conclusion", "references", "appendix"]
            )
            planned_items = list(citation_plan.get(display_name, []))
            assigned_tag_list = [str(item.get("tag") or "") for item in planned_items if item.get("tag")]
            claim_plan_text = agent.format_claim_evidence_plan(planned_items)
            section_id = str(sec.get("section_id") or "")
            section_role = str(
                APP_CONFIG.get("section_reasoning_profiles", {}).get(mode, {}).get(section_id) or ""
            )
            mode_directive = ""
            if section_id in {"theory", "theoretical_framing"}:
                mode_directive = str(APP_CONFIG.get("hypothesis_development_directive") or "")
            elif section_id in {"methodology", "methods"}:
                mode_directive = str(APP_CONFIG.get("methodological_defense_directive") or "")
            elif section_id == "discussion":
                mode_directive = str(APP_CONFIG.get("discussion_depth_directive") or "")
            elif section_id == "conclusion":
                mode_directive = str(APP_CONFIG.get("conclusion_anchor_directive") or "")

            if is_quarantined or section_id in {"results", "findings"}:
                citation_rule = (
                    "NO EXTERNAL LITERATURE CITATIONS ALLOWED IN THIS UNIT. "
                    "Report only frozen empirical/qualitative evidence and the interpretation permitted by the section role."
                )
            else:
                citation_rule = (
                    "Use the citation-first claim-evidence plan below. Every listed item has already been assigned to this unit on evidence relevance: use each assigned item exactly once unless it would contradict the frozen research evidence. "
                    "For each item, write one distinct claim no broader than the supplied evidence and attach exactly its assigned [CITE-XX] tag. "
                    "Never stack sources, substitute tags, or use literature as evidence for this study's own facts.\n"
                    f"<CLAIM_EVIDENCE_PLAN>\n{claim_plan_text}\n</CLAIM_EVIDENCE_PLAN>"
                )
            memory_context = agent._build_memory_context(sec_title, 2500)
            memory_block = f"\n<GLOBAL_MEMORY>\n{memory_context}\n</GLOBAL_MEMORY>\n" if memory_context else ""
            snippets = rag_engine.get_relevant_context(display_name) if rag_engine is not None else ""
            if snippets:
                memory_block += f"\n<TARGETED_RAG>\n{snippets}\n</TARGETED_RAG>\n"
            heading_directive = "Do not output the section heading; it will be inserted by the system."
            if subsection:
                heading_directive += (
                    f" Do not output the subsection heading '{subsection}'; it will also be inserted by the system. "
                    "Use level-3 headings only when analytically necessary."
                )
            elif sec_title.lower() in {"introduction", "conclusion"}:
                heading_directive += " Do not use additional headings."
            else:
                heading_directive += (
                    " Use level-2 or level-3 headings only when they clarify the argument; "
                    "never use level-4 headings or numbered headings."
                )
            table_rule = (
                "Present only frozen statistical evidence in standard Markdown tables with plain-text captions."
                if sec["is_method_or_result"] and mode == "amj" else ""
            )
            concepts = ", ".join(sec.get("concepts") or [])
            theme_evidence = [item for item in (sec.get("theme_evidence") or []) if isinstance(item, dict)]
            evidence_labels = []
            for item in theme_evidence:
                tag = str(item.get("tag") or "").strip()
                stage = str(item.get("stage") or "Not specified").strip()
                role = str(item.get("evidence_role") or "typical").strip()
                case_label = str(item.get("case_label") or "Not specified").strip()
                if tag:
                    evidence_labels.append(f"{tag} [stage={stage}; case={case_label}; role={role}]")
            qualitative_rule = (
                f"Assigned qualitative concepts: {concepts}. Assigned quotation tags: {', '.join(evidence_labels)}. "
                "Use only these assigned quotation tags in this Findings unit and do not repeat a tag used elsewhere. "
                "When invoking a quotation, output only its [QUOTE-XX] tag at the exact position; do not reproduce or paraphrase the quotation because the system inserts the frozen verbatim text. "
                "Preserve any frozen stage/case/deviant role and use deviant evidence to qualify rather than erase the pattern."
                if concepts or evidence_labels else ""
            )
            evidence_packet = (
                f"{base_evidence_contract}\n<FROZEN_RESEARCH_EVIDENCE>\n{user_data_md}\n</FROZEN_RESEARCH_EVIDENCE>"
                if user_data_md else base_evidence_contract
            )
            completed_label = ", ".join(completed_sections)
            anti_repeat = (
                f"Already completed: {completed_label}. Do not restate their background, gap, or contribution."
                if completed_label else ""
            )
            voice = str(APP_CONFIG.get("journal_voice_profiles", {}).get(mode) or "")
            sys_prompt = (
                APP_CONFIG.get("sys_academic_writer", "") + "\n"
                + evidence_packet + "\n" + voice + "\n"
                + APP_CONFIG.get("deep_refinement_directive", "")
            )
            word_budget = APP_CONFIG.get("word_budget", {})
            min_ratio = float(word_budget.get("section_min_ratio", 0.92))
            max_ratio = float(word_budget.get("section_max_ratio", 1.08))
            user_prompt = f"""Write the assigned English manuscript unit.
Section: {sec_title}
Subsection: {subsection or 'None'}
Section reasoning role: {section_role}
Additional mode directive: {mode_directive}
Target length: {int(adjusted_target * min_ratio)}-{int(adjusted_target * max_ratio)} words.
{heading_directive}
{citation_rule}
{table_rule}
{str(sec.get('required_content') or '')}
{qualitative_rule}
{anti_repeat}
<PROVISIONAL_RESEARCH_LOGIC>
{research_logic_contract}
</PROVISIONAL_RESEARCH_LOGIC>
<WORKING_ABSTRACT_ANCHOR>
{working_abstract}
</WORKING_ABSTRACT_ANCHOR>
ANCHOR PRECEDENCE: Frozen research evidence and the PROVISIONAL_RESEARCH_LOGIC override the Working Abstract if any tension appears. The Working Abstract is continuity guidance only and must never force a result.
{memory_block}
Literature synthesis: {synthesis_result}
<FROZEN_RESEARCH_EVIDENCE>
{user_data_md}
</FROZEN_RESEARCH_EVIDENCE>
Previous tail: {context_tail[-2500:]}
Output only final academic prose. Do not report the word count."""
            if agent._count_tokens(sys_prompt) + agent._count_tokens(user_prompt) > 60000:
                user_prompt = agent._truncate_context(user_prompt, 60000 - agent._count_tokens(sys_prompt))

            current_index = len(completed_sections) + 1
            pct = min(88, 15 + int((current_index / max(1, total_steps)) * 72))
            self._progress(pct, display_name)
            prefix = ""
            if sec.get("section_first"):
                prefix += f"# {sec_title}\n\n"
            if sec.get("subsection_first") and subsection:
                prefix += f"## {subsection}\n\n"
            en_raw = self._stream_text(
                agent.stream_deepseek(
                    sys_prompt, user_prompt, max_tokens=7000,
                    model_role="writing", thinking=False, allow_fallback=False,
                ),
                preview_prefix=full_draft_en + prefix,
                preview_stage=display_name,
            )
            en_raw = re.sub(r"<think>.*?</think>", "", en_raw, flags=re.S).strip()
            en_raw = re.sub(rf"(?im)^#\s*{re.escape(sec_title)}\s*\n+", "", en_raw).strip()
            if subsection:
                en_raw = re.sub(rf"(?im)^##\s*{re.escape(subsection)}\s*\n+", "", en_raw).strip()
            en_raw = re.sub(r"(?m)^(#{2,3})\s+[\d\.\s一二三四五六七八九十、]+\s*(.*)", r"\1 \2", en_raw)
            en_raw = re.sub(r"(?m)^#{4,}\s+(.*)", r"**\1**", en_raw)
            en_raw = re.sub(r"(?m)^#\s+(.+)$", r"## \1", en_raw)
            en_raw = agent.critique_and_repair_section(
                en_raw, section_id=section_id, section_name=display_name, mode=mode,
                evidence_contract=evidence_packet,
            )
            en_raw = agent.polish_academic_section(en_raw, display_name, abstract=False, mode=mode)
            # Keep QUOTE tags immutable through critic/style editing, then bind
            # each tag to its frozen verbatim quotation exactly once. This
            # prevents editorial passes from silently paraphrasing evidence.
            if mode == "asq" and user_data_md and "appendix" not in sec_title.lower():
                quote_dict = {tag: quote for quote, tag in re.findall(r'"([^"]+)"\s*\[(QUOTE-\d+)\]', user_data_md)}
                for tag, quote in quote_dict.items():
                    en_raw = en_raw.replace(tag, f'"{quote}"')
            if not is_quarantined and assigned_tag_list:
                en_raw = agent.reinforce_claim_specific_citation_coverage(
                    en_raw, assigned_tag_list, section_name=display_name
                )
            used_before = set(agent.used_citations)
            en_resolved = agent.audit_and_resolve_citations(
                prefix + en_raw, section_name=display_name, quarantine=is_quarantined
            )
            used_after = set(agent.used_citations)
            survived = used_after - used_before
            if assigned_tag_list and not is_quarantined:
                required_survival = min(
                    len(assigned_tag_list),
                    max(1, int(math.ceil(len(assigned_tag_list) * 0.78))),
                )
                if len(survived) < required_survival:
                    missing_items = [
                        item for item in planned_items
                        if str(item.get("tag") or "") not in used_after
                    ]
                    needed = max(0, required_survival - len(survived))
                    supplemental_raw = agent.build_citation_recovery_paragraph(
                        missing_items[:needed], section_name=display_name
                    )
                    if supplemental_raw:
                        supplemental_resolved = agent.audit_and_resolve_citations(
                            supplemental_raw,
                            section_name=f"{display_name} — citation recovery",
                            quarantine=False,
                        )
                        if supplemental_resolved.strip():
                            en_resolved = en_resolved.rstrip() + "\n\n" + supplemental_resolved.strip()
                        used_after = set(agent.used_citations)
                        survived = used_after - used_before
            full_draft_en += en_resolved + "\n\n"
            if assigned_tag_list:
                logging.info(
                    "Citation-first survival: section=%s planned=%s validated_new=%s",
                    display_name, len(assigned_tag_list), len(survived),
                )
            self._publish_preview(full_draft_en, display_name, force=True)
            agent._extract_memory_from_text(en_raw, sec_title)
            context_tail = " ".join(en_raw.split()[-200:])
            actual_words = self._english_count_for_target(en_resolved)
            accumulated_deviation += actual_words - adjusted_target
            completed_sections.append(display_name)
            section_key = str(sec.get('section_id') or sec_title)
            if self._section_is_complete(adaptive_structure, completed_sections, section_key):
                frozen_section = self._extract_top_level_section(full_draft_en, sec_title)
                translation_sections, translation_glossary = self._translate_frozen_unit(
                    agent, unit_key=section_key, unit_name=sec_title, source_text=frozen_section,
                    target_lang=target_lang, translation_sections=translation_sections,
                    translation_glossary=translation_glossary, adaptive_structure=adaptive_structure,
                    include_front=True,
                )
            partial_translation = self._assemble_partial_translation(translation_sections, adaptive_structure, include_front=True)
            state.update(
                {
                    "completed_sections": completed_sections,
                    "full_draft_en": full_draft_en,
                    "full_draft_translated": partial_translation,
                    "translation_sections": translation_sections,
                    "translation_glossary": translation_glossary,
                    "context_tail": context_tail,
                    "accumulated_deviation": accumulated_deviation,
                    "agent_state": agent.export_state(),
                    "data_metadata": data_metadata,
                }
            )
            self._checkpoint(f"section_{len(completed_sections):03d}", state, pct)

        if not state.get("english_finalized"):
            self._progress(91, "english_validation")
            full_draft_en = _clean_trailing_references(full_draft_en, ["Reference", "References", "Reference List"])
            # Validate the completed body while the Working Abstract is still in
            # place. Final front matter must not be generated until all body
            # sections, citation cleanup, and any Discussion rebalance are frozen.
            agent.validate_manuscript_structure(full_draft_en, adaptive_structure, translated=False)
            if mode == "asq":
                repeated_quotes = agent.find_repeated_qualitative_quotes(full_draft_en, data_metadata)
                if repeated_quotes:
                    raise RuntimeError(f"Repeated qualitative quotations: {repeated_quotes[:5]}")
            references = []
            seen_refs = set()
            for tag in sorted(agent.used_citations, key=agent._citation_tag_number):
                ref = str((agent.citation_map.get(tag) or {}).get("ref") or "").strip()
                if ref and ref not in seen_refs:
                    references.append(f"- {ref}")
                    seen_refs.add(ref)
            if len(references) < MIN_FINAL_REFERENCES:
                raise RuntimeError(
                    "Insufficient final cited references after semantic validation: "
                    f"{len(references)}/{MIN_FINAL_REFERENCES}."
                )
            full_draft_en = full_draft_en.rstrip() + "\n\n# References\n" + "\n\n".join(references)
            actual_words = self._english_count_for_target(full_draft_en)
            quality = APP_CONFIG.get("manuscript_validation", {})
            minimum_words = int(target_words * float(quality.get("minimum_total_ratio", 0.97)))
            maximum_words = int(target_words * float(quality.get("maximum_total_ratio", 1.03)))
            if actual_words < minimum_words or actual_words > maximum_words:
                full_draft_en = agent.rebalance_discussion_length(
                    full_draft_en, target_words=target_words, mode=mode, actual_words=actual_words
                )
            body_only, _reference_tail = agent.validate_reference_tail(
                full_draft_en, minimum_references=MIN_FINAL_REFERENCES
            )
            agent.validate_manuscript_structure(body_only, adaptive_structure, translated=False)
            agent.enforce_resolved_claim_specific_citations(body_only)

            # Body is now frozen. Generate both Final Abstract and Final Keywords
            # from the completed manuscript/evidence, then replace the provisional
            # front matter deterministically. No body rebalance may occur afterward.
            self._progress(92, "final_front_matter")
            final_front = agent.generate_final_front_matter(
                topic, keywords, body_only, data_evidence=user_data_md, mode=mode
            )
            final_abstract = str(final_front.get("abstract") or "").strip()
            final_keywords = list(final_front.get("keywords") or [])
            if not final_abstract or not final_keywords:
                raise RuntimeError("final_front_matter_missing")
            full_draft_en = replace_abstract(full_draft_en, final_abstract)
            full_draft_en = self._replace_keywords(full_draft_en, final_keywords)
            body_only, _reference_tail = agent.validate_reference_tail(
                full_draft_en, minimum_references=MIN_FINAL_REFERENCES
            )
            agent.validate_manuscript_structure(body_only, adaptive_structure, translated=False)
            agent.enforce_resolved_claim_specific_citations(body_only)
            self._validate_target_length(full_draft_en, target_words)

            # Final front matter has a different source hash from the Working
            # front matter. Retranslate only __front_matter__; unchanged body
            # translations are retained and reused.
            section_order_for_front = self._unique_top_level_sections(adaptive_structure)
            if section_order_for_front:
                final_front_source = self._extract_front_matter(body_only, section_order_for_front[0][1])
                translation_sections, translation_glossary = self._translate_frozen_unit(
                    agent, unit_key='__front_matter__', unit_name='Final Front Matter',
                    source_text=final_front_source, target_lang=target_lang,
                    translation_sections=translation_sections, translation_glossary=translation_glossary,
                    adaptive_structure=adaptive_structure, include_front=True,
                )
            partial_target = self._assemble_partial_translation(
                translation_sections, adaptive_structure, include_front=True
            )
            self._publish_preview(full_draft_en, "english_complete", force=True, complete=True)
            state.update(
                {
                    "full_draft_en": full_draft_en,
                    "full_draft_translated": partial_target,
                    "translation_sections": translation_sections,
                    "translation_glossary": translation_glossary,
                    "final_abstract": final_abstract,
                    "final_keywords": final_keywords,
                    "front_matter_phase": "final",
                    "english_finalized": True,
                    "translation_complete": False,
                    "agent_state": agent.export_state(),
                }
            )
            self._checkpoint("english_complete", state, 93)
        else:
            full_draft_en = str(state.get("full_draft_en") or "")

        full_draft_translated = str(state.get("full_draft_translated") or "")
        translation_sections = dict(state.get('translation_sections') or translation_sections or {})
        translation_glossary = dict(state.get('translation_glossary') or translation_glossary or {})
        if not state.get("translation_complete") or not full_draft_translated:
            self._progress(94, "translation")
            body_only, reference_tail = agent.validate_reference_tail(
                full_draft_en, minimum_references=MIN_FINAL_REFERENCES
            )
            section_order = self._unique_top_level_sections(adaptive_structure)
            if not section_order:
                raise RuntimeError('translation_section_plan_missing')
            front_matter = self._extract_front_matter(body_only, section_order[0][1])
            final_units = [('__front_matter__', 'Front Matter', front_matter)]
            for key, title in section_order:
                final_units.append((key, title, self._extract_top_level_section(body_only, title)))
            for index, (unit_key, unit_name, source_text) in enumerate(final_units, start=1):
                translation_sections, translation_glossary = self._translate_frozen_unit(
                    agent, unit_key=unit_key, unit_name=unit_name, source_text=source_text,
                    target_lang=target_lang, translation_sections=translation_sections,
                    translation_glossary=translation_glossary, adaptive_structure=adaptive_structure,
                    include_front=True,
                )
                partial = self._assemble_partial_translation(
                    translation_sections, adaptive_structure, include_front=True
                )
                state.update({
                    'full_draft_en': full_draft_en,
                    'full_draft_translated': partial,
                    'translation_sections': translation_sections,
                    'translation_glossary': translation_glossary,
                    'agent_state': agent.export_state(),
                })
                pct = min(96, 94 + int(index * 2 / max(1, len(final_units))))
                self._checkpoint(f'translation_{unit_key}', state, pct)

            full_draft_translated = self._assemble_partial_translation(
                translation_sections, adaptive_structure, include_front=True
            ).rstrip()
            full_draft_translated += (
                f"\n\n# {agent._localized_reference_heading(target_lang)}\n" + reference_tail.strip()
            )
            agent.validate_translation_alignment(full_draft_en, full_draft_translated)
            self._publish_target_preview(
                full_draft_translated, 'translation_complete', force=True, complete=True
            )
            state.update(
                {
                    "full_draft_translated": full_draft_translated,
                    "translation_sections": translation_sections,
                    "translation_glossary": translation_glossary,
                    "translation_complete": True,
                    "agent_state": agent.export_state(),
                }
            )
            self._checkpoint("translation_complete", state, 96)

        existing_output = str(state.get("output_filename") or "").strip()
        if state.get("package_validated") and existing_output:
            existing_path = self.output_dir / os.path.basename(existing_output)
            if existing_path.is_file() and existing_path.stat().st_size > 0:
                self._progress(99, "package_validated")
                return {"output_filename": existing_path.name, "output_path": str(existing_path)}

        self._progress(97, "packaging")
        bj_time = datetime.now(timezone(timedelta(hours=8))).strftime("%Y%m%d_%H%M")
        package = build_validated_package(
            english_markdown=full_draft_en,
            translated_markdown=full_draft_translated,
            english_filename=f"Research_EN_{bj_time}.docx",
            translated_filename=f"Research_{output_suffix}_{bj_time}.docx",
            data_metadata=data_metadata,
            mode=mode,
            citation_audit=agent.citation_audit_log,
            literature_metrics=agent.get_source_metrics(),
        )
        output_filename = f"SmartPaper_{bj_time}.zip"
        pending = self.output_dir / (".pending_" + output_filename)
        pending.write_bytes(package.zip_bytes)
        final = self.output_dir / output_filename
        os.replace(pending, final)
        state.update(
            {
                "output_filename": output_filename,
                "package_validated": True,
                "agent_state": agent.export_state(),
            }
        )
        self._checkpoint("package_validated", state, 99)
        return {"output_filename": output_filename, "output_path": str(final)}

    def _progress_activity(self) -> None:
        # Activity remains Executor-owned. We also snapshot actual request
        # counters so the next Executor heartbeat can persist live UI metrics.
        ds_calls, s2_calls, oa_calls = self._compute_request_counts()
        try:
            self.progress_callback(
                activity_only=True,
                ds_calls=ds_calls,
                s2_calls=s2_calls,
                oa_calls=oa_calls,
            )
        except TypeError:
            pass