Spaces:
Running
Running
| # -*- coding: utf-8 -*- | |
| """R3-Free headless SmartPaper generation runner. | |
| The runner contains no Streamlit calls. It is safe to execute in a subprocess | |
| whose lifetime is independent from Streamlit ScriptRunner reruns and browser | |
| connections. Checkpoints are task-scoped, immutable-by-sequence JSON snapshots. | |
| """ | |
| from __future__ import annotations | |
| import hashlib | |
| import json | |
| import logging | |
| import math | |
| import os | |
| import re | |
| import time | |
| from datetime import datetime, timezone, timedelta | |
| from pathlib import Path | |
| from typing import Callable, Any | |
| from config import APP_CONFIG, AMJ_PROMPTS, ASQ_PROMPTS | |
| from core.checkpoint_store import TaskCheckpointStore | |
| from core.meta_extractor import extract_golden_metadata | |
| from core.pipeline_agent import SSciPipelineAgent | |
| from core.production_guard import build_validated_package, replace_abstract | |
| from core.qualitative_engine import get_qualitative_engine | |
| from core.quantitative_engine import get_quantitative_engine | |
| from core.rag_engine import initialize_rag | |
| from utils.file_parser import extract_text_from_files | |
| ProgressCallback = Callable[..., None] | |
| MIN_FINAL_REFERENCES = 50 | |
| LITERATURE_EVIDENCE_TARGET = 75 | |
| CITATION_PLAN_TARGET = 70 | |
| def _clean_trailing_references(text, keywords=("reference", "references", "reference list", "参考文献")): | |
| lines = str(text or "").split("\n") | |
| for i in range(len(lines) - 1, max(-1, len(lines) - 100), -1): | |
| if any(re.match(rf"^##?\s*{kw}", lines[i], re.IGNORECASE) for kw in keywords): | |
| return "\n".join(lines[:i]).strip() | |
| return str(text or "").strip() | |
| def _clean_markdown_headings(text): | |
| return re.sub(r"(?m)^#+\s+(.*)$", r"**\1**", str(text or "")) | |
| def _canonical_request_hash(request_data: dict[str, Any]) -> str: | |
| material = dict(request_data or {}) | |
| material.pop("request_hash", None) | |
| encoded = json.dumps(material, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode("utf-8") | |
| return hashlib.sha256(encoded).hexdigest() | |
| class GenerationRunner: | |
| def __init__( | |
| self, | |
| *, | |
| task_dir: str | os.PathLike, | |
| request_data: dict[str, Any], | |
| execution_epoch: int, | |
| progress_callback: ProgressCallback | None = None, | |
| cancel_checker: Callable[[], bool] | None = None, | |
| ): | |
| self.task_dir = Path(task_dir) | |
| self.request = dict(request_data) | |
| self.execution_epoch = int(execution_epoch) | |
| self.progress_callback = progress_callback or (lambda **_: None) | |
| self.cancel_checker = cancel_checker or (lambda: False) | |
| expected_hash = str(self.request.get("request_hash") or "") | |
| actual_hash = _canonical_request_hash(self.request) | |
| if not expected_hash or expected_hash != actual_hash: | |
| raise RuntimeError("request_hash_mismatch") | |
| self.request_hash = expected_hash | |
| self.checkpoints = TaskCheckpointStore(self.task_dir, self.request_hash, self.execution_epoch) | |
| self.output_dir = self.task_dir / "outputs" | |
| self.output_dir.mkdir(parents=True, exist_ok=True) | |
| self.agent = None | |
| preview_token = str(self.request.get("preview_token") or "").strip() | |
| self.preview_token = preview_token if re.fullmatch(r"[A-Za-z0-9_-]{32,128}", preview_token) else "" | |
| self.preview_path = None | |
| self._last_preview_write = 0.0 | |
| self._preview_english_text = "" | |
| self._preview_english_stage = "" | |
| self._preview_english_complete = False | |
| self._preview_target_text = "" | |
| self._preview_target_stage = "translation_waiting" | |
| self._preview_target_complete = False | |
| if self.preview_token: | |
| preview_dir = Path(__file__).resolve().parents[1] / "static" / "previews" | |
| preview_dir.mkdir(parents=True, exist_ok=True) | |
| self.preview_path = preview_dir / f"{self.preview_token}.json" | |
| # Best-effort cleanup of stale display-only previews; task/checkpoint | |
| # authority is unaffected by this cache. | |
| cutoff = time.time() - 24 * 3600 | |
| for old_preview in preview_dir.glob("*.json"): | |
| try: | |
| if old_preview.stat().st_mtime < cutoff: | |
| old_preview.unlink() | |
| except OSError: | |
| pass | |
| def _compute_request_counts(self) -> tuple[int, int, int]: | |
| """Return UI metrics: DeepSeek requests and eligible literature results.""" | |
| agent = self.agent | |
| if agent is None: | |
| return 0, 0, 0 | |
| metrics = getattr(agent, "source_metrics", {}) or {} | |
| s2_metric = metrics.get("s2", {}) if isinstance(metrics, dict) else {} | |
| oa_metric = metrics.get("oa", {}) if isinstance(metrics, dict) else {} | |
| return ( | |
| int(getattr(agent, "ds_calls", 0) or 0), | |
| int(s2_metric.get("eligible_result_count", 0) or 0), | |
| int(oa_metric.get("eligible_result_count", 0) or 0), | |
| ) | |
| def _progress(self, pct: int, node: str, *, checkpoint_seq=None, checkpoint_stage=None) -> None: | |
| ds_calls, s2_calls, oa_calls = self._compute_request_counts() | |
| self.progress_callback( | |
| progress_pct=max(0, min(100, int(pct))), | |
| current_node=str(node or "")[:240], | |
| checkpoint_seq=checkpoint_seq, | |
| checkpoint_stage=checkpoint_stage, | |
| ds_calls=ds_calls, | |
| s2_calls=s2_calls, | |
| oa_calls=oa_calls, | |
| ) | |
| def _checkpoint(self, stage: str, state: dict[str, Any], pct: int) -> None: | |
| seq, normalized_stage = self.checkpoints.save(stage, state) | |
| self._progress(pct, normalized_stage, checkpoint_seq=seq, checkpoint_stage=normalized_stage) | |
| def _preview_visible_text(value: str) -> str: | |
| text = str(value or "") | |
| # Never expose hidden reasoning. Raw CITE placeholders are also hidden | |
| # from the user because they have not passed semantic citation QA yet; | |
| # the authoritative manuscript buffer remains untouched. | |
| text = re.sub(r"(?is)<think>.*?(?:</think>|$)", "", text) | |
| text = re.sub(r"\s*\[?CITE-\d+\]?\s*", " ", text, flags=re.I) | |
| text = re.sub(r"[ \t]{2,}", " ", text) | |
| return text.strip() | |
| def _write_preview_payload(self, *, force: bool = False) -> None: | |
| if self.preview_path is None: | |
| return | |
| now = time.monotonic() | |
| if not force and now - self._last_preview_write < 0.45: | |
| return | |
| payload = { | |
| "task_id": str(self.request.get("task_id") or ""), | |
| "request_hash": self.request_hash, | |
| "execution_epoch": self.execution_epoch, | |
| "stage": str(self._preview_english_stage or "")[:240], | |
| "text": self._preview_visible_text(self._preview_english_text), | |
| "complete": bool(self._preview_english_complete), | |
| "target_stage": str(self._preview_target_stage or "")[:240], | |
| "target_text": self._preview_visible_text(self._preview_target_text), | |
| "target_complete": bool(self._preview_target_complete), | |
| "updated_at": datetime.now(timezone.utc).isoformat(), | |
| } | |
| temporary = self.preview_path.with_suffix(".json.tmp") | |
| temporary.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8") | |
| os.replace(temporary, self.preview_path) | |
| self._last_preview_write = now | |
| def _publish_preview(self, text: str, stage: str, *, force: bool = False, complete: bool = False) -> None: | |
| self._preview_english_text = str(text or "") | |
| self._preview_english_stage = str(stage or "") | |
| self._preview_english_complete = bool(complete) | |
| self._write_preview_payload(force=force) | |
| def _publish_target_preview(self, text: str, stage: str, *, force: bool = True, complete: bool = False) -> None: | |
| self._preview_target_text = str(text or "") | |
| self._preview_target_stage = str(stage or "") | |
| self._preview_target_complete = bool(complete) | |
| self._write_preview_payload(force=force) | |
| def _stream_text(self, generator, *, preview_prefix: str | None = None, preview_stage: str = "") -> str: | |
| chunks = [] | |
| for chunk in generator: | |
| value = str(chunk or "") | |
| if not value: | |
| continue | |
| chunks.append(value) | |
| if preview_prefix is not None: | |
| self._publish_preview(preview_prefix + "".join(chunks), preview_stage, force=False) | |
| text = "".join(chunks) | |
| if preview_prefix is not None: | |
| self._publish_preview(preview_prefix + text, preview_stage, force=True) | |
| return text | |
| def _text_hash(value: str) -> str: | |
| normalized = re.sub(r'\s+', ' ', str(value or '')).strip() | |
| return hashlib.sha256(normalized.encode('utf-8')).hexdigest() | |
| def _unique_top_level_sections(tasks): | |
| output = [] | |
| seen = set() | |
| for task in tasks or []: | |
| key = str(task.get('section_id') or task.get('section') or '').strip() | |
| title = str(task.get('section') or '').strip() | |
| if key and title and key not in seen: | |
| seen.add(key); output.append((key, title)) | |
| return output | |
| def _extract_top_level_section(manuscript: str, title: str) -> str: | |
| value = str(manuscript or '') | |
| match = re.search( | |
| rf'(?ms)^#\s+{re.escape(str(title))}\s*$\n(.*?)(?=^#\s+[^#].*?$|\Z)', value | |
| ) | |
| if not match: | |
| return '' | |
| return f"# {title}\n" + match.group(1).strip() | |
| def _extract_front_matter(manuscript: str, first_section_title: str) -> str: | |
| value = str(manuscript or '') | |
| match = re.search(rf'(?m)^#\s+{re.escape(str(first_section_title))}\s*$', value) | |
| return value[:match.start()].strip() if match else '' | |
| def _replace_keywords(manuscript: str, keywords) -> str: | |
| value = str(manuscript or '') | |
| if isinstance(keywords, (list, tuple)): | |
| rendered = ', '.join(str(item).strip() for item in keywords if str(item).strip()) | |
| else: | |
| rendered = str(keywords or '').strip() | |
| pattern = re.compile(r'(?ms)(^##\s+Keywords\s*$\n)(.*?)(?=^#\s+[^#].*?$|\Z)') | |
| matches = list(pattern.finditer(value)) | |
| if len(matches) != 1: | |
| raise RuntimeError(f'keywords_heading_count_invalid:{len(matches)}') | |
| match = matches[0] | |
| return value[:match.start()] + match.group(1) + rendered + '\n\n' + value[match.end():].lstrip('\n') | |
| def _section_is_complete(tasks, completed_display_names, section_key: str) -> bool: | |
| relevant = [ | |
| str(task.get('display_name') or '') for task in (tasks or []) | |
| if str(task.get('section_id') or task.get('section') or '') == str(section_key) | |
| ] | |
| done = set(str(item) for item in (completed_display_names or [])) | |
| return bool(relevant) and all(name in done for name in relevant) | |
| def _glossary_version(glossary) -> str: | |
| encoded = json.dumps(dict(glossary or {}), ensure_ascii=False, sort_keys=True).encode('utf-8') | |
| return hashlib.sha256(encoded).hexdigest()[:16] | |
| def _assemble_partial_translation(translation_sections, adaptive_structure, *, include_front=False) -> str: | |
| mapping = translation_sections if isinstance(translation_sections, dict) else {} | |
| parts = [] | |
| if include_front: | |
| record = mapping.get('__front_matter__') or {} | |
| if record.get('qa_status') == 'passed' and record.get('translated_text'): | |
| parts.append(str(record['translated_text']).strip()) | |
| for key, _title in GenerationRunner._unique_top_level_sections(adaptive_structure): | |
| record = mapping.get(key) or {} | |
| if record.get('qa_status') == 'passed' and record.get('translated_text'): | |
| parts.append(str(record['translated_text']).strip()) | |
| return '\n\n'.join(part for part in parts if part).strip() | |
| def _translate_frozen_unit( | |
| self, agent, *, unit_key: str, unit_name: str, source_text: str, target_lang: str, | |
| translation_sections: dict, translation_glossary: dict, adaptive_structure, include_front=False, | |
| ): | |
| source = str(source_text or '').strip() | |
| if not source: | |
| return translation_sections, translation_glossary | |
| source_hash = self._text_hash(source) | |
| existing = dict((translation_sections or {}).get(unit_key) or {}) | |
| if ( | |
| existing.get('qa_status') == 'passed' | |
| and existing.get('source_hash') == source_hash | |
| and existing.get('target_lang') == target_lang | |
| and existing.get('translated_text') | |
| ): | |
| return translation_sections, translation_glossary | |
| partial = self._assemble_partial_translation(translation_sections, adaptive_structure, include_front=include_front) | |
| self._publish_target_preview(partial, f'translating:{unit_name}', force=True, complete=False) | |
| result = agent.translate_frozen_section( | |
| source, target_lang, glossary=translation_glossary, section_name=unit_name, max_attempts=2 | |
| ) | |
| updated_glossary = dict(result.get('glossary') or translation_glossary or {}) | |
| updated_sections = dict(translation_sections or {}) | |
| updated_sections[unit_key] = { | |
| 'source_hash': source_hash, | |
| 'target_lang': target_lang, | |
| 'glossary_version': self._glossary_version(updated_glossary), | |
| 'translated_text': str(result.get('text') or '').strip(), | |
| 'qa_status': 'passed', | |
| 'attempts': int(result.get('attempts') or 1), | |
| 'completed_at': datetime.now(timezone.utc).isoformat(), | |
| } | |
| partial = self._assemble_partial_translation(updated_sections, adaptive_structure, include_front=include_front) | |
| self._publish_target_preview(partial, f'translation_ready:{unit_name}', force=True, complete=False) | |
| return updated_sections, updated_glossary | |
| def _bounded_target(base_target: int, accumulated_deviation: float, remaining_including_current: int, safe_chunk: int) -> int: | |
| base = max(80, int(base_target)) | |
| correction = -float(accumulated_deviation) / max(1, int(remaining_including_current)) | |
| lower = max(80, int(round(base * 0.75))) | |
| upper = min(int(safe_chunk * 1.5), max(lower, int(round(base * 1.25)))) | |
| return max(lower, min(upper, int(round(base + correction)))) | |
| def _english_count_for_target(manuscript: str) -> int: | |
| body = re.split(r"(?m)^#\s+References\s*$", str(manuscript or ""), maxsplit=1)[0] | |
| body = re.split(r"(?m)^#\s+Appendix: Data Structure and Evidence\s*$", body, maxsplit=1)[0] | |
| # Markdown punctuation is ignored; contractions/hyphenated terms stay one token. | |
| return len(re.findall(r"\b[A-Za-z0-9]+(?:[-'][A-Za-z0-9]+)*\b", body)) | |
| def _validate_target_length(self, manuscript: str, target_words: int) -> None: | |
| quality = APP_CONFIG.get("manuscript_validation", {}) | |
| minimum = int(target_words * float(quality.get("minimum_total_ratio", 0.97))) | |
| maximum = int(target_words * float(quality.get("maximum_total_ratio", 1.03))) | |
| actual = self._english_count_for_target(manuscript) | |
| if actual < minimum or actual > maximum: | |
| raise RuntimeError( | |
| f"English manuscript length outside validated range: actual={actual}, " | |
| f"required={minimum}-{maximum}, target={target_words}." | |
| ) | |
| def _resolve_paths(self, items) -> list[str]: | |
| output = [] | |
| for item in items or []: | |
| path = (self.task_dir / str(item)).resolve() | |
| if self.task_dir.resolve() not in path.parents: | |
| raise RuntimeError("input_path_escape") | |
| if path.is_file(): | |
| output.append(str(path)) | |
| return output | |
| def run(self) -> dict[str, Any]: | |
| req = self.request | |
| mode = str(req.get("mode") or "").lower() | |
| if mode not in {"amj", "asq"}: | |
| raise RuntimeError("invalid_generation_mode") | |
| target_words = int(req.get("target_words") or 0) | |
| if not APP_CONFIG["target_words_min"] <= target_words <= APP_CONFIG["target_words_max"]: | |
| raise RuntimeError("target_words_out_of_range") | |
| ds_key = str(os.environ.get("DEEPSEEK_API_KEY") or "").strip() | |
| s2_key = str(os.environ.get("S2_API_KEY") or "").strip() | |
| if not ds_key: | |
| raise RuntimeError("DEEPSEEK_API_KEY_missing") | |
| if mode == "amj": | |
| APP_CONFIG.update(AMJ_PROMPTS) | |
| else: | |
| APP_CONFIG.update(ASQ_PROMPTS) | |
| agent = SSciPipelineAgent(ds_key, s2_key) | |
| self.agent = agent | |
| agent.set_cancel_checker(self.cancel_checker) | |
| # DeepSeek/search activity marks the executor as genuinely alive; the | |
| # heartbeat thread decides separately when to renew D1's lease. | |
| agent.set_execution_progress_hook(lambda: self._progress_activity()) | |
| safe_chunk_words = int(agent.chunk_words_default) | |
| quant_engine = get_quantitative_engine() | |
| qual_engine = get_qualitative_engine(agent) | |
| state = self.checkpoints.load_latest() or {} | |
| if state: | |
| self._preview_english_text = str(state.get('full_draft_en') or '') | |
| self._preview_target_text = str(state.get('full_draft_translated') or '') | |
| logging.info( | |
| "[SP_EXECUTOR] resume checkpoint seq=%s stage=%s", | |
| state.get("checkpoint_seq"), state.get("checkpoint_stage"), | |
| ) | |
| topic = str(req.get("topic") or "").strip() | |
| keywords = str(req.get("keywords") or "").strip() | |
| target_lang = str(req.get("target_lang") or "").strip() | |
| output_suffix = str(req.get("output_suffix") or "Target").strip() | |
| interface_language = str(req.get("interface_language") or "zh").strip() | |
| reference_paths = self._resolve_paths(req.get("reference_files") or []) | |
| data_paths = self._resolve_paths(req.get("data_files") or []) | |
| interview_paths = self._resolve_paths(req.get("interview_files") or []) | |
| base_evidence_contract = str(APP_CONFIG.get("evidence_integrity_contract") or "") | |
| # Restore agent state before continuing any dependent step. | |
| if isinstance(state.get("agent_state"), dict): | |
| agent.load_state(state["agent_state"]) | |
| user_lit_context = str(state.get("user_lit_context") or "") | |
| user_data_md = str(state.get("user_data_md") or "") | |
| data_metadata = dict(state.get("data_metadata") or {}) | |
| if not state.get("evidence_ready"): | |
| self._progress(3, "inputs_and_evidence") | |
| if reference_paths: | |
| user_lit_context = extract_text_from_files(reference_paths[:20]) | |
| all_paper_texts = [extract_text_from_files([path]) for path in reference_paths[:20]] | |
| agent.global_memory["core_golden_metadata"] = extract_golden_metadata(agent, all_paper_texts) | |
| if mode == "amj": | |
| data_file = data_paths[0] if data_paths else None | |
| _, user_data_md, _, data_metadata = quant_engine.resolve_empirical_data( | |
| data_file, interface_language, topic=topic, keywords=keywords, agent=agent | |
| ) | |
| else: | |
| raw_interviews = extract_text_from_files(interview_paths[:1]) if interview_paths else "" | |
| user_data_md, _, data_metadata = qual_engine.resolve_qualitative_data(topic, keywords, raw_interviews) | |
| state.update( | |
| { | |
| "topic": topic, | |
| "keywords": keywords, | |
| "target_lang": target_lang, | |
| "target_words": target_words, | |
| "mode": mode, | |
| "interface_language": interface_language, | |
| "user_lit_context": user_lit_context, | |
| "user_data_md": user_data_md, | |
| "data_metadata": data_metadata, | |
| "agent_state": agent.export_state(), | |
| "evidence_ready": True, | |
| } | |
| ) | |
| self._checkpoint("evidence_ready", state, 6) | |
| refs_text = str(state.get("refs_text") or "") | |
| if not refs_text: | |
| self._progress(8, "S2 + OpenAlex") | |
| refs_text = agent.fetch_references_parallel( | |
| topic, | |
| keywords, | |
| min_required=LITERATURE_EVIDENCE_TARGET, | |
| progress_callback=lambda: self._progress(9, "S2 + OpenAlex"), | |
| ) | |
| if len(agent.citation_map) < MIN_FINAL_REFERENCES: | |
| raise RuntimeError( | |
| f"Insufficient eligible references: {len(agent.citation_map)}/{MIN_FINAL_REFERENCES}" | |
| ) | |
| if user_lit_context: | |
| core_refs = agent.parse_user_core_literature(user_lit_context) | |
| if core_refs: | |
| refs_text = ( | |
| "【USER PROVIDED CORE LITERATURE】\n" + core_refs | |
| + "\n\n【RETRIEVED LITERATURE】\n" + refs_text | |
| ) | |
| state.update({"refs_text": refs_text, "agent_state": agent.export_state()}) | |
| self._checkpoint("literature_ready", state, 11) | |
| rag_engine = initialize_rag([value for value in (user_lit_context, refs_text) if value]) | |
| synthesis_result = str(state.get("synthesis_result") or "") | |
| if not synthesis_result: | |
| self._progress(12, "literature_synthesis") | |
| synthesis_result = self._stream_text( | |
| agent.stream_synthesize_literature(topic, refs_text, user_core_literature=user_lit_context) | |
| ) | |
| synthesis_result = re.sub(r"<think>.*?</think>", "", synthesis_result, flags=re.DOTALL).strip() | |
| state.update({"synthesis_result": synthesis_result, "agent_state": agent.export_state()}) | |
| self._checkpoint("synthesis_ready", state, 13) | |
| adaptive_structure = state.get("adaptive_structure") | |
| if not isinstance(adaptive_structure, list) or not adaptive_structure: | |
| self._progress(14, "research_logic") | |
| research_logic_response = self._stream_text( | |
| agent.stream_generate_research_logic( | |
| topic, keywords, synthesis_result, refs_text, | |
| data_evidence=user_data_md, mode=mode, | |
| ) | |
| ) | |
| research_logic_contract = re.sub( | |
| r"<think>.*?</think>", "", research_logic_response, flags=re.DOTALL | |
| ).strip() | |
| if not research_logic_contract: | |
| raise RuntimeError("research_logic_contract_missing") | |
| agent.global_memory["research_logic_contract"] = research_logic_contract | |
| self._progress(14, "working_front_matter") | |
| working_front = agent.generate_working_front_matter( | |
| topic, keywords, research_logic_contract, synthesis_result, | |
| data_evidence=user_data_md, mode=mode, | |
| ) | |
| working_abstract = str(working_front.get("abstract") or "").strip() | |
| working_keywords = list(working_front.get("keywords") or []) | |
| if not working_abstract or not working_keywords: | |
| raise RuntimeError("working_front_matter_missing") | |
| agent.global_memory["working_abstract"] = working_abstract | |
| self._progress(14, "outline_planning") | |
| plan = agent.generate_outline(topic, keywords, mode=mode, data_metadata=data_metadata) | |
| en_topic = str(plan.get("en_topic", topic)) | |
| structure = plan.get("structure", []) | |
| full_draft_en = ( | |
| f"# {en_topic}\n\n## Abstract\n{working_abstract}\n\n" | |
| f"## Keywords\n{', '.join(working_keywords)}\n\n" | |
| ) | |
| # Publish the Working Abstract/Keywords immediately, before JIT and | |
| # citation planning, so the user sees the manuscript anchor before | |
| # Introduction while those longer planning stages continue. | |
| self._publish_preview(full_draft_en, "working_front_matter", force=True) | |
| front_budget = max(180, int(APP_CONFIG.get("word_budget", {}).get("front_matter_words", 220))) | |
| body_target_words = max(1000, target_words - front_budget) | |
| adaptive_structure = agent.build_generation_tasks(structure, body_target_words, safe_chunk_words) | |
| for task in adaptive_structure: | |
| task["is_first"] = bool(task.get("section_first")) | |
| jit_texts = agent.prefetch_jit_literature(topic, keywords, adaptive_structure) | |
| if jit_texts: | |
| rag_engine._initialize_chunks([jit_texts]) | |
| self._progress(15, "citation_planning") | |
| citation_plan = agent.build_claim_evidence_plan( | |
| adaptive_structure, topic, synthesis_result, mode=mode, target_unique=CITATION_PLAN_TARGET | |
| ) | |
| planned_unique = { | |
| item.get("tag") | |
| for items in citation_plan.values() | |
| for item in items | |
| if item.get("tag") | |
| } | |
| if len(planned_unique) < MIN_FINAL_REFERENCES: | |
| raise RuntimeError( | |
| "Insufficient claim-evidence plan before drafting: " | |
| f"{len(planned_unique)}/{MIN_FINAL_REFERENCES}." | |
| ) | |
| state.update( | |
| { | |
| "adaptive_structure": adaptive_structure, | |
| "citation_plan": citation_plan, | |
| "research_logic_contract": research_logic_contract, | |
| "working_abstract": working_abstract, | |
| "working_keywords": working_keywords, | |
| "front_matter_phase": "working", | |
| "body_target_words": body_target_words, | |
| "full_draft_en": full_draft_en, | |
| "full_draft_translated": "", | |
| "translation_sections": {}, | |
| "translation_glossary": {}, | |
| "completed_sections": [], | |
| "context_tail": "", | |
| "accumulated_deviation": 0.0, | |
| "agent_state": agent.export_state(), | |
| "outline_ready": True, | |
| } | |
| ) | |
| self._publish_preview(full_draft_en, "outline_ready", force=True) | |
| self._checkpoint("outline_ready", state, 15) | |
| else: | |
| full_draft_en = str(state.get("full_draft_en") or "") | |
| # JIT text itself is already represented in agent citation state; | |
| # refs + user literature rebuild a safe RAG baseline on resume. | |
| completed_sections = list(state.get("completed_sections") or []) | |
| context_tail = str(state.get("context_tail") or "") | |
| accumulated_deviation = float(state.get("accumulated_deviation") or 0.0) | |
| research_logic_contract = str( | |
| state.get("research_logic_contract") or agent.global_memory.get("research_logic_contract") or "" | |
| ) | |
| citation_plan = state.get("citation_plan") if isinstance(state.get("citation_plan"), dict) else {} | |
| translation_sections = dict(state.get('translation_sections') or {}) | |
| translation_glossary = dict(state.get('translation_glossary') or {}) | |
| working_abstract = str(state.get("working_abstract") or "").strip() | |
| working_keywords = list(state.get("working_keywords") or []) | |
| if working_abstract: | |
| agent.global_memory["working_abstract"] = working_abstract | |
| if self._preview_target_text == '': | |
| self._preview_target_text = str(state.get('full_draft_translated') or '') | |
| # Backward-compatible resume path for a pre-R3.2.8.3 checkpoint that has | |
| # an outline but no Working Abstract yet. Generate it once, checkpoint it, | |
| # and reuse it thereafter; never regenerate an unchanged anchor on resume. | |
| if not working_abstract and not state.get("english_finalized"): | |
| working_front = agent.generate_working_front_matter( | |
| topic, keywords, research_logic_contract, synthesis_result, | |
| data_evidence=user_data_md, mode=mode, | |
| ) | |
| working_abstract = str(working_front.get("abstract") or "").strip() | |
| working_keywords = list(working_front.get("keywords") or []) | |
| if not working_abstract or not working_keywords: | |
| raise RuntimeError("working_front_matter_missing") | |
| agent.global_memory["working_abstract"] = working_abstract | |
| full_draft_en = replace_abstract(full_draft_en, working_abstract) | |
| full_draft_en = self._replace_keywords(full_draft_en, working_keywords) | |
| state.update({ | |
| "working_abstract": working_abstract, | |
| "working_keywords": working_keywords, | |
| "front_matter_phase": "working", | |
| "full_draft_en": full_draft_en, | |
| "agent_state": agent.export_state(), | |
| }) | |
| self._publish_preview(full_draft_en, "working_front_matter", force=True) | |
| self._checkpoint("working_front_matter_ready", state, 15) | |
| # Translate the QA-frozen Working front matter before Introduction. The | |
| # same __front_matter__ record is hash-reused on resume and invalidated | |
| # later when Final Abstract/Keywords replace the working front matter. | |
| if working_abstract and not state.get("english_finalized"): | |
| section_order_for_front = self._unique_top_level_sections(adaptive_structure) | |
| if section_order_for_front: | |
| working_front_source = self._extract_front_matter(full_draft_en, section_order_for_front[0][1]) | |
| translation_sections, translation_glossary = self._translate_frozen_unit( | |
| agent, unit_key='__front_matter__', unit_name='Working Front Matter', | |
| source_text=working_front_source, target_lang=target_lang, | |
| translation_sections=translation_sections, translation_glossary=translation_glossary, | |
| adaptive_structure=adaptive_structure, include_front=True, | |
| ) | |
| working_target = self._assemble_partial_translation( | |
| translation_sections, adaptive_structure, include_front=True | |
| ) | |
| state.update({ | |
| "full_draft_en": full_draft_en, | |
| "full_draft_translated": working_target, | |
| "translation_sections": translation_sections, | |
| "translation_glossary": translation_glossary, | |
| "agent_state": agent.export_state(), | |
| }) | |
| self._checkpoint("working_front_matter_translated", state, 16) | |
| if not citation_plan: | |
| self._progress(15, "citation_planning") | |
| citation_plan = agent.build_claim_evidence_plan( | |
| adaptive_structure, topic, synthesis_result, mode=mode, target_unique=CITATION_PLAN_TARGET | |
| ) | |
| state["citation_plan"] = citation_plan | |
| total_steps = len(adaptive_structure) | |
| for sec in adaptive_structure: | |
| display_name = sec["display_name"] | |
| if display_name in completed_sections: | |
| continue | |
| if self.cancel_checker(): | |
| raise RuntimeError("task_cancel_requested") | |
| remaining = sum(1 for item in adaptive_structure if item["display_name"] not in completed_sections) | |
| adjusted_target = self._bounded_target( | |
| int(sec["target_words"]), accumulated_deviation, remaining, safe_chunk_words | |
| ) | |
| sec_title = sec["section"] | |
| subsection = sec.get("subsection") | |
| is_quarantined = any( | |
| word in sec_title.lower() | |
| for word in ["abstract", "keyword", "conclusion", "references", "appendix"] | |
| ) | |
| planned_items = list(citation_plan.get(display_name, [])) | |
| assigned_tag_list = [str(item.get("tag") or "") for item in planned_items if item.get("tag")] | |
| claim_plan_text = agent.format_claim_evidence_plan(planned_items) | |
| section_id = str(sec.get("section_id") or "") | |
| section_role = str( | |
| APP_CONFIG.get("section_reasoning_profiles", {}).get(mode, {}).get(section_id) or "" | |
| ) | |
| mode_directive = "" | |
| if section_id in {"theory", "theoretical_framing"}: | |
| mode_directive = str(APP_CONFIG.get("hypothesis_development_directive") or "") | |
| elif section_id in {"methodology", "methods"}: | |
| mode_directive = str(APP_CONFIG.get("methodological_defense_directive") or "") | |
| elif section_id == "discussion": | |
| mode_directive = str(APP_CONFIG.get("discussion_depth_directive") or "") | |
| elif section_id == "conclusion": | |
| mode_directive = str(APP_CONFIG.get("conclusion_anchor_directive") or "") | |
| if is_quarantined or section_id in {"results", "findings"}: | |
| citation_rule = ( | |
| "NO EXTERNAL LITERATURE CITATIONS ALLOWED IN THIS UNIT. " | |
| "Report only frozen empirical/qualitative evidence and the interpretation permitted by the section role." | |
| ) | |
| else: | |
| citation_rule = ( | |
| "Use the citation-first claim-evidence plan below. Every listed item has already been assigned to this unit on evidence relevance: use each assigned item exactly once unless it would contradict the frozen research evidence. " | |
| "For each item, write one distinct claim no broader than the supplied evidence and attach exactly its assigned [CITE-XX] tag. " | |
| "Never stack sources, substitute tags, or use literature as evidence for this study's own facts.\n" | |
| f"<CLAIM_EVIDENCE_PLAN>\n{claim_plan_text}\n</CLAIM_EVIDENCE_PLAN>" | |
| ) | |
| memory_context = agent._build_memory_context(sec_title, 2500) | |
| memory_block = f"\n<GLOBAL_MEMORY>\n{memory_context}\n</GLOBAL_MEMORY>\n" if memory_context else "" | |
| snippets = rag_engine.get_relevant_context(display_name) if rag_engine is not None else "" | |
| if snippets: | |
| memory_block += f"\n<TARGETED_RAG>\n{snippets}\n</TARGETED_RAG>\n" | |
| heading_directive = "Do not output the section heading; it will be inserted by the system." | |
| if subsection: | |
| heading_directive += ( | |
| f" Do not output the subsection heading '{subsection}'; it will also be inserted by the system. " | |
| "Use level-3 headings only when analytically necessary." | |
| ) | |
| elif sec_title.lower() in {"introduction", "conclusion"}: | |
| heading_directive += " Do not use additional headings." | |
| else: | |
| heading_directive += ( | |
| " Use level-2 or level-3 headings only when they clarify the argument; " | |
| "never use level-4 headings or numbered headings." | |
| ) | |
| table_rule = ( | |
| "Present only frozen statistical evidence in standard Markdown tables with plain-text captions." | |
| if sec["is_method_or_result"] and mode == "amj" else "" | |
| ) | |
| concepts = ", ".join(sec.get("concepts") or []) | |
| theme_evidence = [item for item in (sec.get("theme_evidence") or []) if isinstance(item, dict)] | |
| evidence_labels = [] | |
| for item in theme_evidence: | |
| tag = str(item.get("tag") or "").strip() | |
| stage = str(item.get("stage") or "Not specified").strip() | |
| role = str(item.get("evidence_role") or "typical").strip() | |
| case_label = str(item.get("case_label") or "Not specified").strip() | |
| if tag: | |
| evidence_labels.append(f"{tag} [stage={stage}; case={case_label}; role={role}]") | |
| qualitative_rule = ( | |
| f"Assigned qualitative concepts: {concepts}. Assigned quotation tags: {', '.join(evidence_labels)}. " | |
| "Use only these assigned quotation tags in this Findings unit and do not repeat a tag used elsewhere. " | |
| "When invoking a quotation, output only its [QUOTE-XX] tag at the exact position; do not reproduce or paraphrase the quotation because the system inserts the frozen verbatim text. " | |
| "Preserve any frozen stage/case/deviant role and use deviant evidence to qualify rather than erase the pattern." | |
| if concepts or evidence_labels else "" | |
| ) | |
| evidence_packet = ( | |
| f"{base_evidence_contract}\n<FROZEN_RESEARCH_EVIDENCE>\n{user_data_md}\n</FROZEN_RESEARCH_EVIDENCE>" | |
| if user_data_md else base_evidence_contract | |
| ) | |
| completed_label = ", ".join(completed_sections) | |
| anti_repeat = ( | |
| f"Already completed: {completed_label}. Do not restate their background, gap, or contribution." | |
| if completed_label else "" | |
| ) | |
| voice = str(APP_CONFIG.get("journal_voice_profiles", {}).get(mode) or "") | |
| sys_prompt = ( | |
| APP_CONFIG.get("sys_academic_writer", "") + "\n" | |
| + evidence_packet + "\n" + voice + "\n" | |
| + APP_CONFIG.get("deep_refinement_directive", "") | |
| ) | |
| word_budget = APP_CONFIG.get("word_budget", {}) | |
| min_ratio = float(word_budget.get("section_min_ratio", 0.92)) | |
| max_ratio = float(word_budget.get("section_max_ratio", 1.08)) | |
| user_prompt = f"""Write the assigned English manuscript unit. | |
| Section: {sec_title} | |
| Subsection: {subsection or 'None'} | |
| Section reasoning role: {section_role} | |
| Additional mode directive: {mode_directive} | |
| Target length: {int(adjusted_target * min_ratio)}-{int(adjusted_target * max_ratio)} words. | |
| {heading_directive} | |
| {citation_rule} | |
| {table_rule} | |
| {str(sec.get('required_content') or '')} | |
| {qualitative_rule} | |
| {anti_repeat} | |
| <PROVISIONAL_RESEARCH_LOGIC> | |
| {research_logic_contract} | |
| </PROVISIONAL_RESEARCH_LOGIC> | |
| <WORKING_ABSTRACT_ANCHOR> | |
| {working_abstract} | |
| </WORKING_ABSTRACT_ANCHOR> | |
| ANCHOR PRECEDENCE: Frozen research evidence and the PROVISIONAL_RESEARCH_LOGIC override the Working Abstract if any tension appears. The Working Abstract is continuity guidance only and must never force a result. | |
| {memory_block} | |
| Literature synthesis: {synthesis_result} | |
| <FROZEN_RESEARCH_EVIDENCE> | |
| {user_data_md} | |
| </FROZEN_RESEARCH_EVIDENCE> | |
| Previous tail: {context_tail[-2500:]} | |
| Output only final academic prose. Do not report the word count.""" | |
| if agent._count_tokens(sys_prompt) + agent._count_tokens(user_prompt) > 60000: | |
| user_prompt = agent._truncate_context(user_prompt, 60000 - agent._count_tokens(sys_prompt)) | |
| current_index = len(completed_sections) + 1 | |
| pct = min(88, 15 + int((current_index / max(1, total_steps)) * 72)) | |
| self._progress(pct, display_name) | |
| prefix = "" | |
| if sec.get("section_first"): | |
| prefix += f"# {sec_title}\n\n" | |
| if sec.get("subsection_first") and subsection: | |
| prefix += f"## {subsection}\n\n" | |
| en_raw = self._stream_text( | |
| agent.stream_deepseek( | |
| sys_prompt, user_prompt, max_tokens=7000, | |
| model_role="writing", thinking=False, allow_fallback=False, | |
| ), | |
| preview_prefix=full_draft_en + prefix, | |
| preview_stage=display_name, | |
| ) | |
| en_raw = re.sub(r"<think>.*?</think>", "", en_raw, flags=re.S).strip() | |
| en_raw = re.sub(rf"(?im)^#\s*{re.escape(sec_title)}\s*\n+", "", en_raw).strip() | |
| if subsection: | |
| en_raw = re.sub(rf"(?im)^##\s*{re.escape(subsection)}\s*\n+", "", en_raw).strip() | |
| en_raw = re.sub(r"(?m)^(#{2,3})\s+[\d\.\s一二三四五六七八九十、]+\s*(.*)", r"\1 \2", en_raw) | |
| en_raw = re.sub(r"(?m)^#{4,}\s+(.*)", r"**\1**", en_raw) | |
| en_raw = re.sub(r"(?m)^#\s+(.+)$", r"## \1", en_raw) | |
| en_raw = agent.critique_and_repair_section( | |
| en_raw, section_id=section_id, section_name=display_name, mode=mode, | |
| evidence_contract=evidence_packet, | |
| ) | |
| en_raw = agent.polish_academic_section(en_raw, display_name, abstract=False, mode=mode) | |
| # Keep QUOTE tags immutable through critic/style editing, then bind | |
| # each tag to its frozen verbatim quotation exactly once. This | |
| # prevents editorial passes from silently paraphrasing evidence. | |
| if mode == "asq" and user_data_md and "appendix" not in sec_title.lower(): | |
| quote_dict = {tag: quote for quote, tag in re.findall(r'"([^"]+)"\s*\[(QUOTE-\d+)\]', user_data_md)} | |
| for tag, quote in quote_dict.items(): | |
| en_raw = en_raw.replace(tag, f'"{quote}"') | |
| if not is_quarantined and assigned_tag_list: | |
| en_raw = agent.reinforce_claim_specific_citation_coverage( | |
| en_raw, assigned_tag_list, section_name=display_name | |
| ) | |
| used_before = set(agent.used_citations) | |
| en_resolved = agent.audit_and_resolve_citations( | |
| prefix + en_raw, section_name=display_name, quarantine=is_quarantined | |
| ) | |
| used_after = set(agent.used_citations) | |
| survived = used_after - used_before | |
| if assigned_tag_list and not is_quarantined: | |
| required_survival = min( | |
| len(assigned_tag_list), | |
| max(1, int(math.ceil(len(assigned_tag_list) * 0.78))), | |
| ) | |
| if len(survived) < required_survival: | |
| missing_items = [ | |
| item for item in planned_items | |
| if str(item.get("tag") or "") not in used_after | |
| ] | |
| needed = max(0, required_survival - len(survived)) | |
| supplemental_raw = agent.build_citation_recovery_paragraph( | |
| missing_items[:needed], section_name=display_name | |
| ) | |
| if supplemental_raw: | |
| supplemental_resolved = agent.audit_and_resolve_citations( | |
| supplemental_raw, | |
| section_name=f"{display_name} — citation recovery", | |
| quarantine=False, | |
| ) | |
| if supplemental_resolved.strip(): | |
| en_resolved = en_resolved.rstrip() + "\n\n" + supplemental_resolved.strip() | |
| used_after = set(agent.used_citations) | |
| survived = used_after - used_before | |
| full_draft_en += en_resolved + "\n\n" | |
| if assigned_tag_list: | |
| logging.info( | |
| "Citation-first survival: section=%s planned=%s validated_new=%s", | |
| display_name, len(assigned_tag_list), len(survived), | |
| ) | |
| self._publish_preview(full_draft_en, display_name, force=True) | |
| agent._extract_memory_from_text(en_raw, sec_title) | |
| context_tail = " ".join(en_raw.split()[-200:]) | |
| actual_words = self._english_count_for_target(en_resolved) | |
| accumulated_deviation += actual_words - adjusted_target | |
| completed_sections.append(display_name) | |
| section_key = str(sec.get('section_id') or sec_title) | |
| if self._section_is_complete(adaptive_structure, completed_sections, section_key): | |
| frozen_section = self._extract_top_level_section(full_draft_en, sec_title) | |
| translation_sections, translation_glossary = self._translate_frozen_unit( | |
| agent, unit_key=section_key, unit_name=sec_title, source_text=frozen_section, | |
| target_lang=target_lang, translation_sections=translation_sections, | |
| translation_glossary=translation_glossary, adaptive_structure=adaptive_structure, | |
| include_front=True, | |
| ) | |
| partial_translation = self._assemble_partial_translation(translation_sections, adaptive_structure, include_front=True) | |
| state.update( | |
| { | |
| "completed_sections": completed_sections, | |
| "full_draft_en": full_draft_en, | |
| "full_draft_translated": partial_translation, | |
| "translation_sections": translation_sections, | |
| "translation_glossary": translation_glossary, | |
| "context_tail": context_tail, | |
| "accumulated_deviation": accumulated_deviation, | |
| "agent_state": agent.export_state(), | |
| "data_metadata": data_metadata, | |
| } | |
| ) | |
| self._checkpoint(f"section_{len(completed_sections):03d}", state, pct) | |
| if not state.get("english_finalized"): | |
| self._progress(91, "english_validation") | |
| full_draft_en = _clean_trailing_references(full_draft_en, ["Reference", "References", "Reference List"]) | |
| # Validate the completed body while the Working Abstract is still in | |
| # place. Final front matter must not be generated until all body | |
| # sections, citation cleanup, and any Discussion rebalance are frozen. | |
| agent.validate_manuscript_structure(full_draft_en, adaptive_structure, translated=False) | |
| if mode == "asq": | |
| repeated_quotes = agent.find_repeated_qualitative_quotes(full_draft_en, data_metadata) | |
| if repeated_quotes: | |
| raise RuntimeError(f"Repeated qualitative quotations: {repeated_quotes[:5]}") | |
| references = [] | |
| seen_refs = set() | |
| for tag in sorted(agent.used_citations, key=agent._citation_tag_number): | |
| ref = str((agent.citation_map.get(tag) or {}).get("ref") or "").strip() | |
| if ref and ref not in seen_refs: | |
| references.append(f"- {ref}") | |
| seen_refs.add(ref) | |
| if len(references) < MIN_FINAL_REFERENCES: | |
| raise RuntimeError( | |
| "Insufficient final cited references after semantic validation: " | |
| f"{len(references)}/{MIN_FINAL_REFERENCES}." | |
| ) | |
| full_draft_en = full_draft_en.rstrip() + "\n\n# References\n" + "\n\n".join(references) | |
| actual_words = self._english_count_for_target(full_draft_en) | |
| quality = APP_CONFIG.get("manuscript_validation", {}) | |
| minimum_words = int(target_words * float(quality.get("minimum_total_ratio", 0.97))) | |
| maximum_words = int(target_words * float(quality.get("maximum_total_ratio", 1.03))) | |
| if actual_words < minimum_words or actual_words > maximum_words: | |
| full_draft_en = agent.rebalance_discussion_length( | |
| full_draft_en, target_words=target_words, mode=mode, actual_words=actual_words | |
| ) | |
| body_only, _reference_tail = agent.validate_reference_tail( | |
| full_draft_en, minimum_references=MIN_FINAL_REFERENCES | |
| ) | |
| agent.validate_manuscript_structure(body_only, adaptive_structure, translated=False) | |
| agent.enforce_resolved_claim_specific_citations(body_only) | |
| # Body is now frozen. Generate both Final Abstract and Final Keywords | |
| # from the completed manuscript/evidence, then replace the provisional | |
| # front matter deterministically. No body rebalance may occur afterward. | |
| self._progress(92, "final_front_matter") | |
| final_front = agent.generate_final_front_matter( | |
| topic, keywords, body_only, data_evidence=user_data_md, mode=mode | |
| ) | |
| final_abstract = str(final_front.get("abstract") or "").strip() | |
| final_keywords = list(final_front.get("keywords") or []) | |
| if not final_abstract or not final_keywords: | |
| raise RuntimeError("final_front_matter_missing") | |
| full_draft_en = replace_abstract(full_draft_en, final_abstract) | |
| full_draft_en = self._replace_keywords(full_draft_en, final_keywords) | |
| body_only, _reference_tail = agent.validate_reference_tail( | |
| full_draft_en, minimum_references=MIN_FINAL_REFERENCES | |
| ) | |
| agent.validate_manuscript_structure(body_only, adaptive_structure, translated=False) | |
| agent.enforce_resolved_claim_specific_citations(body_only) | |
| self._validate_target_length(full_draft_en, target_words) | |
| # Final front matter has a different source hash from the Working | |
| # front matter. Retranslate only __front_matter__; unchanged body | |
| # translations are retained and reused. | |
| section_order_for_front = self._unique_top_level_sections(adaptive_structure) | |
| if section_order_for_front: | |
| final_front_source = self._extract_front_matter(body_only, section_order_for_front[0][1]) | |
| translation_sections, translation_glossary = self._translate_frozen_unit( | |
| agent, unit_key='__front_matter__', unit_name='Final Front Matter', | |
| source_text=final_front_source, target_lang=target_lang, | |
| translation_sections=translation_sections, translation_glossary=translation_glossary, | |
| adaptive_structure=adaptive_structure, include_front=True, | |
| ) | |
| partial_target = self._assemble_partial_translation( | |
| translation_sections, adaptive_structure, include_front=True | |
| ) | |
| self._publish_preview(full_draft_en, "english_complete", force=True, complete=True) | |
| state.update( | |
| { | |
| "full_draft_en": full_draft_en, | |
| "full_draft_translated": partial_target, | |
| "translation_sections": translation_sections, | |
| "translation_glossary": translation_glossary, | |
| "final_abstract": final_abstract, | |
| "final_keywords": final_keywords, | |
| "front_matter_phase": "final", | |
| "english_finalized": True, | |
| "translation_complete": False, | |
| "agent_state": agent.export_state(), | |
| } | |
| ) | |
| self._checkpoint("english_complete", state, 93) | |
| else: | |
| full_draft_en = str(state.get("full_draft_en") or "") | |
| full_draft_translated = str(state.get("full_draft_translated") or "") | |
| translation_sections = dict(state.get('translation_sections') or translation_sections or {}) | |
| translation_glossary = dict(state.get('translation_glossary') or translation_glossary or {}) | |
| if not state.get("translation_complete") or not full_draft_translated: | |
| self._progress(94, "translation") | |
| body_only, reference_tail = agent.validate_reference_tail( | |
| full_draft_en, minimum_references=MIN_FINAL_REFERENCES | |
| ) | |
| section_order = self._unique_top_level_sections(adaptive_structure) | |
| if not section_order: | |
| raise RuntimeError('translation_section_plan_missing') | |
| front_matter = self._extract_front_matter(body_only, section_order[0][1]) | |
| final_units = [('__front_matter__', 'Front Matter', front_matter)] | |
| for key, title in section_order: | |
| final_units.append((key, title, self._extract_top_level_section(body_only, title))) | |
| for index, (unit_key, unit_name, source_text) in enumerate(final_units, start=1): | |
| translation_sections, translation_glossary = self._translate_frozen_unit( | |
| agent, unit_key=unit_key, unit_name=unit_name, source_text=source_text, | |
| target_lang=target_lang, translation_sections=translation_sections, | |
| translation_glossary=translation_glossary, adaptive_structure=adaptive_structure, | |
| include_front=True, | |
| ) | |
| partial = self._assemble_partial_translation( | |
| translation_sections, adaptive_structure, include_front=True | |
| ) | |
| state.update({ | |
| 'full_draft_en': full_draft_en, | |
| 'full_draft_translated': partial, | |
| 'translation_sections': translation_sections, | |
| 'translation_glossary': translation_glossary, | |
| 'agent_state': agent.export_state(), | |
| }) | |
| pct = min(96, 94 + int(index * 2 / max(1, len(final_units)))) | |
| self._checkpoint(f'translation_{unit_key}', state, pct) | |
| full_draft_translated = self._assemble_partial_translation( | |
| translation_sections, adaptive_structure, include_front=True | |
| ).rstrip() | |
| full_draft_translated += ( | |
| f"\n\n# {agent._localized_reference_heading(target_lang)}\n" + reference_tail.strip() | |
| ) | |
| agent.validate_translation_alignment(full_draft_en, full_draft_translated) | |
| self._publish_target_preview( | |
| full_draft_translated, 'translation_complete', force=True, complete=True | |
| ) | |
| state.update( | |
| { | |
| "full_draft_translated": full_draft_translated, | |
| "translation_sections": translation_sections, | |
| "translation_glossary": translation_glossary, | |
| "translation_complete": True, | |
| "agent_state": agent.export_state(), | |
| } | |
| ) | |
| self._checkpoint("translation_complete", state, 96) | |
| existing_output = str(state.get("output_filename") or "").strip() | |
| if state.get("package_validated") and existing_output: | |
| existing_path = self.output_dir / os.path.basename(existing_output) | |
| if existing_path.is_file() and existing_path.stat().st_size > 0: | |
| self._progress(99, "package_validated") | |
| return {"output_filename": existing_path.name, "output_path": str(existing_path)} | |
| self._progress(97, "packaging") | |
| bj_time = datetime.now(timezone(timedelta(hours=8))).strftime("%Y%m%d_%H%M") | |
| package = build_validated_package( | |
| english_markdown=full_draft_en, | |
| translated_markdown=full_draft_translated, | |
| english_filename=f"Research_EN_{bj_time}.docx", | |
| translated_filename=f"Research_{output_suffix}_{bj_time}.docx", | |
| data_metadata=data_metadata, | |
| mode=mode, | |
| citation_audit=agent.citation_audit_log, | |
| literature_metrics=agent.get_source_metrics(), | |
| ) | |
| output_filename = f"SmartPaper_{bj_time}.zip" | |
| pending = self.output_dir / (".pending_" + output_filename) | |
| pending.write_bytes(package.zip_bytes) | |
| final = self.output_dir / output_filename | |
| os.replace(pending, final) | |
| state.update( | |
| { | |
| "output_filename": output_filename, | |
| "package_validated": True, | |
| "agent_state": agent.export_state(), | |
| } | |
| ) | |
| self._checkpoint("package_validated", state, 99) | |
| return {"output_filename": output_filename, "output_path": str(final)} | |
| def _progress_activity(self) -> None: | |
| # Activity remains Executor-owned. We also snapshot actual request | |
| # counters so the next Executor heartbeat can persist live UI metrics. | |
| ds_calls, s2_calls, oa_calls = self._compute_request_counts() | |
| try: | |
| self.progress_callback( | |
| activity_only=True, | |
| ds_calls=ds_calls, | |
| s2_calls=s2_calls, | |
| oa_calls=oa_calls, | |
| ) | |
| except TypeError: | |
| pass | |