SmartPaper / app.py
GuXiaopeng's picture
Deploy SmartPaper production v1.1 R1Fix clean
28ce91d verified
Raw History Blame
64.1 kB
# -*- coding: utf-8 -*- """Smart Draft - Dual Track (AMJ Empirical vs ASQ Qualitative)""" from core.meta_extractor import extract_golden_metadata from core.rag_engine import initialize_rag from core.qualitative_engine import get_qualitative_engine from core.quantitative_engine import get_quantitative_engine from core.production_guard import build_validated_package, replace_abstract import streamlit as st st.set_page_config( page_title="Article Agent", page_icon="📊", layout="wide", initial_sidebar_state="collapsed", ) import warnings warnings.filterwarnings("ignore", message="coroutine 'expire_cache' was never awaited") import os import streamlit.components.v1 as components import time import re import math import json import secrets from datetime import datetime, timezone, timedelta from concurrent.futures import ThreadPoolExecutor from config import APP_CONFIG, AMJ_PROMPTS, ASQ_PROMPTS from utils.file_parser import extract_text_from_files from core.pipeline_agent import SSciPipelineAgent from managed_auth import ( require_alipay_login, render_logged_in_account, start_generation_task, heartbeat_generation_task, finish_generation_task, render_payment_purchase, ) from i18n import LANGUAGE_OPTIONS, get_amj_csv_template, get_asq_txt_template, get_output_suffix, get_target_flag, get_target_language, tr os.environ["STREAMLIT_SERVER_ENABLE_CORS"] = "false" os.environ["STREAMLIT_SERVER_ENABLE_XSRF_PROTECTION"] = "false" WORKSPACE_DIR = APP_CONFIG.get("workspace_dir", ".workspace") os.makedirs(WORKSPACE_DIR, exist_ok=True) LEGACY_CHECKPOINT_FILE = os.path.join(WORKSPACE_DIR, "snapshot.json") CHECKPOINT_FILE = LEGACY_CHECKPOINT_FILE META_SECTIONS_BLACKLIST = set(APP_CONFIG["blacklists"]["meta_sections"]) def update_heartbeat(): """Refresh the authoritative Cloudflare/D1 task lease only.""" now = time.time() task = st.session_state.get("sp_generation_task") or {} task_id = str(task.get("task_id") or "").strip() interval = max(3, int(APP_CONFIG.get("task_heartbeat_interval_seconds", 5))) last_remote = float(st.session_state.get("sp_last_task_heartbeat") or 0.0) if not task_id or now - last_remote < interval: return try: heartbeat_generation_task( task_id, current_node=str(st.session_state.get("sp_current_node") or "")[:240], ) st.session_state["sp_last_task_heartbeat"] = now except Exception: # The hidden browser bridge continues heartbeats independently. A single # transient UI heartbeat must not interrupt model generation. pass def is_system_busy(): """D1 is the sole concurrency authority; no duplicate Hugging Face file lock.""" return False def clear_heartbeat(): """Backward-compatible no-op after removal of the duplicate local lock.""" return None def stream_with_heartbeat(generator): """Refresh the D1 lease while streaming model output.""" last_beat = time.time() for chunk in generator: now = time.time() if now - last_beat > 5: update_heartbeat() last_beat = now yield chunk def clean_trailing_references(text, keywords=("reference", "references", "reference list", "参考文献")): lines = text.split('\n') for i in range(len(lines)-1, max(-1, len(lines)-100), -1): if any(re.match(rf'^##?\s*{kw}', lines[i], re.IGNORECASE) for kw in keywords): return '\n'.join(lines[:i]).strip() return text.strip() def clean_markdown_headings(text): return re.sub(r'(?m)^#+\s+(.*)$', r'**\1**', text) def _read_runtime_secret(name): # Read a Hugging Face/Streamlit secret and remove copied whitespace. value = os.environ.get(name, "") if not value: try: value = st.secrets.get(name, "") except Exception: value = "" return str(value or "").strip() global_ds_key = _read_runtime_secret("DEEPSEEK_API_KEY") global_s2_key = _read_runtime_secret("S2_API_KEY") WECHAT_BLOCKER_JS = """<script>try{var ua=navigator.userAgent.toLowerCase();if(ua.indexOf('micromessenger')!==-1){var parentDoc=window.parent.document;if(!parentDoc.getElementById('wx-blocker')){var overlay=parentDoc.createElement('div');overlay.id='wx-blocker';overlay.style.cssText='position:fixed;top:0;left:0;width:100vw;height:100vh;background:#1a365d;z-index:9999999;color:white;display:flex;flex-direction:column;align-items:center;justify-content:center;text-align:center;font-family:sans-serif;padding:30px;box-sizing:border-box;';overlay.innerHTML='<h2 style="font-size:26px;margin-bottom:15px;font-weight:bold;">环境受限 / Restricted</h2><p style="font-size:18px;line-height:1.6;color:#e2e8f0;max-width:100%;">为了保障最终双语 ZIP 压缩包正常下载,<b>本系统禁止在微信内直接运行</b>。<br><br>👉 <b>请点击右上角 [···] 菜单,选择“在浏览器打开” (Open in Browser)</b></p>';parentDoc.body.appendChild(overlay);}}}catch(e){}</script>""" components.html(WECHAT_BLOCKER_JS, height=0, width=0) def inject_file_uploader_i18n(language: str) -> None: """Localize Streamlit uploaders without modifying their React structure.""" payload = { "browse": tr(language, "browse_files"), "uploaders": [ { "title": tr(language, "drop_docs"), "limit": tr(language, "limit_docs"), }, { "title": tr(language, "drop_data"), "limit": tr(language, "limit_data"), }, { "title": tr(language, "drop_docs"), "limit": tr(language, "limit_docs"), }, { "title": tr(language, "drop_interviews"), "limit": tr(language, "limit_interviews"), }, ], } script = f""" <script> (() => {{ const payload = {json.dumps(payload, ensure_ascii=False)}; const parentDocument = window.parent.document; function leafTextElements(root) {{ return Array.from( root.querySelectorAll("p, span, small, div") ).filter((element) => ( element.children.length === 0 && element.textContent.trim().length > 0 )); }} function replaceTextSafely(element, value) {{ if (element && element.textContent.trim() !== value) {{ element.textContent = value; }} }} function applyUploaderTranslations() {{ const zones = parentDocument.querySelectorAll( '[data-testid="stFileUploaderDropzone"]' ); zones.forEach((zone, index) => {{ const config = payload.uploaders[index]; if (!config) return; const instructions = zone.querySelector( '[data-testid="stFileUploaderDropzoneInstructions"]' ); if (instructions) {{ const leaves = leafTextElements(instructions); replaceTextSafely(leaves[0], config.title); replaceTextSafely(leaves[1], config.limit); }} const button = zone.querySelector("button"); if (button) {{ const buttonLeaves = leafTextElements(button); const textTarget = buttonLeaves[buttonLeaves.length - 1]; replaceTextSafely(textTarget, payload.browse); button.setAttribute("aria-label", payload.browse); button.setAttribute("title", payload.browse); }} }}); }} [0, 100, 300, 700, 1400, 2500].forEach((delay) => {{ window.setTimeout(applyUploaderTranslations, delay); }}); }})(); </script> """ components.html(script, height=0, width=0) def inject_wechat_browser_notice() -> None: """Warn immediately when SmartPaper is opened inside WeChat or WeCom.""" script = """ <script> (() => { const parentWindow = window.parent; const parentDocument = parentWindow.document; const userAgent = String( parentWindow.navigator.userAgent || navigator.userAgent || "" ); const isWechat = /MicroMessenger|WeChat|wxwork/i.test(userAgent); const noticeId = "smartpaper-wechat-browser-notice"; const existingNotice = parentDocument.getElementById(noticeId); if (!isWechat) { if (existingNotice) existingNotice.remove(); return; } if (existingNotice) return; const overlay = parentDocument.createElement("div"); overlay.id = noticeId; overlay.style.cssText = [ "position:fixed", "inset:0", "z-index:2147483647", "display:flex", "align-items:center", "justify-content:center", "padding:24px", "background:rgba(8,12,20,.96)", "box-sizing:border-box" ].join(";"); const card = parentDocument.createElement("div"); card.style.cssText = [ "width:min(620px,100%)", "padding:30px", "border:1px solid rgba(148,163,184,.35)", "border-radius:18px", "background:#111827", "color:#f8fafc", "font-family:Arial,'Microsoft YaHei','PingFang SC',sans-serif", "line-height:1.7", "box-shadow:0 20px 50px rgba(0,0,0,.38)" ].join(";"); const title = parentDocument.createElement("h2"); title.textContent = "请在浏览器中打开 / Open in Browser"; title.style.cssText = "margin:0 0 14px;font-size:24px;color:#ffffff;"; const chinese = parentDocument.createElement("p"); chinese.textContent = "微信内置页面可能无法正常完成登录和文件操作。请点击右上角“…”并选择“在浏览器打开”。"; chinese.style.cssText = "margin:0 0 12px;font-size:16px;"; const english = parentDocument.createElement("p"); english.textContent = "The WeChat in-app page may not complete sign-in or file operations correctly. Tap “…” in the top-right corner and choose “Open in Browser.”"; english.style.cssText = "margin:0 0 20px;font-size:15px;color:#cbd5e1;"; const copyButton = parentDocument.createElement("button"); copyButton.type = "button"; copyButton.textContent = "复制网址 / Copy link"; copyButton.style.cssText = [ "width:100%", "padding:12px 16px", "border:0", "border-radius:10px", "background:#1677ff", "color:#ffffff", "font-size:16px", "font-weight:700", "cursor:pointer" ].join(";"); copyButton.addEventListener("click", async () => { const url = parentWindow.location.href; try { await parentWindow.navigator.clipboard.writeText(url); copyButton.textContent = "已复制 / Copied"; } catch (_) { const textarea = parentDocument.createElement("textarea"); textarea.value = url; textarea.style.position = "fixed"; textarea.style.opacity = "0"; parentDocument.body.appendChild(textarea); textarea.focus(); textarea.select(); parentDocument.execCommand("copy"); textarea.remove(); copyButton.textContent = "已复制 / Copied"; } parentWindow.setTimeout(() => { copyButton.textContent = "复制网址 / Copy link"; }, 1800); }); card.appendChild(title); card.appendChild(chinese); card.appendChild(english); card.appendChild(copyButton); overlay.appendChild(card); parentDocument.body.appendChild(overlay); })(); </script> """ components.html(script, height=0, width=0) if "is_running" not in st.session_state: st.session_state.is_running = False if "start_pipeline" not in st.session_state: st.session_state.start_pipeline = False if "resume_pipeline" not in st.session_state: st.session_state.resume_pipeline = False if "zip_buffer" not in st.session_state: st.session_state.zip_buffer = None if "run_mode" not in st.session_state: st.session_state.run_mode = "amj" if "sp_generation_task" not in st.session_state: st.session_state.sp_generation_task = None if "sp_generation_idempotency_key" not in st.session_state: st.session_state.sp_generation_idempotency_key = "" if "sp_last_task_heartbeat" not in st.session_state: st.session_state.sp_last_task_heartbeat = 0.0 if "sp_current_node" not in st.session_state: st.session_state.sp_current_node = "" if "sp_pending_task_finalization" not in st.session_state: st.session_state.sp_pending_task_finalization = None if "sp_pending_package" not in st.session_state: st.session_state.sp_pending_package = None if "sp_terminal_notice" not in st.session_state: st.session_state.sp_terminal_notice = None def _safe_user_workspace_key(value): cleaned = re.sub(r"[^A-Za-z0-9_.-]", "_", str(value or "").strip()) return cleaned[:80] or "anonymous" def _load_recoverable_checkpoint(): """Return a validated checkpoint for the current authenticated user only.""" if not os.path.isfile(CHECKPOINT_FILE): return None try: with open(CHECKPOINT_FILE, "r", encoding="utf-8") as handle: state = json.load(handle) except Exception: return None required = { "topic", "keywords", "target_lang", "words", "agent_state", "adaptive_structure", "completed_sections", } if not isinstance(state, dict) or not required.issubset(state): return None if not isinstance(state.get("adaptive_structure"), list): return None if not isinstance(state.get("completed_sections"), list): return None return state def _new_generation_idempotency_key() -> str: return f"gen_{int(time.time() * 1000)}_{secrets.token_urlsafe(18)}" def _prepare_new_generation_request() -> None: st.session_state["sp_generation_idempotency_key"] = _new_generation_idempotency_key() st.session_state["sp_generation_task"] = None st.session_state["sp_last_task_heartbeat"] = 0.0 st.session_state["sp_current_node"] = "task_start" st.session_state["zip_buffer"] = None st.session_state["zip_filename"] = "" st.session_state["sp_pending_task_finalization"] = None st.session_state["sp_pending_package"] = None st.session_state["sp_terminal_notice"] = None def _begin_managed_generation_task(*, is_resume: bool, loaded_state: dict | None, mode: str, language: str): checkpoint_key = "" if is_resume and isinstance(loaded_state, dict): checkpoint_key = str(loaded_state.get("managed_task_idempotency_key") or "").strip() idempotency_key = checkpoint_key or str( st.session_state.get("sp_generation_idempotency_key") or "" ).strip() if not idempotency_key: idempotency_key = _new_generation_idempotency_key() response = start_generation_task( idempotency_key=idempotency_key, mode=mode, interface_language=language, ) task = dict(response.get("task") or {}) if not task.get("task_id"): raise RuntimeError("task_start_missing_task_id") st.session_state["sp_generation_idempotency_key"] = idempotency_key st.session_state["sp_generation_task"] = task st.session_state["sp_last_task_heartbeat"] = time.time() return task, idempotency_key def _task_output_dir(task_id: str) -> str: root = globals().get("_user_workspace") or os.path.join(WORKSPACE_DIR, "users", "anonymous") path = os.path.join(root, "tasks", _safe_user_workspace_key(task_id), "outputs") os.makedirs(path, exist_ok=True) return path def _checkpoint_state_for_update() -> dict: if not os.path.isfile(CHECKPOINT_FILE): return {} try: with open(CHECKPOINT_FILE, "r", encoding="utf-8") as handle: value = json.load(handle) return value if isinstance(value, dict) else {} except Exception: return {} def _write_checkpoint_state(value: dict) -> None: if not isinstance(value, dict) or not value: return os.makedirs(os.path.dirname(CHECKPOINT_FILE), exist_ok=True) temporary = CHECKPOINT_FILE + ".tmp" with open(temporary, "w", encoding="utf-8") as handle: json.dump(value, handle, ensure_ascii=False) os.replace(temporary, CHECKPOINT_FILE) def _persist_pending_finalization() -> None: pending = st.session_state.get("sp_pending_task_finalization") if not isinstance(pending, dict): return state = _checkpoint_state_for_update() if not state: return state["pending_task_finalization"] = pending package = st.session_state.get("sp_pending_package") if isinstance(package, dict): state["pending_package"] = package _write_checkpoint_state(state) def _restore_pending_finalization() -> None: if isinstance(st.session_state.get("sp_pending_task_finalization"), dict): return state = _checkpoint_state_for_update() pending = state.get("pending_task_finalization") if isinstance(state, dict) else None if not isinstance(pending, dict): return st.session_state["sp_pending_task_finalization"] = pending package = state.get("pending_package") if isinstance(package, dict): st.session_state["sp_pending_package"] = package task_id = str(pending.get("task_id") or "").strip() if task_id and not st.session_state.get("sp_generation_task"): st.session_state["sp_generation_task"] = {"task_id": task_id} def _remove_pending_checkpoint_metadata(*, delete_checkpoint=False, detach_task=False) -> None: if delete_checkpoint: try: os.remove(CHECKPOINT_FILE) except OSError: pass return state = _checkpoint_state_for_update() if not state: return state.pop("pending_task_finalization", None) state.pop("pending_package", None) if detach_task: state["managed_task_idempotency_key"] = "" state["managed_task_id"] = "" _write_checkpoint_state(state) def _prune_user_output_packages(max_files: int = 5) -> None: root = globals().get("_user_workspace") if not root or not os.path.isdir(root): return packages = [] for current_root, _, names in os.walk(os.path.join(root, "tasks")): for name in names: if name.endswith(".zip") and not name.startswith(".pending_"): path = os.path.join(current_root, name) try: packages.append((os.path.getmtime(path), path)) except OSError: pass packages.sort() while len(packages) > max_files: _, old_path = packages.pop(0) try: os.remove(old_path) except OSError: pass def _process_pending_task_finalization(): pending = st.session_state.get("sp_pending_task_finalization") if not isinstance(pending, dict): return None task_id = str(pending.get("task_id") or "").strip() status = str(pending.get("status") or "").strip().lower() if not task_id or status not in {"completed", "failed", "cancelled", "resumable"}: raise RuntimeError("invalid_pending_task_finalization") package_bytes = None package = st.session_state.get("sp_pending_package") if status == "completed": if not isinstance(package, dict): raise RuntimeError("validated_package_metadata_missing") pending_path = str(package.get("pending_path") or "").strip() if not pending_path or not os.path.isfile(pending_path): raise RuntimeError("validated_package_file_missing") with open(pending_path, "rb") as handle: package_bytes = handle.read() if not package_bytes: raise RuntimeError("validated_package_file_empty") response = finish_generation_task( task_id, status=status, safe_error_code=str(pending.get("safe_error_code") or "")[:80], safe_error_detail=str(pending.get("safe_error_detail") or "")[:800], current_node=str(pending.get("current_node") or "")[:240], ) if status == "completed": filename = str(package.get("filename") or "SmartPaper.zip") pending_path = str(package.get("pending_path") or "") final_path = os.path.join(_task_output_dir(task_id), os.path.basename(filename)) try: os.replace(pending_path, final_path) except OSError: # The bytes were read and validated before quota commit, so the user # can still download even if the local archival rename fails. final_path = pending_path st.session_state["zip_buffer"] = package_bytes st.session_state["zip_filename"] = os.path.basename(filename) st.session_state["sp_terminal_notice"] = {"kind": "success"} _prune_user_output_packages() elif status in {"failed", "cancelled"}: st.session_state["sp_terminal_notice"] = { "kind": "error" if status == "failed" else "warning", "message": str(pending.get("safe_error_detail") or "")[:800], } _remove_pending_checkpoint_metadata( delete_checkpoint=bool(pending.get("delete_checkpoint")), detach_task=bool(pending.get("detach_checkpoint")), ) if status == "resumable": st.session_state["sp_generation_task"] = dict(response.get("task") or {"task_id": task_id}) else: st.session_state["sp_generation_task"] = None st.session_state["sp_generation_idempotency_key"] = "" st.session_state["sp_last_task_heartbeat"] = 0.0 st.session_state["sp_pending_task_finalization"] = None st.session_state["sp_pending_package"] = None return response def _finish_managed_generation_task( status: str, *, safe_error_code: str = "", safe_error_detail: str = "", current_node: str = "", delete_checkpoint: bool = False, detach_checkpoint: bool = False, ): task = st.session_state.get("sp_generation_task") or {} task_id = str(task.get("task_id") or "").strip() if not task_id: return None pending = { "task_id": task_id, "status": str(status or "").strip().lower(), "safe_error_code": str(safe_error_code or "")[:80], "safe_error_detail": str(safe_error_detail or "")[:800], "current_node": str(current_node or "")[:240], "delete_checkpoint": bool(delete_checkpoint), "detach_checkpoint": bool(detach_checkpoint), } st.session_state["sp_pending_task_finalization"] = pending _persist_pending_finalization() return _process_pending_task_finalization() def _quota_start_error_message(language: str, error: Exception) -> str: raw = str(error or "") return tr(language, "quota_exhausted") if "payment_required" in raw else raw inject_wechat_browser_notice() DEFAULT_TOPIC = "Generative Artificial Intelligence in Ubiquitous Learning: Evaluating a Chatbot-based Recommendation Engine for Personalized and Context-aware Education" DEFAULT_KEYWORDS = "Ubiquitous computing; Context awareness; Deep learning; Personalized educational; Educational technologies; HCI; Recommendation system; LLMs; Generative AI; ChatGPT; Prompting engineering; Few-shot prompting" _fragment = getattr(st, "fragment", None) or getattr(st, "experimental_fragment", None) if _fragment is None: def _fragment(function): return function def _current_language_code() -> str: choice = st.session_state.get("interface_language_choice") return LANGUAGE_OPTIONS.get(choice, "zh") # Public access uses Alipay only. Password, legacy, and pilot login paths are not exposed. authenticated_user = require_alipay_login(_current_language_code()) # Checkpoints are user-scoped. The former global snapshot.json could cause one # user or an old run to trigger a false recovery notice for another session. _user_workspace = os.path.join( WORKSPACE_DIR, "users", _safe_user_workspace_key(authenticated_user.get("id")), ) os.makedirs(_user_workspace, exist_ok=True) CHECKPOINT_FILE = os.path.join(_user_workspace, "snapshot.json") _restore_pending_finalization() _process_pending_task_finalization() @_fragment def render_workspace_controls(user): """Render controls in an isolated rerun scope for fast language switching.""" top_toolbar = st.container() with top_toolbar: language_col, account_col = st.columns([5.4, 4.6]) with language_col: lang_choice = st.radio( "Language", list(LANGUAGE_OPTIONS.keys()), index=0, horizontal=True, label_visibility="collapsed", key="interface_language_choice", ) lang_code = LANGUAGE_OPTIONS[lang_choice] st.session_state["sp_lang_code"] = lang_code st.session_state["sp_target_lang"] = get_target_language(lang_code) st.session_state["sp_output_suffix"] = get_output_suffix(lang_code) st.session_state["sp_target_flag"] = get_target_flag(lang_code) with account_col: render_logged_in_account(lang_code, user) # Cross-device global busy check remains unchanged in function, but all visible text is localized. is_my_session = st.session_state.get("is_running", False) or \ st.session_state.get("start_pipeline", False) or \ st.session_state.get("resume_pipeline", False) ui_blocked = (not is_my_session and is_system_busy()) st.session_state["sp_ui_blocked"] = ui_blocked if ui_blocked: st.markdown( f"<h3 style='text-align:center;color:#e6a23c;'>{tr(lang_code, 'system_busy_title')}</h3>", unsafe_allow_html=True, ) st.warning(tr(lang_code, "system_busy_warning")) st.info(tr(lang_code, "system_busy_info")) return st.markdown(f"<div class='main-title'><h2>{tr(lang_code, 'title')}</h2></div>", unsafe_allow_html=True) if _load_recoverable_checkpoint() is not None and not (st.session_state.is_running or st.session_state.resume_pipeline or st.session_state.start_pipeline): st.warning(tr(lang_code, "checkpoint_warning")) colA, colB = st.columns([1, 1]) if colA.button(tr(lang_code, "resume"), key="resume_task"): if is_system_busy(): st.warning(tr(lang_code, "busy_short")) else: checkpoint_state = _load_recoverable_checkpoint() or {} checkpoint_key = str(checkpoint_state.get("managed_task_idempotency_key") or "").strip() st.session_state["sp_generation_idempotency_key"] = checkpoint_key or _new_generation_idempotency_key() st.session_state["sp_generation_task"] = None st.session_state["sp_last_task_heartbeat"] = 0.0 update_heartbeat() st.session_state.resume_pipeline = True st.session_state.start_pipeline = False st.rerun() if colB.button(tr(lang_code, "clear"), key="clear_task"): if is_system_busy(): st.warning(tr(lang_code, "busy_short")) else: checkpoint_state = _load_recoverable_checkpoint() or {} checkpoint_task_id = str(checkpoint_state.get("managed_task_id") or "").strip() if checkpoint_task_id: st.session_state["sp_generation_task"] = {"task_id": checkpoint_task_id} try: _finish_managed_generation_task( "cancelled", safe_error_code="E_USER_CLEARED_CHECKPOINT", safe_error_detail="User cleared the recoverable checkpoint.", current_node="checkpoint_clear", delete_checkpoint=True, ) except Exception as exc: st.error(str(exc)) return else: try: os.remove(CHECKPOINT_FILE) except OSError: pass clear_heartbeat() st.rerun() st.markdown("---") st.text_input( tr(lang_code, "topic_label"), value=DEFAULT_TOPIC, disabled=st.session_state.is_running, key="sp_topic", ) st.text_input( tr(lang_code, "keywords_label"), value=DEFAULT_KEYWORDS, disabled=st.session_state.is_running, key="sp_keywords", ) st.number_input( tr(lang_code, "words_label"), min_value=12000, max_value=25000, value=20000, step=1000, disabled=st.session_state.is_running, key="sp_words", ) tab_amj, tab_asq = st.tabs([tr(lang_code, "tab_amj"), tr(lang_code, "tab_asq")]) with tab_amj: st.markdown(tr(lang_code, "amj_config")) st.info(tr(lang_code, "amj_info")) st.file_uploader( tr(lang_code, "upload_reference"), type=["pdf", "doc", "docx"], accept_multiple_files=True, key="up_amj_ref", disabled=st.session_state.is_running, ) st.file_uploader( tr(lang_code, "upload_stats"), type=["csv", "xlsx", "xls"], key="up_amj_data", disabled=st.session_state.is_running, ) with st.expander(tr(lang_code, "amj_expander")): st.markdown(tr(lang_code, "amj_hint")) amj_csv_template = get_amj_csv_template(lang_code) st.code(amj_csv_template, language="csv") st.download_button( label=tr(lang_code, "download_amj_template"), data=amj_csv_template, file_name="AMJ_Data_Template.csv", mime="text/csv", use_container_width=True, ) if st.button(tr(lang_code, "start_amj"), disabled=st.session_state.is_running, key="btn_amj"): if is_system_busy(): st.warning(tr(lang_code, "busy_short")) else: _prepare_new_generation_request() update_heartbeat() st.session_state.run_mode = "amj" st.session_state.start_pipeline = True st.rerun() with tab_asq: st.markdown(tr(lang_code, "asq_config")) st.info(tr(lang_code, "asq_info")) st.file_uploader( tr(lang_code, "upload_reference"), type=["pdf", "doc", "docx"], accept_multiple_files=True, key="up_asq_ref", disabled=st.session_state.is_running, ) st.file_uploader( tr(lang_code, "upload_interviews"), type=["txt", "pdf", "doc", "docx"], key="up_asq_data", disabled=st.session_state.is_running, ) with st.expander(tr(lang_code, "asq_expander")): st.markdown(tr(lang_code, "asq_hint")) asq_txt_template = get_asq_txt_template(lang_code) st.code(asq_txt_template, language="text") st.download_button( label=tr(lang_code, "download_asq_template"), data=asq_txt_template, file_name="ASQ_Interview_Template.txt", mime="text/plain", use_container_width=True, ) if st.button(tr(lang_code, "start_asq"), disabled=st.session_state.is_running, key="btn_asq"): if is_system_busy(): st.warning(tr(lang_code, "busy_short")) else: _prepare_new_generation_request() update_heartbeat() st.session_state.run_mode = "asq" st.session_state.start_pipeline = True st.rerun() # Localize Streamlit built-in uploader text after the uploaders are rendered. inject_file_uploader_i18n(lang_code) render_workspace_controls(authenticated_user) lang_code = str(st.session_state.get("sp_lang_code") or _current_language_code()) target_lang = str(st.session_state.get("sp_target_lang") or get_target_language(lang_code)) output_suffix = str(st.session_state.get("sp_output_suffix") or get_output_suffix(lang_code)) target_flag = str(st.session_state.get("sp_target_flag") or get_target_flag(lang_code)) topic = str(st.session_state.get("sp_topic") or DEFAULT_TOPIC) keywords = str(st.session_state.get("sp_keywords") or DEFAULT_KEYWORDS) words = int(st.session_state.get("sp_words") or 20000) uploaded_papers_amj = st.session_state.get("up_amj_ref") or [] uploaded_data_amj = st.session_state.get("up_amj_data") uploaded_papers_asq = st.session_state.get("up_asq_ref") or [] uploaded_interviews = st.session_state.get("up_asq_data") if st.session_state.get("sp_ui_blocked"): st.stop() if st.session_state.get("sp_payment_required") and not (st.session_state.start_pipeline or st.session_state.resume_pipeline): render_payment_purchase(lang_code) _terminal_notice = st.session_state.pop("sp_terminal_notice", None) if isinstance(_terminal_notice, dict) and _terminal_notice.get("kind") in {"error", "warning"}: _message = str(_terminal_notice.get("message") or "").strip() if _message: (st.error if _terminal_notice.get("kind") == "error" else st.warning)(_message) progress_bar = st.empty() monitor_ui = st.empty() def render_monitor(agent, phase, current_sec="", details=""): st.session_state["sp_current_node"] = str(current_sec or phase or "")[:240] monitor_ui.markdown(f""" <div class='log-box'> <div><b>{tr(lang_code, 'monitor_pipeline')}:</b> {phase}</div><div><b>{tr(lang_code, 'monitor_node')}:</b> {current_sec}</div><div><b>{tr(lang_code, 'monitor_log')}:</b> {details}</div><hr> <div><b>{tr(lang_code, 'monitor_compute')}:</b></div> <div style='display: flex; gap: 8px;'> <span style='background: #fef3c7; padding: 4px 8px; border-radius: 4px; color: black;'>🧠 DeepSeek: <b>{agent.ds_calls}</b></span> <span style='background: #dbeafe; padding: 4px 8px; border-radius: 4px; color: black;'>📚 S2: <b>{agent.s2_calls}</b></span> <span style='background: #d1fae5; padding: 4px 8px; border-radius: 4px; color: black;'>🌐 OpenAlex: <b>{agent.oa_calls}</b></span> </div> </div> """, unsafe_allow_html=True) FLAG_MAP = {"Simplified Chinese (简体中文)": "🇨🇳", "Russian (Русский)": "🇷🇺", "Kazakh (Қазақша)": "🇰🇿"} if st.session_state.start_pipeline or st.session_state.resume_pipeline: if not global_ds_key: st.error(tr(lang_code, "api_key_missing")) st.stop() if st.session_state.get("is_running", False): recoverable_state = _load_recoverable_checkpoint() st.session_state.is_running = False st.session_state.start_pipeline = False st.session_state.resume_pipeline = bool(recoverable_state) if recoverable_state: st.toast(tr(lang_code, "resume_toast"), icon="⚡") update_heartbeat() st.rerun() # A normal page rerun without a valid checkpoint is not a recovery event. # Keep the already-rendered writing interface and show no recovery toast. clear_heartbeat() st.stop() is_resume = st.session_state.resume_pipeline loaded_state = _load_recoverable_checkpoint() if is_resume else None if is_resume and loaded_state is None: is_resume = False try: managed_task, managed_task_key = _begin_managed_generation_task( is_resume=is_resume, loaded_state=loaded_state, mode=st.session_state.run_mode, language=lang_code, ) except Exception as exc: clear_heartbeat() st.session_state.is_running = False st.session_state.start_pipeline = False st.session_state.resume_pipeline = False st.error(_quota_start_error_message(lang_code, exc)) if "payment_required" in str(exc): st.session_state["sp_payment_required"] = True render_payment_purchase(lang_code) st.stop() st.session_state.is_running = True agent = SSciPipelineAgent(global_ds_key, global_s2_key) SAFE_CHUNK_WORDS = agent.chunk_words_default qual_engine = get_qualitative_engine(agent) quant_engine = get_quantitative_engine() completion_finalization_attempted = False try: update_heartbeat() if is_resume and loaded_state: pb = progress_bar.progress(50, text=tr(lang_code, "resume_loading")) state = loaded_state state["managed_task_idempotency_key"] = managed_task_key state["managed_task_id"] = managed_task.get("task_id", "") topic = state["topic"] keywords = state["keywords"] target_lang = state["target_lang"] words = state["words"] user_data_md = state["user_data_md"] data_metadata = state.get("data_metadata", {}) user_lit_context = state["user_lit_context"] agent.load_state(state["agent_state"]) adaptive_structure = state["adaptive_structure"] completed_sections = state["completed_sections"] full_draft_en = state["full_draft_en"] full_draft_translated = state["full_draft_translated"] synthesis_result = state["synthesis_result"] refs_text = state["refs_text"] context_tail = state["context_tail"] st.session_state.run_mode = state.get("run_mode", "amj") if st.session_state.run_mode == "amj": APP_CONFIG.update(AMJ_PROMPTS) else: APP_CONFIG.update(ASQ_PROMPTS) st.success(tr(lang_code, "resume_success", done=len(completed_sections), total=len(adaptive_structure))) render_monitor(agent, tr(lang_code, "resume_phase"), tr(lang_code, "state_check"), tr(lang_code, "history_restored")) else: pb = progress_bar.progress(1, text=tr(lang_code, "pipeline_start")) user_lit_context = "" user_data_md = "" data_metadata = {} if st.session_state.run_mode == "amj": if uploaded_papers_amj: user_lit_context = extract_text_from_files(uploaded_papers_amj[:20]) with st.status(tr(lang_code, "literature_extract"), expanded=True) as status: ph = st.empty() all_paper_texts = [extract_text_from_files([f]) for f in uploaded_papers_amj[:20]] future = ThreadPoolExecutor(max_workers=1).submit(extract_golden_metadata, agent, all_paper_texts) dot_count = 0 while not future.done(): update_heartbeat() ph.markdown(tr(lang_code, "literature_deep") + " ⏳" * (dot_count % 4)) time.sleep(1) dot_count += 1 agent.global_memory["core_golden_metadata"] = future.result() st.session_state.rag_engine = initialize_rag(all_paper_texts) status.update(label=tr(lang_code, "literature_done"), state="complete", expanded=False) with st.spinner(tr(lang_code, "extract_quant")): update_heartbeat() _, user_data_md, msg, data_metadata = quant_engine.resolve_empirical_data(uploaded_data_amj, lang_code) st.info(msg) elif st.session_state.run_mode == "asq": if uploaded_papers_asq: user_lit_context = extract_text_from_files(uploaded_papers_asq[:20]) with st.status(tr(lang_code, "literature_extract"), expanded=True) as status: ph = st.empty() all_paper_texts = [extract_text_from_files([f]) for f in uploaded_papers_asq[:20]] future = ThreadPoolExecutor(max_workers=1).submit(extract_golden_metadata, agent, all_paper_texts) dot_count = 0 while not future.done(): update_heartbeat() ph.markdown(tr(lang_code, "literature_deep") + " ⏳" * (dot_count % 4)) time.sleep(1) dot_count += 1 agent.global_memory["core_golden_metadata"] = future.result() st.session_state.rag_engine = initialize_rag(all_paper_texts) status.update(label=tr(lang_code, "literature_done"), state="complete", expanded=False) update_heartbeat() raw_interviews = extract_text_from_files([uploaded_interviews]) if uploaded_interviews else "" user_data_md, msg, data_metadata = qual_engine.resolve_qualitative_data(topic, keywords, raw_interviews) st.info(msg or tr(lang_code, "qualitative_ready")) state = { "topic": topic, "keywords": keywords, "target_lang": target_lang, "words": words, "user_data_md": user_data_md, "data_metadata": data_metadata, "user_lit_context": user_lit_context, "completed_sections": [], "full_draft_en": "", "full_draft_translated": "", "context_tail": "", "run_mode": st.session_state.run_mode, "interface_language": lang_code, "managed_task_idempotency_key": managed_task_key, "managed_task_id": managed_task.get("task_id", ""), } pb.progress(5, text=tr(lang_code, "config_route")) if st.session_state.run_mode == "amj": APP_CONFIG.update(AMJ_PROMPTS) state["prompt_config"] = AMJ_PROMPTS else: APP_CONFIG.update(ASQ_PROMPTS) state["prompt_config"] = ASQ_PROMPTS pb.progress(10, text=tr(lang_code, "literature_search")) def sync_monitor_ui(): update_heartbeat() render_monitor(agent, tr(lang_code, "search_phase"), "S2 + OpenAlex", tr(lang_code, "search_detail")) target_refs_count = 50 refs_text = agent.fetch_references_parallel(topic, keywords, min_required=target_refs_count, progress_callback=sync_monitor_ui) if len(agent.citation_map) < target_refs_count: raise RuntimeError( tr(lang_code, "refs_insufficient", count=len(agent.citation_map), required=target_refs_count) ) else: st.success(tr(lang_code, "refs_success", count=len(agent.citation_map))) with st.spinner(tr(lang_code, "rag_sync")): update_heartbeat() if hasattr(st.session_state, "rag_engine") and st.session_state.rag_engine is not None: st.session_state.rag_engine._initialize_chunks([refs_text]) else: st.session_state.rag_engine = initialize_rag([refs_text]) st.success(tr(lang_code, "rag_done")) if user_lit_context: pb.progress(11, text=tr(lang_code, "core_literature_parsing")) update_heartbeat() core_refs = agent.parse_user_core_literature(user_lit_context) if core_refs: refs_text = "【USER PROVIDED CORE LITERATURE】\n" + core_refs + "\n\n【RETRIEVED LITERATURE】\n" + refs_text st.success(tr(lang_code, "core_refs_done")) state["refs_text"] = refs_text pb.progress(12, text=tr(lang_code, "synthesis_progress")) with st.status(tr(lang_code, "synthesis_status"), expanded=True) as status: ph = st.empty() synthesis_result = ph.write_stream(stream_with_heartbeat(agent.stream_synthesize_literature(topic, refs_text, user_core_literature=user_lit_context))) synthesis_result = re.sub(r'<think>.*?</think>', '', synthesis_result, flags=re.DOTALL).strip() status.update(label=tr(lang_code, "synthesis_done"), state="complete", expanded=False) state["synthesis_result"] = synthesis_result pb.progress(14, text=tr(lang_code, "anchor_progress")) with st.status(tr(lang_code, "anchor_status"), expanded=True) as status: ph = st.empty() anchor_response = ph.write_stream(stream_with_heartbeat(agent.stream_generate_anchors(topic, keywords, synthesis_result, refs_text, data_evidence=user_data_md, mode=st.session_state.run_mode))) status.update(label=tr(lang_code, "anchor_done"), state="complete", expanded=False) # ========================================================== # [核心修复] 必须先剔除 <think> 标签!防止正则匹配到模型在思考时引用提示词的空占位符 # ========================================================== anchor_response_clean = re.sub(r'<think>.*?</think>', '', anchor_response, flags=re.DOTALL).strip() abstract_match = re.search(r'-+\s*ABSTRACT\s*-+\s*(.*?)(?=\s*-+\s*CONCLUSION\s*-+|$)', anchor_response_clean, re.DOTALL | re.IGNORECASE) conclusion_match = re.search(r'-+\s*CONCLUSION\s*-+\s*(.*?)$', anchor_response_clean, re.DOTALL | re.IGNORECASE) abstract_anchor = abstract_match.group(1).strip() if abstract_match else anchor_response_clean.strip() conclusion_anchor = conclusion_match.group(1).strip() if conclusion_match else "" agent.global_memory["abstract_anchor"] = abstract_anchor agent.global_memory["conclusion_anchor"] = conclusion_anchor # ========================================================== pb.progress(15, text=tr(lang_code, "outline_progress")) update_heartbeat() plan = agent.generate_outline(topic, keywords, mode=st.session_state.run_mode, data_metadata=data_metadata) en_topic = str(plan.get("en_topic", topic)) structure = plan.get("structure", []) en_abstract_clean = clean_markdown_headings(re.sub(r'(?i)^abstract[:\s]*', '', str(abstract_anchor)).strip()) trans_abstract_clean = "" loc_abs = {"Simplified Chinese (简体中文)": "摘要", "Russian (Русский)": "Аннотация", "Kazakh (Қазақша)": "Түйіндеме"}.get(target_lang, "Abstract") loc_kw = {"Simplified Chinese (简体中文)": "关键词", "Russian (Русский)": "Ключевые слова", "Kazakh (Қазақша)": "Кілт сөздер"}.get(target_lang, "Keywords") full_draft_en = f"# {en_topic}\n\n## Abstract\n{en_abstract_clean}\n\n## Keywords\n{keywords}\n\n" full_draft_translated = f"# {topic}\n\n## {loc_abs}\n{trans_abstract_clean}\n\n## {loc_kw}\n{keywords}\n\n" adaptive_structure = agent.build_generation_tasks(structure, int(words), SAFE_CHUNK_WORDS) for task in adaptive_structure: task["is_first"] = bool(task.get("section_first")) with st.spinner(tr(lang_code, "jit_status")): update_heartbeat() jit_texts = agent.prefetch_jit_literature(topic, keywords, adaptive_structure) if jit_texts: if hasattr(st.session_state, "rag_engine") and st.session_state.rag_engine is not None: st.session_state.rag_engine._initialize_chunks([jit_texts]) else: st.session_state.rag_engine = initialize_rag([jit_texts]) st.success(tr(lang_code, "jit_done")) state["adaptive_structure"] = adaptive_structure state["agent_state"] = agent.export_state() state["full_draft_en"] = full_draft_en state["full_draft_translated"] = full_draft_translated completed_sections = [] context_tail = "" os.makedirs(WORKSPACE_DIR, exist_ok=True) with open(CHECKPOINT_FILE, "w", encoding="utf-8") as f: json.dump(state, f, ensure_ascii=False) total_steps = len(adaptive_structure) current_step = len(completed_sections) st.markdown("---") ui_blocks = [] target_flag = FLAG_MAP.get(target_lang, "🌐") for sec in adaptive_structure: display_name = sec["display_name"] st.markdown(f"### ⚙️ {display_name}") col_en, col_trans = st.columns(2) with col_en: st.markdown(tr(lang_code, "english_draft")) en_ph = st.empty() with col_trans: st.markdown(tr(lang_code, "target_draft", flag=target_flag)) trans_ph = st.empty() if display_name in completed_sections: en_ph.success(tr(lang_code, "restored")) trans_ph.info(tr(lang_code, "translation_waiting")) else: en_ph.info(tr(lang_code, "waiting")) trans_ph.info(tr(lang_code, "translation_waiting")) ui_blocks.append({"sec": sec, "en_ph": en_ph, "trans_ph": trans_ph}) generation_aborted = False generation_failure_code = "" generation_failure_detail = "" accumulated_deviation = 0 remaining_sections = sum(1 for item in adaptive_structure if item["display_name"] not in completed_sections) all_cite_keys = list(agent.citation_map.keys()) valid_chunks = [item for item in adaptive_structure if not any(word in item["section"].lower() for word in ["abstract","keyword","appendix"])] allocation_map = {} current_cite_idx = 0 for idx, item in enumerate(valid_chunks): count = max(1, math.ceil(len(all_cite_keys) / max(1, len(valid_chunks)))) allocation_map[item["display_name"]] = all_cite_keys[current_cite_idx:] if idx == len(valid_chunks)-1 else all_cite_keys[current_cite_idx:current_cite_idx+count] current_cite_idx += count for block in ui_blocks: if generation_aborted: break sec = block["sec"] en_ph = block["en_ph"] trans_ph = block["trans_ph"] sec_title = sec["section"] display_name = sec["display_name"] if display_name in completed_sections: continue remaining_sections = max(0, remaining_sections - 1) adjustment = -accumulated_deviation / remaining_sections if remaining_sections else 0 adjusted_target = min(int(SAFE_CHUNK_WORDS * 1.5), max(80, int(sec["target_words"] + adjustment))) is_quarantined = any(word in sec_title.lower() for word in ["abstract","keyword","conclusion","references","appendix"]) assigned_tags = ", ".join(allocation_map.get(display_name, [])) or "the single most relevant source" citation_rule = "NO IN-TEXT CITATIONS ALLOWED IN THIS SECTION." if is_quarantined else f"Use claim-specific citation. Each sentence may contain at most one source tag. Candidate tags: {assigned_tags}. <AVAILABLE_CITATIONS>{refs_text}</AVAILABLE_CITATIONS>" memory_context = agent._build_memory_context(sec_title, 2500) memory_block = f"\n<GLOBAL_MEMORY>\n{memory_context}\n</GLOBAL_MEMORY>\n" if memory_context else "" if getattr(st.session_state, "rag_engine", None) is not None: snippets = st.session_state.rag_engine.get_relevant_context(display_name) if snippets: memory_block += f"\n<TARGETED_RAG>\n{snippets}\n</TARGETED_RAG>\n" subsection = sec.get("subsection") heading_directive = "Do not output the section heading; it will be inserted by the system." if subsection: heading_directive += f" Do not output the subsection heading '{subsection}'; it will also be inserted by the system. Use level-3 headings only when analytically necessary." elif sec_title.lower() in {"introduction", "conclusion"}: heading_directive += " Do not use additional headings." else: heading_directive += " Use level-2 or level-3 headings only when they clarify the argument; never use level-4 headings or numbered headings." table_rule = "Present statistical evidence in standard Markdown tables with plain-text captions." if sec["is_method_or_result"] and st.session_state.run_mode == "amj" else "" required_content = str(sec.get("required_content") or "") concepts = ", ".join(sec.get("concepts") or []) qualitative_rule = f"Assigned qualitative concepts: {concepts}. Use only their assigned representative quotations and do not repeat a quotation already used elsewhere." if concepts else "" data_injection = f"\n<RESEARCH_EVIDENCE>\n{user_data_md}\n</RESEARCH_EVIDENCE>\n" if user_data_md and sec["is_method_or_result"] else "" completed_label = ", ".join(completed_sections) anti_repeat = f"Already completed: {completed_label}. Do not restate their background, gap, or contribution." if completed_label else "" sys_prompt = APP_CONFIG.get("sys_academic_writer", "") + "\n" + APP_CONFIG.get("deep_refinement_directive", "") user_prompt = f"""Write the assigned English manuscript unit. Section: {sec_title} Subsection: {subsection or 'None'} Target length: {int(adjusted_target*0.9)}-{int(adjusted_target*1.1)} words. {heading_directive} {citation_rule} {table_rule} {required_content} {qualitative_rule} {anti_repeat} {memory_block} Literature synthesis: {synthesis_result} {data_injection} Previous tail: {context_tail[-2500:]} Output only final academic prose. Do not report the word count.""" if agent._count_tokens(sys_prompt) + agent._count_tokens(user_prompt) > 60000: user_prompt = agent._truncate_context(user_prompt, 60000 - agent._count_tokens(sys_prompt)) render_monitor(agent, tr(lang_code, "drafting_phase", current=current_step+1, total=total_steps), display_name, tr(lang_code, "target_words", words=adjusted_target, engine=tr(lang_code, "engine_amj") if st.session_state.run_mode=="amj" else tr(lang_code, "engine_asq"))) try: with st.status(tr(lang_code, "draft_status", name=display_name), expanded=True) as status: update_heartbeat() en_raw = en_ph.write_stream(stream_with_heartbeat(agent.stream_deepseek(sys_prompt, user_prompt, max_tokens=7000, model_role="writing", thinking=False, allow_fallback=False))) status.update(label=tr(lang_code, "draft_done", name=display_name), state="complete", expanded=False) en_raw = re.sub(r'<think>.*?</think>', '', str(en_raw), flags=re.S).strip() en_raw = re.sub(rf'(?im)^#\s*{re.escape(sec_title)}\s*\n+', '', en_raw).strip() if subsection: en_raw = re.sub(rf'(?im)^##\s*{re.escape(subsection)}\s*\n+', '', en_raw).strip() en_raw = re.sub(r'(?m)^(#{2,3})\s+[\d\.\s一二三四五六七八九十、]+\s*(.*)', r'\1 \2', en_raw) en_raw = re.sub(r'(?m)^#{4,}\s+(.*)', r'**\1**', en_raw) en_raw = re.sub(r'(?m)^#\s+(.+)$', r'## \1', en_raw) if st.session_state.run_mode == "asq" and user_data_md and "appendix" not in sec_title.lower(): quote_dict = {tag: quote for quote, tag in re.findall(r'"([^"]+)"\s*\[(QUOTE-\d+)\]', user_data_md)} for tag, quote in quote_dict.items(): en_raw = en_raw.replace(tag, f'"{quote}"') en_raw = agent.polish_academic_section(en_raw, display_name, abstract=False) prefix = "" if sec.get("section_first"): prefix += f"# {sec_title}\n\n" if sec.get("subsection_first") and subsection: prefix += f"## {subsection}\n\n" en_with_headings = prefix + en_raw en_resolved = agent.audit_and_resolve_citations(en_with_headings, section_name=display_name, quarantine=is_quarantined) en_ph.markdown(en_resolved) trans_ph.info(tr(lang_code, "translation_waiting")) full_draft_en += en_resolved + "\n\n" agent._extract_memory_from_text(en_raw, sec_title) words_tail = en_raw.split() context_tail = " ".join(words_tail[-200:]) actual_words = len(en_raw.split()) accumulated_deviation += actual_words - adjusted_target current_step += 1 completed_sections.append(display_name) state.update({ "completed_sections": completed_sections, "full_draft_en": full_draft_en, "full_draft_translated": "", "context_tail": context_tail, "agent_state": agent.export_state(), "data_metadata": data_metadata, }) with open(CHECKPOINT_FILE, "w", encoding="utf-8") as checkpoint: json.dump(state, checkpoint, ensure_ascii=False) st.caption(tr(lang_code, "word_count", actual=actual_words, target=adjusted_target, deviation=accumulated_deviation)) pb.progress(min(15 + int((current_step/max(1,total_steps))*75), 90), text=tr(lang_code, "section_complete_progress", name=display_name)) except Exception as error: en_ph.error(tr(lang_code, "node_error")) st.error(tr(lang_code, "node_error_detail", error=error)) generation_aborted = True generation_failure_code = "E_GENERATION_NODE_FAILED" generation_failure_detail = str(error)[:800] if generation_aborted: _finish_managed_generation_task( "failed", safe_error_code=generation_failure_code or "E_GENERATION_INTERRUPTED", safe_error_detail=generation_failure_detail or "Generation stopped; quota was released and the checkpoint was retained for a fresh retry.", current_node=str(st.session_state.get("sp_current_node") or "generation"), detach_checkpoint=True, ) if not generation_aborted and full_draft_en.strip(): pb.progress(91, text=tr(lang_code, "pack_docx")) update_heartbeat() full_draft_en = clean_trailing_references(full_draft_en, ["Reference","References","Reference List"]) # The final abstract is derived from the completed English manuscript and frozen evidence. final_abstract = agent.generate_final_abstract(topic, keywords, full_draft_en, data_evidence=user_data_md, mode=st.session_state.run_mode) full_draft_en = replace_abstract(full_draft_en, final_abstract) agent.validate_manuscript_structure(full_draft_en, adaptive_structure, translated=False) if st.session_state.run_mode == "asq": repeated_quotes = agent.find_repeated_qualitative_quotes(full_draft_en, data_metadata) if repeated_quotes: raise RuntimeError(f"Repeated qualitative quotations: {repeated_quotes[:5]}") # Only sources actually cited in the manuscript enter the reference list. references = [] seen_refs = set() for tag in sorted(agent.used_citations, key=agent._citation_tag_number): ref = str((agent.citation_map.get(tag) or {}).get("ref") or "").strip() if ref and ref not in seen_refs: references.append(f"- {ref}") seen_refs.add(ref) if not references: raise RuntimeError("No cited references remained after citation validation.") full_draft_en = full_draft_en.rstrip() + "\n\n# References\n" + "\n\n".join(references) pb.progress(94, text=tr(lang_code, "translation_running")) update_heartbeat() full_draft_translated = agent.translate_locked_manuscript(full_draft_en, target_lang) agent.validate_translation_alignment(full_draft_en, full_draft_translated) bj_time = datetime.now(timezone(timedelta(hours=8))).strftime("%Y%m%d_%H%M") package = build_validated_package( english_markdown=full_draft_en, translated_markdown=full_draft_translated, english_filename=f"Research_EN_{bj_time}.docx", translated_filename=f"Research_{output_suffix}_{bj_time}.docx", data_metadata=data_metadata, mode=st.session_state.run_mode, citation_audit=agent.citation_audit_log, literature_metrics=agent.get_source_metrics(), ) generated_zip_bytes = package.zip_bytes generated_zip_filename = f"SmartPaper_{bj_time}.zip" # Persist the already validated package in the current user's task # directory before asking Worker/D1 to commit quota. The download is # exposed only after the browser receives completed+committed. task_id = str((st.session_state.get("sp_generation_task") or {}).get("task_id") or "").strip() if not task_id: raise RuntimeError("package_task_id_missing") pending_path = os.path.join(_task_output_dir(task_id), ".pending_" + generated_zip_filename) with open(pending_path, "wb") as output_file: output_file.write(generated_zip_bytes) st.session_state["sp_pending_package"] = { "pending_path": pending_path, "filename": generated_zip_filename, } completion_finalization_attempted = True _finish_managed_generation_task( "completed", current_node="package_validated", delete_checkpoint=True, ) pb.progress(100, text=tr(lang_code, "complete")) render_monitor(agent, tr(lang_code, "complete"), "✅", tr(lang_code, "document_generated")) except Exception as e: if not completion_finalization_attempted: try: _finish_managed_generation_task( "failed", safe_error_code="E_GENERATION_FAILED", safe_error_detail=str(e)[:800], current_node=str(st.session_state.get("sp_current_node") or "generation"), detach_checkpoint=True, ) except Exception: pass st.error(tr(lang_code, "startup_error", error=e)) finally: clear_heartbeat() st.session_state.is_running = False st.session_state.start_pipeline = False st.session_state.resume_pipeline = False if st.session_state.zip_buffer: st.success(tr(lang_code, "success")) st.download_button(label=tr(lang_code, "download"), data=st.session_state.zip_buffer, file_name=st.session_state.zip_filename, mime="application/zip")
# -*- coding: utf-8 -*-
"""Smart Draft - Dual Track (AMJ Empirical vs ASQ Qualitative)"""
from core.meta_extractor import extract_golden_metadata
from core.rag_engine import initialize_rag
from core.qualitative_engine import get_qualitative_engine
from core.quantitative_engine import get_quantitative_engine
from core.production_guard import build_validated_package, replace_abstract
import streamlit as st
st.set_page_config(
page_title="Article Agent",
page_icon="📊",
layout="wide",
initial_sidebar_state="collapsed",
)
import warnings
warnings.filterwarnings("ignore", message="coroutine 'expire_cache' was never awaited")
import os
import streamlit.components.v1 as components
import time
import re
import math
import json
import secrets
from datetime import datetime, timezone, timedelta
from concurrent.futures import ThreadPoolExecutor
from config import APP_CONFIG, AMJ_PROMPTS, ASQ_PROMPTS
from utils.file_parser import extract_text_from_files
from core.pipeline_agent import SSciPipelineAgent
from managed_auth import (
require_alipay_login,
render_logged_in_account,
start_generation_task,
heartbeat_generation_task,
finish_generation_task,
render_payment_purchase,
)
from i18n import LANGUAGE_OPTIONS, get_amj_csv_template, get_asq_txt_template, get_output_suffix, get_target_flag, get_target_language, tr
os.environ["STREAMLIT_SERVER_ENABLE_CORS"] = "false"
os.environ["STREAMLIT_SERVER_ENABLE_XSRF_PROTECTION"] = "false"
WORKSPACE_DIR = APP_CONFIG.get("workspace_dir", ".workspace")
os.makedirs(WORKSPACE_DIR, exist_ok=True)
LEGACY_CHECKPOINT_FILE = os.path.join(WORKSPACE_DIR, "snapshot.json")
CHECKPOINT_FILE = LEGACY_CHECKPOINT_FILE
META_SECTIONS_BLACKLIST = set(APP_CONFIG["blacklists"]["meta_sections"])
def update_heartbeat():
"""Refresh the authoritative Cloudflare/D1 task lease only."""
now = time.time()
task = st.session_state.get("sp_generation_task") or {}
task_id = str(task.get("task_id") or "").strip()
interval = max(3, int(APP_CONFIG.get("task_heartbeat_interval_seconds", 5)))
last_remote = float(st.session_state.get("sp_last_task_heartbeat") or 0.0)
if not task_id or now - last_remote < interval:
return
try:
heartbeat_generation_task(
task_id,
current_node=str(st.session_state.get("sp_current_node") or "")[:240],
)
st.session_state["sp_last_task_heartbeat"] = now
except Exception:
# The hidden browser bridge continues heartbeats independently. A single
# transient UI heartbeat must not interrupt model generation.
pass
def is_system_busy():
"""D1 is the sole concurrency authority; no duplicate Hugging Face file lock."""
return False
def clear_heartbeat():
"""Backward-compatible no-op after removal of the duplicate local lock."""
return None
def stream_with_heartbeat(generator):
"""Refresh the D1 lease while streaming model output."""
last_beat = time.time()
for chunk in generator:
now = time.time()
if now - last_beat > 5:
update_heartbeat()
last_beat = now
yield chunk
def clean_trailing_references(text, keywords=("reference", "references", "reference list", "参考文献")):
lines = text.split('\n')
for i in range(len(lines)-1, max(-1, len(lines)-100), -1):
if any(re.match(rf'^##?\s*{kw}', lines[i], re.IGNORECASE) for kw in keywords):
return '\n'.join(lines[:i]).strip()