# -*- coding: utf-8 -*-
"""Smart Draft - Dual Track (AMJ Empirical vs ASQ Qualitative)"""
from core.meta_extractor import extract_golden_metadata
from core.rag_engine import initialize_rag
from core.qualitative_engine import get_qualitative_engine
from core.quantitative_engine import get_quantitative_engine
from core.production_guard import build_validated_package, replace_abstract
import streamlit as st
st.set_page_config(
page_title="Article Agent",
page_icon="📊",
layout="wide",
initial_sidebar_state="collapsed",
)
import warnings
warnings.filterwarnings("ignore", message="coroutine 'expire_cache' was never awaited")
import os
import streamlit.components.v1 as components
import time
import re
import math
import json
import secrets
from datetime import datetime, timezone, timedelta
from concurrent.futures import ThreadPoolExecutor
from config import APP_CONFIG, AMJ_PROMPTS, ASQ_PROMPTS
from utils.file_parser import extract_text_from_files
from core.pipeline_agent import SSciPipelineAgent
from managed_auth import (
require_alipay_login,
render_logged_in_account,
start_generation_task,
heartbeat_generation_task,
finish_generation_task,
render_payment_purchase,
)
from i18n import LANGUAGE_OPTIONS, get_amj_csv_template, get_asq_txt_template, get_output_suffix, get_target_flag, get_target_language, tr
os.environ["STREAMLIT_SERVER_ENABLE_CORS"] = "false"
os.environ["STREAMLIT_SERVER_ENABLE_XSRF_PROTECTION"] = "false"
WORKSPACE_DIR = APP_CONFIG.get("workspace_dir", ".workspace")
os.makedirs(WORKSPACE_DIR, exist_ok=True)
LEGACY_CHECKPOINT_FILE = os.path.join(WORKSPACE_DIR, "snapshot.json")
CHECKPOINT_FILE = LEGACY_CHECKPOINT_FILE
META_SECTIONS_BLACKLIST = set(APP_CONFIG["blacklists"]["meta_sections"])
def update_heartbeat():
"""Refresh the authoritative Cloudflare/D1 task lease only."""
now = time.time()
task = st.session_state.get("sp_generation_task") or {}
task_id = str(task.get("task_id") or "").strip()
interval = max(3, int(APP_CONFIG.get("task_heartbeat_interval_seconds", 5)))
last_remote = float(st.session_state.get("sp_last_task_heartbeat") or 0.0)
if not task_id or now - last_remote < interval:
return
try:
heartbeat_generation_task(
task_id,
current_node=str(st.session_state.get("sp_current_node") or "")[:240],
)
st.session_state["sp_last_task_heartbeat"] = now
except Exception:
# The hidden browser bridge continues heartbeats independently. A single
# transient UI heartbeat must not interrupt model generation.
pass
def is_system_busy():
"""D1 is the sole concurrency authority; no duplicate Hugging Face file lock."""
return False
def clear_heartbeat():
"""Backward-compatible no-op after removal of the duplicate local lock."""
return None
def stream_with_heartbeat(generator):
"""Refresh the D1 lease while streaming model output."""
last_beat = time.time()
for chunk in generator:
now = time.time()
if now - last_beat > 5:
update_heartbeat()
last_beat = now
yield chunk
def clean_trailing_references(text, keywords=("reference", "references", "reference list", "参考文献")):
lines = text.split('\n')
for i in range(len(lines)-1, max(-1, len(lines)-100), -1):
if any(re.match(rf'^##?\s*{kw}', lines[i], re.IGNORECASE) for kw in keywords):
return '\n'.join(lines[:i]).strip()
return text.strip()
def clean_markdown_headings(text):
return re.sub(r'(?m)^#+\s+(.*)$', r'**\1**', text)
def _read_runtime_secret(name):
# Read a Hugging Face/Streamlit secret and remove copied whitespace.
value = os.environ.get(name, "")
if not value:
try:
value = st.secrets.get(name, "")
except Exception:
value = ""
return str(value or "").strip()
global_ds_key = _read_runtime_secret("DEEPSEEK_API_KEY")
global_s2_key = _read_runtime_secret("S2_API_KEY")
WECHAT_BLOCKER_JS = """"""
components.html(WECHAT_BLOCKER_JS, height=0, width=0)
def inject_file_uploader_i18n(language: str) -> None:
"""Localize Streamlit uploaders without modifying their React structure."""
payload = {
"browse": tr(language, "browse_files"),
"uploaders": [
{
"title": tr(language, "drop_docs"),
"limit": tr(language, "limit_docs"),
},
{
"title": tr(language, "drop_data"),
"limit": tr(language, "limit_data"),
},
{
"title": tr(language, "drop_docs"),
"limit": tr(language, "limit_docs"),
},
{
"title": tr(language, "drop_interviews"),
"limit": tr(language, "limit_interviews"),
},
],
}
script = f"""
"""
components.html(script, height=0, width=0)
def inject_wechat_browser_notice() -> None:
"""Warn immediately when SmartPaper is opened inside WeChat or WeCom."""
script = """
"""
components.html(script, height=0, width=0)
if "is_running" not in st.session_state: st.session_state.is_running = False
if "start_pipeline" not in st.session_state: st.session_state.start_pipeline = False
if "resume_pipeline" not in st.session_state: st.session_state.resume_pipeline = False
if "zip_buffer" not in st.session_state: st.session_state.zip_buffer = None
if "run_mode" not in st.session_state: st.session_state.run_mode = "amj"
if "sp_generation_task" not in st.session_state: st.session_state.sp_generation_task = None
if "sp_generation_idempotency_key" not in st.session_state: st.session_state.sp_generation_idempotency_key = ""
if "sp_last_task_heartbeat" not in st.session_state: st.session_state.sp_last_task_heartbeat = 0.0
if "sp_current_node" not in st.session_state: st.session_state.sp_current_node = ""
if "sp_pending_task_finalization" not in st.session_state: st.session_state.sp_pending_task_finalization = None
if "sp_pending_package" not in st.session_state: st.session_state.sp_pending_package = None
if "sp_terminal_notice" not in st.session_state: st.session_state.sp_terminal_notice = None
def _safe_user_workspace_key(value):
cleaned = re.sub(r"[^A-Za-z0-9_.-]", "_", str(value or "").strip())
return cleaned[:80] or "anonymous"
def _load_recoverable_checkpoint():
"""Return a validated checkpoint for the current authenticated user only."""
if not os.path.isfile(CHECKPOINT_FILE):
return None
try:
with open(CHECKPOINT_FILE, "r", encoding="utf-8") as handle:
state = json.load(handle)
except Exception:
return None
required = {
"topic", "keywords", "target_lang", "words",
"agent_state", "adaptive_structure", "completed_sections",
}
if not isinstance(state, dict) or not required.issubset(state):
return None
if not isinstance(state.get("adaptive_structure"), list):
return None
if not isinstance(state.get("completed_sections"), list):
return None
return state
def _new_generation_idempotency_key() -> str:
return f"gen_{int(time.time() * 1000)}_{secrets.token_urlsafe(18)}"
def _prepare_new_generation_request() -> None:
st.session_state["sp_generation_idempotency_key"] = _new_generation_idempotency_key()
st.session_state["sp_generation_task"] = None
st.session_state["sp_last_task_heartbeat"] = 0.0
st.session_state["sp_current_node"] = "task_start"
st.session_state["zip_buffer"] = None
st.session_state["zip_filename"] = ""
st.session_state["sp_pending_task_finalization"] = None
st.session_state["sp_pending_package"] = None
st.session_state["sp_terminal_notice"] = None
def _begin_managed_generation_task(*, is_resume: bool, loaded_state: dict | None, mode: str, language: str):
checkpoint_key = ""
if is_resume and isinstance(loaded_state, dict):
checkpoint_key = str(loaded_state.get("managed_task_idempotency_key") or "").strip()
idempotency_key = checkpoint_key or str(
st.session_state.get("sp_generation_idempotency_key") or ""
).strip()
if not idempotency_key:
idempotency_key = _new_generation_idempotency_key()
response = start_generation_task(
idempotency_key=idempotency_key,
mode=mode,
interface_language=language,
)
task = dict(response.get("task") or {})
if not task.get("task_id"):
raise RuntimeError("task_start_missing_task_id")
st.session_state["sp_generation_idempotency_key"] = idempotency_key
st.session_state["sp_generation_task"] = task
st.session_state["sp_last_task_heartbeat"] = time.time()
return task, idempotency_key
def _task_output_dir(task_id: str) -> str:
root = globals().get("_user_workspace") or os.path.join(WORKSPACE_DIR, "users", "anonymous")
path = os.path.join(root, "tasks", _safe_user_workspace_key(task_id), "outputs")
os.makedirs(path, exist_ok=True)
return path
def _checkpoint_state_for_update() -> dict:
if not os.path.isfile(CHECKPOINT_FILE):
return {}
try:
with open(CHECKPOINT_FILE, "r", encoding="utf-8") as handle:
value = json.load(handle)
return value if isinstance(value, dict) else {}
except Exception:
return {}
def _write_checkpoint_state(value: dict) -> None:
if not isinstance(value, dict) or not value:
return
os.makedirs(os.path.dirname(CHECKPOINT_FILE), exist_ok=True)
temporary = CHECKPOINT_FILE + ".tmp"
with open(temporary, "w", encoding="utf-8") as handle:
json.dump(value, handle, ensure_ascii=False)
os.replace(temporary, CHECKPOINT_FILE)
def _persist_pending_finalization() -> None:
pending = st.session_state.get("sp_pending_task_finalization")
if not isinstance(pending, dict):
return
state = _checkpoint_state_for_update()
if not state:
return
state["pending_task_finalization"] = pending
package = st.session_state.get("sp_pending_package")
if isinstance(package, dict):
state["pending_package"] = package
_write_checkpoint_state(state)
def _restore_pending_finalization() -> None:
if isinstance(st.session_state.get("sp_pending_task_finalization"), dict):
return
state = _checkpoint_state_for_update()
pending = state.get("pending_task_finalization") if isinstance(state, dict) else None
if not isinstance(pending, dict):
return
st.session_state["sp_pending_task_finalization"] = pending
package = state.get("pending_package")
if isinstance(package, dict):
st.session_state["sp_pending_package"] = package
task_id = str(pending.get("task_id") or "").strip()
if task_id and not st.session_state.get("sp_generation_task"):
st.session_state["sp_generation_task"] = {"task_id": task_id}
def _remove_pending_checkpoint_metadata(*, delete_checkpoint=False, detach_task=False) -> None:
if delete_checkpoint:
try:
os.remove(CHECKPOINT_FILE)
except OSError:
pass
return
state = _checkpoint_state_for_update()
if not state:
return
state.pop("pending_task_finalization", None)
state.pop("pending_package", None)
if detach_task:
state["managed_task_idempotency_key"] = ""
state["managed_task_id"] = ""
_write_checkpoint_state(state)
def _prune_user_output_packages(max_files: int = 5) -> None:
root = globals().get("_user_workspace")
if not root or not os.path.isdir(root):
return
packages = []
for current_root, _, names in os.walk(os.path.join(root, "tasks")):
for name in names:
if name.endswith(".zip") and not name.startswith(".pending_"):
path = os.path.join(current_root, name)
try:
packages.append((os.path.getmtime(path), path))
except OSError:
pass
packages.sort()
while len(packages) > max_files:
_, old_path = packages.pop(0)
try:
os.remove(old_path)
except OSError:
pass
def _process_pending_task_finalization():
pending = st.session_state.get("sp_pending_task_finalization")
if not isinstance(pending, dict):
return None
task_id = str(pending.get("task_id") or "").strip()
status = str(pending.get("status") or "").strip().lower()
if not task_id or status not in {"completed", "failed", "cancelled", "resumable"}:
raise RuntimeError("invalid_pending_task_finalization")
package_bytes = None
package = st.session_state.get("sp_pending_package")
if status == "completed":
if not isinstance(package, dict):
raise RuntimeError("validated_package_metadata_missing")
pending_path = str(package.get("pending_path") or "").strip()
if not pending_path or not os.path.isfile(pending_path):
raise RuntimeError("validated_package_file_missing")
with open(pending_path, "rb") as handle:
package_bytes = handle.read()
if not package_bytes:
raise RuntimeError("validated_package_file_empty")
response = finish_generation_task(
task_id,
status=status,
safe_error_code=str(pending.get("safe_error_code") or "")[:80],
safe_error_detail=str(pending.get("safe_error_detail") or "")[:800],
current_node=str(pending.get("current_node") or "")[:240],
)
if status == "completed":
filename = str(package.get("filename") or "SmartPaper.zip")
pending_path = str(package.get("pending_path") or "")
final_path = os.path.join(_task_output_dir(task_id), os.path.basename(filename))
try:
os.replace(pending_path, final_path)
except OSError:
# The bytes were read and validated before quota commit, so the user
# can still download even if the local archival rename fails.
final_path = pending_path
st.session_state["zip_buffer"] = package_bytes
st.session_state["zip_filename"] = os.path.basename(filename)
st.session_state["sp_terminal_notice"] = {"kind": "success"}
_prune_user_output_packages()
elif status in {"failed", "cancelled"}:
st.session_state["sp_terminal_notice"] = {
"kind": "error" if status == "failed" else "warning",
"message": str(pending.get("safe_error_detail") or "")[:800],
}
_remove_pending_checkpoint_metadata(
delete_checkpoint=bool(pending.get("delete_checkpoint")),
detach_task=bool(pending.get("detach_checkpoint")),
)
if status == "resumable":
st.session_state["sp_generation_task"] = dict(response.get("task") or {"task_id": task_id})
else:
st.session_state["sp_generation_task"] = None
st.session_state["sp_generation_idempotency_key"] = ""
st.session_state["sp_last_task_heartbeat"] = 0.0
st.session_state["sp_pending_task_finalization"] = None
st.session_state["sp_pending_package"] = None
return response
def _finish_managed_generation_task(
status: str,
*,
safe_error_code: str = "",
safe_error_detail: str = "",
current_node: str = "",
delete_checkpoint: bool = False,
detach_checkpoint: bool = False,
):
task = st.session_state.get("sp_generation_task") or {}
task_id = str(task.get("task_id") or "").strip()
if not task_id:
return None
pending = {
"task_id": task_id,
"status": str(status or "").strip().lower(),
"safe_error_code": str(safe_error_code or "")[:80],
"safe_error_detail": str(safe_error_detail or "")[:800],
"current_node": str(current_node or "")[:240],
"delete_checkpoint": bool(delete_checkpoint),
"detach_checkpoint": bool(detach_checkpoint),
}
st.session_state["sp_pending_task_finalization"] = pending
_persist_pending_finalization()
return _process_pending_task_finalization()
def _quota_start_error_message(language: str, error: Exception) -> str:
raw = str(error or "")
return tr(language, "quota_exhausted") if "payment_required" in raw else raw
inject_wechat_browser_notice()
DEFAULT_TOPIC = "Generative Artificial Intelligence in Ubiquitous Learning: Evaluating a Chatbot-based Recommendation Engine for Personalized and Context-aware Education"
DEFAULT_KEYWORDS = "Ubiquitous computing; Context awareness; Deep learning; Personalized educational; Educational technologies; HCI; Recommendation system; LLMs; Generative AI; ChatGPT; Prompting engineering; Few-shot prompting"
_fragment = getattr(st, "fragment", None) or getattr(st, "experimental_fragment", None)
if _fragment is None:
def _fragment(function):
return function
def _current_language_code() -> str:
choice = st.session_state.get("interface_language_choice")
return LANGUAGE_OPTIONS.get(choice, "zh")
# Public access uses Alipay only. Password, legacy, and pilot login paths are not exposed.
authenticated_user = require_alipay_login(_current_language_code())
# Checkpoints are user-scoped. The former global snapshot.json could cause one
# user or an old run to trigger a false recovery notice for another session.
_user_workspace = os.path.join(
WORKSPACE_DIR,
"users",
_safe_user_workspace_key(authenticated_user.get("id")),
)
os.makedirs(_user_workspace, exist_ok=True)
CHECKPOINT_FILE = os.path.join(_user_workspace, "snapshot.json")
_restore_pending_finalization()
_process_pending_task_finalization()
@_fragment
def render_workspace_controls(user):
"""Render controls in an isolated rerun scope for fast language switching."""
top_toolbar = st.container()
with top_toolbar:
language_col, account_col = st.columns([5.4, 4.6])
with language_col:
lang_choice = st.radio(
"Language",
list(LANGUAGE_OPTIONS.keys()),
index=0,
horizontal=True,
label_visibility="collapsed",
key="interface_language_choice",
)
lang_code = LANGUAGE_OPTIONS[lang_choice]
st.session_state["sp_lang_code"] = lang_code
st.session_state["sp_target_lang"] = get_target_language(lang_code)
st.session_state["sp_output_suffix"] = get_output_suffix(lang_code)
st.session_state["sp_target_flag"] = get_target_flag(lang_code)
with account_col:
render_logged_in_account(lang_code, user)
# Cross-device global busy check remains unchanged in function, but all visible text is localized.
is_my_session = st.session_state.get("is_running", False) or \
st.session_state.get("start_pipeline", False) or \
st.session_state.get("resume_pipeline", False)
ui_blocked = (not is_my_session and is_system_busy())
st.session_state["sp_ui_blocked"] = ui_blocked
if ui_blocked:
st.markdown(
f"
{tr(lang_code, 'system_busy_title')}
",
unsafe_allow_html=True,
)
st.warning(tr(lang_code, "system_busy_warning"))
st.info(tr(lang_code, "system_busy_info"))
return
st.markdown(f"{tr(lang_code, 'title')}
", unsafe_allow_html=True)
if _load_recoverable_checkpoint() is not None and not (st.session_state.is_running or st.session_state.resume_pipeline or st.session_state.start_pipeline):
st.warning(tr(lang_code, "checkpoint_warning"))
colA, colB = st.columns([1, 1])
if colA.button(tr(lang_code, "resume"), key="resume_task"):
if is_system_busy():
st.warning(tr(lang_code, "busy_short"))
else:
checkpoint_state = _load_recoverable_checkpoint() or {}
checkpoint_key = str(checkpoint_state.get("managed_task_idempotency_key") or "").strip()
st.session_state["sp_generation_idempotency_key"] = checkpoint_key or _new_generation_idempotency_key()
st.session_state["sp_generation_task"] = None
st.session_state["sp_last_task_heartbeat"] = 0.0
update_heartbeat()
st.session_state.resume_pipeline = True
st.session_state.start_pipeline = False
st.rerun()
if colB.button(tr(lang_code, "clear"), key="clear_task"):
if is_system_busy():
st.warning(tr(lang_code, "busy_short"))
else:
checkpoint_state = _load_recoverable_checkpoint() or {}
checkpoint_task_id = str(checkpoint_state.get("managed_task_id") or "").strip()
if checkpoint_task_id:
st.session_state["sp_generation_task"] = {"task_id": checkpoint_task_id}
try:
_finish_managed_generation_task(
"cancelled",
safe_error_code="E_USER_CLEARED_CHECKPOINT",
safe_error_detail="User cleared the recoverable checkpoint.",
current_node="checkpoint_clear",
delete_checkpoint=True,
)
except Exception as exc:
st.error(str(exc))
return
else:
try:
os.remove(CHECKPOINT_FILE)
except OSError:
pass
clear_heartbeat()
st.rerun()
st.markdown("---")
st.text_input(
tr(lang_code, "topic_label"),
value=DEFAULT_TOPIC,
disabled=st.session_state.is_running,
key="sp_topic",
)
st.text_input(
tr(lang_code, "keywords_label"),
value=DEFAULT_KEYWORDS,
disabled=st.session_state.is_running,
key="sp_keywords",
)
st.number_input(
tr(lang_code, "words_label"),
min_value=12000,
max_value=25000,
value=20000,
step=1000,
disabled=st.session_state.is_running,
key="sp_words",
)
tab_amj, tab_asq = st.tabs([tr(lang_code, "tab_amj"), tr(lang_code, "tab_asq")])
with tab_amj:
st.markdown(tr(lang_code, "amj_config"))
st.info(tr(lang_code, "amj_info"))
st.file_uploader(
tr(lang_code, "upload_reference"),
type=["pdf", "doc", "docx"],
accept_multiple_files=True,
key="up_amj_ref",
disabled=st.session_state.is_running,
)
st.file_uploader(
tr(lang_code, "upload_stats"),
type=["csv", "xlsx", "xls"],
key="up_amj_data",
disabled=st.session_state.is_running,
)
with st.expander(tr(lang_code, "amj_expander")):
st.markdown(tr(lang_code, "amj_hint"))
amj_csv_template = get_amj_csv_template(lang_code)
st.code(amj_csv_template, language="csv")
st.download_button(
label=tr(lang_code, "download_amj_template"),
data=amj_csv_template,
file_name="AMJ_Data_Template.csv",
mime="text/csv",
use_container_width=True,
)
if st.button(tr(lang_code, "start_amj"), disabled=st.session_state.is_running, key="btn_amj"):
if is_system_busy():
st.warning(tr(lang_code, "busy_short"))
else:
_prepare_new_generation_request()
update_heartbeat()
st.session_state.run_mode = "amj"
st.session_state.start_pipeline = True
st.rerun()
with tab_asq:
st.markdown(tr(lang_code, "asq_config"))
st.info(tr(lang_code, "asq_info"))
st.file_uploader(
tr(lang_code, "upload_reference"),
type=["pdf", "doc", "docx"],
accept_multiple_files=True,
key="up_asq_ref",
disabled=st.session_state.is_running,
)
st.file_uploader(
tr(lang_code, "upload_interviews"),
type=["txt", "pdf", "doc", "docx"],
key="up_asq_data",
disabled=st.session_state.is_running,
)
with st.expander(tr(lang_code, "asq_expander")):
st.markdown(tr(lang_code, "asq_hint"))
asq_txt_template = get_asq_txt_template(lang_code)
st.code(asq_txt_template, language="text")
st.download_button(
label=tr(lang_code, "download_asq_template"),
data=asq_txt_template,
file_name="ASQ_Interview_Template.txt",
mime="text/plain",
use_container_width=True,
)
if st.button(tr(lang_code, "start_asq"), disabled=st.session_state.is_running, key="btn_asq"):
if is_system_busy():
st.warning(tr(lang_code, "busy_short"))
else:
_prepare_new_generation_request()
update_heartbeat()
st.session_state.run_mode = "asq"
st.session_state.start_pipeline = True
st.rerun()
# Localize Streamlit built-in uploader text after the uploaders are rendered.
inject_file_uploader_i18n(lang_code)
render_workspace_controls(authenticated_user)
lang_code = str(st.session_state.get("sp_lang_code") or _current_language_code())
target_lang = str(st.session_state.get("sp_target_lang") or get_target_language(lang_code))
output_suffix = str(st.session_state.get("sp_output_suffix") or get_output_suffix(lang_code))
target_flag = str(st.session_state.get("sp_target_flag") or get_target_flag(lang_code))
topic = str(st.session_state.get("sp_topic") or DEFAULT_TOPIC)
keywords = str(st.session_state.get("sp_keywords") or DEFAULT_KEYWORDS)
words = int(st.session_state.get("sp_words") or 20000)
uploaded_papers_amj = st.session_state.get("up_amj_ref") or []
uploaded_data_amj = st.session_state.get("up_amj_data")
uploaded_papers_asq = st.session_state.get("up_asq_ref") or []
uploaded_interviews = st.session_state.get("up_asq_data")
if st.session_state.get("sp_ui_blocked"):
st.stop()
if st.session_state.get("sp_payment_required") and not (st.session_state.start_pipeline or st.session_state.resume_pipeline):
render_payment_purchase(lang_code)
_terminal_notice = st.session_state.pop("sp_terminal_notice", None)
if isinstance(_terminal_notice, dict) and _terminal_notice.get("kind") in {"error", "warning"}:
_message = str(_terminal_notice.get("message") or "").strip()
if _message:
(st.error if _terminal_notice.get("kind") == "error" else st.warning)(_message)
progress_bar = st.empty()
monitor_ui = st.empty()
def render_monitor(agent, phase, current_sec="", details=""):
st.session_state["sp_current_node"] = str(current_sec or phase or "")[:240]
monitor_ui.markdown(f"""
{tr(lang_code, 'monitor_pipeline')}: {phase}
{tr(lang_code, 'monitor_node')}: {current_sec}
{tr(lang_code, 'monitor_log')}: {details}
{tr(lang_code, 'monitor_compute')}:
🧠 DeepSeek: {agent.ds_calls}
📚 S2: {agent.s2_calls}
🌐 OpenAlex: {agent.oa_calls}
""", unsafe_allow_html=True)
FLAG_MAP = {"Simplified Chinese (简体中文)": "🇨🇳", "Russian (Русский)": "🇷🇺", "Kazakh (Қазақша)": "🇰🇿"}
if st.session_state.start_pipeline or st.session_state.resume_pipeline:
if not global_ds_key:
st.error(tr(lang_code, "api_key_missing"))
st.stop()
if st.session_state.get("is_running", False):
recoverable_state = _load_recoverable_checkpoint()
st.session_state.is_running = False
st.session_state.start_pipeline = False
st.session_state.resume_pipeline = bool(recoverable_state)
if recoverable_state:
st.toast(tr(lang_code, "resume_toast"), icon="⚡")
update_heartbeat()
st.rerun()
# A normal page rerun without a valid checkpoint is not a recovery event.
# Keep the already-rendered writing interface and show no recovery toast.
clear_heartbeat()
st.stop()
is_resume = st.session_state.resume_pipeline
loaded_state = _load_recoverable_checkpoint() if is_resume else None
if is_resume and loaded_state is None:
is_resume = False
try:
managed_task, managed_task_key = _begin_managed_generation_task(
is_resume=is_resume,
loaded_state=loaded_state,
mode=st.session_state.run_mode,
language=lang_code,
)
except Exception as exc:
clear_heartbeat()
st.session_state.is_running = False
st.session_state.start_pipeline = False
st.session_state.resume_pipeline = False
st.error(_quota_start_error_message(lang_code, exc))
if "payment_required" in str(exc):
st.session_state["sp_payment_required"] = True
render_payment_purchase(lang_code)
st.stop()
st.session_state.is_running = True
agent = SSciPipelineAgent(global_ds_key, global_s2_key)
SAFE_CHUNK_WORDS = agent.chunk_words_default
qual_engine = get_qualitative_engine(agent)
quant_engine = get_quantitative_engine()
completion_finalization_attempted = False
try:
update_heartbeat()
if is_resume and loaded_state:
pb = progress_bar.progress(50, text=tr(lang_code, "resume_loading"))
state = loaded_state
state["managed_task_idempotency_key"] = managed_task_key
state["managed_task_id"] = managed_task.get("task_id", "")
topic = state["topic"]
keywords = state["keywords"]
target_lang = state["target_lang"]
words = state["words"]
user_data_md = state["user_data_md"]
data_metadata = state.get("data_metadata", {})
user_lit_context = state["user_lit_context"]
agent.load_state(state["agent_state"])
adaptive_structure = state["adaptive_structure"]
completed_sections = state["completed_sections"]
full_draft_en = state["full_draft_en"]
full_draft_translated = state["full_draft_translated"]
synthesis_result = state["synthesis_result"]
refs_text = state["refs_text"]
context_tail = state["context_tail"]
st.session_state.run_mode = state.get("run_mode", "amj")
if st.session_state.run_mode == "amj":
APP_CONFIG.update(AMJ_PROMPTS)
else:
APP_CONFIG.update(ASQ_PROMPTS)
st.success(tr(lang_code, "resume_success", done=len(completed_sections), total=len(adaptive_structure)))
render_monitor(agent, tr(lang_code, "resume_phase"), tr(lang_code, "state_check"), tr(lang_code, "history_restored"))
else:
pb = progress_bar.progress(1, text=tr(lang_code, "pipeline_start"))
user_lit_context = ""
user_data_md = ""
data_metadata = {}
if st.session_state.run_mode == "amj":
if uploaded_papers_amj:
user_lit_context = extract_text_from_files(uploaded_papers_amj[:20])
with st.status(tr(lang_code, "literature_extract"), expanded=True) as status:
ph = st.empty()
all_paper_texts = [extract_text_from_files([f]) for f in uploaded_papers_amj[:20]]
future = ThreadPoolExecutor(max_workers=1).submit(extract_golden_metadata, agent, all_paper_texts)
dot_count = 0
while not future.done():
update_heartbeat()
ph.markdown(tr(lang_code, "literature_deep") + " ⏳" * (dot_count % 4))
time.sleep(1)
dot_count += 1
agent.global_memory["core_golden_metadata"] = future.result()
st.session_state.rag_engine = initialize_rag(all_paper_texts)
status.update(label=tr(lang_code, "literature_done"), state="complete", expanded=False)
with st.spinner(tr(lang_code, "extract_quant")):
update_heartbeat()
_, user_data_md, msg, data_metadata = quant_engine.resolve_empirical_data(uploaded_data_amj, lang_code)
st.info(msg)
elif st.session_state.run_mode == "asq":
if uploaded_papers_asq:
user_lit_context = extract_text_from_files(uploaded_papers_asq[:20])
with st.status(tr(lang_code, "literature_extract"), expanded=True) as status:
ph = st.empty()
all_paper_texts = [extract_text_from_files([f]) for f in uploaded_papers_asq[:20]]
future = ThreadPoolExecutor(max_workers=1).submit(extract_golden_metadata, agent, all_paper_texts)
dot_count = 0
while not future.done():
update_heartbeat()
ph.markdown(tr(lang_code, "literature_deep") + " ⏳" * (dot_count % 4))
time.sleep(1)
dot_count += 1
agent.global_memory["core_golden_metadata"] = future.result()
st.session_state.rag_engine = initialize_rag(all_paper_texts)
status.update(label=tr(lang_code, "literature_done"), state="complete", expanded=False)
update_heartbeat()
raw_interviews = extract_text_from_files([uploaded_interviews]) if uploaded_interviews else ""
user_data_md, msg, data_metadata = qual_engine.resolve_qualitative_data(topic, keywords, raw_interviews)
st.info(msg or tr(lang_code, "qualitative_ready"))
state = {
"topic": topic, "keywords": keywords, "target_lang": target_lang, "words": words,
"user_data_md": user_data_md, "data_metadata": data_metadata, "user_lit_context": user_lit_context,
"completed_sections": [], "full_draft_en": "", "full_draft_translated": "",
"context_tail": "", "run_mode": st.session_state.run_mode, "interface_language": lang_code,
"managed_task_idempotency_key": managed_task_key,
"managed_task_id": managed_task.get("task_id", ""),
}
pb.progress(5, text=tr(lang_code, "config_route"))
if st.session_state.run_mode == "amj":
APP_CONFIG.update(AMJ_PROMPTS)
state["prompt_config"] = AMJ_PROMPTS
else:
APP_CONFIG.update(ASQ_PROMPTS)
state["prompt_config"] = ASQ_PROMPTS
pb.progress(10, text=tr(lang_code, "literature_search"))
def sync_monitor_ui():
update_heartbeat()
render_monitor(agent, tr(lang_code, "search_phase"), "S2 + OpenAlex", tr(lang_code, "search_detail"))
target_refs_count = 50
refs_text = agent.fetch_references_parallel(topic, keywords, min_required=target_refs_count, progress_callback=sync_monitor_ui)
if len(agent.citation_map) < target_refs_count:
raise RuntimeError(
tr(lang_code, "refs_insufficient", count=len(agent.citation_map), required=target_refs_count)
)
else:
st.success(tr(lang_code, "refs_success", count=len(agent.citation_map)))
with st.spinner(tr(lang_code, "rag_sync")):
update_heartbeat()
if hasattr(st.session_state, "rag_engine") and st.session_state.rag_engine is not None:
st.session_state.rag_engine._initialize_chunks([refs_text])
else:
st.session_state.rag_engine = initialize_rag([refs_text])
st.success(tr(lang_code, "rag_done"))
if user_lit_context:
pb.progress(11, text=tr(lang_code, "core_literature_parsing"))
update_heartbeat()
core_refs = agent.parse_user_core_literature(user_lit_context)
if core_refs:
refs_text = "【USER PROVIDED CORE LITERATURE】\n" + core_refs + "\n\n【RETRIEVED LITERATURE】\n" + refs_text
st.success(tr(lang_code, "core_refs_done"))
state["refs_text"] = refs_text
pb.progress(12, text=tr(lang_code, "synthesis_progress"))
with st.status(tr(lang_code, "synthesis_status"), expanded=True) as status:
ph = st.empty()
synthesis_result = ph.write_stream(stream_with_heartbeat(agent.stream_synthesize_literature(topic, refs_text, user_core_literature=user_lit_context)))
synthesis_result = re.sub(r'.*?', '', synthesis_result, flags=re.DOTALL).strip()
status.update(label=tr(lang_code, "synthesis_done"), state="complete", expanded=False)
state["synthesis_result"] = synthesis_result
pb.progress(14, text=tr(lang_code, "anchor_progress"))
with st.status(tr(lang_code, "anchor_status"), expanded=True) as status:
ph = st.empty()
anchor_response = ph.write_stream(stream_with_heartbeat(agent.stream_generate_anchors(topic, keywords, synthesis_result, refs_text, data_evidence=user_data_md, mode=st.session_state.run_mode)))
status.update(label=tr(lang_code, "anchor_done"), state="complete", expanded=False)
# ==========================================================
# [核心修复] 必须先剔除 标签!防止正则匹配到模型在思考时引用提示词的空占位符
# ==========================================================
anchor_response_clean = re.sub(r'.*?', '', anchor_response, flags=re.DOTALL).strip()
abstract_match = re.search(r'-+\s*ABSTRACT\s*-+\s*(.*?)(?=\s*-+\s*CONCLUSION\s*-+|$)', anchor_response_clean, re.DOTALL | re.IGNORECASE)
conclusion_match = re.search(r'-+\s*CONCLUSION\s*-+\s*(.*?)$', anchor_response_clean, re.DOTALL | re.IGNORECASE)
abstract_anchor = abstract_match.group(1).strip() if abstract_match else anchor_response_clean.strip()
conclusion_anchor = conclusion_match.group(1).strip() if conclusion_match else ""
agent.global_memory["abstract_anchor"] = abstract_anchor
agent.global_memory["conclusion_anchor"] = conclusion_anchor
# ==========================================================
pb.progress(15, text=tr(lang_code, "outline_progress"))
update_heartbeat()
plan = agent.generate_outline(topic, keywords, mode=st.session_state.run_mode, data_metadata=data_metadata)
en_topic = str(plan.get("en_topic", topic))
structure = plan.get("structure", [])
en_abstract_clean = clean_markdown_headings(re.sub(r'(?i)^abstract[:\s]*', '', str(abstract_anchor)).strip())
trans_abstract_clean = ""
loc_abs = {"Simplified Chinese (简体中文)": "摘要", "Russian (Русский)": "Аннотация", "Kazakh (Қазақша)": "Түйіндеме"}.get(target_lang, "Abstract")
loc_kw = {"Simplified Chinese (简体中文)": "关键词", "Russian (Русский)": "Ключевые слова", "Kazakh (Қазақша)": "Кілт сөздер"}.get(target_lang, "Keywords")
full_draft_en = f"# {en_topic}\n\n## Abstract\n{en_abstract_clean}\n\n## Keywords\n{keywords}\n\n"
full_draft_translated = f"# {topic}\n\n## {loc_abs}\n{trans_abstract_clean}\n\n## {loc_kw}\n{keywords}\n\n"
adaptive_structure = agent.build_generation_tasks(structure, int(words), SAFE_CHUNK_WORDS)
for task in adaptive_structure:
task["is_first"] = bool(task.get("section_first"))
with st.spinner(tr(lang_code, "jit_status")):
update_heartbeat()
jit_texts = agent.prefetch_jit_literature(topic, keywords, adaptive_structure)
if jit_texts:
if hasattr(st.session_state, "rag_engine") and st.session_state.rag_engine is not None:
st.session_state.rag_engine._initialize_chunks([jit_texts])
else:
st.session_state.rag_engine = initialize_rag([jit_texts])
st.success(tr(lang_code, "jit_done"))
state["adaptive_structure"] = adaptive_structure
state["agent_state"] = agent.export_state()
state["full_draft_en"] = full_draft_en
state["full_draft_translated"] = full_draft_translated
completed_sections = []
context_tail = ""
os.makedirs(WORKSPACE_DIR, exist_ok=True)
with open(CHECKPOINT_FILE, "w", encoding="utf-8") as f:
json.dump(state, f, ensure_ascii=False)
total_steps = len(adaptive_structure)
current_step = len(completed_sections)
st.markdown("---")
ui_blocks = []
target_flag = FLAG_MAP.get(target_lang, "🌐")
for sec in adaptive_structure:
display_name = sec["display_name"]
st.markdown(f"### ⚙️ {display_name}")
col_en, col_trans = st.columns(2)
with col_en:
st.markdown(tr(lang_code, "english_draft"))
en_ph = st.empty()
with col_trans:
st.markdown(tr(lang_code, "target_draft", flag=target_flag))
trans_ph = st.empty()
if display_name in completed_sections:
en_ph.success(tr(lang_code, "restored"))
trans_ph.info(tr(lang_code, "translation_waiting"))
else:
en_ph.info(tr(lang_code, "waiting"))
trans_ph.info(tr(lang_code, "translation_waiting"))
ui_blocks.append({"sec": sec, "en_ph": en_ph, "trans_ph": trans_ph})
generation_aborted = False
generation_failure_code = ""
generation_failure_detail = ""
accumulated_deviation = 0
remaining_sections = sum(1 for item in adaptive_structure if item["display_name"] not in completed_sections)
all_cite_keys = list(agent.citation_map.keys())
valid_chunks = [item for item in adaptive_structure if not any(word in item["section"].lower() for word in ["abstract","keyword","appendix"])]
allocation_map = {}
current_cite_idx = 0
for idx, item in enumerate(valid_chunks):
count = max(1, math.ceil(len(all_cite_keys) / max(1, len(valid_chunks))))
allocation_map[item["display_name"]] = all_cite_keys[current_cite_idx:] if idx == len(valid_chunks)-1 else all_cite_keys[current_cite_idx:current_cite_idx+count]
current_cite_idx += count
for block in ui_blocks:
if generation_aborted:
break
sec = block["sec"]
en_ph = block["en_ph"]
trans_ph = block["trans_ph"]
sec_title = sec["section"]
display_name = sec["display_name"]
if display_name in completed_sections:
continue
remaining_sections = max(0, remaining_sections - 1)
adjustment = -accumulated_deviation / remaining_sections if remaining_sections else 0
adjusted_target = min(int(SAFE_CHUNK_WORDS * 1.5), max(80, int(sec["target_words"] + adjustment)))
is_quarantined = any(word in sec_title.lower() for word in ["abstract","keyword","conclusion","references","appendix"])
assigned_tags = ", ".join(allocation_map.get(display_name, [])) or "the single most relevant source"
citation_rule = "NO IN-TEXT CITATIONS ALLOWED IN THIS SECTION." if is_quarantined else f"Use claim-specific citation. Each sentence may contain at most one source tag. Candidate tags: {assigned_tags}. {refs_text}"
memory_context = agent._build_memory_context(sec_title, 2500)
memory_block = f"\n\n{memory_context}\n\n" if memory_context else ""
if getattr(st.session_state, "rag_engine", None) is not None:
snippets = st.session_state.rag_engine.get_relevant_context(display_name)
if snippets:
memory_block += f"\n\n{snippets}\n\n"
subsection = sec.get("subsection")
heading_directive = "Do not output the section heading; it will be inserted by the system."
if subsection:
heading_directive += f" Do not output the subsection heading '{subsection}'; it will also be inserted by the system. Use level-3 headings only when analytically necessary."
elif sec_title.lower() in {"introduction", "conclusion"}:
heading_directive += " Do not use additional headings."
else:
heading_directive += " Use level-2 or level-3 headings only when they clarify the argument; never use level-4 headings or numbered headings."
table_rule = "Present statistical evidence in standard Markdown tables with plain-text captions." if sec["is_method_or_result"] and st.session_state.run_mode == "amj" else ""
required_content = str(sec.get("required_content") or "")
concepts = ", ".join(sec.get("concepts") or [])
qualitative_rule = f"Assigned qualitative concepts: {concepts}. Use only their assigned representative quotations and do not repeat a quotation already used elsewhere." if concepts else ""
data_injection = f"\n\n{user_data_md}\n\n" if user_data_md and sec["is_method_or_result"] else ""
completed_label = ", ".join(completed_sections)
anti_repeat = f"Already completed: {completed_label}. Do not restate their background, gap, or contribution." if completed_label else ""
sys_prompt = APP_CONFIG.get("sys_academic_writer", "") + "\n" + APP_CONFIG.get("deep_refinement_directive", "")
user_prompt = f"""Write the assigned English manuscript unit.
Section: {sec_title}
Subsection: {subsection or 'None'}
Target length: {int(adjusted_target*0.9)}-{int(adjusted_target*1.1)} words.
{heading_directive}
{citation_rule}
{table_rule}
{required_content}
{qualitative_rule}
{anti_repeat}
{memory_block}
Literature synthesis: {synthesis_result}
{data_injection}
Previous tail: {context_tail[-2500:]}
Output only final academic prose. Do not report the word count."""
if agent._count_tokens(sys_prompt) + agent._count_tokens(user_prompt) > 60000:
user_prompt = agent._truncate_context(user_prompt, 60000 - agent._count_tokens(sys_prompt))
render_monitor(agent, tr(lang_code, "drafting_phase", current=current_step+1, total=total_steps), display_name, tr(lang_code, "target_words", words=adjusted_target, engine=tr(lang_code, "engine_amj") if st.session_state.run_mode=="amj" else tr(lang_code, "engine_asq")))
try:
with st.status(tr(lang_code, "draft_status", name=display_name), expanded=True) as status:
update_heartbeat()
en_raw = en_ph.write_stream(stream_with_heartbeat(agent.stream_deepseek(sys_prompt, user_prompt, max_tokens=7000, model_role="writing", thinking=False, allow_fallback=False)))
status.update(label=tr(lang_code, "draft_done", name=display_name), state="complete", expanded=False)
en_raw = re.sub(r'.*?', '', str(en_raw), flags=re.S).strip()
en_raw = re.sub(rf'(?im)^#\s*{re.escape(sec_title)}\s*\n+', '', en_raw).strip()
if subsection:
en_raw = re.sub(rf'(?im)^##\s*{re.escape(subsection)}\s*\n+', '', en_raw).strip()
en_raw = re.sub(r'(?m)^(#{2,3})\s+[\d\.\s一二三四五六七八九十、]+\s*(.*)', r'\1 \2', en_raw)
en_raw = re.sub(r'(?m)^#{4,}\s+(.*)', r'**\1**', en_raw)
en_raw = re.sub(r'(?m)^#\s+(.+)$', r'## \1', en_raw)
if st.session_state.run_mode == "asq" and user_data_md and "appendix" not in sec_title.lower():
quote_dict = {tag: quote for quote, tag in re.findall(r'"([^"]+)"\s*\[(QUOTE-\d+)\]', user_data_md)}
for tag, quote in quote_dict.items():
en_raw = en_raw.replace(tag, f'"{quote}"')
en_raw = agent.polish_academic_section(en_raw, display_name, abstract=False)
prefix = ""
if sec.get("section_first"):
prefix += f"# {sec_title}\n\n"
if sec.get("subsection_first") and subsection:
prefix += f"## {subsection}\n\n"
en_with_headings = prefix + en_raw
en_resolved = agent.audit_and_resolve_citations(en_with_headings, section_name=display_name, quarantine=is_quarantined)
en_ph.markdown(en_resolved)
trans_ph.info(tr(lang_code, "translation_waiting"))
full_draft_en += en_resolved + "\n\n"
agent._extract_memory_from_text(en_raw, sec_title)
words_tail = en_raw.split()
context_tail = " ".join(words_tail[-200:])
actual_words = len(en_raw.split())
accumulated_deviation += actual_words - adjusted_target
current_step += 1
completed_sections.append(display_name)
state.update({
"completed_sections": completed_sections,
"full_draft_en": full_draft_en,
"full_draft_translated": "",
"context_tail": context_tail,
"agent_state": agent.export_state(),
"data_metadata": data_metadata,
})
with open(CHECKPOINT_FILE, "w", encoding="utf-8") as checkpoint:
json.dump(state, checkpoint, ensure_ascii=False)
st.caption(tr(lang_code, "word_count", actual=actual_words, target=adjusted_target, deviation=accumulated_deviation))
pb.progress(min(15 + int((current_step/max(1,total_steps))*75), 90), text=tr(lang_code, "section_complete_progress", name=display_name))
except Exception as error:
en_ph.error(tr(lang_code, "node_error"))
st.error(tr(lang_code, "node_error_detail", error=error))
generation_aborted = True
generation_failure_code = "E_GENERATION_NODE_FAILED"
generation_failure_detail = str(error)[:800]
if generation_aborted:
_finish_managed_generation_task(
"failed",
safe_error_code=generation_failure_code or "E_GENERATION_INTERRUPTED",
safe_error_detail=generation_failure_detail or "Generation stopped; quota was released and the checkpoint was retained for a fresh retry.",
current_node=str(st.session_state.get("sp_current_node") or "generation"),
detach_checkpoint=True,
)
if not generation_aborted and full_draft_en.strip():
pb.progress(91, text=tr(lang_code, "pack_docx"))
update_heartbeat()
full_draft_en = clean_trailing_references(full_draft_en, ["Reference","References","Reference List"])
# The final abstract is derived from the completed English manuscript and frozen evidence.
final_abstract = agent.generate_final_abstract(topic, keywords, full_draft_en, data_evidence=user_data_md, mode=st.session_state.run_mode)
full_draft_en = replace_abstract(full_draft_en, final_abstract)
agent.validate_manuscript_structure(full_draft_en, adaptive_structure, translated=False)
if st.session_state.run_mode == "asq":
repeated_quotes = agent.find_repeated_qualitative_quotes(full_draft_en, data_metadata)
if repeated_quotes:
raise RuntimeError(f"Repeated qualitative quotations: {repeated_quotes[:5]}")
# Only sources actually cited in the manuscript enter the reference list.
references = []
seen_refs = set()
for tag in sorted(agent.used_citations, key=agent._citation_tag_number):
ref = str((agent.citation_map.get(tag) or {}).get("ref") or "").strip()
if ref and ref not in seen_refs:
references.append(f"- {ref}")
seen_refs.add(ref)
if not references:
raise RuntimeError("No cited references remained after citation validation.")
full_draft_en = full_draft_en.rstrip() + "\n\n# References\n" + "\n\n".join(references)
pb.progress(94, text=tr(lang_code, "translation_running"))
update_heartbeat()
full_draft_translated = agent.translate_locked_manuscript(full_draft_en, target_lang)
agent.validate_translation_alignment(full_draft_en, full_draft_translated)
bj_time = datetime.now(timezone(timedelta(hours=8))).strftime("%Y%m%d_%H%M")
package = build_validated_package(
english_markdown=full_draft_en,
translated_markdown=full_draft_translated,
english_filename=f"Research_EN_{bj_time}.docx",
translated_filename=f"Research_{output_suffix}_{bj_time}.docx",
data_metadata=data_metadata,
mode=st.session_state.run_mode,
citation_audit=agent.citation_audit_log,
literature_metrics=agent.get_source_metrics(),
)
generated_zip_bytes = package.zip_bytes
generated_zip_filename = f"SmartPaper_{bj_time}.zip"
# Persist the already validated package in the current user's task
# directory before asking Worker/D1 to commit quota. The download is
# exposed only after the browser receives completed+committed.
task_id = str((st.session_state.get("sp_generation_task") or {}).get("task_id") or "").strip()
if not task_id:
raise RuntimeError("package_task_id_missing")
pending_path = os.path.join(_task_output_dir(task_id), ".pending_" + generated_zip_filename)
with open(pending_path, "wb") as output_file:
output_file.write(generated_zip_bytes)
st.session_state["sp_pending_package"] = {
"pending_path": pending_path,
"filename": generated_zip_filename,
}
completion_finalization_attempted = True
_finish_managed_generation_task(
"completed",
current_node="package_validated",
delete_checkpoint=True,
)
pb.progress(100, text=tr(lang_code, "complete"))
render_monitor(agent, tr(lang_code, "complete"), "✅", tr(lang_code, "document_generated"))
except Exception as e:
if not completion_finalization_attempted:
try:
_finish_managed_generation_task(
"failed",
safe_error_code="E_GENERATION_FAILED",
safe_error_detail=str(e)[:800],
current_node=str(st.session_state.get("sp_current_node") or "generation"),
detach_checkpoint=True,
)
except Exception:
pass
st.error(tr(lang_code, "startup_error", error=e))
finally:
clear_heartbeat()
st.session_state.is_running = False
st.session_state.start_pipeline = False
st.session_state.resume_pipeline = False
if st.session_state.zip_buffer:
st.success(tr(lang_code, "success"))
st.download_button(label=tr(lang_code, "download"), data=st.session_state.zip_buffer, file_name=st.session_state.zip_filename, mime="application/zip")