Deploy SmartPaper v1.1 R3-Free

#6
app.py CHANGED
@@ -1,132 +1,47 @@
1
  # -*- coding: utf-8 -*-
2
- """Smart Draft - Dual Track (AMJ Empirical vs ASQ Qualitative)"""
3
- from core.meta_extractor import extract_golden_metadata
4
- from core.rag_engine import initialize_rag
5
- from core.qualitative_engine import get_qualitative_engine
6
- from core.quantitative_engine import get_quantitative_engine
7
- from core.production_guard import build_validated_package, replace_abstract
8
- import streamlit as st
9
-
10
- st.set_page_config(
11
- page_title="Article Agent",
12
- page_icon="📊",
13
- layout="wide",
14
- initial_sidebar_state="collapsed",
15
- )
16
-
17
- import warnings
18
- warnings.filterwarnings("ignore", message="coroutine 'expire_cache' was never awaited")
19
-
20
- import os
21
- import streamlit.components.v1 as components
22
- import time
23
  import logging
24
- import threading
25
  import re
26
- import math
27
- import json
28
  import secrets
29
- from datetime import datetime, timezone, timedelta
30
- from concurrent.futures import ThreadPoolExecutor
 
 
 
 
 
 
 
 
31
 
32
- from config import APP_CONFIG, AMJ_PROMPTS, ASQ_PROMPTS
33
- from utils.file_parser import extract_text_from_files
34
- from core.pipeline_agent import SSciPipelineAgent
35
  from managed_auth import (
 
 
36
  require_alipay_login,
37
  render_logged_in_account,
38
- start_generation_task,
39
- heartbeat_generation_task,
40
- finish_generation_task,
41
  render_payment_purchase,
 
 
 
 
 
42
  )
43
- from i18n import LANGUAGE_OPTIONS, get_amj_csv_template, get_asq_txt_template, get_output_suffix, get_target_flag, get_target_language, tr
44
-
45
- os.environ["STREAMLIT_SERVER_ENABLE_CORS"] = "false"
46
- os.environ["STREAMLIT_SERVER_ENABLE_XSRF_PROTECTION"] = "false"
47
 
48
- # Streamlit executes the app on its ScriptRunner thread rather than Python's
49
- # process main thread. Capture that thread so background worker threads cannot
50
- # emit browser/D1 heartbeats on behalf of work they do not own.
51
- STREAMLIT_SCRIPT_THREAD_ID = threading.get_ident()
 
 
52
 
53
  WORKSPACE_DIR = APP_CONFIG.get("workspace_dir", ".workspace")
54
  os.makedirs(WORKSPACE_DIR, exist_ok=True)
55
- LEGACY_CHECKPOINT_FILE = os.path.join(WORKSPACE_DIR, "snapshot.json")
56
- CHECKPOINT_FILE = LEGACY_CHECKPOINT_FILE
57
-
58
- META_SECTIONS_BLACKLIST = set(APP_CONFIG["blacklists"]["meta_sections"])
59
-
60
-
61
- def update_heartbeat():
62
- """Refresh D1 only while the Python generation process is actually active."""
63
- if threading.get_ident() != STREAMLIT_SCRIPT_THREAD_ID:
64
- return
65
- now = time.time()
66
- task = st.session_state.get("sp_generation_task") or {}
67
- task_id = str(task.get("task_id") or "").strip()
68
- interval = max(3, int(APP_CONFIG.get("task_heartbeat_interval_seconds", 5)))
69
- last_remote = float(st.session_state.get("sp_last_task_heartbeat") or 0.0)
70
- if not task_id or now - last_remote < interval:
71
- return
72
- try:
73
- heartbeat_generation_task(
74
- task_id,
75
- current_node=str(st.session_state.get("sp_current_node") or "")[:240],
76
- )
77
- st.session_state["sp_last_task_heartbeat"] = now
78
- except Exception as exc:
79
- logging.warning(
80
- "[SP_TASK] heartbeat_enqueue_failed task_id=%s error=%s",
81
- task_id,
82
- type(exc).__name__,
83
- )
84
-
85
-
86
- def is_system_busy():
87
- """D1 is the sole concurrency authority; no duplicate Hugging Face file lock."""
88
- return False
89
-
90
-
91
- def clear_heartbeat():
92
- """Backward-compatible no-op after removal of the duplicate local lock."""
93
- return None
94
-
95
-
96
- def stream_with_heartbeat(generator):
97
- """Refresh the D1 lease while streaming model output."""
98
- last_beat = time.time()
99
- for chunk in generator:
100
- now = time.time()
101
- if now - last_beat > 5:
102
- update_heartbeat()
103
- last_beat = now
104
- yield chunk
105
-
106
-
107
- def clean_trailing_references(text, keywords=("reference", "references", "reference list", "参考文献")):
108
- lines = text.split('\n')
109
- for i in range(len(lines)-1, max(-1, len(lines)-100), -1):
110
- if any(re.match(rf'^##?\s*{kw}', lines[i], re.IGNORECASE) for kw in keywords):
111
- return '\n'.join(lines[:i]).strip()
112
- return text.strip()
113
-
114
- def clean_markdown_headings(text):
115
- return re.sub(r'(?m)^#+\s+(.*)$', r'**\1**', text)
116
-
117
- def _read_runtime_secret(name):
118
- # Read a Hugging Face/Streamlit secret and remove copied whitespace.
119
- value = os.environ.get(name, "")
120
- if not value:
121
- try:
122
- value = st.secrets.get(name, "")
123
- except Exception:
124
- value = ""
125
- return str(value or "").strip()
126
-
127
-
128
- global_ds_key = _read_runtime_secret("DEEPSEEK_API_KEY")
129
- global_s2_key = _read_runtime_secret("S2_API_KEY")
130
 
131
  WECHAT_BLOCKER_JS = """<script>try{var ua=navigator.userAgent.toLowerCase();if(ua.indexOf('micromessenger')!==-1){var parentDoc=window.parent.document;if(!parentDoc.getElementById('wx-blocker')){var overlay=parentDoc.createElement('div');overlay.id='wx-blocker';overlay.style.cssText='position:fixed;top:0;left:0;width:100vw;height:100vh;background:#1a365d;z-index:9999999;color:white;display:flex;flex-direction:column;align-items:center;justify-content:center;text-align:center;font-family:sans-serif;padding:30px;box-sizing:border-box;';overlay.innerHTML='<h2 style="font-size:26px;margin-bottom:15px;font-weight:bold;">环境受限 / Restricted</h2><p style="font-size:18px;line-height:1.6;color:#e2e8f0;max-width:100%;">为了保障最终双语 ZIP 压缩包正常下载,<b>本系统禁止在微信内直接运行</b>。<br><br>👉 <b>请点击右上角 [···] 菜单,选择“在浏览器打开” (Open in Browser)</b></p>';parentDoc.body.appendChild(overlay);}}}catch(e){}</script>"""
132
  components.html(WECHAT_BLOCKER_JS, height=0, width=0)
@@ -338,301 +253,268 @@ def inject_wechat_browser_notice() -> None:
338
  components.html(script, height=0, width=0)
339
 
340
 
341
- if "is_running" not in st.session_state: st.session_state.is_running = False
342
- if "start_pipeline" not in st.session_state: st.session_state.start_pipeline = False
343
- if "resume_pipeline" not in st.session_state: st.session_state.resume_pipeline = False
344
- if "zip_buffer" not in st.session_state: st.session_state.zip_buffer = None
345
- if "run_mode" not in st.session_state: st.session_state.run_mode = "amj"
346
- if "sp_generation_task" not in st.session_state: st.session_state.sp_generation_task = None
347
- if "sp_generation_idempotency_key" not in st.session_state: st.session_state.sp_generation_idempotency_key = ""
348
- if "sp_last_task_heartbeat" not in st.session_state: st.session_state.sp_last_task_heartbeat = 0.0
349
- if "sp_current_node" not in st.session_state: st.session_state.sp_current_node = ""
350
- if "sp_pending_task_finalization" not in st.session_state: st.session_state.sp_pending_task_finalization = None
351
- if "sp_pending_package" not in st.session_state: st.session_state.sp_pending_package = None
352
- if "sp_terminal_notice" not in st.session_state: st.session_state.sp_terminal_notice = None
353
-
354
 
355
- def _safe_user_workspace_key(value):
356
- cleaned = re.sub(r"[^A-Za-z0-9_.-]", "_", str(value or "").strip())
357
- return cleaned[:80] or "anonymous"
 
 
 
 
 
 
 
 
 
 
 
358
 
359
 
360
- def _load_recoverable_checkpoint():
361
- """Return a validated checkpoint for the current authenticated user only."""
362
- if not os.path.isfile(CHECKPOINT_FILE):
363
- return None
364
- try:
365
- with open(CHECKPOINT_FILE, "r", encoding="utf-8") as handle:
366
- state = json.load(handle)
367
- except Exception:
368
- return None
369
- required = {
370
- "topic", "keywords", "target_lang", "words",
371
- "agent_state", "adaptive_structure", "completed_sections",
372
- }
373
- if not isinstance(state, dict) or not required.issubset(state):
374
- return None
375
- if not isinstance(state.get("adaptive_structure"), list):
376
- return None
377
- if not isinstance(state.get("completed_sections"), list):
378
- return None
379
- return state
380
 
381
 
382
  def _new_generation_idempotency_key() -> str:
383
  return f"gen_{int(time.time() * 1000)}_{secrets.token_urlsafe(18)}"
384
 
385
 
386
- def _prepare_new_generation_request() -> None:
387
- st.session_state["sp_generation_idempotency_key"] = _new_generation_idempotency_key()
388
- st.session_state["sp_generation_task"] = None
389
- st.session_state["sp_last_task_heartbeat"] = 0.0
390
- st.session_state["sp_current_node"] = "task_start"
391
- st.session_state["zip_buffer"] = None
392
- st.session_state["zip_filename"] = ""
393
- st.session_state["sp_pending_task_finalization"] = None
394
- st.session_state["sp_pending_package"] = None
395
- st.session_state["sp_terminal_notice"] = None
396
-
397
-
398
- def _begin_managed_generation_task(*, is_resume: bool, loaded_state: dict | None, mode: str, language: str):
399
- checkpoint_key = ""
400
- if is_resume and isinstance(loaded_state, dict):
401
- checkpoint_key = str(loaded_state.get("managed_task_idempotency_key") or "").strip()
402
-
403
- idempotency_key = checkpoint_key or str(
404
- st.session_state.get("sp_generation_idempotency_key") or ""
405
- ).strip()
406
- if not idempotency_key:
407
- idempotency_key = _new_generation_idempotency_key()
408
-
409
- response = start_generation_task(
410
- idempotency_key=idempotency_key,
411
- mode=mode,
412
- interface_language=language,
413
- )
414
- task = dict(response.get("task") or {})
415
- if not task.get("task_id"):
416
- raise RuntimeError("task_start_missing_task_id")
417
 
418
- st.session_state["sp_generation_idempotency_key"] = idempotency_key
419
- st.session_state["sp_generation_task"] = task
420
- st.session_state["sp_last_task_heartbeat"] = time.time()
421
- logging.info(
422
- "[SP_TASK] task_started task_id=%s mode=%s language=%s",
423
- str(task.get("task_id") or ""),
424
- mode,
425
- language,
426
- )
427
- return task, idempotency_key
428
 
 
 
 
 
 
429
 
430
- def _task_output_dir(task_id: str) -> str:
431
- root = globals().get("_user_workspace") or os.path.join(WORKSPACE_DIR, "users", "anonymous")
432
- path = os.path.join(root, "tasks", _safe_user_workspace_key(task_id), "outputs")
433
- os.makedirs(path, exist_ok=True)
434
- return path
435
 
 
 
436
 
437
- def _checkpoint_state_for_update() -> dict:
438
- if not os.path.isfile(CHECKPOINT_FILE):
439
- return {}
440
- try:
441
- with open(CHECKPOINT_FILE, "r", encoding="utf-8") as handle:
442
- value = json.load(handle)
443
- return value if isinstance(value, dict) else {}
444
- except Exception:
445
- return {}
446
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
447
 
448
- def _write_checkpoint_state(value: dict) -> None:
449
- if not isinstance(value, dict) or not value:
450
- return
451
- os.makedirs(os.path.dirname(CHECKPOINT_FILE), exist_ok=True)
452
- temporary = CHECKPOINT_FILE + ".tmp"
453
- with open(temporary, "w", encoding="utf-8") as handle:
454
- json.dump(value, handle, ensure_ascii=False)
455
- os.replace(temporary, CHECKPOINT_FILE)
456
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
457
 
458
- def _persist_pending_finalization() -> None:
459
- pending = st.session_state.get("sp_pending_task_finalization")
460
- if not isinstance(pending, dict):
461
- return
462
- state = _checkpoint_state_for_update()
463
- if not state:
464
- return
465
- state["pending_task_finalization"] = pending
466
- package = st.session_state.get("sp_pending_package")
467
- if isinstance(package, dict):
468
- state["pending_package"] = package
469
- _write_checkpoint_state(state)
 
 
 
 
 
 
470
 
471
 
472
- def _restore_pending_finalization() -> None:
473
- if isinstance(st.session_state.get("sp_pending_task_finalization"), dict):
474
- return
475
- state = _checkpoint_state_for_update()
476
- pending = state.get("pending_task_finalization") if isinstance(state, dict) else None
477
- if not isinstance(pending, dict):
478
- return
479
- st.session_state["sp_pending_task_finalization"] = pending
480
- package = state.get("pending_package")
481
- if isinstance(package, dict):
482
- st.session_state["sp_pending_package"] = package
483
- task_id = str(pending.get("task_id") or "").strip()
484
- if task_id and not st.session_state.get("sp_generation_task"):
485
- st.session_state["sp_generation_task"] = {"task_id": task_id}
486
 
487
 
488
- def _remove_pending_checkpoint_metadata(*, delete_checkpoint=False, detach_task=False) -> None:
489
- if delete_checkpoint:
490
- try:
491
- os.remove(CHECKPOINT_FILE)
492
- except OSError:
493
- pass
494
- return
495
- state = _checkpoint_state_for_update()
496
- if not state:
497
- return
498
- state.pop("pending_task_finalization", None)
499
- state.pop("pending_package", None)
500
- if detach_task:
501
- state["managed_task_idempotency_key"] = ""
502
- state["managed_task_id"] = ""
503
- _write_checkpoint_state(state)
504
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
505
 
506
- def _prune_user_output_packages(max_files: int = 5) -> None:
507
- root = globals().get("_user_workspace")
508
- if not root or not os.path.isdir(root):
509
- return
510
- packages = []
511
- for current_root, _, names in os.walk(os.path.join(root, "tasks")):
512
- for name in names:
513
- if name.endswith(".zip") and not name.startswith(".pending_"):
514
- path = os.path.join(current_root, name)
 
 
 
 
 
 
 
 
 
 
 
 
515
  try:
516
- packages.append((os.path.getmtime(path), path))
517
  except OSError:
518
  pass
519
- packages.sort()
520
- while len(packages) > max_files:
521
- _, old_path = packages.pop(0)
522
- try:
523
- os.remove(old_path)
524
- except OSError:
525
  pass
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
526
 
527
 
528
- def _process_pending_task_finalization():
529
- pending = st.session_state.get("sp_pending_task_finalization")
530
- if not isinstance(pending, dict):
531
- return None
532
- task_id = str(pending.get("task_id") or "").strip()
533
- status = str(pending.get("status") or "").strip().lower()
534
- if not task_id or status not in {"completed", "failed", "cancelled", "resumable"}:
535
- raise RuntimeError("invalid_pending_task_finalization")
536
-
537
- package_bytes = None
538
- package = st.session_state.get("sp_pending_package")
539
- if status == "completed":
540
- if not isinstance(package, dict):
541
- raise RuntimeError("validated_package_metadata_missing")
542
- pending_path = str(package.get("pending_path") or "").strip()
543
- if not pending_path or not os.path.isfile(pending_path):
544
- raise RuntimeError("validated_package_file_missing")
545
- with open(pending_path, "rb") as handle:
546
- package_bytes = handle.read()
547
- if not package_bytes:
548
- raise RuntimeError("validated_package_file_empty")
549
-
550
- logging.info(
551
- "[SP_TASK] task_finalization_request task_id=%s status=%s node=%s",
552
- task_id,
553
- status,
554
- str(pending.get("current_node") or "")[:120],
555
- )
556
- response = finish_generation_task(
557
- task_id,
558
- status=status,
559
- safe_error_code=str(pending.get("safe_error_code") or "")[:80],
560
- safe_error_detail=str(pending.get("safe_error_detail") or "")[:800],
561
- current_node=str(pending.get("current_node") or "")[:240],
562
- )
563
 
564
- logging.info(
565
- "[SP_TASK] task_finalization_confirmed task_id=%s status=%s reservation=%s",
566
- task_id,
567
- status,
568
- str((response.get("reservation") or {}).get("status") or ""),
569
- )
570
 
571
- if status == "completed":
572
- filename = str(package.get("filename") or "SmartPaper.zip")
573
- pending_path = str(package.get("pending_path") or "")
574
- final_path = os.path.join(_task_output_dir(task_id), os.path.basename(filename))
575
- try:
576
- os.replace(pending_path, final_path)
577
- except OSError:
578
- # The bytes were read and validated before quota commit, so the user
579
- # can still download even if the local archival rename fails.
580
- final_path = pending_path
581
- st.session_state["zip_buffer"] = package_bytes
582
- st.session_state["zip_filename"] = os.path.basename(filename)
583
- st.session_state["sp_terminal_notice"] = {"kind": "success"}
584
- _prune_user_output_packages()
585
- elif status in {"failed", "cancelled"}:
586
  st.session_state["sp_terminal_notice"] = {
587
- "kind": "error" if status == "failed" else "warning",
588
- "message": str(pending.get("safe_error_detail") or "")[:800],
589
  }
 
 
 
 
590
 
591
- _remove_pending_checkpoint_metadata(
592
- delete_checkpoint=bool(pending.get("delete_checkpoint")),
593
- detach_task=bool(pending.get("detach_checkpoint")),
594
- )
595
- if status == "resumable":
596
- st.session_state["sp_generation_task"] = dict(response.get("task") or {"task_id": task_id})
597
- else:
598
- st.session_state["sp_generation_task"] = None
599
- st.session_state["sp_generation_idempotency_key"] = ""
600
- st.session_state["sp_last_task_heartbeat"] = 0.0
601
- st.session_state["sp_pending_task_finalization"] = None
602
- st.session_state["sp_pending_package"] = None
603
- return response
604
-
605
-
606
- def _finish_managed_generation_task(
607
- status: str,
608
- *,
609
- safe_error_code: str = "",
610
- safe_error_detail: str = "",
611
- current_node: str = "",
612
- delete_checkpoint: bool = False,
613
- detach_checkpoint: bool = False,
614
- ):
615
- task = st.session_state.get("sp_generation_task") or {}
616
  task_id = str(task.get("task_id") or "").strip()
617
  if not task_id:
618
- return None
619
- pending = {
620
- "task_id": task_id,
621
- "status": str(status or "").strip().lower(),
622
- "safe_error_code": str(safe_error_code or "")[:80],
623
- "safe_error_detail": str(safe_error_detail or "")[:800],
624
- "current_node": str(current_node or "")[:240],
625
- "delete_checkpoint": bool(delete_checkpoint),
626
- "detach_checkpoint": bool(detach_checkpoint),
627
- }
628
- st.session_state["sp_pending_task_finalization"] = pending
629
- _persist_pending_finalization()
630
- return _process_pending_task_finalization()
631
-
632
-
633
- def _quota_start_error_message(language: str, error: Exception) -> str:
634
- raw = str(error or "")
635
- return tr(language, "quota_exhausted") if "payment_required" in raw else raw
636
 
637
 
638
  inject_wechat_browser_notice()
@@ -642,8 +524,10 @@ DEFAULT_KEYWORDS = "Ubiquitous computing; Context awareness; Deep learning; Pers
642
 
643
  _fragment = getattr(st, "fragment", None) or getattr(st, "experimental_fragment", None)
644
  if _fragment is None:
645
- def _fragment(function):
646
- return function
 
 
647
 
648
 
649
  def _current_language_code() -> str:
@@ -651,777 +535,161 @@ def _current_language_code() -> str:
651
  return LANGUAGE_OPTIONS.get(choice, "zh")
652
 
653
 
654
- # Public access uses Alipay only. Password, legacy, and pilot login paths are not exposed.
655
  authenticated_user = require_alipay_login(_current_language_code())
656
-
657
- # Checkpoints are user-scoped. The former global snapshot.json could cause one
658
- # user or an old run to trigger a false recovery notice for another session.
659
- _user_workspace = os.path.join(
660
- WORKSPACE_DIR,
661
- "users",
662
- _safe_user_workspace_key(authenticated_user.get("id")),
663
- )
664
  os.makedirs(_user_workspace, exist_ok=True)
665
- CHECKPOINT_FILE = os.path.join(_user_workspace, "snapshot.json")
666
- _restore_pending_finalization()
667
- _process_pending_task_finalization()
668
 
669
 
670
  @_fragment
671
  def render_workspace_controls(user):
672
- """Render controls in an isolated rerun scope for fast language switching."""
673
  top_toolbar = st.container()
674
-
675
  with top_toolbar:
676
  language_col, account_col = st.columns([5.4, 4.6])
677
-
678
  with language_col:
679
  lang_choice = st.radio(
680
- "Language",
681
- list(LANGUAGE_OPTIONS.keys()),
682
- index=0,
683
- horizontal=True,
684
- label_visibility="collapsed",
685
- key="interface_language_choice",
686
  )
687
-
688
  lang_code = LANGUAGE_OPTIONS[lang_choice]
689
  st.session_state["sp_lang_code"] = lang_code
690
  st.session_state["sp_target_lang"] = get_target_language(lang_code)
691
  st.session_state["sp_output_suffix"] = get_output_suffix(lang_code)
692
  st.session_state["sp_target_flag"] = get_target_flag(lang_code)
693
-
694
  with account_col:
695
  render_logged_in_account(lang_code, user)
696
 
697
- # Cross-device global busy check remains unchanged in function, but all visible text is localized.
698
- is_my_session = st.session_state.get("is_running", False) or \
699
- st.session_state.get("start_pipeline", False) or \
700
- st.session_state.get("resume_pipeline", False)
701
- ui_blocked = (not is_my_session and is_system_busy())
702
- st.session_state["sp_ui_blocked"] = ui_blocked
703
- if ui_blocked:
704
- st.markdown(
705
- f"<h3 style='text-align:center;color:#e6a23c;'>{tr(lang_code, 'system_busy_title')}</h3>",
706
- unsafe_allow_html=True,
707
- )
708
- st.warning(tr(lang_code, "system_busy_warning"))
709
- st.info(tr(lang_code, "system_busy_info"))
710
- return
711
 
712
  st.markdown(f"<div class='main-title'><h2>{tr(lang_code, 'title')}</h2></div>", unsafe_allow_html=True)
713
 
714
- if _load_recoverable_checkpoint() is not None and not (st.session_state.is_running or st.session_state.resume_pipeline or st.session_state.start_pipeline):
715
  st.warning(tr(lang_code, "checkpoint_warning"))
716
  colA, colB = st.columns([1, 1])
717
  if colA.button(tr(lang_code, "resume"), key="resume_task"):
718
- if is_system_busy():
719
- st.warning(tr(lang_code, "busy_short"))
720
- else:
721
- checkpoint_state = _load_recoverable_checkpoint() or {}
722
- checkpoint_key = str(checkpoint_state.get("managed_task_idempotency_key") or "").strip()
723
- st.session_state["sp_generation_idempotency_key"] = checkpoint_key or _new_generation_idempotency_key()
724
- st.session_state["sp_generation_task"] = None
725
- st.session_state["sp_last_task_heartbeat"] = 0.0
726
- update_heartbeat()
727
- st.session_state.resume_pipeline = True
728
- st.session_state.start_pipeline = False
729
- st.rerun()
730
-
731
  if colB.button(tr(lang_code, "clear"), key="clear_task"):
732
- if is_system_busy():
733
- st.warning(tr(lang_code, "busy_short"))
734
- else:
735
- checkpoint_state = _load_recoverable_checkpoint() or {}
736
- checkpoint_task_id = str(checkpoint_state.get("managed_task_id") or "").strip()
737
- if checkpoint_task_id:
738
- st.session_state["sp_generation_task"] = {"task_id": checkpoint_task_id}
739
- try:
740
- _finish_managed_generation_task(
741
- "cancelled",
742
- safe_error_code="E_USER_CLEARED_CHECKPOINT",
743
- safe_error_detail="User cleared the recoverable checkpoint.",
744
- current_node="checkpoint_clear",
745
- delete_checkpoint=True,
746
- )
747
- except Exception as exc:
748
- st.error(str(exc))
749
- return
750
- else:
751
- try:
752
- os.remove(CHECKPOINT_FILE)
753
- except OSError:
754
- pass
755
- clear_heartbeat()
756
- st.rerun()
757
  st.markdown("---")
758
 
759
- st.text_input(
760
- tr(lang_code, "topic_label"),
761
- value=DEFAULT_TOPIC,
762
- disabled=st.session_state.is_running,
763
- key="sp_topic",
764
- )
765
- st.text_input(
766
- tr(lang_code, "keywords_label"),
767
- value=DEFAULT_KEYWORDS,
768
- disabled=st.session_state.is_running,
769
- key="sp_keywords",
770
- )
771
  st.number_input(
772
  tr(lang_code, "words_label"),
773
- min_value=12000,
774
- max_value=25000,
775
- value=20000,
776
  step=1000,
777
- disabled=st.session_state.is_running,
778
  key="sp_words",
779
  )
780
 
781
  tab_amj, tab_asq = st.tabs([tr(lang_code, "tab_amj"), tr(lang_code, "tab_asq")])
782
-
783
  with tab_amj:
784
  st.markdown(tr(lang_code, "amj_config"))
785
  st.info(tr(lang_code, "amj_info"))
786
- st.file_uploader(
787
- tr(lang_code, "upload_reference"),
788
- type=["pdf", "doc", "docx"],
789
- accept_multiple_files=True,
790
- key="up_amj_ref",
791
- disabled=st.session_state.is_running,
792
- )
793
- st.file_uploader(
794
- tr(lang_code, "upload_stats"),
795
- type=["csv", "xlsx", "xls"],
796
- key="up_amj_data",
797
- disabled=st.session_state.is_running,
798
- )
799
-
800
  with st.expander(tr(lang_code, "amj_expander")):
801
  st.markdown(tr(lang_code, "amj_hint"))
802
- amj_csv_template = get_amj_csv_template(lang_code)
803
- st.code(amj_csv_template, language="csv")
804
- st.download_button(
805
- label=tr(lang_code, "download_amj_template"),
806
- data=amj_csv_template,
807
- file_name="AMJ_Data_Template.csv",
808
- mime="text/csv",
809
- use_container_width=True,
810
- )
811
-
812
- if st.button(tr(lang_code, "start_amj"), disabled=st.session_state.is_running, key="btn_amj"):
813
- if is_system_busy():
814
- st.warning(tr(lang_code, "busy_short"))
815
- else:
816
- _prepare_new_generation_request()
817
- update_heartbeat()
818
- st.session_state.run_mode = "amj"
819
- st.session_state.start_pipeline = True
820
- st.rerun()
821
 
822
  with tab_asq:
823
  st.markdown(tr(lang_code, "asq_config"))
824
  st.info(tr(lang_code, "asq_info"))
825
- st.file_uploader(
826
- tr(lang_code, "upload_reference"),
827
- type=["pdf", "doc", "docx"],
828
- accept_multiple_files=True,
829
- key="up_asq_ref",
830
- disabled=st.session_state.is_running,
831
- )
832
- st.file_uploader(
833
- tr(lang_code, "upload_interviews"),
834
- type=["txt", "pdf", "doc", "docx"],
835
- key="up_asq_data",
836
- disabled=st.session_state.is_running,
837
- )
838
-
839
  with st.expander(tr(lang_code, "asq_expander")):
840
  st.markdown(tr(lang_code, "asq_hint"))
841
- asq_txt_template = get_asq_txt_template(lang_code)
842
- st.code(asq_txt_template, language="text")
843
- st.download_button(
844
- label=tr(lang_code, "download_asq_template"),
845
- data=asq_txt_template,
846
- file_name="ASQ_Interview_Template.txt",
847
- mime="text/plain",
848
- use_container_width=True,
849
- )
850
 
851
- if st.button(tr(lang_code, "start_asq"), disabled=st.session_state.is_running, key="btn_asq"):
852
- if is_system_busy():
853
- st.warning(tr(lang_code, "busy_short"))
854
- else:
855
- _prepare_new_generation_request()
856
- update_heartbeat()
857
- st.session_state.run_mode = "asq"
858
- st.session_state.start_pipeline = True
859
- st.rerun()
860
-
861
- # Localize Streamlit built-in uploader text after the uploaders are rendered.
862
  inject_file_uploader_i18n(lang_code)
863
 
864
 
865
  render_workspace_controls(authenticated_user)
866
-
867
  lang_code = str(st.session_state.get("sp_lang_code") or _current_language_code())
868
- target_lang = str(st.session_state.get("sp_target_lang") or get_target_language(lang_code))
869
- output_suffix = str(st.session_state.get("sp_output_suffix") or get_output_suffix(lang_code))
870
- target_flag = str(st.session_state.get("sp_target_flag") or get_target_flag(lang_code))
871
-
872
- topic = str(st.session_state.get("sp_topic") or DEFAULT_TOPIC)
873
- keywords = str(st.session_state.get("sp_keywords") or DEFAULT_KEYWORDS)
874
- words = int(st.session_state.get("sp_words") or 20000)
875
- uploaded_papers_amj = st.session_state.get("up_amj_ref") or []
876
- uploaded_data_amj = st.session_state.get("up_amj_data")
877
- uploaded_papers_asq = st.session_state.get("up_asq_ref") or []
878
- uploaded_interviews = st.session_state.get("up_asq_data")
879
-
880
- if st.session_state.get("sp_ui_blocked"):
881
- st.stop()
882
-
883
- if st.session_state.get("sp_payment_required") and not (st.session_state.start_pipeline or st.session_state.resume_pipeline):
884
  render_payment_purchase(lang_code)
885
 
886
- _terminal_notice = st.session_state.pop("sp_terminal_notice", None)
887
- if isinstance(_terminal_notice, dict) and _terminal_notice.get("kind") in {"error", "warning"}:
888
- _message = str(_terminal_notice.get("message") or "").strip()
889
- if _message:
890
- (st.error if _terminal_notice.get("kind") == "error" else st.warning)(_message)
891
-
892
- progress_bar = st.empty()
893
- monitor_ui = st.empty()
894
-
895
- def render_monitor(agent, phase, current_sec="", details=""):
896
- node = str(current_sec or phase or "")[:240]
897
- st.session_state["sp_current_node"] = node
898
- if node and node != st.session_state.get("sp_last_logged_node"):
899
- task_id = str((st.session_state.get("sp_generation_task") or {}).get("task_id") or "")
900
- logging.info("[SP_TASK] node_enter task_id=%s node=%s", task_id, node)
901
- st.session_state["sp_last_logged_node"] = node
902
- monitor_ui.markdown(f"""
903
  <div class='log-box'>
904
- <div><b>{tr(lang_code, 'monitor_pipeline')}:</b> {phase}</div><div><b>{tr(lang_code, 'monitor_node')}:</b> {current_sec}</div><div><b>{tr(lang_code, 'monitor_log')}:</b> {details}</div><hr>
905
  <div><b>{tr(lang_code, 'monitor_compute')}:</b></div>
906
  <div style='display: flex; gap: 8px;'>
907
- <span style='background: #fef3c7; padding: 4px 8px; border-radius: 4px; color: black;'>🧠 DeepSeek: <b>{agent.ds_calls}</b></span>
908
- <span style='background: #dbeafe; padding: 4px 8px; border-radius: 4px; color: black;'>📚 S2: <b>{agent.s2_calls}</b></span>
909
- <span style='background: #d1fae5; padding: 4px 8px; border-radius: 4px; color: black;'>🌐 OpenAlex: <b>{agent.oa_calls}</b></span>
910
  </div>
911
  </div>
912
  """, unsafe_allow_html=True)
913
 
914
- FLAG_MAP = {"Simplified Chinese (简体中文)": "🇨🇳", "Russian (Русский)": "🇷🇺", "Kazakh (Қазақша)": "🇰🇿"}
915
-
916
- if st.session_state.start_pipeline or st.session_state.resume_pipeline:
917
- if not global_ds_key:
918
- st.error(tr(lang_code, "api_key_missing"))
919
- st.stop()
920
-
921
- if st.session_state.get("is_running", False):
922
- recoverable_state = _load_recoverable_checkpoint()
923
- st.session_state.is_running = False
924
- st.session_state.start_pipeline = False
925
- st.session_state.resume_pipeline = bool(recoverable_state)
926
- if recoverable_state:
927
- st.toast(tr(lang_code, "resume_toast"), icon="⚡")
928
- update_heartbeat()
929
- st.rerun()
930
- # A normal page rerun without a valid checkpoint is not a recovery event.
931
- # Keep the already-rendered writing interface and show no recovery toast.
932
- clear_heartbeat()
933
- st.stop()
934
-
935
- is_resume = st.session_state.resume_pipeline
936
- loaded_state = _load_recoverable_checkpoint() if is_resume else None
937
- if is_resume and loaded_state is None:
938
- is_resume = False
939
-
940
- try:
941
- managed_task, managed_task_key = _begin_managed_generation_task(
942
- is_resume=is_resume,
943
- loaded_state=loaded_state,
944
- mode=st.session_state.run_mode,
945
- language=lang_code,
946
- )
947
- except Exception as exc:
948
- clear_heartbeat()
949
- st.session_state.is_running = False
950
- st.session_state.start_pipeline = False
951
- st.session_state.resume_pipeline = False
952
- st.error(_quota_start_error_message(lang_code, exc))
953
- if "payment_required" in str(exc):
954
- st.session_state["sp_payment_required"] = True
955
- render_payment_purchase(lang_code)
956
- st.stop()
957
-
958
- st.session_state.is_running = True
959
- agent = SSciPipelineAgent(global_ds_key, global_s2_key)
960
- agent.set_execution_progress_hook(update_heartbeat)
961
- SAFE_CHUNK_WORDS = agent.chunk_words_default
962
- qual_engine = get_qualitative_engine(agent)
963
- quant_engine = get_quantitative_engine()
964
- completion_finalization_attempted = False
965
 
966
- try:
967
- update_heartbeat()
968
-
969
- if is_resume and loaded_state:
970
- pb = progress_bar.progress(50, text=tr(lang_code, "resume_loading"))
971
- state = loaded_state
972
- state["managed_task_idempotency_key"] = managed_task_key
973
- state["managed_task_id"] = managed_task.get("task_id", "")
974
- topic = state["topic"]
975
- keywords = state["keywords"]
976
- target_lang = state["target_lang"]
977
- words = state["words"]
978
- user_data_md = state["user_data_md"]
979
- data_metadata = state.get("data_metadata", {})
980
- user_lit_context = state["user_lit_context"]
981
-
982
- agent.load_state(state["agent_state"])
983
- adaptive_structure = state["adaptive_structure"]
984
- completed_sections = state["completed_sections"]
985
- full_draft_en = state["full_draft_en"]
986
- full_draft_translated = state["full_draft_translated"]
987
- synthesis_result = state["synthesis_result"]
988
- refs_text = state["refs_text"]
989
- context_tail = state["context_tail"]
990
-
991
- st.session_state.run_mode = state.get("run_mode", "amj")
992
- if st.session_state.run_mode == "amj":
993
- APP_CONFIG.update(AMJ_PROMPTS)
994
- else:
995
- APP_CONFIG.update(ASQ_PROMPTS)
996
-
997
- st.success(tr(lang_code, "resume_success", done=len(completed_sections), total=len(adaptive_structure)))
998
- render_monitor(agent, tr(lang_code, "resume_phase"), tr(lang_code, "state_check"), tr(lang_code, "history_restored"))
999
-
1000
  else:
1001
- pb = progress_bar.progress(1, text=tr(lang_code, "pipeline_start"))
1002
- user_lit_context = ""
1003
- user_data_md = ""
1004
- data_metadata = {}
1005
-
1006
- if st.session_state.run_mode == "amj":
1007
- if uploaded_papers_amj:
1008
- user_lit_context = extract_text_from_files(uploaded_papers_amj[:20])
1009
- with st.status(tr(lang_code, "literature_extract"), expanded=True) as status:
1010
- ph = st.empty()
1011
- all_paper_texts = [extract_text_from_files([f]) for f in uploaded_papers_amj[:20]]
1012
-
1013
- future = ThreadPoolExecutor(max_workers=1).submit(extract_golden_metadata, agent, all_paper_texts)
1014
- dot_count = 0
1015
- while not future.done():
1016
- update_heartbeat()
1017
- ph.markdown(tr(lang_code, "literature_deep") + " ⏳" * (dot_count % 4))
1018
- time.sleep(1)
1019
- dot_count += 1
1020
-
1021
- agent.global_memory["core_golden_metadata"] = future.result()
1022
- st.session_state.rag_engine = initialize_rag(all_paper_texts)
1023
- status.update(label=tr(lang_code, "literature_done"), state="complete", expanded=False)
1024
-
1025
- with st.spinner(tr(lang_code, "extract_quant")):
1026
- update_heartbeat()
1027
- _, user_data_md, msg, data_metadata = quant_engine.resolve_empirical_data(uploaded_data_amj, lang_code)
1028
- st.info(msg)
1029
-
1030
- elif st.session_state.run_mode == "asq":
1031
- if uploaded_papers_asq:
1032
- user_lit_context = extract_text_from_files(uploaded_papers_asq[:20])
1033
- with st.status(tr(lang_code, "literature_extract"), expanded=True) as status:
1034
- ph = st.empty()
1035
- all_paper_texts = [extract_text_from_files([f]) for f in uploaded_papers_asq[:20]]
1036
-
1037
- future = ThreadPoolExecutor(max_workers=1).submit(extract_golden_metadata, agent, all_paper_texts)
1038
- dot_count = 0
1039
- while not future.done():
1040
- update_heartbeat()
1041
- ph.markdown(tr(lang_code, "literature_deep") + " ⏳" * (dot_count % 4))
1042
- time.sleep(1)
1043
- dot_count += 1
1044
-
1045
- agent.global_memory["core_golden_metadata"] = future.result()
1046
- st.session_state.rag_engine = initialize_rag(all_paper_texts)
1047
- status.update(label=tr(lang_code, "literature_done"), state="complete", expanded=False)
1048
-
1049
- update_heartbeat()
1050
- raw_interviews = extract_text_from_files([uploaded_interviews]) if uploaded_interviews else ""
1051
- user_data_md, msg, data_metadata = qual_engine.resolve_qualitative_data(topic, keywords, raw_interviews)
1052
- st.info(msg or tr(lang_code, "qualitative_ready"))
1053
-
1054
- state = {
1055
- "topic": topic, "keywords": keywords, "target_lang": target_lang, "words": words,
1056
- "user_data_md": user_data_md, "data_metadata": data_metadata, "user_lit_context": user_lit_context,
1057
- "completed_sections": [], "full_draft_en": "", "full_draft_translated": "",
1058
- "context_tail": "", "run_mode": st.session_state.run_mode, "interface_language": lang_code,
1059
- "managed_task_idempotency_key": managed_task_key,
1060
- "managed_task_id": managed_task.get("task_id", ""),
1061
- }
1062
-
1063
- pb.progress(5, text=tr(lang_code, "config_route"))
1064
-
1065
- if st.session_state.run_mode == "amj":
1066
- APP_CONFIG.update(AMJ_PROMPTS)
1067
- state["prompt_config"] = AMJ_PROMPTS
1068
- else:
1069
- APP_CONFIG.update(ASQ_PROMPTS)
1070
- state["prompt_config"] = ASQ_PROMPTS
1071
-
1072
- pb.progress(10, text=tr(lang_code, "literature_search"))
1073
- def sync_monitor_ui():
1074
- update_heartbeat()
1075
- render_monitor(agent, tr(lang_code, "search_phase"), "S2 + OpenAlex", tr(lang_code, "search_detail"))
1076
-
1077
- target_refs_count = 50
1078
- refs_text = agent.fetch_references_parallel(topic, keywords, min_required=target_refs_count, progress_callback=sync_monitor_ui)
1079
-
1080
- if len(agent.citation_map) < target_refs_count:
1081
- raise RuntimeError(
1082
- tr(lang_code, "refs_insufficient", count=len(agent.citation_map), required=target_refs_count)
1083
- )
1084
- else:
1085
- st.success(tr(lang_code, "refs_success", count=len(agent.citation_map)))
1086
-
1087
- with st.spinner(tr(lang_code, "rag_sync")):
1088
- update_heartbeat()
1089
- if hasattr(st.session_state, "rag_engine") and st.session_state.rag_engine is not None:
1090
- st.session_state.rag_engine._initialize_chunks([refs_text])
1091
- else:
1092
- st.session_state.rag_engine = initialize_rag([refs_text])
1093
- st.success(tr(lang_code, "rag_done"))
1094
-
1095
- if user_lit_context:
1096
- pb.progress(11, text=tr(lang_code, "core_literature_parsing"))
1097
- update_heartbeat()
1098
- core_refs = agent.parse_user_core_literature(user_lit_context)
1099
- if core_refs:
1100
- refs_text = "【USER PROVIDED CORE LITERATURE】\n" + core_refs + "\n\n【RETRIEVED LITERATURE】\n" + refs_text
1101
- st.success(tr(lang_code, "core_refs_done"))
1102
-
1103
- state["refs_text"] = refs_text
1104
-
1105
- pb.progress(12, text=tr(lang_code, "synthesis_progress"))
1106
- with st.status(tr(lang_code, "synthesis_status"), expanded=True) as status:
1107
- ph = st.empty()
1108
- synthesis_result = ph.write_stream(stream_with_heartbeat(agent.stream_synthesize_literature(topic, refs_text, user_core_literature=user_lit_context)))
1109
- synthesis_result = re.sub(r'<think>.*?</think>', '', synthesis_result, flags=re.DOTALL).strip()
1110
- status.update(label=tr(lang_code, "synthesis_done"), state="complete", expanded=False)
1111
- state["synthesis_result"] = synthesis_result
1112
-
1113
- pb.progress(14, text=tr(lang_code, "anchor_progress"))
1114
- with st.status(tr(lang_code, "anchor_status"), expanded=True) as status:
1115
- ph = st.empty()
1116
- anchor_response = ph.write_stream(stream_with_heartbeat(agent.stream_generate_anchors(topic, keywords, synthesis_result, refs_text, data_evidence=user_data_md, mode=st.session_state.run_mode)))
1117
- status.update(label=tr(lang_code, "anchor_done"), state="complete", expanded=False)
1118
-
1119
- # ==========================================================
1120
- # [核心修复] 必须先剔除 <think> 标签!防止正则匹配到模型在思考时引用提示词的空占位符
1121
- # ==========================================================
1122
- anchor_response_clean = re.sub(r'<think>.*?</think>', '', anchor_response, flags=re.DOTALL).strip()
1123
-
1124
- abstract_match = re.search(r'-+\s*ABSTRACT\s*-+\s*(.*?)(?=\s*-+\s*CONCLUSION\s*-+|$)', anchor_response_clean, re.DOTALL | re.IGNORECASE)
1125
- conclusion_match = re.search(r'-+\s*CONCLUSION\s*-+\s*(.*?)$', anchor_response_clean, re.DOTALL | re.IGNORECASE)
1126
- abstract_anchor = abstract_match.group(1).strip() if abstract_match else anchor_response_clean.strip()
1127
- conclusion_anchor = conclusion_match.group(1).strip() if conclusion_match else ""
1128
- agent.global_memory["abstract_anchor"] = abstract_anchor
1129
- agent.global_memory["conclusion_anchor"] = conclusion_anchor
1130
- # ==========================================================
1131
-
1132
- pb.progress(15, text=tr(lang_code, "outline_progress"))
1133
- update_heartbeat()
1134
- plan = agent.generate_outline(topic, keywords, mode=st.session_state.run_mode, data_metadata=data_metadata)
1135
- en_topic = str(plan.get("en_topic", topic))
1136
- structure = plan.get("structure", [])
1137
-
1138
- en_abstract_clean = clean_markdown_headings(re.sub(r'(?i)^abstract[:\s]*', '', str(abstract_anchor)).strip())
1139
- trans_abstract_clean = ""
1140
-
1141
- loc_abs = {"Simplified Chinese (简体中文)": "摘要", "Russian (Русский)": "Аннотация", "Kazakh (Қазақша)": "Түйіндеме"}.get(target_lang, "Abstract")
1142
- loc_kw = {"Simplified Chinese (简体中文)": "关键词", "Russian (Русский)": "Ключевые слова", "Kazakh (Қазақша)": "Кілт сөздер"}.get(target_lang, "Keywords")
1143
-
1144
- full_draft_en = f"# {en_topic}\n\n## Abstract\n{en_abstract_clean}\n\n## Keywords\n{keywords}\n\n"
1145
- full_draft_translated = f"# {topic}\n\n## {loc_abs}\n{trans_abstract_clean}\n\n## {loc_kw}\n{keywords}\n\n"
1146
-
1147
- adaptive_structure = agent.build_generation_tasks(structure, int(words), SAFE_CHUNK_WORDS)
1148
- for task in adaptive_structure:
1149
- task["is_first"] = bool(task.get("section_first"))
1150
-
1151
- with st.spinner(tr(lang_code, "jit_status")):
1152
- update_heartbeat()
1153
- jit_texts = agent.prefetch_jit_literature(topic, keywords, adaptive_structure)
1154
- if jit_texts:
1155
- if hasattr(st.session_state, "rag_engine") and st.session_state.rag_engine is not None:
1156
- st.session_state.rag_engine._initialize_chunks([jit_texts])
1157
- else:
1158
- st.session_state.rag_engine = initialize_rag([jit_texts])
1159
- st.success(tr(lang_code, "jit_done"))
1160
-
1161
- state["adaptive_structure"] = adaptive_structure
1162
- state["agent_state"] = agent.export_state()
1163
- state["full_draft_en"] = full_draft_en
1164
- state["full_draft_translated"] = full_draft_translated
1165
- completed_sections = []
1166
- context_tail = ""
1167
-
1168
- os.makedirs(WORKSPACE_DIR, exist_ok=True)
1169
- with open(CHECKPOINT_FILE, "w", encoding="utf-8") as f:
1170
- json.dump(state, f, ensure_ascii=False)
1171
-
1172
- total_steps = len(adaptive_structure)
1173
- current_step = len(completed_sections)
1174
- st.markdown("---")
1175
- ui_blocks = []
1176
- target_flag = FLAG_MAP.get(target_lang, "🌐")
1177
-
1178
- for sec in adaptive_structure:
1179
- display_name = sec["display_name"]
1180
- st.markdown(f"### ⚙️ {display_name}")
1181
- col_en, col_trans = st.columns(2)
1182
- with col_en:
1183
- st.markdown(tr(lang_code, "english_draft"))
1184
- en_ph = st.empty()
1185
- with col_trans:
1186
- st.markdown(tr(lang_code, "target_draft", flag=target_flag))
1187
- trans_ph = st.empty()
1188
- if display_name in completed_sections:
1189
- en_ph.success(tr(lang_code, "restored"))
1190
- trans_ph.info(tr(lang_code, "translation_waiting"))
1191
- else:
1192
- en_ph.info(tr(lang_code, "waiting"))
1193
- trans_ph.info(tr(lang_code, "translation_waiting"))
1194
- ui_blocks.append({"sec": sec, "en_ph": en_ph, "trans_ph": trans_ph})
1195
-
1196
- generation_aborted = False
1197
- generation_failure_code = ""
1198
- generation_failure_detail = ""
1199
- accumulated_deviation = 0
1200
- remaining_sections = sum(1 for item in adaptive_structure if item["display_name"] not in completed_sections)
1201
- all_cite_keys = list(agent.citation_map.keys())
1202
- valid_chunks = [item for item in adaptive_structure if not any(word in item["section"].lower() for word in ["abstract","keyword","appendix"])]
1203
- allocation_map = {}
1204
- current_cite_idx = 0
1205
- for idx, item in enumerate(valid_chunks):
1206
- count = max(1, math.ceil(len(all_cite_keys) / max(1, len(valid_chunks))))
1207
- allocation_map[item["display_name"]] = all_cite_keys[current_cite_idx:] if idx == len(valid_chunks)-1 else all_cite_keys[current_cite_idx:current_cite_idx+count]
1208
- current_cite_idx += count
1209
-
1210
- for block in ui_blocks:
1211
- if generation_aborted:
1212
- break
1213
- sec = block["sec"]
1214
- en_ph = block["en_ph"]
1215
- trans_ph = block["trans_ph"]
1216
- sec_title = sec["section"]
1217
- display_name = sec["display_name"]
1218
- if display_name in completed_sections:
1219
- continue
1220
- remaining_sections = max(0, remaining_sections - 1)
1221
- adjustment = -accumulated_deviation / remaining_sections if remaining_sections else 0
1222
- adjusted_target = min(int(SAFE_CHUNK_WORDS * 1.5), max(80, int(sec["target_words"] + adjustment)))
1223
- is_quarantined = any(word in sec_title.lower() for word in ["abstract","keyword","conclusion","references","appendix"])
1224
- assigned_tags = ", ".join(allocation_map.get(display_name, [])) or "the single most relevant source"
1225
- citation_rule = "NO IN-TEXT CITATIONS ALLOWED IN THIS SECTION." if is_quarantined else f"Use claim-specific citation. Each sentence may contain at most one source tag. Candidate tags: {assigned_tags}. <AVAILABLE_CITATIONS>{refs_text}</AVAILABLE_CITATIONS>"
1226
- memory_context = agent._build_memory_context(sec_title, 2500)
1227
- memory_block = f"\n<GLOBAL_MEMORY>\n{memory_context}\n</GLOBAL_MEMORY>\n" if memory_context else ""
1228
- if getattr(st.session_state, "rag_engine", None) is not None:
1229
- snippets = st.session_state.rag_engine.get_relevant_context(display_name)
1230
- if snippets:
1231
- memory_block += f"\n<TARGETED_RAG>\n{snippets}\n</TARGETED_RAG>\n"
1232
- subsection = sec.get("subsection")
1233
- heading_directive = "Do not output the section heading; it will be inserted by the system."
1234
- if subsection:
1235
- heading_directive += f" Do not output the subsection heading '{subsection}'; it will also be inserted by the system. Use level-3 headings only when analytically necessary."
1236
- elif sec_title.lower() in {"introduction", "conclusion"}:
1237
- heading_directive += " Do not use additional headings."
1238
- else:
1239
- heading_directive += " Use level-2 or level-3 headings only when they clarify the argument; never use level-4 headings or numbered headings."
1240
- table_rule = "Present statistical evidence in standard Markdown tables with plain-text captions." if sec["is_method_or_result"] and st.session_state.run_mode == "amj" else ""
1241
- required_content = str(sec.get("required_content") or "")
1242
- concepts = ", ".join(sec.get("concepts") or [])
1243
- qualitative_rule = f"Assigned qualitative concepts: {concepts}. Use only their assigned representative quotations and do not repeat a quotation already used elsewhere." if concepts else ""
1244
- data_injection = f"\n<RESEARCH_EVIDENCE>\n{user_data_md}\n</RESEARCH_EVIDENCE>\n" if user_data_md and sec["is_method_or_result"] else ""
1245
- completed_label = ", ".join(completed_sections)
1246
- anti_repeat = f"Already completed: {completed_label}. Do not restate their background, gap, or contribution." if completed_label else ""
1247
- sys_prompt = APP_CONFIG.get("sys_academic_writer", "") + "\n" + APP_CONFIG.get("deep_refinement_directive", "")
1248
- user_prompt = f"""Write the assigned English manuscript unit.
1249
- Section: {sec_title}
1250
- Subsection: {subsection or 'None'}
1251
- Target length: {int(adjusted_target*0.9)}-{int(adjusted_target*1.1)} words.
1252
- {heading_directive}
1253
- {citation_rule}
1254
- {table_rule}
1255
- {required_content}
1256
- {qualitative_rule}
1257
- {anti_repeat}
1258
- {memory_block}
1259
- Literature synthesis: {synthesis_result}
1260
- {data_injection}
1261
- Previous tail: {context_tail[-2500:]}
1262
- Output only final academic prose. Do not report the word count."""
1263
- if agent._count_tokens(sys_prompt) + agent._count_tokens(user_prompt) > 60000:
1264
- user_prompt = agent._truncate_context(user_prompt, 60000 - agent._count_tokens(sys_prompt))
1265
- render_monitor(agent, tr(lang_code, "drafting_phase", current=current_step+1, total=total_steps), display_name, tr(lang_code, "target_words", words=adjusted_target, engine=tr(lang_code, "engine_amj") if st.session_state.run_mode=="amj" else tr(lang_code, "engine_asq")))
1266
- try:
1267
- with st.status(tr(lang_code, "draft_status", name=display_name), expanded=True) as status:
1268
- update_heartbeat()
1269
- en_raw = en_ph.write_stream(stream_with_heartbeat(agent.stream_deepseek(sys_prompt, user_prompt, max_tokens=7000, model_role="writing", thinking=False, allow_fallback=False)))
1270
- status.update(label=tr(lang_code, "draft_done", name=display_name), state="complete", expanded=False)
1271
- en_raw = re.sub(r'<think>.*?</think>', '', str(en_raw), flags=re.S).strip()
1272
- en_raw = re.sub(rf'(?im)^#\s*{re.escape(sec_title)}\s*\n+', '', en_raw).strip()
1273
- if subsection:
1274
- en_raw = re.sub(rf'(?im)^##\s*{re.escape(subsection)}\s*\n+', '', en_raw).strip()
1275
- en_raw = re.sub(r'(?m)^(#{2,3})\s+[\d\.\s一二三四五六七八九十、]+\s*(.*)', r'\1 \2', en_raw)
1276
- en_raw = re.sub(r'(?m)^#{4,}\s+(.*)', r'**\1**', en_raw)
1277
- en_raw = re.sub(r'(?m)^#\s+(.+)$', r'## \1', en_raw)
1278
- if st.session_state.run_mode == "asq" and user_data_md and "appendix" not in sec_title.lower():
1279
- quote_dict = {tag: quote for quote, tag in re.findall(r'"([^"]+)"\s*\[(QUOTE-\d+)\]', user_data_md)}
1280
- for tag, quote in quote_dict.items():
1281
- en_raw = en_raw.replace(tag, f'"{quote}"')
1282
- en_raw = agent.polish_academic_section(en_raw, display_name, abstract=False)
1283
- prefix = ""
1284
- if sec.get("section_first"):
1285
- prefix += f"# {sec_title}\n\n"
1286
- if sec.get("subsection_first") and subsection:
1287
- prefix += f"## {subsection}\n\n"
1288
- en_with_headings = prefix + en_raw
1289
- en_resolved = agent.audit_and_resolve_citations(en_with_headings, section_name=display_name, quarantine=is_quarantined)
1290
- en_ph.markdown(en_resolved)
1291
- trans_ph.info(tr(lang_code, "translation_waiting"))
1292
- full_draft_en += en_resolved + "\n\n"
1293
- agent._extract_memory_from_text(en_raw, sec_title)
1294
- words_tail = en_raw.split()
1295
- context_tail = " ".join(words_tail[-200:])
1296
- actual_words = len(en_raw.split())
1297
- accumulated_deviation += actual_words - adjusted_target
1298
- current_step += 1
1299
- completed_sections.append(display_name)
1300
- state.update({
1301
- "completed_sections": completed_sections,
1302
- "full_draft_en": full_draft_en,
1303
- "full_draft_translated": "",
1304
- "context_tail": context_tail,
1305
- "agent_state": agent.export_state(),
1306
- "data_metadata": data_metadata,
1307
- })
1308
- with open(CHECKPOINT_FILE, "w", encoding="utf-8") as checkpoint:
1309
- json.dump(state, checkpoint, ensure_ascii=False)
1310
- st.caption(tr(lang_code, "word_count", actual=actual_words, target=adjusted_target, deviation=accumulated_deviation))
1311
- pb.progress(min(15 + int((current_step/max(1,total_steps))*75), 90), text=tr(lang_code, "section_complete_progress", name=display_name))
1312
- except Exception as error:
1313
- en_ph.error(tr(lang_code, "node_error"))
1314
- st.error(tr(lang_code, "node_error_detail", error=error))
1315
- generation_aborted = True
1316
- generation_failure_code = "E_GENERATION_NODE_FAILED"
1317
- generation_failure_detail = str(error)[:800]
1318
-
1319
- if generation_aborted:
1320
- _finish_managed_generation_task(
1321
- "failed",
1322
- safe_error_code=generation_failure_code or "E_GENERATION_INTERRUPTED",
1323
- safe_error_detail=generation_failure_detail or "Generation stopped; quota was released and the checkpoint was retained for a fresh retry.",
1324
- current_node=str(st.session_state.get("sp_current_node") or "generation"),
1325
- detach_checkpoint=True,
1326
- )
1327
-
1328
- if not generation_aborted and full_draft_en.strip():
1329
- pb.progress(91, text=tr(lang_code, "pack_docx"))
1330
- update_heartbeat()
1331
- full_draft_en = clean_trailing_references(full_draft_en, ["Reference","References","Reference List"])
1332
- # The final abstract is derived from the completed English manuscript and frozen evidence.
1333
- final_abstract = agent.generate_final_abstract(topic, keywords, full_draft_en, data_evidence=user_data_md, mode=st.session_state.run_mode)
1334
- full_draft_en = replace_abstract(full_draft_en, final_abstract)
1335
- agent.validate_manuscript_structure(full_draft_en, adaptive_structure, translated=False)
1336
- if st.session_state.run_mode == "asq":
1337
- repeated_quotes = agent.find_repeated_qualitative_quotes(full_draft_en, data_metadata)
1338
- if repeated_quotes:
1339
- raise RuntimeError(f"Repeated qualitative quotations: {repeated_quotes[:5]}")
1340
- # Only sources actually cited in the manuscript enter the reference list.
1341
- references = []
1342
- seen_refs = set()
1343
- for tag in sorted(agent.used_citations, key=agent._citation_tag_number):
1344
- ref = str((agent.citation_map.get(tag) or {}).get("ref") or "").strip()
1345
- if ref and ref not in seen_refs:
1346
- references.append(f"- {ref}")
1347
- seen_refs.add(ref)
1348
- if not references:
1349
- raise RuntimeError("No cited references remained after citation validation.")
1350
- full_draft_en = full_draft_en.rstrip() + "\n\n# References\n" + "\n\n".join(references)
1351
-
1352
- pb.progress(94, text=tr(lang_code, "translation_running"))
1353
- update_heartbeat()
1354
- full_draft_translated = agent.translate_locked_manuscript(full_draft_en, target_lang)
1355
- agent.validate_translation_alignment(full_draft_en, full_draft_translated)
1356
-
1357
- bj_time = datetime.now(timezone(timedelta(hours=8))).strftime("%Y%m%d_%H%M")
1358
- package = build_validated_package(
1359
- english_markdown=full_draft_en,
1360
- translated_markdown=full_draft_translated,
1361
- english_filename=f"Research_EN_{bj_time}.docx",
1362
- translated_filename=f"Research_{output_suffix}_{bj_time}.docx",
1363
- data_metadata=data_metadata,
1364
- mode=st.session_state.run_mode,
1365
- citation_audit=agent.citation_audit_log,
1366
- literature_metrics=agent.get_source_metrics(),
1367
- )
1368
- generated_zip_bytes = package.zip_bytes
1369
- generated_zip_filename = f"SmartPaper_{bj_time}.zip"
1370
-
1371
- # Persist the already validated package in the current user's task
1372
- # directory before asking Worker/D1 to commit quota. The download is
1373
- # exposed only after the browser receives completed+committed.
1374
- task_id = str((st.session_state.get("sp_generation_task") or {}).get("task_id") or "").strip()
1375
- if not task_id:
1376
- raise RuntimeError("package_task_id_missing")
1377
- pending_path = os.path.join(_task_output_dir(task_id), ".pending_" + generated_zip_filename)
1378
- with open(pending_path, "wb") as output_file:
1379
- output_file.write(generated_zip_bytes)
1380
- st.session_state["sp_pending_package"] = {
1381
- "pending_path": pending_path,
1382
- "filename": generated_zip_filename,
1383
- }
1384
- completion_finalization_attempted = True
1385
- _finish_managed_generation_task(
1386
- "completed",
1387
- current_node="package_validated",
1388
- delete_checkpoint=True,
1389
- )
1390
- pb.progress(100, text=tr(lang_code, "complete"))
1391
- render_monitor(agent, tr(lang_code, "complete"), "✅", tr(lang_code, "document_generated"))
1392
-
1393
- except Exception as e:
1394
- task_id = str((st.session_state.get("sp_generation_task") or {}).get("task_id") or "")
1395
- logging.exception(
1396
- "[SP_TASK] generation_exception task_id=%s node=%s error_type=%s",
1397
- task_id,
1398
- str(st.session_state.get("sp_current_node") or "generation")[:120],
1399
- type(e).__name__,
1400
- )
1401
- if not completion_finalization_attempted:
1402
- try:
1403
- _finish_managed_generation_task(
1404
- "failed",
1405
- safe_error_code="E_GENERATION_FAILED",
1406
- safe_error_detail=str(e)[:800],
1407
- current_node=str(st.session_state.get("sp_current_node") or "generation"),
1408
- detach_checkpoint=True,
1409
- )
1410
- except Exception:
1411
- pass
1412
- st.error(tr(lang_code, "startup_error", error=e))
1413
- finally:
1414
- task_id = str((st.session_state.get("sp_generation_task") or {}).get("task_id") or "")
1415
- logging.info(
1416
- "[SP_TASK] pipeline_exit task_id=%s node=%s",
1417
- task_id,
1418
- str(st.session_state.get("sp_current_node") or "")[:120],
1419
- )
1420
- clear_heartbeat()
1421
- st.session_state.is_running = False
1422
- st.session_state.start_pipeline = False
1423
- st.session_state.resume_pipeline = False
1424
-
1425
- if st.session_state.zip_buffer:
1426
- st.success(tr(lang_code, "success"))
1427
- st.download_button(label=tr(lang_code, "download"), data=st.session_state.zip_buffer, file_name=st.session_state.zip_filename, mime="application/zip")
 
1
  # -*- coding: utf-8 -*-
2
+ """SmartPaper R3-Free UI: Streamlit is an observer, never the paper executor."""
3
+ import hashlib
4
+ import json
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5
  import logging
6
+ import os
7
  import re
 
 
8
  import secrets
9
+ import shutil
10
+ import signal
11
+ import subprocess
12
+ import sys
13
+ import time
14
+ from datetime import datetime, timezone
15
+ from pathlib import Path
16
+
17
+ import streamlit as st
18
+ import streamlit.components.v1 as components
19
 
20
+ from config import APP_CONFIG
 
 
21
  from managed_auth import (
22
+ cancel_generation_task,
23
+ generation_task_status,
24
  require_alipay_login,
25
  render_logged_in_account,
 
 
 
26
  render_payment_purchase,
27
+ start_generation_task,
28
+ )
29
+ from i18n import (
30
+ LANGUAGE_OPTIONS, get_amj_csv_template, get_asq_txt_template,
31
+ get_output_suffix, get_target_flag, get_target_language, tr,
32
  )
 
 
 
 
33
 
34
+ st.set_page_config(
35
+ page_title="Article Agent",
36
+ page_icon="📊",
37
+ layout="wide",
38
+ initial_sidebar_state="collapsed",
39
+ )
40
 
41
  WORKSPACE_DIR = APP_CONFIG.get("workspace_dir", ".workspace")
42
  os.makedirs(WORKSPACE_DIR, exist_ok=True)
43
+ os.environ["STREAMLIT_SERVER_ENABLE_CORS"] = "false"
44
+ os.environ["STREAMLIT_SERVER_ENABLE_XSRF_PROTECTION"] = "false"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
45
 
46
  WECHAT_BLOCKER_JS = """<script>try{var ua=navigator.userAgent.toLowerCase();if(ua.indexOf('micromessenger')!==-1){var parentDoc=window.parent.document;if(!parentDoc.getElementById('wx-blocker')){var overlay=parentDoc.createElement('div');overlay.id='wx-blocker';overlay.style.cssText='position:fixed;top:0;left:0;width:100vw;height:100vh;background:#1a365d;z-index:9999999;color:white;display:flex;flex-direction:column;align-items:center;justify-content:center;text-align:center;font-family:sans-serif;padding:30px;box-sizing:border-box;';overlay.innerHTML='<h2 style="font-size:26px;margin-bottom:15px;font-weight:bold;">环境受限 / Restricted</h2><p style="font-size:18px;line-height:1.6;color:#e2e8f0;max-width:100%;">为了保障最终双语 ZIP 压缩包正常下载,<b>本系统禁止在微信内直接运行</b>。<br><br>👉 <b>请点击右上角 [···] 菜单,选择“在浏览器打开” (Open in Browser)</b></p>';parentDoc.body.appendChild(overlay);}}}catch(e){}</script>"""
47
  components.html(WECHAT_BLOCKER_JS, height=0, width=0)
 
253
  components.html(script, height=0, width=0)
254
 
255
 
 
 
 
 
 
 
 
 
 
 
 
 
 
256
 
257
+ for key, default in {
258
+ "is_running": False,
259
+ "zip_buffer": None,
260
+ "zip_filename": "",
261
+ "run_mode": "amj",
262
+ "sp_generation_task": None,
263
+ "sp_pending_launch": None,
264
+ "sp_pending_resume_task_id": "",
265
+ "sp_pending_cancel_task_id": "",
266
+ "sp_payment_required": False,
267
+ "sp_terminal_notice": None,
268
+ }.items():
269
+ if key not in st.session_state:
270
+ st.session_state[key] = default
271
 
272
 
273
+ def _safe_key(value: str) -> str:
274
+ cleaned = re.sub(r"[^A-Za-z0-9_.-]", "_", str(value or "").strip())
275
+ return cleaned[:100] or "anonymous"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
276
 
277
 
278
  def _new_generation_idempotency_key() -> str:
279
  return f"gen_{int(time.time() * 1000)}_{secrets.token_urlsafe(18)}"
280
 
281
 
282
+ def _atomic_json(path: Path, value: dict) -> None:
283
+ path.parent.mkdir(parents=True, exist_ok=True)
284
+ temporary = path.with_suffix(path.suffix + ".tmp")
285
+ temporary.write_text(json.dumps(value, ensure_ascii=False), encoding="utf-8")
286
+ os.replace(temporary, path)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
287
 
 
 
 
 
 
 
 
 
 
 
288
 
289
+ def _canonical_request_hash(value: dict) -> str:
290
+ material = dict(value or {})
291
+ material.pop("request_hash", None)
292
+ encoded = json.dumps(material, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
293
+ return hashlib.sha256(encoded).hexdigest()
294
 
 
 
 
 
 
295
 
296
+ def _task_dir(task_id: str) -> Path:
297
+ return Path(_user_workspace) / "tasks" / _safe_key(task_id)
298
 
 
 
 
 
 
 
 
 
 
299
 
300
+ def _task_output_path(task: dict) -> Path | None:
301
+ task_id = str(task.get("task_id") or "").strip()
302
+ filename = os.path.basename(str(task.get("output_filename") or "").strip())
303
+ if not task_id or not filename:
304
+ return None
305
+ return _task_dir(task_id) / "outputs" / filename
306
+
307
+
308
+ def _save_uploaded(items, inputs_dir: Path, prefix: str) -> list[str]:
309
+ saved = []
310
+ for index, item in enumerate(items or []):
311
+ if item is None:
312
+ continue
313
+ original = os.path.basename(str(getattr(item, "name", f"input_{index}")))
314
+ safe_name = re.sub(r"[^A-Za-z0-9_.-]", "_", original)[:140] or f"input_{index}"
315
+ filename = f"{prefix}_{index:02d}_{safe_name}"
316
+ path = inputs_dir / filename
317
+ data = item.getvalue() if hasattr(item, "getvalue") else item.read()
318
+ path.write_bytes(bytes(data or b""))
319
+ saved.append(str(Path("inputs") / filename))
320
+ try:
321
+ item.seek(0)
322
+ except Exception:
323
+ pass
324
+ return saved
325
 
 
 
 
 
 
 
 
 
326
 
327
+ def _persist_task_request(task: dict, mode: str, lang_code: str) -> dict:
328
+ task_id = str(task.get("task_id") or "").strip()
329
+ numeric_id = (st.session_state.get("sp_user") or {}).get("numeric_id")
330
+ if not task_id or not isinstance(numeric_id, int) or numeric_id <= 0:
331
+ raise RuntimeError("task_identity_missing")
332
+ task_dir = _task_dir(task_id)
333
+ inputs_dir = task_dir / "inputs"
334
+ inputs_dir.mkdir(parents=True, exist_ok=True)
335
+
336
+ if mode == "amj":
337
+ refs = _save_uploaded(st.session_state.get("up_amj_ref") or [], inputs_dir, "ref")
338
+ data = _save_uploaded([st.session_state.get("up_amj_data")], inputs_dir, "data")
339
+ interviews = []
340
+ else:
341
+ refs = _save_uploaded(st.session_state.get("up_asq_ref") or [], inputs_dir, "ref")
342
+ data = []
343
+ interviews = _save_uploaded([st.session_state.get("up_asq_data")], inputs_dir, "interview")
344
 
345
+ request_data = {
346
+ "task_id": task_id,
347
+ "user_id": numeric_id,
348
+ "mode": mode,
349
+ "interface_language": lang_code,
350
+ "target_lang": get_target_language(lang_code),
351
+ "output_suffix": get_output_suffix(lang_code),
352
+ "topic": str(st.session_state.get("sp_topic") or DEFAULT_TOPIC),
353
+ "keywords": str(st.session_state.get("sp_keywords") or DEFAULT_KEYWORDS),
354
+ "target_words": int(st.session_state.get("sp_words") or APP_CONFIG["target_words_default"]),
355
+ "reference_files": refs,
356
+ "data_files": data,
357
+ "interview_files": interviews,
358
+ "created_at": datetime.now(timezone.utc).isoformat(),
359
+ }
360
+ request_data["request_hash"] = _canonical_request_hash(request_data)
361
+ _atomic_json(task_dir / "request.json", request_data)
362
+ return request_data
363
 
364
 
365
+ def _pid_alive(pid: int) -> bool:
366
+ if not isinstance(pid, int) or pid <= 0:
367
+ return False
368
+ try:
369
+ os.kill(pid, 0)
370
+ return True
371
+ except OSError:
372
+ return False
 
 
 
 
 
 
373
 
374
 
375
+ def _pid_is_task_executor(pid: int, task_id: str, task_dir: Path) -> bool:
376
+ """Best-effort Linux identity check before trusting/killing a launch PID.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
377
 
378
+ Hugging Face Spaces run on Linux. A stale PID can be reused, so a bare
379
+ ``os.kill(pid, 0)`` is never enough to identify our executor.
380
+ """
381
+ if not _pid_alive(pid):
382
+ return False
383
+ try:
384
+ raw = (Path("/proc") / str(pid) / "cmdline").read_bytes()
385
+ command = raw.replace(b"\x00", b" ").decode("utf-8", errors="ignore")
386
+ except Exception:
387
+ return False
388
+ return (
389
+ "task_executor.py" in command
390
+ and f"--task-id {task_id}" in command
391
+ and str(task_dir.resolve()) in command
392
+ )
393
 
394
+
395
+ def _launch_executor(task_id: str, *, force: bool = False) -> int:
396
+ task_dir = _task_dir(task_id)
397
+ request_path = task_dir / "request.json"
398
+ if not request_path.is_file():
399
+ raise RuntimeError("task_request_manifest_missing")
400
+ request_data = json.loads(request_path.read_text(encoding="utf-8"))
401
+ numeric_id = int(request_data.get("user_id") or 0)
402
+ marker_path = task_dir / "executor_launch.json"
403
+ if marker_path.is_file():
404
+ try:
405
+ marker = json.loads(marker_path.read_text(encoding="utf-8"))
406
+ pid = int(marker.get("pid") or 0)
407
+ verified_executor = _pid_is_task_executor(pid, task_id, task_dir)
408
+ if verified_executor and not force:
409
+ return pid
410
+ if verified_executor and force:
411
+ # Resume means D1 has already made the previous execution
412
+ # reclaimable. Stop only a PID whose /proc command line proves
413
+ # it is this task's old executor; a new epoch will be the sole
414
+ # authority after claim.
415
  try:
416
+ os.kill(pid, signal.SIGTERM)
417
  except OSError:
418
  pass
419
+ except Exception:
 
 
 
 
 
420
  pass
421
+ log_path = task_dir / "executor.log"
422
+ task_dir.mkdir(parents=True, exist_ok=True)
423
+ command = [
424
+ sys.executable,
425
+ str(Path(__file__).resolve().parent / "task_executor.py"),
426
+ "--task-dir", str(task_dir.resolve()),
427
+ "--task-id", str(task_id),
428
+ "--user-id", str(numeric_id),
429
+ ]
430
+ with log_path.open("ab") as log_handle:
431
+ process = subprocess.Popen(
432
+ command,
433
+ cwd=str(Path(__file__).resolve().parent),
434
+ stdin=subprocess.DEVNULL,
435
+ stdout=log_handle,
436
+ stderr=subprocess.STDOUT,
437
+ close_fds=True,
438
+ start_new_session=True,
439
+ env=os.environ.copy(),
440
+ )
441
+ _atomic_json(marker_path, {"pid": process.pid, "launched_at": datetime.now(timezone.utc).isoformat()})
442
+ logging.info("[SP_UI] executor_launched task_id=%s pid=%s", task_id, process.pid)
443
+ return int(process.pid)
444
 
445
 
446
+ def _process_pending_cancel() -> None:
447
+ task_id = str(st.session_state.get("sp_pending_cancel_task_id") or "").strip()
448
+ if not task_id:
449
+ return
450
+ result = cancel_generation_task(task_id)
451
+ if result.get("cancelled"):
452
+ shutil.rmtree(_task_dir(task_id), ignore_errors=True)
453
+ st.session_state["sp_pending_cancel_task_id"] = ""
454
+ st.session_state["sp_generation_task"] = dict(result.get("task") or {}) or None
455
+ st.rerun()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
456
 
 
 
 
 
 
 
457
 
458
+ def _process_pending_resume() -> None:
459
+ task_id = str(st.session_state.get("sp_pending_resume_task_id") or "").strip()
460
+ if not task_id:
461
+ return
462
+ if not (_task_dir(task_id) / "request.json").is_file():
463
+ st.session_state["sp_pending_resume_task_id"] = ""
464
+ st.session_state["sp_pending_cancel_task_id"] = task_id
 
 
 
 
 
 
 
 
465
  st.session_state["sp_terminal_notice"] = {
466
+ "kind": "error",
467
+ "message": "Local checkpoint/request files are unavailable after runtime replacement; the reserved quota will be released.",
468
  }
469
+ st.rerun()
470
+ _launch_executor(task_id, force=True)
471
+ st.session_state["sp_pending_resume_task_id"] = ""
472
+ st.rerun()
473
 
474
+
475
+ def _process_pending_launch(lang_code: str) -> None:
476
+ pending = st.session_state.get("sp_pending_launch")
477
+ if not isinstance(pending, dict):
478
+ return
479
+ mode = str(pending.get("mode") or "").strip().lower()
480
+ idempotency_key = str(pending.get("idempotency_key") or "").strip()
481
+ try:
482
+ response = start_generation_task(
483
+ idempotency_key=idempotency_key,
484
+ mode=mode,
485
+ interface_language=lang_code,
486
+ target_words=int(st.session_state.get("sp_words") or APP_CONFIG["target_words_default"]),
487
+ )
488
+ except Exception as exc:
489
+ st.session_state["sp_pending_launch"] = None
490
+ raw = str(exc or "")
491
+ if "payment_required" in raw:
492
+ st.session_state["sp_payment_required"] = True
493
+ st.error(tr(lang_code, "quota_exhausted"))
494
+ render_payment_purchase(lang_code)
495
+ else:
496
+ st.error(raw)
497
+ return
498
+ task = dict(response.get("task") or {})
499
  task_id = str(task.get("task_id") or "").strip()
500
  if not task_id:
501
+ st.session_state["sp_pending_launch"] = None
502
+ st.error("task_start_missing_task_id")
503
+ return
504
+ st.session_state["sp_generation_task"] = task
505
+ try:
506
+ request_path = _task_dir(task_id) / "request.json"
507
+ if not request_path.is_file():
508
+ _persist_task_request(task, mode, lang_code)
509
+ _launch_executor(task_id)
510
+ except Exception as exc:
511
+ st.session_state["sp_pending_launch"] = None
512
+ st.session_state["sp_pending_cancel_task_id"] = task_id
513
+ st.session_state["sp_terminal_notice"] = {"kind": "error", "message": str(exc)[:800]}
514
+ st.rerun()
515
+ st.session_state["sp_pending_launch"] = None
516
+ st.session_state["sp_payment_required"] = False
517
+ st.rerun()
 
518
 
519
 
520
  inject_wechat_browser_notice()
 
524
 
525
  _fragment = getattr(st, "fragment", None) or getattr(st, "experimental_fragment", None)
526
  if _fragment is None:
527
+ def _fragment(function=None, **_kwargs):
528
+ if function is not None:
529
+ return function
530
+ return lambda fn: fn
531
 
532
 
533
  def _current_language_code() -> str:
 
535
  return LANGUAGE_OPTIONS.get(choice, "zh")
536
 
537
 
 
538
  authenticated_user = require_alipay_login(_current_language_code())
539
+ _user_workspace = os.path.join(WORKSPACE_DIR, "users", _safe_key(authenticated_user.get("id")))
 
 
 
 
 
 
 
540
  os.makedirs(_user_workspace, exist_ok=True)
 
 
 
541
 
542
 
543
  @_fragment
544
  def render_workspace_controls(user):
 
545
  top_toolbar = st.container()
 
546
  with top_toolbar:
547
  language_col, account_col = st.columns([5.4, 4.6])
 
548
  with language_col:
549
  lang_choice = st.radio(
550
+ "Language", list(LANGUAGE_OPTIONS.keys()), index=0, horizontal=True,
551
+ label_visibility="collapsed", key="interface_language_choice",
 
 
 
 
552
  )
 
553
  lang_code = LANGUAGE_OPTIONS[lang_choice]
554
  st.session_state["sp_lang_code"] = lang_code
555
  st.session_state["sp_target_lang"] = get_target_language(lang_code)
556
  st.session_state["sp_output_suffix"] = get_output_suffix(lang_code)
557
  st.session_state["sp_target_flag"] = get_target_flag(lang_code)
 
558
  with account_col:
559
  render_logged_in_account(lang_code, user)
560
 
561
+ task = st.session_state.get("sp_generation_task") or {}
562
+ task_status = str(task.get("status") or "")
563
+ active = task_status in {"created", "running", "resumable", "cancelling"}
564
+ st.session_state["is_running"] = active
 
 
 
 
 
 
 
 
 
 
565
 
566
  st.markdown(f"<div class='main-title'><h2>{tr(lang_code, 'title')}</h2></div>", unsafe_allow_html=True)
567
 
568
+ if task_status == "resumable":
569
  st.warning(tr(lang_code, "checkpoint_warning"))
570
  colA, colB = st.columns([1, 1])
571
  if colA.button(tr(lang_code, "resume"), key="resume_task"):
572
+ st.session_state["sp_pending_resume_task_id"] = str(task.get("task_id") or "")
573
+ st.rerun()
 
 
 
 
 
 
 
 
 
 
 
574
  if colB.button(tr(lang_code, "clear"), key="clear_task"):
575
+ st.session_state["sp_pending_cancel_task_id"] = str(task.get("task_id") or "")
576
+ st.rerun()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
577
  st.markdown("---")
578
 
579
+ st.text_input(tr(lang_code, "topic_label"), value=DEFAULT_TOPIC, disabled=active, key="sp_topic")
580
+ st.text_input(tr(lang_code, "keywords_label"), value=DEFAULT_KEYWORDS, disabled=active, key="sp_keywords")
 
 
 
 
 
 
 
 
 
 
581
  st.number_input(
582
  tr(lang_code, "words_label"),
583
+ min_value=int(APP_CONFIG["target_words_min"]),
584
+ max_value=int(APP_CONFIG["target_words_max"]),
585
+ value=int(APP_CONFIG["target_words_default"]),
586
  step=1000,
587
+ disabled=active,
588
  key="sp_words",
589
  )
590
 
591
  tab_amj, tab_asq = st.tabs([tr(lang_code, "tab_amj"), tr(lang_code, "tab_asq")])
 
592
  with tab_amj:
593
  st.markdown(tr(lang_code, "amj_config"))
594
  st.info(tr(lang_code, "amj_info"))
595
+ st.file_uploader(tr(lang_code, "upload_reference"), type=["pdf", "doc", "docx"], accept_multiple_files=True, key="up_amj_ref", disabled=active)
596
+ st.file_uploader(tr(lang_code, "upload_stats"), type=["csv", "xlsx", "xls"], key="up_amj_data", disabled=active)
 
 
 
 
 
 
 
 
 
 
 
 
597
  with st.expander(tr(lang_code, "amj_expander")):
598
  st.markdown(tr(lang_code, "amj_hint"))
599
+ template = get_amj_csv_template(lang_code)
600
+ st.code(template, language="csv")
601
+ st.download_button(label=tr(lang_code, "download_amj_template"), data=template, file_name="AMJ_Data_Template.csv", mime="text/csv", use_container_width=True)
602
+ if st.button(tr(lang_code, "start_amj"), disabled=active, key="btn_amj"):
603
+ st.session_state["run_mode"] = "amj"
604
+ st.session_state["sp_pending_launch"] = {"mode": "amj", "idempotency_key": _new_generation_idempotency_key()}
605
+ st.rerun()
 
 
 
 
 
 
 
 
 
 
 
 
606
 
607
  with tab_asq:
608
  st.markdown(tr(lang_code, "asq_config"))
609
  st.info(tr(lang_code, "asq_info"))
610
+ st.file_uploader(tr(lang_code, "upload_reference"), type=["pdf", "doc", "docx"], accept_multiple_files=True, key="up_asq_ref", disabled=active)
611
+ st.file_uploader(tr(lang_code, "upload_interviews"), type=["txt", "pdf", "doc", "docx"], key="up_asq_data", disabled=active)
 
 
 
 
 
 
 
 
 
 
 
 
612
  with st.expander(tr(lang_code, "asq_expander")):
613
  st.markdown(tr(lang_code, "asq_hint"))
614
+ template = get_asq_txt_template(lang_code)
615
+ st.code(template, language="text")
616
+ st.download_button(label=tr(lang_code, "download_asq_template"), data=template, file_name="ASQ_Interview_Template.txt", mime="text/plain", use_container_width=True)
617
+ if st.button(tr(lang_code, "start_asq"), disabled=active, key="btn_asq"):
618
+ st.session_state["run_mode"] = "asq"
619
+ st.session_state["sp_pending_launch"] = {"mode": "asq", "idempotency_key": _new_generation_idempotency_key()}
620
+ st.rerun()
 
 
621
 
 
 
 
 
 
 
 
 
 
 
 
622
  inject_file_uploader_i18n(lang_code)
623
 
624
 
625
  render_workspace_controls(authenticated_user)
 
626
  lang_code = str(st.session_state.get("sp_lang_code") or _current_language_code())
627
+
628
+ _notice = st.session_state.pop("sp_terminal_notice", None)
629
+ if isinstance(_notice, dict) and _notice.get("message"):
630
+ (st.error if _notice.get("kind") == "error" else st.warning)(str(_notice.get("message")))
631
+
632
+ _process_pending_cancel()
633
+ _process_pending_resume()
634
+ _process_pending_launch(lang_code)
635
+
636
+ if st.session_state.get("sp_payment_required"):
 
 
 
 
 
 
637
  render_payment_purchase(lang_code)
638
 
639
+
640
+ def _render_monitor_box(task: dict) -> None:
641
+ status = str(task.get("status") or "")
642
+ node = str(task.get("current_node") or status or "")
643
+ progress = max(0, min(100, int(task.get("progress_pct") or 0)))
644
+ if status in {"created", "running", "resumable", "cancelling"}:
645
+ st.progress(progress, text=f"{progress}% — {node}")
646
+ st.markdown(f"""
 
 
 
 
 
 
 
 
 
647
  <div class='log-box'>
648
+ <div><b>{tr(lang_code, 'monitor_pipeline')}:</b> {status}</div><div><b>{tr(lang_code, 'monitor_node')}:</b> {node}</div><div><b>{tr(lang_code, 'monitor_log')}:</b> checkpoint {int(task.get('checkpoint_seq') or 0)} · {str(task.get('checkpoint_stage') or '')}</div><hr>
649
  <div><b>{tr(lang_code, 'monitor_compute')}:</b></div>
650
  <div style='display: flex; gap: 8px;'>
651
+ <span style='background: #fef3c7; padding: 4px 8px; border-radius: 4px; color: black;'>🧠 DeepSeek: <b>{int(task.get('ds_calls') or 0)}</b></span>
652
+ <span style='background: #dbeafe; padding: 4px 8px; border-radius: 4px; color: black;'>📚 S2: <b>{int(task.get('s2_calls') or 0)}</b></span>
653
+ <span style='background: #d1fae5; padding: 4px 8px; border-radius: 4px; color: black;'>🌐 OpenAlex: <b>{int(task.get('oa_calls') or 0)}</b></span>
654
  </div>
655
  </div>
656
  """, unsafe_allow_html=True)
657
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
658
 
659
+ def _render_task_status_once() -> None:
660
+ task = st.session_state.get("sp_generation_task") or {}
661
+ task_id = str(task.get("task_id") or "").strip()
662
+ if not task_id:
663
+ return
664
+ status = str(task.get("status") or "")
665
+ if status in {"created", "running", "resumable", "cancelling"}:
666
+ try:
667
+ result = generation_task_status(task_id)
668
+ task = dict(result.get("task") or task)
669
+ st.session_state["sp_generation_task"] = task
670
+ status = str(task.get("status") or "")
671
+ except Exception as exc:
672
+ logging.warning("[SP_UI] status_poll_error task_id=%s error=%s", task_id, type(exc).__name__)
673
+ _render_monitor_box(task)
674
+ if status == "completed":
675
+ output_path = _task_output_path(task)
676
+ if output_path and output_path.is_file():
677
+ st.success(tr(lang_code, "success"))
678
+ st.download_button(label=tr(lang_code, "download"), data=output_path.read_bytes(), file_name=output_path.name, mime="application/zip")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
679
  else:
680
+ st.warning("The task completed, but the local output file is unavailable after runtime replacement.")
681
+ elif status == "failed":
682
+ st.error(str(task.get("safe_error_detail") or task.get("safe_error_code") or tr(lang_code, "node_error")))
683
+ elif status == "cancelled":
684
+ detail = str(task.get("safe_error_detail") or "")
685
+ if detail:
686
+ st.warning(detail)
687
+
688
+
689
+ if getattr(st, "fragment", None) is not None:
690
+ @st.fragment(run_every=5)
691
+ def _task_monitor_fragment():
692
+ _render_task_status_once()
693
+ _task_monitor_fragment()
694
+ else:
695
+ _render_task_status_once()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
config.py CHANGED
@@ -8,6 +8,9 @@ def load_config():
8
  "lit_per_page": int(os.environ.get("LIT_PER_PAGE", "100")),
9
  "lit_max_workers": int(os.environ.get("LIT_MAX_WORKERS", "5")),
10
  "chunk_words_default": int(os.environ.get("CHUNK_WORDS_DEFAULT", "1500")),
 
 
 
11
  "search_max_retries": max(1, int(os.environ.get("SEARCH_MAX_RETRIES", "4"))),
12
  "translation_batch_tokens": max(800, int(os.environ.get("TRANSLATION_BATCH_TOKENS", "2500"))),
13
  "translation_glossary_term_limit": max(10, int(os.environ.get("TRANSLATION_GLOSSARY_TERM_LIMIT", "40"))),
@@ -124,11 +127,15 @@ def load_config():
124
  int(os.environ.get("CITATION_SEMANTIC_JSON_REPAIR_RETRIES", "1")),
125
  ),
126
  },
127
- # Task/lease settings are consumed by the HF client. The Worker remains
128
- # the authoritative source for task ownership and lock expiry.
129
- "task_heartbeat_interval_seconds": max(
130
- 3,
131
- int(os.environ.get("SMARTPAPER_TASK_HEARTBEAT_SECONDS", "5")),
 
 
 
 
132
  ),
133
  "manuscript_quality": {
134
  "abstract_max_words": int(os.environ.get("ABSTRACT_MAX_WORDS", "200")),
@@ -147,7 +154,7 @@ def load_config():
147
 
148
  # --- UI 多语言标签 ---
149
  "ui_labels": {
150
- "中文": {"title": "📊 工商管理论文智能体", "btn_generate": "🚀 生成初稿", "btn_download": "📥 下载文件", "topic_label": "📝 论文名称", "keywords_label": "🔍 关键字", "words_label": "📏 目标总单词数 (12000-25000单词)", "success": "🎉 流水线推演完毕!双语文档已就绪。"},
151
  "English": {"title": "📊 Business Administration Article Agent", "btn_generate": "🚀 Generate Draft", "btn_download": "📥 Download", "topic_label": "📝 Article Title", "keywords_label": "🔍 Keywords", "words_label": "📏 Target Word Count", "success": "🎉 Task complete!"},
152
  "Русский": {"title": "📊 Агент статей по бизнес-администрированию", "btn_generate": "🚀 Черновик", "btn_download": "📥 Скачать", "topic_label": "📝 Тема", "keywords_label": "🔍 Ключевые слова", "words_label": "📏 Количество слов", "success": "🎉 Задача выполнена!"},
153
  "Қазақша": {"title": "📊 Бизнес-басқару жөніндегі ғылыми мақала агенті", "btn_generate": "🚀 Жоба дайындау", "btn_download": "📥 Жүктеу", "topic_label": "📝 Мақала атауы", "keywords_label": "🔍 Түйін сөздер", "words_label": "📏 Мақсатты сөз саны", "success": "🎉 Тапсырма аяқталды!"}
 
8
  "lit_per_page": int(os.environ.get("LIT_PER_PAGE", "100")),
9
  "lit_max_workers": int(os.environ.get("LIT_MAX_WORKERS", "5")),
10
  "chunk_words_default": int(os.environ.get("CHUNK_WORDS_DEFAULT", "1500")),
11
+ "target_words_min": 5000,
12
+ "target_words_default": 5000,
13
+ "target_words_max": 20000,
14
  "search_max_retries": max(1, int(os.environ.get("SEARCH_MAX_RETRIES", "4"))),
15
  "translation_batch_tokens": max(800, int(os.environ.get("TRANSLATION_BATCH_TOKENS", "2500"))),
16
  "translation_glossary_term_limit": max(10, int(os.environ.get("TRANSLATION_GLOSSARY_TERM_LIMIT", "40"))),
 
127
  int(os.environ.get("CITATION_SEMANTIC_JSON_REPAIR_RETRIES", "1")),
128
  ),
129
  },
130
+ # R3-Free task execution runs outside Streamlit. The executor renews
131
+ # the Worker lease only while real Python/model progress is recent.
132
+ "executor_heartbeat_interval_seconds": max(
133
+ 10,
134
+ int(os.environ.get("SMARTPAPER_EXECUTOR_HEARTBEAT_SECONDS", "15")),
135
+ ),
136
+ "executor_activity_grace_seconds": max(
137
+ 30,
138
+ int(os.environ.get("SMARTPAPER_EXECUTOR_ACTIVITY_GRACE_SECONDS", "45")),
139
  ),
140
  "manuscript_quality": {
141
  "abstract_max_words": int(os.environ.get("ABSTRACT_MAX_WORDS", "200")),
 
154
 
155
  # --- UI 多语言标签 ---
156
  "ui_labels": {
157
+ "中文": {"title": "📊 工商管理论文智能体", "btn_generate": "🚀 生成初稿", "btn_download": "📥 下载文件", "topic_label": "📝 论文名称", "keywords_label": "🔍 关键字", "words_label": "📏 目标总单词数 (5000-20000单词)", "success": "🎉 流水线推演完毕!双语文档已就绪。"},
158
  "English": {"title": "📊 Business Administration Article Agent", "btn_generate": "🚀 Generate Draft", "btn_download": "📥 Download", "topic_label": "📝 Article Title", "keywords_label": "🔍 Keywords", "words_label": "📏 Target Word Count", "success": "🎉 Task complete!"},
159
  "Русский": {"title": "📊 Агент статей по бизнес-администрированию", "btn_generate": "🚀 Черновик", "btn_download": "📥 Скачать", "topic_label": "📝 Тема", "keywords_label": "🔍 Ключевые слова", "words_label": "📏 Количество слов", "success": "🎉 Задача выполнена!"},
160
  "Қазақша": {"title": "📊 Бизнес-басқару жөніндегі ғылыми мақала агенті", "btn_generate": "🚀 Жоба дайындау", "btn_download": "📥 Жүктеу", "topic_label": "📝 Мақала атауы", "keywords_label": "🔍 Түйін сөздер", "words_label": "📏 Мақсатты сөз саны", "success": "🎉 Тапсырма аяқталды!"}
core/generation_runner.py ADDED
@@ -0,0 +1,592 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # -*- coding: utf-8 -*-
2
+ """R3-Free headless SmartPaper generation runner.
3
+
4
+ The runner contains no Streamlit calls. It is safe to execute in a subprocess
5
+ whose lifetime is independent from Streamlit ScriptRunner reruns and browser
6
+ connections. Checkpoints are task-scoped, immutable-by-sequence JSON snapshots.
7
+ """
8
+ from __future__ import annotations
9
+
10
+ import hashlib
11
+ import json
12
+ import logging
13
+ import math
14
+ import os
15
+ import re
16
+ from datetime import datetime, timezone, timedelta
17
+ from pathlib import Path
18
+ from typing import Callable, Any
19
+
20
+ from config import APP_CONFIG, AMJ_PROMPTS, ASQ_PROMPTS
21
+ from core.meta_extractor import extract_golden_metadata
22
+ from core.pipeline_agent import SSciPipelineAgent
23
+ from core.production_guard import build_validated_package, replace_abstract
24
+ from core.qualitative_engine import get_qualitative_engine
25
+ from core.quantitative_engine import get_quantitative_engine
26
+ from core.rag_engine import initialize_rag
27
+ from utils.file_parser import extract_text_from_files
28
+
29
+ ProgressCallback = Callable[..., None]
30
+
31
+
32
+ def _atomic_json(path: Path, value: dict[str, Any]) -> None:
33
+ path.parent.mkdir(parents=True, exist_ok=True)
34
+ temporary = path.with_suffix(path.suffix + ".tmp")
35
+ temporary.write_text(json.dumps(value, ensure_ascii=False), encoding="utf-8")
36
+ os.replace(temporary, path)
37
+
38
+
39
+ def _clean_trailing_references(text, keywords=("reference", "references", "reference list", "参考文献")):
40
+ lines = str(text or "").split("\n")
41
+ for i in range(len(lines) - 1, max(-1, len(lines) - 100), -1):
42
+ if any(re.match(rf"^##?\s*{kw}", lines[i], re.IGNORECASE) for kw in keywords):
43
+ return "\n".join(lines[:i]).strip()
44
+ return str(text or "").strip()
45
+
46
+
47
+ def _clean_markdown_headings(text):
48
+ return re.sub(r"(?m)^#+\s+(.*)$", r"**\1**", str(text or ""))
49
+
50
+
51
+ def _canonical_request_hash(request_data: dict[str, Any]) -> str:
52
+ material = dict(request_data or {})
53
+ material.pop("request_hash", None)
54
+ encoded = json.dumps(material, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
55
+ return hashlib.sha256(encoded).hexdigest()
56
+
57
+
58
+ class TaskCheckpointStore:
59
+ def __init__(self, task_dir: Path, request_hash: str, execution_epoch: int):
60
+ self.task_dir = Path(task_dir)
61
+ self.request_hash = str(request_hash)
62
+ self.execution_epoch = int(execution_epoch)
63
+ self.directory = self.task_dir / "checkpoints"
64
+ self.directory.mkdir(parents=True, exist_ok=True)
65
+ self.sequence = 0
66
+
67
+ def load_latest(self) -> dict[str, Any] | None:
68
+ candidates: list[tuple[int, Path, dict[str, Any]]] = []
69
+ for path in self.directory.glob("checkpoint_*.json"):
70
+ try:
71
+ payload = json.loads(path.read_text(encoding="utf-8"))
72
+ if not isinstance(payload, dict):
73
+ continue
74
+ if payload.get("request_hash") != self.request_hash:
75
+ continue
76
+ seq = int(payload.get("checkpoint_seq") or 0)
77
+ if seq <= 0:
78
+ continue
79
+ candidates.append((seq, path, payload))
80
+ except Exception:
81
+ continue
82
+ if not candidates:
83
+ return None
84
+ seq, _, payload = max(candidates, key=lambda item: item[0])
85
+ self.sequence = seq
86
+ return payload
87
+
88
+ def save(self, stage: str, state: dict[str, Any]) -> tuple[int, str]:
89
+ self.sequence += 1
90
+ stage_name = re.sub(r"[^A-Za-z0-9_.-]", "_", str(stage or "checkpoint"))[:80]
91
+ payload = dict(state or {})
92
+ payload.update(
93
+ {
94
+ "request_hash": self.request_hash,
95
+ "checkpoint_seq": self.sequence,
96
+ "checkpoint_stage": stage_name,
97
+ "checkpoint_execution_epoch": self.execution_epoch,
98
+ "checkpoint_saved_at": datetime.now(timezone.utc).isoformat(),
99
+ }
100
+ )
101
+ path = self.directory / f"checkpoint_{self.sequence:05d}_{stage_name}.json"
102
+ _atomic_json(path, payload)
103
+ latest = self.directory / "latest.json"
104
+ _atomic_json(latest, payload)
105
+ return self.sequence, stage_name
106
+
107
+
108
+ class GenerationRunner:
109
+ def __init__(
110
+ self,
111
+ *,
112
+ task_dir: str | os.PathLike,
113
+ request_data: dict[str, Any],
114
+ execution_epoch: int,
115
+ progress_callback: ProgressCallback | None = None,
116
+ cancel_checker: Callable[[], bool] | None = None,
117
+ ):
118
+ self.task_dir = Path(task_dir)
119
+ self.request = dict(request_data)
120
+ self.execution_epoch = int(execution_epoch)
121
+ self.progress_callback = progress_callback or (lambda **_: None)
122
+ self.cancel_checker = cancel_checker or (lambda: False)
123
+ expected_hash = str(self.request.get("request_hash") or "")
124
+ actual_hash = _canonical_request_hash(self.request)
125
+ if not expected_hash or expected_hash != actual_hash:
126
+ raise RuntimeError("request_hash_mismatch")
127
+ self.request_hash = expected_hash
128
+ self.checkpoints = TaskCheckpointStore(self.task_dir, self.request_hash, self.execution_epoch)
129
+ self.output_dir = self.task_dir / "outputs"
130
+ self.output_dir.mkdir(parents=True, exist_ok=True)
131
+ self.agent = None
132
+
133
+ def _progress(self, pct: int, node: str, *, checkpoint_seq=None, checkpoint_stage=None) -> None:
134
+ agent = self.agent
135
+ self.progress_callback(
136
+ progress_pct=max(0, min(100, int(pct))),
137
+ current_node=str(node or "")[:240],
138
+ checkpoint_seq=checkpoint_seq,
139
+ checkpoint_stage=checkpoint_stage,
140
+ ds_calls=int(getattr(agent, "ds_calls", 0) or 0),
141
+ s2_calls=int(getattr(agent, "s2_calls", 0) or 0),
142
+ oa_calls=int(getattr(agent, "oa_calls", 0) or 0),
143
+ )
144
+
145
+ def _checkpoint(self, stage: str, state: dict[str, Any], pct: int) -> None:
146
+ seq, normalized_stage = self.checkpoints.save(stage, state)
147
+ self._progress(pct, normalized_stage, checkpoint_seq=seq, checkpoint_stage=normalized_stage)
148
+
149
+ @staticmethod
150
+ def _stream_text(generator) -> str:
151
+ return "".join(str(chunk or "") for chunk in generator)
152
+
153
+ @staticmethod
154
+ def _bounded_target(base_target: int, accumulated_deviation: float, remaining_including_current: int, safe_chunk: int) -> int:
155
+ base = max(80, int(base_target))
156
+ correction = -float(accumulated_deviation) / max(1, int(remaining_including_current))
157
+ lower = max(80, int(round(base * 0.75)))
158
+ upper = min(int(safe_chunk * 1.5), max(lower, int(round(base * 1.25))))
159
+ return max(lower, min(upper, int(round(base + correction))))
160
+
161
+ @staticmethod
162
+ def _english_count_for_target(manuscript: str) -> int:
163
+ body = re.split(r"(?m)^#\s+References\s*$", str(manuscript or ""), maxsplit=1)[0]
164
+ # Markdown punctuation is ignored; contractions/hyphenated terms stay one token.
165
+ return len(re.findall(r"\b[A-Za-z0-9]+(?:[-'][A-Za-z0-9]+)*\b", body))
166
+
167
+ def _validate_target_length(self, manuscript: str, target_words: int) -> None:
168
+ quality = APP_CONFIG.get("manuscript_validation", {})
169
+ minimum = int(target_words * float(quality.get("minimum_total_ratio", 0.85)))
170
+ maximum = int(target_words * float(quality.get("maximum_total_ratio", 1.15)))
171
+ actual = self._english_count_for_target(manuscript)
172
+ if actual < minimum or actual > maximum:
173
+ raise RuntimeError(
174
+ f"English manuscript length outside validated range: actual={actual}, "
175
+ f"required={minimum}-{maximum}, target={target_words}."
176
+ )
177
+
178
+ def _resolve_paths(self, items) -> list[str]:
179
+ output = []
180
+ for item in items or []:
181
+ path = (self.task_dir / str(item)).resolve()
182
+ if self.task_dir.resolve() not in path.parents:
183
+ raise RuntimeError("input_path_escape")
184
+ if path.is_file():
185
+ output.append(str(path))
186
+ return output
187
+
188
+ def run(self) -> dict[str, Any]:
189
+ req = self.request
190
+ mode = str(req.get("mode") or "").lower()
191
+ if mode not in {"amj", "asq"}:
192
+ raise RuntimeError("invalid_generation_mode")
193
+ target_words = int(req.get("target_words") or 0)
194
+ if not APP_CONFIG["target_words_min"] <= target_words <= APP_CONFIG["target_words_max"]:
195
+ raise RuntimeError("target_words_out_of_range")
196
+
197
+ ds_key = str(os.environ.get("DEEPSEEK_API_KEY") or "").strip()
198
+ s2_key = str(os.environ.get("S2_API_KEY") or "").strip()
199
+ if not ds_key:
200
+ raise RuntimeError("DEEPSEEK_API_KEY_missing")
201
+
202
+ if mode == "amj":
203
+ APP_CONFIG.update(AMJ_PROMPTS)
204
+ else:
205
+ APP_CONFIG.update(ASQ_PROMPTS)
206
+
207
+ agent = SSciPipelineAgent(ds_key, s2_key)
208
+ self.agent = agent
209
+ agent.set_cancel_checker(self.cancel_checker)
210
+ # DeepSeek/search activity marks the executor as genuinely alive; the
211
+ # heartbeat thread decides separately when to renew D1's lease.
212
+ agent.set_execution_progress_hook(lambda: self._progress_activity())
213
+ safe_chunk_words = int(agent.chunk_words_default)
214
+ quant_engine = get_quantitative_engine()
215
+ qual_engine = get_qualitative_engine(agent)
216
+
217
+ state = self.checkpoints.load_latest() or {}
218
+ if state:
219
+ logging.info(
220
+ "[SP_EXECUTOR] resume checkpoint seq=%s stage=%s",
221
+ state.get("checkpoint_seq"), state.get("checkpoint_stage"),
222
+ )
223
+
224
+ topic = str(req.get("topic") or "").strip()
225
+ keywords = str(req.get("keywords") or "").strip()
226
+ target_lang = str(req.get("target_lang") or "").strip()
227
+ output_suffix = str(req.get("output_suffix") or "Target").strip()
228
+ interface_language = str(req.get("interface_language") or "zh").strip()
229
+ reference_paths = self._resolve_paths(req.get("reference_files") or [])
230
+ data_paths = self._resolve_paths(req.get("data_files") or [])
231
+ interview_paths = self._resolve_paths(req.get("interview_files") or [])
232
+
233
+ # Restore agent state before continuing any dependent step.
234
+ if isinstance(state.get("agent_state"), dict):
235
+ agent.load_state(state["agent_state"])
236
+
237
+ user_lit_context = str(state.get("user_lit_context") or "")
238
+ user_data_md = str(state.get("user_data_md") or "")
239
+ data_metadata = dict(state.get("data_metadata") or {})
240
+
241
+ if not state.get("evidence_ready"):
242
+ self._progress(3, "inputs_and_evidence")
243
+ if reference_paths:
244
+ user_lit_context = extract_text_from_files(reference_paths[:20])
245
+ all_paper_texts = [extract_text_from_files([path]) for path in reference_paths[:20]]
246
+ agent.global_memory["core_golden_metadata"] = extract_golden_metadata(agent, all_paper_texts)
247
+ if mode == "amj":
248
+ data_file = data_paths[0] if data_paths else None
249
+ _, user_data_md, _, data_metadata = quant_engine.resolve_empirical_data(data_file, interface_language)
250
+ else:
251
+ raw_interviews = extract_text_from_files(interview_paths[:1]) if interview_paths else ""
252
+ user_data_md, _, data_metadata = qual_engine.resolve_qualitative_data(topic, keywords, raw_interviews)
253
+ state.update(
254
+ {
255
+ "topic": topic,
256
+ "keywords": keywords,
257
+ "target_lang": target_lang,
258
+ "target_words": target_words,
259
+ "mode": mode,
260
+ "interface_language": interface_language,
261
+ "user_lit_context": user_lit_context,
262
+ "user_data_md": user_data_md,
263
+ "data_metadata": data_metadata,
264
+ "agent_state": agent.export_state(),
265
+ "evidence_ready": True,
266
+ }
267
+ )
268
+ self._checkpoint("evidence_ready", state, 6)
269
+
270
+ refs_text = str(state.get("refs_text") or "")
271
+ if not refs_text:
272
+ self._progress(8, "S2 + OpenAlex")
273
+ refs_text = agent.fetch_references_parallel(
274
+ topic,
275
+ keywords,
276
+ min_required=50,
277
+ progress_callback=lambda: self._progress(9, "S2 + OpenAlex"),
278
+ )
279
+ if len(agent.citation_map) < 50:
280
+ raise RuntimeError(f"Insufficient eligible references: {len(agent.citation_map)}/50")
281
+ if user_lit_context:
282
+ core_refs = agent.parse_user_core_literature(user_lit_context)
283
+ if core_refs:
284
+ refs_text = (
285
+ "【USER PROVIDED CORE LITERATURE】\n" + core_refs
286
+ + "\n\n【RETRIEVED LITERATURE】\n" + refs_text
287
+ )
288
+ state.update({"refs_text": refs_text, "agent_state": agent.export_state()})
289
+ self._checkpoint("literature_ready", state, 11)
290
+
291
+ rag_engine = initialize_rag([value for value in (user_lit_context, refs_text) if value])
292
+
293
+ synthesis_result = str(state.get("synthesis_result") or "")
294
+ if not synthesis_result:
295
+ self._progress(12, "literature_synthesis")
296
+ synthesis_result = self._stream_text(
297
+ agent.stream_synthesize_literature(topic, refs_text, user_core_literature=user_lit_context)
298
+ )
299
+ synthesis_result = re.sub(r"<think>.*?</think>", "", synthesis_result, flags=re.DOTALL).strip()
300
+ state.update({"synthesis_result": synthesis_result, "agent_state": agent.export_state()})
301
+ self._checkpoint("synthesis_ready", state, 13)
302
+
303
+ adaptive_structure = state.get("adaptive_structure")
304
+ if not isinstance(adaptive_structure, list) or not adaptive_structure:
305
+ self._progress(14, "anchors_and_outline")
306
+ anchor_response = self._stream_text(
307
+ agent.stream_generate_anchors(
308
+ topic, keywords, synthesis_result, refs_text,
309
+ data_evidence=user_data_md, mode=mode,
310
+ )
311
+ )
312
+ anchor_clean = re.sub(r"<think>.*?</think>", "", anchor_response, flags=re.DOTALL).strip()
313
+ abstract_match = re.search(
314
+ r"-+\s*ABSTRACT\s*-+\s*(.*?)(?=\s*-+\s*CONCLUSION\s*-+|$)",
315
+ anchor_clean, re.DOTALL | re.IGNORECASE,
316
+ )
317
+ conclusion_match = re.search(
318
+ r"-+\s*CONCLUSION\s*-+\s*(.*?)$", anchor_clean, re.DOTALL | re.IGNORECASE,
319
+ )
320
+ abstract_anchor = abstract_match.group(1).strip() if abstract_match else anchor_clean
321
+ conclusion_anchor = conclusion_match.group(1).strip() if conclusion_match else ""
322
+ agent.global_memory["abstract_anchor"] = abstract_anchor
323
+ agent.global_memory["conclusion_anchor"] = conclusion_anchor
324
+
325
+ plan = agent.generate_outline(topic, keywords, mode=mode, data_metadata=data_metadata)
326
+ en_topic = str(plan.get("en_topic", topic))
327
+ structure = plan.get("structure", [])
328
+ en_abstract_clean = _clean_markdown_headings(
329
+ re.sub(r"(?i)^abstract[:\s]*", "", str(abstract_anchor)).strip()
330
+ )
331
+ full_draft_en = f"# {en_topic}\n\n## Abstract\n{en_abstract_clean}\n\n## Keywords\n{keywords}\n\n"
332
+ adaptive_structure = agent.build_generation_tasks(structure, target_words, safe_chunk_words)
333
+ for task in adaptive_structure:
334
+ task["is_first"] = bool(task.get("section_first"))
335
+ jit_texts = agent.prefetch_jit_literature(topic, keywords, adaptive_structure)
336
+ if jit_texts:
337
+ rag_engine._initialize_chunks([jit_texts])
338
+ state.update(
339
+ {
340
+ "adaptive_structure": adaptive_structure,
341
+ "full_draft_en": full_draft_en,
342
+ "full_draft_translated": "",
343
+ "completed_sections": [],
344
+ "context_tail": "",
345
+ "accumulated_deviation": 0.0,
346
+ "agent_state": agent.export_state(),
347
+ "outline_ready": True,
348
+ }
349
+ )
350
+ self._checkpoint("outline_ready", state, 15)
351
+ else:
352
+ full_draft_en = str(state.get("full_draft_en") or "")
353
+ # JIT text itself is already represented in agent citation state;
354
+ # refs + user literature rebuild a safe RAG baseline on resume.
355
+
356
+ completed_sections = list(state.get("completed_sections") or [])
357
+ context_tail = str(state.get("context_tail") or "")
358
+ accumulated_deviation = float(state.get("accumulated_deviation") or 0.0)
359
+ all_cite_keys = list(agent.citation_map.keys())
360
+ valid_chunks = [
361
+ item for item in adaptive_structure
362
+ if not any(word in item["section"].lower() for word in ["abstract", "keyword", "appendix"])
363
+ ]
364
+ allocation_map: dict[str, list[str]] = {}
365
+ current_cite_idx = 0
366
+ for idx, item in enumerate(valid_chunks):
367
+ count = max(1, math.ceil(len(all_cite_keys) / max(1, len(valid_chunks))))
368
+ allocation_map[item["display_name"]] = (
369
+ all_cite_keys[current_cite_idx:]
370
+ if idx == len(valid_chunks) - 1
371
+ else all_cite_keys[current_cite_idx:current_cite_idx + count]
372
+ )
373
+ current_cite_idx += count
374
+
375
+ total_steps = len(adaptive_structure)
376
+ for sec in adaptive_structure:
377
+ display_name = sec["display_name"]
378
+ if display_name in completed_sections:
379
+ continue
380
+ if self.cancel_checker():
381
+ raise RuntimeError("task_cancel_requested")
382
+ remaining = sum(1 for item in adaptive_structure if item["display_name"] not in completed_sections)
383
+ adjusted_target = self._bounded_target(
384
+ int(sec["target_words"]), accumulated_deviation, remaining, safe_chunk_words
385
+ )
386
+ sec_title = sec["section"]
387
+ subsection = sec.get("subsection")
388
+ is_quarantined = any(
389
+ word in sec_title.lower()
390
+ for word in ["abstract", "keyword", "conclusion", "references", "appendix"]
391
+ )
392
+ assigned_tags = ", ".join(allocation_map.get(display_name, [])) or "the single most relevant source"
393
+ citation_rule = (
394
+ "NO IN-TEXT CITATIONS ALLOWED IN THIS SECTION."
395
+ if is_quarantined
396
+ else "Use claim-specific citation. Each sentence may contain at most one source tag. "
397
+ f"Candidate tags: {assigned_tags}. <AVAILABLE_CITATIONS>{refs_text}</AVAILABLE_CITATIONS>"
398
+ )
399
+ memory_context = agent._build_memory_context(sec_title, 2500)
400
+ memory_block = f"\n<GLOBAL_MEMORY>\n{memory_context}\n</GLOBAL_MEMORY>\n" if memory_context else ""
401
+ snippets = rag_engine.get_relevant_context(display_name) if rag_engine is not None else ""
402
+ if snippets:
403
+ memory_block += f"\n<TARGETED_RAG>\n{snippets}\n</TARGETED_RAG>\n"
404
+ heading_directive = "Do not output the section heading; it will be inserted by the system."
405
+ if subsection:
406
+ heading_directive += (
407
+ f" Do not output the subsection heading '{subsection}'; it will also be inserted by the system. "
408
+ "Use level-3 headings only when analytically necessary."
409
+ )
410
+ elif sec_title.lower() in {"introduction", "conclusion"}:
411
+ heading_directive += " Do not use additional headings."
412
+ else:
413
+ heading_directive += (
414
+ " Use level-2 or level-3 headings only when they clarify the argument; "
415
+ "never use level-4 headings or numbered headings."
416
+ )
417
+ table_rule = (
418
+ "Present statistical evidence in standard Markdown tables with plain-text captions."
419
+ if sec["is_method_or_result"] and mode == "amj" else ""
420
+ )
421
+ concepts = ", ".join(sec.get("concepts") or [])
422
+ qualitative_rule = (
423
+ f"Assigned qualitative concepts: {concepts}. Use only their assigned representative quotations "
424
+ "and do not repeat a quotation already used elsewhere."
425
+ if concepts else ""
426
+ )
427
+ data_injection = (
428
+ f"\n<RESEARCH_EVIDENCE>\n{user_data_md}\n</RESEARCH_EVIDENCE>\n"
429
+ if user_data_md and sec["is_method_or_result"] else ""
430
+ )
431
+ completed_label = ", ".join(completed_sections)
432
+ anti_repeat = (
433
+ f"Already completed: {completed_label}. Do not restate their background, gap, or contribution."
434
+ if completed_label else ""
435
+ )
436
+ sys_prompt = APP_CONFIG.get("sys_academic_writer", "") + "\n" + APP_CONFIG.get("deep_refinement_directive", "")
437
+ user_prompt = f"""Write the assigned English manuscript unit.
438
+ Section: {sec_title}
439
+ Subsection: {subsection or 'None'}
440
+ Target length: {int(adjusted_target * 0.9)}-{int(adjusted_target * 1.1)} words.
441
+ {heading_directive}
442
+ {citation_rule}
443
+ {table_rule}
444
+ {str(sec.get('required_content') or '')}
445
+ {qualitative_rule}
446
+ {anti_repeat}
447
+ {memory_block}
448
+ Literature synthesis: {synthesis_result}
449
+ {data_injection}
450
+ Previous tail: {context_tail[-2500:]}
451
+ Output only final academic prose. Do not report the word count."""
452
+ if agent._count_tokens(sys_prompt) + agent._count_tokens(user_prompt) > 60000:
453
+ user_prompt = agent._truncate_context(user_prompt, 60000 - agent._count_tokens(sys_prompt))
454
+
455
+ current_index = len(completed_sections) + 1
456
+ pct = min(88, 15 + int((current_index / max(1, total_steps)) * 72))
457
+ self._progress(pct, display_name)
458
+ en_raw = self._stream_text(
459
+ agent.stream_deepseek(
460
+ sys_prompt, user_prompt, max_tokens=7000,
461
+ model_role="writing", thinking=False, allow_fallback=False,
462
+ )
463
+ )
464
+ en_raw = re.sub(r"<think>.*?</think>", "", en_raw, flags=re.S).strip()
465
+ en_raw = re.sub(rf"(?im)^#\s*{re.escape(sec_title)}\s*\n+", "", en_raw).strip()
466
+ if subsection:
467
+ en_raw = re.sub(rf"(?im)^##\s*{re.escape(subsection)}\s*\n+", "", en_raw).strip()
468
+ en_raw = re.sub(r"(?m)^(#{2,3})\s+[\d\.\s一二三四五六七八九十、]+\s*(.*)", r"\1 \2", en_raw)
469
+ en_raw = re.sub(r"(?m)^#{4,}\s+(.*)", r"**\1**", en_raw)
470
+ en_raw = re.sub(r"(?m)^#\s+(.+)$", r"## \1", en_raw)
471
+ if mode == "asq" and user_data_md and "appendix" not in sec_title.lower():
472
+ quote_dict = {tag: quote for quote, tag in re.findall(r'"([^"]+)"\s*\[(QUOTE-\d+)\]', user_data_md)}
473
+ for tag, quote in quote_dict.items():
474
+ en_raw = en_raw.replace(tag, f'"{quote}"')
475
+ en_raw = agent.polish_academic_section(en_raw, display_name, abstract=False)
476
+ prefix = ""
477
+ if sec.get("section_first"):
478
+ prefix += f"# {sec_title}\n\n"
479
+ if sec.get("subsection_first") and subsection:
480
+ prefix += f"## {subsection}\n\n"
481
+ en_resolved = agent.audit_and_resolve_citations(
482
+ prefix + en_raw, section_name=display_name, quarantine=is_quarantined
483
+ )
484
+ full_draft_en += en_resolved + "\n\n"
485
+ agent._extract_memory_from_text(en_raw, sec_title)
486
+ context_tail = " ".join(en_raw.split()[-200:])
487
+ actual_words = len(en_raw.split())
488
+ accumulated_deviation += actual_words - adjusted_target
489
+ completed_sections.append(display_name)
490
+ state.update(
491
+ {
492
+ "completed_sections": completed_sections,
493
+ "full_draft_en": full_draft_en,
494
+ "full_draft_translated": "",
495
+ "context_tail": context_tail,
496
+ "accumulated_deviation": accumulated_deviation,
497
+ "agent_state": agent.export_state(),
498
+ "data_metadata": data_metadata,
499
+ }
500
+ )
501
+ self._checkpoint(f"section_{len(completed_sections):03d}", state, pct)
502
+
503
+ if not state.get("english_finalized"):
504
+ self._progress(91, "english_validation")
505
+ full_draft_en = _clean_trailing_references(full_draft_en, ["Reference", "References", "Reference List"])
506
+ final_abstract = agent.generate_final_abstract(
507
+ topic, keywords, full_draft_en, data_evidence=user_data_md, mode=mode
508
+ )
509
+ full_draft_en = replace_abstract(full_draft_en, final_abstract)
510
+ agent.validate_manuscript_structure(full_draft_en, adaptive_structure, translated=False)
511
+ if mode == "asq":
512
+ repeated_quotes = agent.find_repeated_qualitative_quotes(full_draft_en, data_metadata)
513
+ if repeated_quotes:
514
+ raise RuntimeError(f"Repeated qualitative quotations: {repeated_quotes[:5]}")
515
+ references = []
516
+ seen_refs = set()
517
+ for tag in sorted(agent.used_citations, key=agent._citation_tag_number):
518
+ ref = str((agent.citation_map.get(tag) or {}).get("ref") or "").strip()
519
+ if ref and ref not in seen_refs:
520
+ references.append(f"- {ref}")
521
+ seen_refs.add(ref)
522
+ if not references:
523
+ raise RuntimeError("No cited references remained after citation validation.")
524
+ full_draft_en = full_draft_en.rstrip() + "\n\n# References\n" + "\n\n".join(references)
525
+ self._validate_target_length(full_draft_en, target_words)
526
+ state.update(
527
+ {
528
+ "full_draft_en": full_draft_en,
529
+ "english_finalized": True,
530
+ "agent_state": agent.export_state(),
531
+ }
532
+ )
533
+ self._checkpoint("english_complete", state, 93)
534
+ else:
535
+ full_draft_en = str(state.get("full_draft_en") or "")
536
+
537
+ full_draft_translated = str(state.get("full_draft_translated") or "")
538
+ if not state.get("translation_complete") or not full_draft_translated:
539
+ self._progress(95, "translation")
540
+ full_draft_translated = agent.translate_locked_manuscript(full_draft_en, target_lang)
541
+ agent.validate_translation_alignment(full_draft_en, full_draft_translated)
542
+ state.update(
543
+ {
544
+ "full_draft_translated": full_draft_translated,
545
+ "translation_complete": True,
546
+ "agent_state": agent.export_state(),
547
+ }
548
+ )
549
+ self._checkpoint("translation_complete", state, 96)
550
+
551
+ existing_output = str(state.get("output_filename") or "").strip()
552
+ if state.get("package_validated") and existing_output:
553
+ existing_path = self.output_dir / os.path.basename(existing_output)
554
+ if existing_path.is_file() and existing_path.stat().st_size > 0:
555
+ self._progress(99, "package_validated")
556
+ return {"output_filename": existing_path.name, "output_path": str(existing_path)}
557
+
558
+ self._progress(97, "packaging")
559
+ bj_time = datetime.now(timezone(timedelta(hours=8))).strftime("%Y%m%d_%H%M")
560
+ package = build_validated_package(
561
+ english_markdown=full_draft_en,
562
+ translated_markdown=full_draft_translated,
563
+ english_filename=f"Research_EN_{bj_time}.docx",
564
+ translated_filename=f"Research_{output_suffix}_{bj_time}.docx",
565
+ data_metadata=data_metadata,
566
+ mode=mode,
567
+ citation_audit=agent.citation_audit_log,
568
+ literature_metrics=agent.get_source_metrics(),
569
+ )
570
+ output_filename = f"SmartPaper_{bj_time}.zip"
571
+ pending = self.output_dir / (".pending_" + output_filename)
572
+ pending.write_bytes(package.zip_bytes)
573
+ final = self.output_dir / output_filename
574
+ os.replace(pending, final)
575
+ state.update(
576
+ {
577
+ "output_filename": output_filename,
578
+ "package_validated": True,
579
+ "agent_state": agent.export_state(),
580
+ }
581
+ )
582
+ self._checkpoint("package_validated", state, 99)
583
+ return {"output_filename": output_filename, "output_path": str(final)}
584
+
585
+ def _progress_activity(self) -> None:
586
+ # This call intentionally does not update D1 directly. The executor's
587
+ # heartbeat thread uses it only to prove that real backend work is still
588
+ # occurring, which avoids browser-driven or timer-only zombie liveness.
589
+ try:
590
+ self.progress_callback(activity_only=True)
591
+ except TypeError:
592
+ pass
core/pipeline_agent.py CHANGED
@@ -1,6 +1,5 @@
1
  # -*- coding: utf-8 -*-
2
  import os
3
- import streamlit as st
4
  import time
5
  import re
6
  import json
@@ -63,15 +62,11 @@ class SSciPipelineAgent:
63
  "DEEPSEEK_API_KEY is empty. Configure it in Hugging Face "
64
  "Space Settings > Variables and secrets."
65
  )
66
- try:
67
- self.s2_key = st.secrets.get("S2_API_KEY", s2_key)
68
- except:
69
- self.s2_key = os.environ.get("S2_API_KEY", s2_key)
70
-
71
- try:
72
- self.openalex_key = st.secrets.get("OA_API_KEY", "")
73
- except:
74
- self.openalex_key = os.environ.get("OA_API_KEY", "")
75
  self.openalex_email = str(APP_CONFIG.get("openalex_email") or "").strip()
76
 
77
  self.ds_calls = 0
@@ -2916,6 +2911,24 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
2916
  raise DeepSeekRequestError(
2917
  f'Translation content coverage is too low: {ratio:.3f}.'
2918
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2919
  return True
2920
 
2921
  def enforce_claim_specific_citations(self, text):
 
1
  # -*- coding: utf-8 -*-
2
  import os
 
3
  import time
4
  import re
5
  import json
 
62
  "DEEPSEEK_API_KEY is empty. Configure it in Hugging Face "
63
  "Space Settings > Variables and secrets."
64
  )
65
+ # R3-Free: the generation engine is headless. Hugging Face Secrets are
66
+ # exposed to the process as environment variables; Streamlit is not a
67
+ # dependency of the executor or pipeline.
68
+ self.s2_key = str(os.environ.get("S2_API_KEY", s2_key) or "").strip()
69
+ self.openalex_key = str(os.environ.get("OA_API_KEY", "") or "").strip()
 
 
 
 
70
  self.openalex_email = str(APP_CONFIG.get("openalex_email") or "").strip()
71
 
72
  self.ds_calls = 0
 
2911
  raise DeepSeekRequestError(
2912
  f'Translation content coverage is too low: {ratio:.3f}.'
2913
  )
2914
+
2915
+ # Russian and Kazakh both use Cyrillic, so a generic Cyrillic check is
2916
+ # insufficient. Long Kazakh manuscripts must contain a meaningful
2917
+ # number of Kazakh-specific letters; Russian manuscripts must not be
2918
+ # dominated by those letters. This is deterministic and adds no API cost.
2919
+ kazakh_specific = re.findall(r'[ӘәҒғҚқҢңӨөҰұҮүҺһІі]', target_body)
2920
+ cyrillic = re.findall(r'[А-Яа-яЁёӘәҒғҚқҢңӨөҰұҮүҺһІі]', target_body)
2921
+ if re.search(r'(?m)^#\s+Пайдаланылған әдебиеттер\s*$', target):
2922
+ minimum_kazakh = max(8, int(len(cyrillic) * 0.003))
2923
+ if len(kazakh_specific) < minimum_kazakh:
2924
+ raise DeepSeekRequestError(
2925
+ 'Kazakh language validation failed: insufficient Kazakh-specific letters.'
2926
+ )
2927
+ elif re.search(r'(?m)^#\s+Литература\s*$', target):
2928
+ if cyrillic and len(kazakh_specific) / len(cyrillic) > 0.02:
2929
+ raise DeepSeekRequestError(
2930
+ 'Russian language validation failed: excessive Kazakh-specific letters.'
2931
+ )
2932
  return True
2933
 
2934
  def enforce_claim_specific_citations(self, text):
core/qualitative_engine.py CHANGED
@@ -6,7 +6,6 @@ from __future__ import annotations
6
  import logging
7
  import re
8
 
9
- import streamlit as st
10
 
11
  from config import APP_CONFIG
12
 
@@ -69,35 +68,18 @@ Rules:
69
  def resolve_qualitative_data(self, topic, keywords, raw_interviews=""):
70
  has_user_data = bool(raw_interviews and len(raw_interviews) > 100)
71
  prompt = self._build_request(topic, keywords, raw_interviews)
72
- model_label = self.agent.model_for("reasoning")
73
-
74
- if has_user_data:
75
- status_label = (
76
- f"🧠 正在使用 {model_label} 分析访谈材料并冻结研究事实…"
77
- )
78
- complete_label = "✅ 访谈材料、研究事实与编码结构已完成"
79
- else:
80
- status_label = (
81
- f"🧠 正在使用 {model_label} 构建一致的定性研究证据包…"
82
- )
83
- complete_label = "✅ 定性研究事实与编码结构已完成"
84
-
85
- with st.status(status_label, expanded=True) as status:
86
- placeholder = st.empty()
87
- response = placeholder.write_stream(
88
- self.agent.stream_deepseek(
89
- "You are a rigorous qualitative research analyst.",
90
- prompt,
91
- model_role="reasoning",
92
- thinking=True,
93
- allow_fallback=False,
94
- )
95
- )
96
- status.update(
97
- label=complete_label,
98
- state="complete",
99
- expanded=False,
100
  )
 
101
 
102
  response = re.sub(
103
  r"<think>.*?</think>",
 
6
  import logging
7
  import re
8
 
 
9
 
10
  from config import APP_CONFIG
11
 
 
68
  def resolve_qualitative_data(self, topic, keywords, raw_interviews=""):
69
  has_user_data = bool(raw_interviews and len(raw_interviews) > 100)
70
  prompt = self._build_request(topic, keywords, raw_interviews)
71
+ # R3-Free engine code is headless. Streamlit rendering belongs to app.py;
72
+ # this method only produces deterministic evidence + metadata.
73
+ response = "".join(
74
+ str(chunk or "")
75
+ for chunk in self.agent.stream_deepseek(
76
+ "You are a rigorous qualitative research analyst.",
77
+ prompt,
78
+ model_role="reasoning",
79
+ thinking=True,
80
+ allow_fallback=False,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
81
  )
82
+ )
83
 
84
  response = re.sub(
85
  r"<think>.*?</think>",
i18n.py CHANGED
@@ -208,7 +208,7 @@ TEXTS: dict[str, dict[str, str]] = {
208
  "title": "📊 工商管理论文智能体",
209
  "topic_label": "📝 论文名称",
210
  "keywords_label": "🔍 关键字",
211
- "words_label": "📏 目标总单词数(12000–25000单词)",
212
  "success": "🎉 流水线推演完毕!双语文档已就绪。",
213
  "download": "📥 下载文件",
214
  "system_busy_title": "⏳ 系统忙碌",
@@ -361,7 +361,7 @@ TEXTS: dict[str, dict[str, str]] = {
361
  "title": "📊 Business Administration Article Agent",
362
  "topic_label": "📝 Article Title",
363
  "keywords_label": "🔍 Keywords",
364
- "words_label": "📏 Target Word Count (12,000–25,000 words)",
365
  "success": "🎉 The pipeline is complete. The bilingual documents are ready.",
366
  "download": "📥 Download Files",
367
  "system_busy_title": "⏳ System Busy",
@@ -514,7 +514,7 @@ TEXTS: dict[str, dict[str, str]] = {
514
  "title": "📊 Интеллектуальный агент для научных статей по управлению бизнесом",
515
  "topic_label": "📝 Название статьи",
516
  "keywords_label": "🔍 Ключевые слова",
517
- "words_label": "📏 Целевой объём (12 000–25 000 слов)",
518
  "success": "🎉 Работа конвейера завершена. Двуязычные документы готовы.",
519
  "download": "📥 Скачать файлы",
520
  "system_busy_title": "⏳ Система занята",
@@ -667,7 +667,7 @@ TEXTS: dict[str, dict[str, str]] = {
667
  "title": "📊 Бизнес-басқару жөніндегі ғылыми мақала интеллектуалды агенті",
668
  "topic_label": "📝 Мақала атауы",
669
  "keywords_label": "🔍 Түйін сөздер",
670
- "words_label": "📏 Мақсатты көлем (12 000–25 000 сөз)",
671
  "success": "🎉 Жұмыс үдерісі аяқталды. Екі тілдегі құжаттар дайын.",
672
  "download": "📥 Файлдарды жүктеу",
673
  "system_busy_title": "⏳ Жүйе бос емес",
 
208
  "title": "📊 工商管理论文智能体",
209
  "topic_label": "📝 论文名称",
210
  "keywords_label": "🔍 关键字",
211
+ "words_label": "📏 目标总单词数(5000–20000单词)",
212
  "success": "🎉 流水线推演完毕!双语文档已就绪。",
213
  "download": "📥 下载文件",
214
  "system_busy_title": "⏳ 系统忙碌",
 
361
  "title": "📊 Business Administration Article Agent",
362
  "topic_label": "📝 Article Title",
363
  "keywords_label": "🔍 Keywords",
364
+ "words_label": "📏 Target Word Count (5,000–20,000 words)",
365
  "success": "🎉 The pipeline is complete. The bilingual documents are ready.",
366
  "download": "📥 Download Files",
367
  "system_busy_title": "⏳ System Busy",
 
514
  "title": "📊 Интеллектуальный агент для научных статей по управлению бизнесом",
515
  "topic_label": "📝 Название статьи",
516
  "keywords_label": "🔍 Ключевые слова",
517
+ "words_label": "📏 Целевой объём (5 000–20 000 слов)",
518
  "success": "🎉 Работа конвейера завершена. Двуязычные документы готовы.",
519
  "download": "📥 Скачать файлы",
520
  "system_busy_title": "⏳ Система занята",
 
667
  "title": "📊 Бизнес-басқару жөніндегі ғылыми мақала интеллектуалды агенті",
668
  "topic_label": "📝 Мақала атауы",
669
  "keywords_label": "🔍 Түйін сөздер",
670
+ "words_label": "📏 Мақсатты көлем (5 000–20 000 сөз)",
671
  "success": "🎉 Жұмыс үдерісі аяқталды. Екі тілдегі құжаттар дайын.",
672
  "download": "📥 Файлдарды жүктеу",
673
  "system_busy_title": "⏳ Жүйе бос емес",
managed_auth.py CHANGED
@@ -29,39 +29,6 @@ from smartpaper_restore_bridge import edge_bridge
29
  EDGE_ASSERTION_COOKIE_NAME = "smartpaper_edge_assertion"
30
  EDGE_ASSERTION_AUDIENCE = "smartpaper-streamlit"
31
  ASSERTION_CLOCK_SKEW_SECONDS = 30
32
- EXECUTION_HEARTBEAT_PROOF_VERSION = "sp-execution-heartbeat-v1"
33
-
34
-
35
- def _execution_heartbeat_proof(task_id: str, current_node: str) -> tuple[int, str]:
36
- """Create a short-lived HMAC proof that only the HF backend can mint."""
37
- user = st.session_state.get("sp_user") or {}
38
- numeric_id = user.get("numeric_id")
39
- normalized_task = str(task_id or "").strip()
40
- normalized_node = str(current_node or "")[:240]
41
- if (
42
- not isinstance(numeric_id, int)
43
- or isinstance(numeric_id, bool)
44
- or numeric_id <= 0
45
- or not normalized_task
46
- ):
47
- raise RuntimeError("execution_heartbeat_identity_required")
48
- user_id = str(numeric_id)
49
- issued_at = int(time.time())
50
- task_hash = hashlib.sha256(normalized_task.encode("utf-8")).hexdigest()
51
- user_hash = hashlib.sha256(user_id.encode("utf-8")).hexdigest()
52
- node_hash = hashlib.sha256(normalized_node.encode("utf-8")).hexdigest()
53
- canonical = (
54
- f"{EXECUTION_HEARTBEAT_PROOF_VERSION}:"
55
- f"{task_hash}:{user_hash}:{node_hash}:{issued_at}"
56
- )
57
- signature = hmac.new(
58
- _service_key().encode("utf-8"),
59
- canonical.encode("ascii"),
60
- hashlib.sha256,
61
- ).hexdigest()
62
- return issued_at, signature
63
-
64
-
65
  def _runtime_value(name: str) -> str:
66
  value = os.environ.get(name, "")
67
  if not value:
@@ -162,7 +129,7 @@ def _verify_edge_assertion(token: str) -> dict[str, Any]:
162
 
163
  def _clear_generation_session_state() -> None:
164
  for key in (
165
- "sp_generation_task", "sp_current_node", "sp_last_task_heartbeat",
166
  "sp_generation_idempotency_key", "zip_buffer", "zip_filename",
167
  "is_running", "start_pipeline", "resume_pipeline", "run_mode",
168
  "rag_engine", "up_amj_ref", "up_amj_data", "up_asq_ref",
@@ -212,7 +179,6 @@ def _edge_call(
212
  command_id: str,
213
  payload: dict[str, Any] | None = None,
214
  *,
215
- fire_and_forget: bool = False,
216
  key_prefix: str = "sp_edge",
217
  ) -> dict[str, Any] | None:
218
  return edge_bridge(
@@ -220,7 +186,6 @@ def _edge_call(
220
  command_id=command_id,
221
  api_origin=_api_base(),
222
  payload=payload or {},
223
- fire_and_forget=fire_and_forget,
224
  key=_bridge_key(key_prefix, command_id),
225
  )
226
 
@@ -274,10 +239,18 @@ def get_account_snapshot(*, block: bool = False) -> dict[str, Any] | None:
274
  user = result.get("user")
275
  if isinstance(user, dict):
276
  st.session_state["sp_user"] = dict(user)
 
 
 
 
 
 
 
 
277
  return result
278
 
279
 
280
- def start_generation_task(*, idempotency_key: str, mode: str, interface_language: str) -> dict[str, Any]:
281
  normalized = str(idempotency_key or "").strip()
282
  if not normalized:
283
  raise RuntimeError("idempotency_key_required")
@@ -288,6 +261,7 @@ def start_generation_task(*, idempotency_key: str, mode: str, interface_language
288
  "idempotency_key": normalized,
289
  "mode": str(mode or "").strip().lower(),
290
  "interface_language": str(interface_language or "zh").strip(),
 
291
  },
292
  key_prefix="sp_task_start",
293
  )
@@ -309,29 +283,6 @@ def recover_generation_task() -> dict[str, Any]:
309
  return dict(result)
310
 
311
 
312
- def heartbeat_generation_task(task_id: str, *, current_node: str = "") -> dict[str, Any]:
313
- normalized = str(task_id or "").strip()
314
- if not normalized:
315
- raise RuntimeError("task_id_required")
316
- normalized_node = str(current_node or "")[:240]
317
- issued_at, proof = _execution_heartbeat_proof(normalized, normalized_node)
318
- bucket = int(time.time() // 20)
319
- _edge_call(
320
- "task_heartbeat",
321
- f"heartbeat:{normalized}:{bucket}",
322
- {
323
- "task_id": normalized,
324
- "current_node": normalized_node,
325
- "heartbeat_source": "execution",
326
- "heartbeat_issued_at": issued_at,
327
- "heartbeat_proof": proof,
328
- },
329
- fire_and_forget=True,
330
- key_prefix="sp_task_heartbeat",
331
- )
332
- return {"ok": True, "queued": True, "cancel_requested": False}
333
-
334
-
335
  def generation_task_status(task_id: str) -> dict[str, Any]:
336
  normalized = str(task_id or "").strip()
337
  command_id = f"status:{normalized}:{int(time.time() // 5)}"
@@ -342,57 +293,10 @@ def generation_task_status(task_id: str) -> dict[str, Any]:
342
  st.stop()
343
  if not result.get("ok"):
344
  _raise_edge_error(result, "task_status_failed")
345
- return dict(result)
346
-
347
-
348
- def finish_generation_task(
349
- task_id: str,
350
- *,
351
- status: str,
352
- safe_error_code: str = "",
353
- safe_error_detail: str = "",
354
- current_node: str = "",
355
- ) -> dict[str, Any]:
356
- """Finalize a task only after the browser receives Worker/D1 confirmation."""
357
- normalized_task = str(task_id or "").strip()
358
- normalized_status = str(status or "").strip().lower()
359
- if not normalized_task:
360
- raise RuntimeError("task_id_required")
361
- if normalized_status not in {"completed", "failed", "cancelled", "resumable"}:
362
- raise RuntimeError("invalid_finish_status")
363
- command_id = f"finish:{normalized_task}:{normalized_status}"
364
- result = _edge_call(
365
- "task_finish",
366
- command_id,
367
- {
368
- "task_id": normalized_task,
369
- "status": normalized_status,
370
- "safe_error_code": str(safe_error_code or "")[:80],
371
- "safe_error_detail": str(safe_error_detail or "")[:800],
372
- "current_node": str(current_node or "")[:240],
373
- },
374
- fire_and_forget=False,
375
- key_prefix="sp_task_finish",
376
- )
377
- if result is None:
378
- st.stop()
379
- if not result.get("ok"):
380
- _raise_edge_error(result, "task_finish_failed")
381
- task = dict(result.get("task") or {})
382
- reservation = dict(result.get("reservation") or {})
383
- if str(task.get("task_id") or "") != normalized_task:
384
- raise RuntimeError("task_finish_confirmation_mismatch")
385
- if str(task.get("status") or "").lower() != normalized_status:
386
- raise RuntimeError("task_finish_status_not_confirmed")
387
- expected_reservation = {
388
- "completed": "committed",
389
- "failed": "released",
390
- "cancelled": "released",
391
- "resumable": "reserved",
392
- }[normalized_status]
393
- if str(reservation.get("status") or "").lower() != expected_reservation:
394
- raise RuntimeError("task_quota_finalization_not_confirmed")
395
  _merge_quota_into_session(result)
 
 
 
396
  return dict(result)
397
 
398
 
@@ -404,7 +308,6 @@ def cancel_generation_task(task_id: str | None = None) -> dict[str, Any]:
404
  "task_cancel",
405
  command_id,
406
  {"task_id": normalized} if normalized else {},
407
- fire_and_forget=False,
408
  key_prefix="sp_task_cancel",
409
  )
410
  if result is None:
 
29
  EDGE_ASSERTION_COOKIE_NAME = "smartpaper_edge_assertion"
30
  EDGE_ASSERTION_AUDIENCE = "smartpaper-streamlit"
31
  ASSERTION_CLOCK_SKEW_SECONDS = 30
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
32
  def _runtime_value(name: str) -> str:
33
  value = os.environ.get(name, "")
34
  if not value:
 
129
 
130
  def _clear_generation_session_state() -> None:
131
  for key in (
132
+ "sp_generation_task", "sp_current_node",
133
  "sp_generation_idempotency_key", "zip_buffer", "zip_filename",
134
  "is_running", "start_pipeline", "resume_pipeline", "run_mode",
135
  "rag_engine", "up_amj_ref", "up_amj_data", "up_asq_ref",
 
179
  command_id: str,
180
  payload: dict[str, Any] | None = None,
181
  *,
 
182
  key_prefix: str = "sp_edge",
183
  ) -> dict[str, Any] | None:
184
  return edge_bridge(
 
186
  command_id=command_id,
187
  api_origin=_api_base(),
188
  payload=payload or {},
 
189
  key=_bridge_key(key_prefix, command_id),
190
  )
191
 
 
239
  user = result.get("user")
240
  if isinstance(user, dict):
241
  st.session_state["sp_user"] = dict(user)
242
+ task = result.get("task")
243
+ latest_task = result.get("latest_task")
244
+ if isinstance(task, dict) and task.get("task_id"):
245
+ st.session_state["sp_generation_task"] = dict(task)
246
+ elif isinstance(latest_task, dict) and latest_task.get("task_id"):
247
+ st.session_state["sp_generation_task"] = dict(latest_task)
248
+ else:
249
+ st.session_state["sp_generation_task"] = None
250
  return result
251
 
252
 
253
+ def start_generation_task(*, idempotency_key: str, mode: str, interface_language: str, target_words: int) -> dict[str, Any]:
254
  normalized = str(idempotency_key or "").strip()
255
  if not normalized:
256
  raise RuntimeError("idempotency_key_required")
 
261
  "idempotency_key": normalized,
262
  "mode": str(mode or "").strip().lower(),
263
  "interface_language": str(interface_language or "zh").strip(),
264
+ "target_words": int(target_words),
265
  },
266
  key_prefix="sp_task_start",
267
  )
 
283
  return dict(result)
284
 
285
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
286
  def generation_task_status(task_id: str) -> dict[str, Any]:
287
  normalized = str(task_id or "").strip()
288
  command_id = f"status:{normalized}:{int(time.time() // 5)}"
 
293
  st.stop()
294
  if not result.get("ok"):
295
  _raise_edge_error(result, "task_status_failed")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
296
  _merge_quota_into_session(result)
297
+ task = result.get("task")
298
+ if isinstance(task, dict):
299
+ st.session_state["sp_generation_task"] = dict(task)
300
  return dict(result)
301
 
302
 
 
308
  "task_cancel",
309
  command_id,
310
  {"task_id": normalized} if normalized else {},
 
311
  key_prefix="sp_task_cancel",
312
  )
313
  if result is None:
smartpaper_restore_bridge/__init__.py CHANGED
@@ -41,7 +41,6 @@ def restore_bridge(
41
  expires_at=str(expires_at or ""),
42
  api_origin="",
43
  payload={},
44
- fire_and_forget=False,
45
  height=0,
46
  default=None,
47
  key=key,
@@ -55,16 +54,13 @@ def edge_bridge(
55
  command_id: str,
56
  api_origin: str,
57
  payload: dict[str, Any] | None = None,
58
- fire_and_forget: bool = False,
59
  key: str = "sp_edge_bridge",
60
  ) -> dict[str, Any] | None:
61
  """Execute one authenticated same-origin Worker command in the browser.
62
 
63
- The first synchronous invocation usually returns ``None``. Streamlit reruns
64
- after the browser completes the request and then returns the result. Only
65
- non-terminal heartbeat calls may use ``fire_and_forget=True``. Task
66
- completion, failure, cancellation, payment, and account mutations remain
67
- synchronous so Python receives the Worker/D1 postcondition.
68
  """
69
  value = _COMPONENT(
70
  action=str(action or ""),
@@ -73,7 +69,6 @@ def edge_bridge(
73
  expires_at="",
74
  api_origin=str(api_origin or ""),
75
  payload=dict(payload or {}),
76
- fire_and_forget=bool(fire_and_forget),
77
  height=0,
78
  default=None,
79
  key=key,
 
41
  expires_at=str(expires_at or ""),
42
  api_origin="",
43
  payload={},
 
44
  height=0,
45
  default=None,
46
  key=key,
 
54
  command_id: str,
55
  api_origin: str,
56
  payload: dict[str, Any] | None = None,
 
57
  key: str = "sp_edge_bridge",
58
  ) -> dict[str, Any] | None:
59
  """Execute one authenticated same-origin Worker command in the browser.
60
 
61
+ R3-Free keeps the browser bridge synchronous and limited to user-facing
62
+ operations. Execution liveness and terminal success/failure are owned by
63
+ the independent Python executor, never by this component.
 
 
64
  """
65
  value = _COMPONENT(
66
  action=str(action or ""),
 
69
  expires_at="",
70
  api_origin=str(api_origin or ""),
71
  payload=dict(payload or {}),
 
72
  height=0,
73
  default=None,
74
  key=key,
smartpaper_restore_bridge/frontend/index.html CHANGED
@@ -15,20 +15,14 @@
15
 
16
  const RESTORE_KEY = "smartpaper.restore_token.v1";
17
  const LEGACY_COOKIE_NAME = "__Host-smartpaper_restore";
18
- const ACTIVE_TASK_KEY = "smartpaper.edge.active_task.v3";
19
- const PENDING_OPS_KEY = "smartpaper.edge.pending_ops.v3";
20
  const RESULT_PREFIX = "smartpaper.edge.result.v3:";
21
  const MAX_TOKEN_LENGTH = 4096;
22
  const MAX_RESULT_AGE_MS = 10 * 60 * 1000;
23
- const HEARTBEAT_MS = 25000;
24
  const REQUEST_TIMEOUT_MS = 20000;
25
  let lastResponseSignature = "";
26
- let heartbeatTimer = null;
27
 
28
  const ACTION_PATHS = Object.freeze({
29
  task_start: "/edge/task/start",
30
- task_heartbeat: "/edge/task/heartbeat",
31
- task_finish: "/edge/task/finish",
32
  task_cancel: "/edge/task/cancel",
33
  task_status: "/edge/task/status",
34
  task_recover: "/edge/task/recover",
@@ -168,48 +162,6 @@
168
  } finally { window.clearTimeout(timer); }
169
  }
170
 
171
- function pendingOperations() {
172
- const value = readJson(PENDING_OPS_KEY, []);
173
- return Array.isArray(value) ? value : [];
174
- }
175
-
176
- function storePending(items) { writeJson(PENDING_OPS_KEY, items.slice(-30)); }
177
-
178
- function queuePending(operation) {
179
- const items = pendingOperations();
180
- if (!items.some(item => item.command_id === operation.command_id)) {
181
- items.push(operation); storePending(items);
182
- }
183
- }
184
-
185
- function removePending(commandId) {
186
- storePending(pendingOperations().filter(item => item.command_id !== commandId));
187
- }
188
-
189
- async function drainPending() {
190
- for (const operation of pendingOperations()) {
191
- const result = await postJson(operation.api_origin, operation.path, operation.payload);
192
- if (result.ok) {
193
- removePending(operation.command_id);
194
- } else if ([401,403].includes(Number(result.http_status || 0))) {
195
- // Authentication may be restored by a new Alipay session. Keep the
196
- // transient heartbeat operation out of the terminal task state.
197
- removePending(operation.command_id);
198
- }
199
- }
200
- }
201
-
202
- function startBackgroundWork() {
203
- // R2: the browser may retry commands that Python already emitted, but it
204
- // must never manufacture task liveness merely because the tab remains open.
205
- // Only real backend execution progress is allowed to enqueue task_heartbeat.
206
- if (heartbeatTimer) return;
207
- heartbeatTimer = window.setInterval(() => {
208
- drainPending();
209
- }, HEARTBEAT_MS);
210
- drainPending();
211
- }
212
-
213
  function cachedResult(commandId) {
214
  const cached = readJson(RESULT_PREFIX + commandId, null);
215
  if (!cached || Date.now() - Number(cached.saved_at || 0) > MAX_RESULT_AGE_MS) return null;
@@ -226,13 +178,6 @@
226
  const apiOrigin = normalizeOrigin(args.api_origin || "");
227
  if (!apiOrigin) return { ok:false, error:"invalid_api_origin", action, command_id:commandId };
228
  const payload = args.payload && typeof args.payload === "object" ? args.payload : {};
229
- const fireAndForget = Boolean(args.fire_and_forget);
230
-
231
- if (fireAndForget) {
232
- queuePending({ command_id:commandId, api_origin:apiOrigin, path, payload });
233
- drainPending();
234
- return { ok:true, queued:true, action, command_id:commandId };
235
- }
236
 
237
  const cached = cachedResult(commandId);
238
  if (cached) return cached;
@@ -241,12 +186,6 @@
241
  // be retryable on the next Streamlit rerun with the same idempotency key.
242
  if (result.ok) saveResult(commandId, result);
243
 
244
- if (action === "task_start" && result.ok && result.task && result.task.task_id) {
245
- writeJson(ACTIVE_TASK_KEY, { api_origin:apiOrigin, task_id:result.task.task_id, current_node:"generation" });
246
- }
247
- if (["task_finish", "task_cancel"].includes(action) && result.ok) {
248
- window.localStorage.removeItem(ACTIVE_TASK_KEY);
249
- }
250
  return result;
251
  }
252
 
@@ -272,7 +211,6 @@
272
  window.addEventListener("message", onRender);
273
  sendMessage("streamlit:componentReady", { apiVersion:1 });
274
  setFrameHeight();
275
- startBackgroundWork();
276
  })();
277
  </script>
278
  </body>
 
15
 
16
  const RESTORE_KEY = "smartpaper.restore_token.v1";
17
  const LEGACY_COOKIE_NAME = "__Host-smartpaper_restore";
 
 
18
  const RESULT_PREFIX = "smartpaper.edge.result.v3:";
19
  const MAX_TOKEN_LENGTH = 4096;
20
  const MAX_RESULT_AGE_MS = 10 * 60 * 1000;
 
21
  const REQUEST_TIMEOUT_MS = 20000;
22
  let lastResponseSignature = "";
 
23
 
24
  const ACTION_PATHS = Object.freeze({
25
  task_start: "/edge/task/start",
 
 
26
  task_cancel: "/edge/task/cancel",
27
  task_status: "/edge/task/status",
28
  task_recover: "/edge/task/recover",
 
162
  } finally { window.clearTimeout(timer); }
163
  }
164
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
165
  function cachedResult(commandId) {
166
  const cached = readJson(RESULT_PREFIX + commandId, null);
167
  if (!cached || Date.now() - Number(cached.saved_at || 0) > MAX_RESULT_AGE_MS) return null;
 
178
  const apiOrigin = normalizeOrigin(args.api_origin || "");
179
  if (!apiOrigin) return { ok:false, error:"invalid_api_origin", action, command_id:commandId };
180
  const payload = args.payload && typeof args.payload === "object" ? args.payload : {};
 
 
 
 
 
 
 
181
 
182
  const cached = cachedResult(commandId);
183
  if (cached) return cached;
 
186
  // be retryable on the next Streamlit rerun with the same idempotency key.
187
  if (result.ok) saveResult(commandId, result);
188
 
 
 
 
 
 
 
189
  return result;
190
  }
191
 
 
211
  window.addEventListener("message", onRender);
212
  sendMessage("streamlit:componentReady", { apiVersion:1 });
213
  setFrameHeight();
 
214
  })();
215
  </script>
216
  </body>
task_executor.py ADDED
@@ -0,0 +1,391 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # -*- coding: utf-8 -*-
2
+ """SmartPaper R3-Free independent task executor.
3
+
4
+ Launched by Streamlit as a detached subprocess. It claims a D1 task with a
5
+ server-to-server service key, runs the headless generation pipeline, renews the
6
+ lease only while real execution activity is recent, and atomically asks the
7
+ Worker to commit/release quota at terminal completion.
8
+ """
9
+ from __future__ import annotations
10
+
11
+ import argparse
12
+ import json
13
+ import logging
14
+ import os
15
+ import re
16
+ import sys
17
+ import threading
18
+ import time
19
+ from pathlib import Path
20
+ from urllib.parse import urlsplit
21
+
22
+ import requests
23
+
24
+ from config import APP_CONFIG
25
+ from core.generation_runner import GenerationRunner
26
+ from core.pipeline_agent import (
27
+ DeepSeekAPIError, DeepSeekAuthenticationError, DeepSeekBalanceError, DeepSeekRequestError,
28
+ )
29
+
30
+ logging.basicConfig(
31
+ level=logging.INFO,
32
+ format="%(asctime)s [%(levelname)s] %(message)s",
33
+ handlers=[logging.StreamHandler(sys.stdout)],
34
+ )
35
+
36
+
37
+ def _safe(value, limit=500):
38
+ text = str(value or "")
39
+ text = re.sub(r"(?i)(api[_ -]?key|authorization|bearer|token)\s*[:=]\s*[^\s,;]+", r"\1=[redacted]", text)
40
+ return text[:limit]
41
+
42
+
43
+ def _api_origin() -> str:
44
+ value = str(os.environ.get("SMARTPAPER_AUTH_API") or "").strip().rstrip("/")
45
+ parsed = urlsplit(value)
46
+ if parsed.scheme != "https" or not parsed.netloc or parsed.path not in ("", "/") or parsed.query or parsed.fragment:
47
+ raise RuntimeError("SMARTPAPER_AUTH_API_invalid")
48
+ return f"https://{parsed.netloc}"
49
+
50
+
51
+ def _service_key() -> str:
52
+ value = str(os.environ.get("SMARTPAPER_SERVICE_KEY") or "").strip()
53
+ if not value:
54
+ raise RuntimeError("SMARTPAPER_SERVICE_KEY_missing")
55
+ return value
56
+
57
+
58
+ class ExecutorApiError(RuntimeError):
59
+ def __init__(self, error: str, status_code: int = 0, payload: dict | None = None):
60
+ super().__init__(str(error or "executor_api_error"))
61
+ self.error = str(error or "executor_api_error")
62
+ self.status_code = int(status_code or 0)
63
+ self.payload = dict(payload or {})
64
+
65
+
66
+ def _is_recoverable_generation_failure(exc: Exception) -> bool:
67
+ """Return True only for transient execution failures that can use checkpoints.
68
+
69
+ Authentication/balance/request-validation failures are terminal and release
70
+ quota immediately. Transport/stream failures can be resumed from the last
71
+ durable task checkpoint without consuming a second quota unit.
72
+ """
73
+ if isinstance(exc, ExecutorApiError):
74
+ return exc.status_code == 429 or exc.status_code >= 500
75
+ if isinstance(exc, (DeepSeekAuthenticationError, DeepSeekBalanceError, DeepSeekRequestError)):
76
+ return False
77
+ if isinstance(exc, (requests.exceptions.Timeout, requests.exceptions.ConnectionError)):
78
+ return True
79
+ if isinstance(exc, DeepSeekAPIError):
80
+ return True
81
+ text = str(exc or "").lower()
82
+ transient_markers = (
83
+ "connection", "timed out", "timeout", "temporarily unavailable",
84
+ "streaming failed after", "api failed after", "stream was interrupted",
85
+ "insufficient eligible references",
86
+ )
87
+ return any(marker in text for marker in transient_markers)
88
+
89
+
90
+ class ExecutorApi:
91
+ def __init__(self):
92
+ self.origin = _api_origin()
93
+ self.headers = {
94
+ "Authorization": f"Bearer {_service_key()}",
95
+ "Accept": "application/json",
96
+ "Content-Type": "application/json",
97
+ "User-Agent": "SmartPaper-R3-Free-Executor/1.0",
98
+ }
99
+
100
+ def post(self, path: str, payload: dict, *, timeout=20) -> dict:
101
+ response = requests.post(
102
+ self.origin + path,
103
+ headers=self.headers,
104
+ json=payload,
105
+ timeout=timeout,
106
+ )
107
+ try:
108
+ data = response.json()
109
+ except Exception as exc:
110
+ raise RuntimeError(f"executor_worker_non_json_http_{response.status_code}") from exc
111
+ if not response.ok or data.get("ok") is False:
112
+ error = str(data.get("error") or data.get("reason") or f"HTTP_{response.status_code}")
113
+ raise ExecutorApiError(error, response.status_code, data)
114
+ return data
115
+
116
+ def claim(self, *, task_id: str, user_id: int, request_hash: str) -> dict:
117
+ return self.post(
118
+ "/executor/task/claim",
119
+ {"task_id": task_id, "user_id": user_id, "request_hash": request_hash},
120
+ )
121
+
122
+ def heartbeat(self, payload: dict) -> dict:
123
+ return self.post("/executor/task/heartbeat", payload)
124
+
125
+ def finish(self, payload: dict) -> dict:
126
+ return self.post("/executor/task/finish", payload, timeout=30)
127
+
128
+
129
+ class ExecutionState:
130
+ def __init__(self, *, task_id: str, user_id: int, epoch: int, instance_id: str, api: ExecutorApi):
131
+ self.task_id = task_id
132
+ self.user_id = int(user_id)
133
+ self.epoch = int(epoch)
134
+ self.instance_id = str(instance_id)
135
+ self.api = api
136
+ self._lock = threading.Lock()
137
+ self._last_activity = time.monotonic()
138
+ self._progress_pct = 0
139
+ self._current_node = "executor_claimed"
140
+ self._checkpoint_seq = 0
141
+ self._checkpoint_stage = ""
142
+ self._ds_calls = 0
143
+ self._s2_calls = 0
144
+ self._oa_calls = 0
145
+ self._cancel = False
146
+ self._stop = threading.Event()
147
+ self._thread = None
148
+
149
+ def touch(self, *, progress_pct=None, current_node=None, checkpoint_seq=None, checkpoint_stage=None, ds_calls=None, s2_calls=None, oa_calls=None, activity_only=False):
150
+ with self._lock:
151
+ self._last_activity = time.monotonic()
152
+ if not activity_only:
153
+ if progress_pct is not None:
154
+ self._progress_pct = max(self._progress_pct, max(0, min(100, int(progress_pct))))
155
+ if current_node:
156
+ self._current_node = str(current_node)[:240]
157
+ if checkpoint_seq is not None and int(checkpoint_seq) >= self._checkpoint_seq:
158
+ self._checkpoint_seq = int(checkpoint_seq)
159
+ self._checkpoint_stage = str(checkpoint_stage or self._checkpoint_stage)[:120]
160
+ if ds_calls is not None:
161
+ self._ds_calls = max(self._ds_calls, int(ds_calls))
162
+ if s2_calls is not None:
163
+ self._s2_calls = max(self._s2_calls, int(s2_calls))
164
+ if oa_calls is not None:
165
+ self._oa_calls = max(self._oa_calls, int(oa_calls))
166
+
167
+ def cancelled(self) -> bool:
168
+ with self._lock:
169
+ return bool(self._cancel)
170
+
171
+ def start_heartbeat(self):
172
+ self._thread = threading.Thread(target=self._heartbeat_loop, name="smartpaper-executor-heartbeat", daemon=True)
173
+ self._thread.start()
174
+
175
+ def stop_heartbeat(self):
176
+ self._stop.set()
177
+ if self._thread and self._thread.is_alive():
178
+ self._thread.join(timeout=2)
179
+
180
+ def _payload(self):
181
+ with self._lock:
182
+ return {
183
+ "task_id": self.task_id,
184
+ "user_id": self.user_id,
185
+ "execution_epoch": self.epoch,
186
+ "executor_instance_id": self.instance_id,
187
+ "current_node": self._current_node,
188
+ "progress_pct": self._progress_pct,
189
+ "checkpoint_seq": self._checkpoint_seq,
190
+ "checkpoint_stage": self._checkpoint_stage,
191
+ "ds_calls": self._ds_calls,
192
+ "s2_calls": self._s2_calls,
193
+ "oa_calls": self._oa_calls,
194
+ }, time.monotonic() - self._last_activity
195
+
196
+ def heartbeat_once(self):
197
+ payload, _ = self._payload()
198
+ result = self.api.heartbeat(payload)
199
+ if result.get("cancel_requested"):
200
+ with self._lock:
201
+ self._cancel = True
202
+ return result
203
+
204
+ def _heartbeat_loop(self):
205
+ interval = int(APP_CONFIG.get("executor_heartbeat_interval_seconds", 15))
206
+ grace = int(APP_CONFIG.get("executor_activity_grace_seconds", 45))
207
+ # Claim itself already set the first heartbeat; delay before the next one.
208
+ while not self._stop.wait(interval):
209
+ payload, idle_seconds = self._payload()
210
+ if idle_seconds > grace:
211
+ logging.warning(
212
+ "[SP_EXECUTOR] heartbeat paused task_id=%s idle_s=%.1f node=%s",
213
+ self.task_id, idle_seconds, payload.get("current_node"),
214
+ )
215
+ continue
216
+ try:
217
+ result = self.api.heartbeat(payload)
218
+ if result.get("cancel_requested"):
219
+ with self._lock:
220
+ self._cancel = True
221
+ return
222
+ except ExecutorApiError as exc:
223
+ if exc.error in {
224
+ "executor_superseded", "generation_lock_lost",
225
+ "task_already_finalized", "task_cancel_requested",
226
+ } or exc.status_code in {404, 409, 410}:
227
+ with self._lock:
228
+ self._cancel = True
229
+ logging.warning(
230
+ "[SP_EXECUTOR] executor_no_longer_authoritative task_id=%s error=%s",
231
+ self.task_id, _safe(exc.error, 180),
232
+ )
233
+ return
234
+ logging.warning(
235
+ "[SP_EXECUTOR] heartbeat_error task_id=%s error=%s",
236
+ self.task_id, _safe(exc, 180),
237
+ )
238
+ except Exception as exc:
239
+ # A transport outage alone does not authorize another local
240
+ # writer. D1 lease/epoch ownership remains authoritative.
241
+ logging.warning(
242
+ "[SP_EXECUTOR] heartbeat_error task_id=%s error=%s",
243
+ self.task_id, _safe(exc, 180),
244
+ )
245
+
246
+
247
+ def _load_request(task_dir: Path) -> dict:
248
+ path = task_dir / "request.json"
249
+ if not path.is_file():
250
+ raise RuntimeError("request_manifest_missing")
251
+ value = json.loads(path.read_text(encoding="utf-8"))
252
+ if not isinstance(value, dict):
253
+ raise RuntimeError("request_manifest_invalid")
254
+ return value
255
+
256
+
257
+ def main() -> int:
258
+ parser = argparse.ArgumentParser()
259
+ parser.add_argument("--task-dir", required=True)
260
+ parser.add_argument("--task-id", required=True)
261
+ parser.add_argument("--user-id", required=True, type=int)
262
+ args = parser.parse_args()
263
+
264
+ task_dir = Path(args.task_dir).resolve()
265
+ request_data = _load_request(task_dir)
266
+ if str(request_data.get("task_id") or "") != args.task_id:
267
+ raise RuntimeError("request_task_id_mismatch")
268
+ if int(request_data.get("user_id") or 0) != int(args.user_id):
269
+ raise RuntimeError("request_user_id_mismatch")
270
+ request_hash = str(request_data.get("request_hash") or "").strip()
271
+ if not request_hash:
272
+ raise RuntimeError("request_hash_missing")
273
+
274
+ api = ExecutorApi()
275
+ claim = api.claim(task_id=args.task_id, user_id=args.user_id, request_hash=request_hash)
276
+ task = dict(claim.get("task") or {})
277
+ execution = dict(claim.get("execution") or {})
278
+ epoch = int(execution.get("execution_epoch") or task.get("execution_epoch") or 0)
279
+ instance_id = str(execution.get("executor_instance_id") or "")
280
+ if epoch <= 0 or not instance_id:
281
+ raise RuntimeError("executor_claim_identity_missing")
282
+ if int(task.get("target_words") or 0) != int(request_data.get("target_words") or 0):
283
+ raise RuntimeError("target_words_claim_mismatch")
284
+
285
+ state = ExecutionState(
286
+ task_id=args.task_id,
287
+ user_id=args.user_id,
288
+ epoch=epoch,
289
+ instance_id=instance_id,
290
+ api=api,
291
+ )
292
+ state.start_heartbeat()
293
+ logging.info(
294
+ "[SP_EXECUTOR] claimed task_id=%s epoch=%s instance=%s",
295
+ args.task_id, epoch, instance_id[:12],
296
+ )
297
+
298
+ # If D1 says a prior checkpoint exists but the free Space's local disk no
299
+ # longer has it (container rebuild), do not pretend resume is possible.
300
+ remote_checkpoint_seq = int(task.get("checkpoint_seq") or 0)
301
+ local_checkpoints = list((task_dir / "checkpoints").glob("checkpoint_*.json")) if (task_dir / "checkpoints").is_dir() else []
302
+ if remote_checkpoint_seq > 0 and not local_checkpoints:
303
+ try:
304
+ api.finish(
305
+ {
306
+ "task_id": args.task_id,
307
+ "user_id": args.user_id,
308
+ "execution_epoch": epoch,
309
+ "executor_instance_id": instance_id,
310
+ "status": "failed",
311
+ "safe_error_code": "E_CHECKPOINT_MISSING",
312
+ "safe_error_detail": "Checkpoint metadata exists but local checkpoint files are unavailable after runtime replacement; quota released.",
313
+ "current_node": "checkpoint_missing",
314
+ }
315
+ )
316
+ finally:
317
+ state.stop_heartbeat()
318
+ return 2
319
+
320
+ try:
321
+ runner = GenerationRunner(
322
+ task_dir=task_dir,
323
+ request_data=request_data,
324
+ execution_epoch=epoch,
325
+ progress_callback=state.touch,
326
+ cancel_checker=state.cancelled,
327
+ )
328
+ result = runner.run()
329
+ state.touch(progress_pct=100, current_node="package_validated")
330
+ # One last accepted heartbeat publishes the final checkpoint metadata
331
+ # before the atomic terminal transaction.
332
+ try:
333
+ state.heartbeat_once()
334
+ except Exception as exc:
335
+ logging.warning("[SP_EXECUTOR] final_heartbeat_error task_id=%s error=%s", args.task_id, _safe(exc, 180))
336
+ if state.cancelled():
337
+ raise RuntimeError("task_cancel_requested")
338
+ api.finish(
339
+ {
340
+ "task_id": args.task_id,
341
+ "user_id": args.user_id,
342
+ "execution_epoch": epoch,
343
+ "executor_instance_id": instance_id,
344
+ "status": "completed",
345
+ "current_node": "package_validated",
346
+ "output_filename": str(result.get("output_filename") or "")[:240],
347
+ }
348
+ )
349
+ logging.info("[SP_EXECUTOR] completed task_id=%s epoch=%s", args.task_id, epoch)
350
+ return 0
351
+ except Exception as exc:
352
+ if state.cancelled():
353
+ logging.warning(
354
+ "[SP_EXECUTOR] stopped_non_authoritative task_id=%s epoch=%s",
355
+ args.task_id, epoch,
356
+ )
357
+ return 3
358
+ if _is_recoverable_generation_failure(exc):
359
+ # Do not terminally fail or release quota. Stopping the real-execution
360
+ # heartbeat lets the Worker move this epoch to `resumable`; the next
361
+ # epoch continues from the latest immutable local checkpoint.
362
+ logging.exception(
363
+ "[SP_EXECUTOR] recoverable_interruption task_id=%s epoch=%s error=%s",
364
+ args.task_id, epoch, _safe(exc, 240),
365
+ )
366
+ return 75
367
+ logging.exception("[SP_EXECUTOR] failed task_id=%s epoch=%s error=%s", args.task_id, epoch, _safe(exc, 240))
368
+ try:
369
+ api.finish(
370
+ {
371
+ "task_id": args.task_id,
372
+ "user_id": args.user_id,
373
+ "execution_epoch": epoch,
374
+ "executor_instance_id": instance_id,
375
+ "status": "failed",
376
+ "safe_error_code": "E_GENERATION_FAILED",
377
+ "safe_error_detail": _safe(exc, 800),
378
+ "current_node": "generation_failed",
379
+ }
380
+ )
381
+ except Exception as finish_exc:
382
+ # A newer epoch or user cancellation may already own/finalize the
383
+ # task. Never override it from a stale executor.
384
+ logging.warning("[SP_EXECUTOR] failure_finalize_rejected task_id=%s error=%s", args.task_id, _safe(finish_exc, 220))
385
+ return 1
386
+ finally:
387
+ state.stop_heartbeat()
388
+
389
+
390
+ if __name__ == "__main__":
391
+ raise SystemExit(main())
utils/file_parser.py CHANGED
@@ -1,5 +1,13 @@
1
  # -*- coding: utf-8 -*-
2
- import streamlit as st
 
 
 
 
 
 
 
 
3
  from docx import Document
4
  from config import APP_CONFIG
5
 
@@ -10,65 +18,114 @@ except ImportError:
10
  PANDAS_AVAILABLE = False
11
 
12
  try:
13
- import fitz # PyMuPDF 的导入名称是 fitz
14
  PYMUPDF_AVAILABLE = True
15
  except ImportError:
16
  PYMUPDF_AVAILABLE = False
17
 
18
- def extract_text_from_files(uploaded_files, max_chars=None):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
19
  if max_chars is None:
20
  max_chars = APP_CONFIG.get("max_extract_chars", 150000)
21
-
22
  extracted_text = ""
23
- for file in uploaded_files:
 
 
24
  try:
25
- file.seek(0)
26
- if file.name.lower().endswith('.pdf') and PYMUPDF_AVAILABLE:
27
- # 从内存中读取文件字节流
28
- pdf_bytes = file.read()
29
- # 使用上下文管理器确保内存被及时释放
30
- with fitz.open(stream=pdf_bytes, filetype="pdf") as pdf_doc:
31
  for page in pdf_doc:
32
  text = page.get_text()
33
- if text:
34
  extracted_text += text + "\n"
35
- if len(extracted_text) > max_chars:
36
  break
37
- elif file.name.lower().endswith(('.docx', '.doc')):
38
- doc = Document(file)
 
39
  for para in doc.paragraphs:
40
  extracted_text += para.text + "\n"
41
- if len(extracted_text) > max_chars:
42
  break
43
  else:
44
- content = file.getvalue().decode('utf-8', errors='ignore')
45
- extracted_text += content + "\n"
46
-
47
  if len(extracted_text) > max_chars:
48
- st.warning(f"⚠️ 文件 {file.name} 已安全截断至 {max_chars} 字符。")
49
  break
50
- except Exception as e:
51
- st.warning(f"⚠️ 解析失败: {e}")
52
  finally:
53
- file.seek(0)
 
 
 
 
54
  return extracted_text[:max_chars]
55
 
 
56
  def extract_data_from_files(uploaded_file, max_chars=20000):
57
- if not uploaded_file:
58
  return ""
59
-
 
 
60
  try:
61
- uploaded_file.seek(0)
62
  if PANDAS_AVAILABLE:
63
- df = pd.read_csv(uploaded_file) if uploaded_file.name.lower().endswith('.csv') else pd.read_excel(uploaded_file)
 
 
 
 
 
 
64
  if len(df) > 100:
65
- st.warning(f"⚠️ 数据表超长,安全截取前100行。")
66
  df = df.head(100)
67
  return df.to_markdown(index=False)
68
- else:
69
- return uploaded_file.getvalue().decode('utf-8', errors='ignore')[:max_chars]
70
- except Exception as e:
71
- st.warning(f"⚠️ 解析失败: {e}")
72
  return ""
73
  finally:
74
- uploaded_file.seek(0)
 
 
 
 
 
1
  # -*- coding: utf-8 -*-
2
+ """Headless file parsing helpers used by both Streamlit and R3-Free executor."""
3
+ from __future__ import annotations
4
+
5
+ import io
6
+ import logging
7
+ import os
8
+ from pathlib import Path
9
+ from typing import BinaryIO, Iterable
10
+
11
  from docx import Document
12
  from config import APP_CONFIG
13
 
 
18
  PANDAS_AVAILABLE = False
19
 
20
  try:
21
+ import pymupdf
22
  PYMUPDF_AVAILABLE = True
23
  except ImportError:
24
  PYMUPDF_AVAILABLE = False
25
 
26
+
27
+ def _name(file) -> str:
28
+ if isinstance(file, (str, os.PathLike, Path)):
29
+ return os.fspath(file)
30
+ return str(getattr(file, "name", "uploaded_file"))
31
+
32
+
33
+ def _read_bytes(file) -> bytes:
34
+ if isinstance(file, (str, os.PathLike, Path)):
35
+ return Path(file).read_bytes()
36
+ try:
37
+ file.seek(0)
38
+ except Exception:
39
+ pass
40
+ if hasattr(file, "getvalue"):
41
+ data = file.getvalue()
42
+ else:
43
+ data = file.read()
44
+ try:
45
+ file.seek(0)
46
+ except Exception:
47
+ pass
48
+ return bytes(data or b"")
49
+
50
+
51
+ def _binary_stream(file) -> io.BytesIO | BinaryIO:
52
+ if isinstance(file, (str, os.PathLike, Path)):
53
+ return io.BytesIO(Path(file).read_bytes())
54
+ try:
55
+ file.seek(0)
56
+ except Exception:
57
+ pass
58
+ return file
59
+
60
+
61
+ def extract_text_from_files(uploaded_files: Iterable, max_chars=None):
62
  if max_chars is None:
63
  max_chars = APP_CONFIG.get("max_extract_chars", 150000)
64
+
65
  extracted_text = ""
66
+ for file in uploaded_files or []:
67
+ filename = _name(file)
68
+ lower_name = filename.lower()
69
  try:
70
+ if lower_name.endswith(".pdf") and PYMUPDF_AVAILABLE:
71
+ pdf_bytes = _read_bytes(file)
72
+ with pymupdf.open(stream=pdf_bytes, filetype="pdf") as pdf_doc:
 
 
 
73
  for page in pdf_doc:
74
  text = page.get_text()
75
+ if text:
76
  extracted_text += text + "\n"
77
+ if len(extracted_text) > max_chars:
78
  break
79
+ elif lower_name.endswith((".docx", ".doc")):
80
+ stream = _binary_stream(file)
81
+ doc = Document(stream)
82
  for para in doc.paragraphs:
83
  extracted_text += para.text + "\n"
84
+ if len(extracted_text) > max_chars:
85
  break
86
  else:
87
+ extracted_text += _read_bytes(file).decode("utf-8", errors="ignore") + "\n"
88
+
 
89
  if len(extracted_text) > max_chars:
90
+ logging.warning("File %s was truncated safely to %s characters.", filename, max_chars)
91
  break
92
+ except Exception as exc:
93
+ logging.warning("File parse failed for %s: %s", filename, exc)
94
  finally:
95
+ if not isinstance(file, (str, os.PathLike, Path)):
96
+ try:
97
+ file.seek(0)
98
+ except Exception:
99
+ pass
100
  return extracted_text[:max_chars]
101
 
102
+
103
  def extract_data_from_files(uploaded_file, max_chars=20000):
104
+ if not uploaded_file:
105
  return ""
106
+
107
+ filename = _name(uploaded_file)
108
+ lower_name = filename.lower()
109
  try:
 
110
  if PANDAS_AVAILABLE:
111
+ source = os.fspath(uploaded_file) if isinstance(uploaded_file, (str, os.PathLike, Path)) else uploaded_file
112
+ if not isinstance(uploaded_file, (str, os.PathLike, Path)):
113
+ try:
114
+ uploaded_file.seek(0)
115
+ except Exception:
116
+ pass
117
+ df = pd.read_csv(source) if lower_name.endswith(".csv") else pd.read_excel(source)
118
  if len(df) > 100:
119
+ logging.warning("Data file %s truncated safely to first 100 rows.", filename)
120
  df = df.head(100)
121
  return df.to_markdown(index=False)
122
+ return _read_bytes(uploaded_file).decode("utf-8", errors="ignore")[:max_chars]
123
+ except Exception as exc:
124
+ logging.warning("Data parse failed for %s: %s", filename, exc)
 
125
  return ""
126
  finally:
127
+ if not isinstance(uploaded_file, (str, os.PathLike, Path)):
128
+ try:
129
+ uploaded_file.seek(0)
130
+ except Exception:
131
+ pass