Spaces:
Running
Running
Step 4A-6-C-R3.2.5 Generation UX Reference Metrics & Translation Integrity Fix
#15
by GuXiaopeng - opened
- .gitattributes +1 -0
- __pycache__/app.cpython-313.pyc +0 -0
- __pycache__/config.cpython-313.pyc +0 -0
- __pycache__/i18n.cpython-313.pyc +0 -0
- __pycache__/managed_auth.cpython-313.pyc +0 -0
- __pycache__/task_executor.cpython-313.pyc +0 -0
- app.py +91 -30
- core/__pycache__/checkpoint_store.cpython-313.pyc +0 -0
- core/__pycache__/generation_runner.cpython-313.pyc +0 -0
- core/__pycache__/meta_extractor.cpython-313.pyc +0 -0
- core/__pycache__/pipeline_agent.cpython-313.pyc +2 -2
- core/__pycache__/production_guard.cpython-313.pyc +0 -0
- core/__pycache__/qualitative_engine.cpython-313.pyc +0 -0
- core/__pycache__/quantitative_engine.cpython-313.pyc +0 -0
- core/__pycache__/rag_engine.cpython-313.pyc +0 -0
- core/generation_runner.py +29 -13
- core/pipeline_agent.py +153 -8
- smartpaper_restore_bridge/__pycache__/__init__.cpython-313.pyc +0 -0
- smartpaper_restore_bridge/frontend/index.html +47 -6
- utils/__pycache__/docx_builder.cpython-313.pyc +0 -0
- utils/__pycache__/file_parser.cpython-313.pyc +0 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
core/__pycache__/pipeline_agent.cpython-313.pyc filter=lfs diff=lfs merge=lfs -text
|
__pycache__/app.cpython-313.pyc
CHANGED
|
Binary files a/__pycache__/app.cpython-313.pyc and b/__pycache__/app.cpython-313.pyc differ
|
|
|
__pycache__/config.cpython-313.pyc
CHANGED
|
Binary files a/__pycache__/config.cpython-313.pyc and b/__pycache__/config.cpython-313.pyc differ
|
|
|
__pycache__/i18n.cpython-313.pyc
CHANGED
|
Binary files a/__pycache__/i18n.cpython-313.pyc and b/__pycache__/i18n.cpython-313.pyc differ
|
|
|
__pycache__/managed_auth.cpython-313.pyc
CHANGED
|
Binary files a/__pycache__/managed_auth.cpython-313.pyc and b/__pycache__/managed_auth.cpython-313.pyc differ
|
|
|
__pycache__/task_executor.cpython-313.pyc
CHANGED
|
Binary files a/__pycache__/task_executor.cpython-313.pyc and b/__pycache__/task_executor.cpython-313.pyc differ
|
|
|
app.py
CHANGED
|
@@ -595,7 +595,10 @@ def _render_monitor_box(task: dict, lang_code: str) -> None:
|
|
| 595 |
<div class='log-box' id='sp-live-monitor'>
|
| 596 |
<div><b>{tr(lang_code, 'monitor_pipeline')}:</b> <span id='sp-live-status'>{status}</span></div>
|
| 597 |
<div><b>{tr(lang_code, 'monitor_node')}:</b> <span id='sp-live-node'>{node}</span></div>
|
| 598 |
-
<div><b>{tr(lang_code, 'monitor_log')}:</b> checkpoint <span id='sp-live-checkpoint-seq'>{checkpoint_seq}</span> · <span id='sp-live-checkpoint-stage'>{checkpoint_stage}</span></div>
|
|
|
|
|
|
|
|
|
|
| 599 |
<div><b>{tr(lang_code, 'monitor_compute')}:</b></div>
|
| 600 |
<div style='display:flex;gap:8px;'>
|
| 601 |
<span style='background: #fef3c7;padding:4px 8px;border-radius:4px;color:black;'>🧠 DeepSeek: <b id='sp-live-ds'>{ds_calls}</b></span>
|
|
@@ -606,38 +609,85 @@ def _render_monitor_box(task: dict, lang_code: str) -> None:
|
|
| 606 |
""", unsafe_allow_html=True)
|
| 607 |
|
| 608 |
|
| 609 |
-
def _watch_task_status_live(task: dict) -> None:
|
| 610 |
-
"""Start browser-only read polling for
|
| 611 |
|
| 612 |
-
|
| 613 |
-
|
| 614 |
-
|
|
|
|
| 615 |
"""
|
| 616 |
task_id = str(task.get("task_id") or "").strip()
|
| 617 |
status = str(task.get("status") or "")
|
| 618 |
if not task_id or status not in LIVE_TASK_STATUSES:
|
| 619 |
-
return
|
| 620 |
api_origin = str(st.session_state.get("sp_auth_api_origin") or "").strip()
|
| 621 |
if not api_origin:
|
| 622 |
logging.warning("[SP_UI] live_watch_missing_api_origin task_id=%s", task_id)
|
| 623 |
-
return
|
| 624 |
try:
|
| 625 |
watch_result = edge_bridge(
|
| 626 |
action="task_watch",
|
| 627 |
command_id=f"watch:{task_id}",
|
| 628 |
api_origin=api_origin,
|
| 629 |
-
payload={
|
|
|
|
|
|
|
|
|
|
|
|
|
| 630 |
key=f"sp_task_watch_{_safe_key(task_id)}",
|
| 631 |
)
|
| 632 |
boundary_task = watch_result.get("task") if isinstance(watch_result, dict) and watch_result.get("boundary") else None
|
| 633 |
if isinstance(boundary_task, dict) and str(boundary_task.get("task_id") or "") == task_id:
|
| 634 |
-
# Consume the watcher's authoritative boundary snapshot directly so
|
| 635 |
-
# the ordinary 5-second task-status cache cannot delay terminal UI.
|
| 636 |
st.session_state["sp_generation_task"] = dict(boundary_task)
|
| 637 |
st.session_state["sp_last_ui_task_status"] = str(boundary_task.get("status") or "")
|
| 638 |
-
|
|
|
|
|
|
|
| 639 |
except Exception as exc:
|
| 640 |
logging.warning("[SP_UI] live_watch_error task_id=%s error=%s", task_id, type(exc).__name__)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 641 |
|
| 642 |
|
| 643 |
def _render_task_status_once(lang_code: str) -> None:
|
|
@@ -645,7 +695,6 @@ def _render_task_status_once(lang_code: str) -> None:
|
|
| 645 |
task_id = str(task.get("task_id") or "").strip()
|
| 646 |
if not task_id:
|
| 647 |
return
|
| 648 |
-
previous_status = str(st.session_state.get("sp_last_ui_task_status") or str(task.get("status") or ""))
|
| 649 |
status = str(task.get("status") or "")
|
| 650 |
if status in ACTIVE_TASK_STATUSES:
|
| 651 |
try:
|
|
@@ -655,10 +704,19 @@ def _render_task_status_once(lang_code: str) -> None:
|
|
| 655 |
status = str(task.get("status") or "")
|
| 656 |
except Exception as exc:
|
| 657 |
logging.warning("[SP_UI] status_poll_error task_id=%s error=%s", task_id, type(exc).__name__)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 658 |
st.session_state["sp_last_ui_task_status"] = status
|
| 659 |
_render_monitor_box(task, lang_code)
|
| 660 |
-
|
| 661 |
-
|
|
|
|
|
|
|
| 662 |
elif status == "completed":
|
| 663 |
output_path = _task_output_path(task)
|
| 664 |
if output_path and output_path.is_file():
|
|
@@ -673,19 +731,10 @@ def _render_task_status_once(lang_code: str) -> None:
|
|
| 673 |
if detail:
|
| 674 |
st.warning(detail)
|
| 675 |
|
| 676 |
-
# Only terminal/resume boundaries may rerun the whole Streamlit app. Live
|
| 677 |
-
# progress is DOM-only and therefore cannot reintroduce periodic flicker.
|
| 678 |
-
if previous_status and status != previous_status and status in {"resumable", "completed", "failed", "cancelled"}:
|
| 679 |
-
st.rerun()
|
| 680 |
-
|
| 681 |
|
| 682 |
-
def
|
| 683 |
-
"""
|
| 684 |
-
|
| 685 |
-
return
|
| 686 |
-
st.session_state["sp_scroll_to_generation_action"] = False
|
| 687 |
-
mode = str(st.session_state.get("run_mode") or "amj").lower()
|
| 688 |
-
target_id = "sp-generation-action-asq" if mode == "asq" else "sp-generation-action-amj"
|
| 689 |
components.html(
|
| 690 |
f"""
|
| 691 |
<script>
|
|
@@ -700,7 +749,7 @@ def _scroll_to_generation_action_once() -> None:
|
|
| 700 |
done = true;
|
| 701 |
target.scrollIntoView({{behavior:'auto', block:'center'}});
|
| 702 |
}};
|
| 703 |
-
[
|
| 704 |
}})();
|
| 705 |
</script>
|
| 706 |
""",
|
|
@@ -709,6 +758,14 @@ def _scroll_to_generation_action_once() -> None:
|
|
| 709 |
)
|
| 710 |
|
| 711 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 712 |
def render_workspace_controls(user):
|
| 713 |
top_toolbar = st.container()
|
| 714 |
with top_toolbar:
|
|
@@ -772,7 +829,9 @@ def render_workspace_controls(user):
|
|
| 772 |
if st.button(tr(lang_code, "start_amj"), disabled=active, key="btn_amj"):
|
| 773 |
st.session_state["run_mode"] = "amj"
|
| 774 |
st.session_state["sp_pending_launch"] = {"mode": "amj", "idempotency_key": _new_generation_idempotency_key()}
|
| 775 |
-
st.
|
|
|
|
|
|
|
| 776 |
|
| 777 |
with tab_asq:
|
| 778 |
st.markdown(tr(lang_code, "asq_config"))
|
|
@@ -788,7 +847,9 @@ def render_workspace_controls(user):
|
|
| 788 |
if st.button(tr(lang_code, "start_asq"), disabled=active, key="btn_asq"):
|
| 789 |
st.session_state["run_mode"] = "asq"
|
| 790 |
st.session_state["sp_pending_launch"] = {"mode": "asq", "idempotency_key": _new_generation_idempotency_key()}
|
| 791 |
-
st.
|
|
|
|
|
|
|
| 792 |
|
| 793 |
inject_file_uploader_i18n(lang_code)
|
| 794 |
|
|
|
|
| 595 |
<div class='log-box' id='sp-live-monitor'>
|
| 596 |
<div><b>{tr(lang_code, 'monitor_pipeline')}:</b> <span id='sp-live-status'>{status}</span></div>
|
| 597 |
<div><b>{tr(lang_code, 'monitor_node')}:</b> <span id='sp-live-node'>{node}</span></div>
|
| 598 |
+
<div><b>{tr(lang_code, 'monitor_log')}:</b> checkpoint <span id='sp-live-checkpoint-seq'>{checkpoint_seq}</span> · <span id='sp-live-checkpoint-stage'>{checkpoint_stage}</span></div>
|
| 599 |
+
<div id='sp-live-activity-feed' style='margin:8px 0 10px 0;max-height:150px;overflow:auto;line-height:1.55;'>
|
| 600 |
+
<div>• {node}</div>
|
| 601 |
+
</div><hr>
|
| 602 |
<div><b>{tr(lang_code, 'monitor_compute')}:</b></div>
|
| 603 |
<div style='display:flex;gap:8px;'>
|
| 604 |
<span style='background: #fef3c7;padding:4px 8px;border-radius:4px;color:black;'>🧠 DeepSeek: <b id='sp-live-ds'>{ds_calls}</b></span>
|
|
|
|
| 609 |
""", unsafe_allow_html=True)
|
| 610 |
|
| 611 |
|
| 612 |
+
def _watch_task_status_live(task: dict) -> dict | None:
|
| 613 |
+
"""Start browser-only read polling for live UI state.
|
| 614 |
|
| 615 |
+
DOM updates remain read-only and never participate in task liveness. A
|
| 616 |
+
Streamlit boundary is emitted only when a durable checkpoint advances or a
|
| 617 |
+
terminal/resumable state appears. Checkpoint boundaries let the UI reveal
|
| 618 |
+
newly completed manuscript text without periodic full-page reruns.
|
| 619 |
"""
|
| 620 |
task_id = str(task.get("task_id") or "").strip()
|
| 621 |
status = str(task.get("status") or "")
|
| 622 |
if not task_id or status not in LIVE_TASK_STATUSES:
|
| 623 |
+
return None
|
| 624 |
api_origin = str(st.session_state.get("sp_auth_api_origin") or "").strip()
|
| 625 |
if not api_origin:
|
| 626 |
logging.warning("[SP_UI] live_watch_missing_api_origin task_id=%s", task_id)
|
| 627 |
+
return None
|
| 628 |
try:
|
| 629 |
watch_result = edge_bridge(
|
| 630 |
action="task_watch",
|
| 631 |
command_id=f"watch:{task_id}",
|
| 632 |
api_origin=api_origin,
|
| 633 |
+
payload={
|
| 634 |
+
"task_id": task_id,
|
| 635 |
+
"poll_interval_ms": 2000,
|
| 636 |
+
"initial_checkpoint_seq": int(task.get("checkpoint_seq") or 0),
|
| 637 |
+
},
|
| 638 |
key=f"sp_task_watch_{_safe_key(task_id)}",
|
| 639 |
)
|
| 640 |
boundary_task = watch_result.get("task") if isinstance(watch_result, dict) and watch_result.get("boundary") else None
|
| 641 |
if isinstance(boundary_task, dict) and str(boundary_task.get("task_id") or "") == task_id:
|
|
|
|
|
|
|
| 642 |
st.session_state["sp_generation_task"] = dict(boundary_task)
|
| 643 |
st.session_state["sp_last_ui_task_status"] = str(boundary_task.get("status") or "")
|
| 644 |
+
if str(watch_result.get("boundary_type") or "") == "checkpoint":
|
| 645 |
+
st.session_state["sp_scroll_to_generation_action"] = True
|
| 646 |
+
return dict(boundary_task)
|
| 647 |
except Exception as exc:
|
| 648 |
logging.warning("[SP_UI] live_watch_error task_id=%s error=%s", task_id, type(exc).__name__)
|
| 649 |
+
return None
|
| 650 |
+
|
| 651 |
+
|
| 652 |
+
def _load_confirmed_checkpoint(task: dict) -> dict | None:
|
| 653 |
+
"""Load only the checkpoint sequence already acknowledged by Worker/D1."""
|
| 654 |
+
task_id = str(task.get("task_id") or "").strip()
|
| 655 |
+
checkpoint_seq = int(task.get("checkpoint_seq") or 0)
|
| 656 |
+
if not task_id or checkpoint_seq <= 0:
|
| 657 |
+
return None
|
| 658 |
+
task_dir = _task_dir(task_id)
|
| 659 |
+
request_path = task_dir / "request.json"
|
| 660 |
+
if not request_path.is_file():
|
| 661 |
+
return None
|
| 662 |
+
try:
|
| 663 |
+
request_payload = json.loads(request_path.read_text(encoding="utf-8"))
|
| 664 |
+
expected_hash = str(request_payload.get("request_hash") or "")
|
| 665 |
+
candidates = sorted((task_dir / "checkpoints").glob(f"checkpoint_{checkpoint_seq:05d}_*.json"))
|
| 666 |
+
if not candidates:
|
| 667 |
+
return None
|
| 668 |
+
payload = json.loads(candidates[-1].read_text(encoding="utf-8"))
|
| 669 |
+
if not isinstance(payload, dict):
|
| 670 |
+
return None
|
| 671 |
+
if expected_hash and str(payload.get("request_hash") or "") != expected_hash:
|
| 672 |
+
return None
|
| 673 |
+
if int(payload.get("checkpoint_seq") or 0) != checkpoint_seq:
|
| 674 |
+
return None
|
| 675 |
+
return payload
|
| 676 |
+
except Exception as exc:
|
| 677 |
+
logging.warning("[SP_UI] checkpoint_preview_error task_id=%s error=%s", task_id, type(exc).__name__)
|
| 678 |
+
return None
|
| 679 |
+
|
| 680 |
+
|
| 681 |
+
def _render_checkpoint_preview(task: dict, lang_code: str) -> None:
|
| 682 |
+
"""Show the durable English draft accumulated through the latest checkpoint."""
|
| 683 |
+
checkpoint = _load_confirmed_checkpoint(task)
|
| 684 |
+
if not checkpoint:
|
| 685 |
+
return
|
| 686 |
+
draft = str(checkpoint.get("full_draft_en") or "").strip()
|
| 687 |
+
if not draft:
|
| 688 |
+
return
|
| 689 |
+
st.markdown(tr(lang_code, "english_draft"))
|
| 690 |
+
st.markdown(draft)
|
| 691 |
|
| 692 |
|
| 693 |
def _render_task_status_once(lang_code: str) -> None:
|
|
|
|
| 695 |
task_id = str(task.get("task_id") or "").strip()
|
| 696 |
if not task_id:
|
| 697 |
return
|
|
|
|
| 698 |
status = str(task.get("status") or "")
|
| 699 |
if status in ACTIVE_TASK_STATUSES:
|
| 700 |
try:
|
|
|
|
| 704 |
status = str(task.get("status") or "")
|
| 705 |
except Exception as exc:
|
| 706 |
logging.warning("[SP_UI] status_poll_error task_id=%s error=%s", task_id, type(exc).__name__)
|
| 707 |
+
|
| 708 |
+
if status in LIVE_TASK_STATUSES:
|
| 709 |
+
boundary_task = _watch_task_status_live(task)
|
| 710 |
+
if isinstance(boundary_task, dict):
|
| 711 |
+
task = boundary_task
|
| 712 |
+
status = str(task.get("status") or "")
|
| 713 |
+
|
| 714 |
st.session_state["sp_last_ui_task_status"] = status
|
| 715 |
_render_monitor_box(task, lang_code)
|
| 716 |
+
_render_checkpoint_preview(task, lang_code)
|
| 717 |
+
|
| 718 |
+
if status == "resumable":
|
| 719 |
+
st.warning(tr(lang_code, "checkpoint_warning"))
|
| 720 |
elif status == "completed":
|
| 721 |
output_path = _task_output_path(task)
|
| 722 |
if output_path and output_path.is_file():
|
|
|
|
| 731 |
if detail:
|
| 732 |
st.warning(detail)
|
| 733 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 734 |
|
| 735 |
+
def _inject_generation_action_scroll(mode: str) -> None:
|
| 736 |
+
"""Scroll immediately to the active generation action without a server wait."""
|
| 737 |
+
target_id = "sp-generation-action-asq" if str(mode or "").lower() == "asq" else "sp-generation-action-amj"
|
|
|
|
|
|
|
|
|
|
|
|
|
| 738 |
components.html(
|
| 739 |
f"""
|
| 740 |
<script>
|
|
|
|
| 749 |
done = true;
|
| 750 |
target.scrollIntoView({{behavior:'auto', block:'center'}});
|
| 751 |
}};
|
| 752 |
+
[0, 40, 100, 220, 450, 800].forEach((delay) => window.parent.setTimeout(locate, delay));
|
| 753 |
}})();
|
| 754 |
</script>
|
| 755 |
""",
|
|
|
|
| 758 |
)
|
| 759 |
|
| 760 |
|
| 761 |
+
def _scroll_to_generation_action_once() -> None:
|
| 762 |
+
"""One-time post-launch/resume/checkpoint scroll to the generation action."""
|
| 763 |
+
if not bool(st.session_state.get("sp_scroll_to_generation_action")):
|
| 764 |
+
return
|
| 765 |
+
st.session_state["sp_scroll_to_generation_action"] = False
|
| 766 |
+
_inject_generation_action_scroll(str(st.session_state.get("run_mode") or "amj"))
|
| 767 |
+
|
| 768 |
+
|
| 769 |
def render_workspace_controls(user):
|
| 770 |
top_toolbar = st.container()
|
| 771 |
with top_toolbar:
|
|
|
|
| 829 |
if st.button(tr(lang_code, "start_amj"), disabled=active, key="btn_amj"):
|
| 830 |
st.session_state["run_mode"] = "amj"
|
| 831 |
st.session_state["sp_pending_launch"] = {"mode": "amj", "idempotency_key": _new_generation_idempotency_key()}
|
| 832 |
+
st.session_state["sp_scroll_to_generation_action"] = True
|
| 833 |
+
st.info(tr(lang_code, "pipeline_start"))
|
| 834 |
+
_inject_generation_action_scroll("amj")
|
| 835 |
|
| 836 |
with tab_asq:
|
| 837 |
st.markdown(tr(lang_code, "asq_config"))
|
|
|
|
| 847 |
if st.button(tr(lang_code, "start_asq"), disabled=active, key="btn_asq"):
|
| 848 |
st.session_state["run_mode"] = "asq"
|
| 849 |
st.session_state["sp_pending_launch"] = {"mode": "asq", "idempotency_key": _new_generation_idempotency_key()}
|
| 850 |
+
st.session_state["sp_scroll_to_generation_action"] = True
|
| 851 |
+
st.info(tr(lang_code, "pipeline_start"))
|
| 852 |
+
_inject_generation_action_scroll("asq")
|
| 853 |
|
| 854 |
inject_file_uploader_i18n(lang_code)
|
| 855 |
|
core/__pycache__/checkpoint_store.cpython-313.pyc
CHANGED
|
Binary files a/core/__pycache__/checkpoint_store.cpython-313.pyc and b/core/__pycache__/checkpoint_store.cpython-313.pyc differ
|
|
|
core/__pycache__/generation_runner.cpython-313.pyc
CHANGED
|
Binary files a/core/__pycache__/generation_runner.cpython-313.pyc and b/core/__pycache__/generation_runner.cpython-313.pyc differ
|
|
|
core/__pycache__/meta_extractor.cpython-313.pyc
CHANGED
|
Binary files a/core/__pycache__/meta_extractor.cpython-313.pyc and b/core/__pycache__/meta_extractor.cpython-313.pyc differ
|
|
|
core/__pycache__/pipeline_agent.cpython-313.pyc
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f86e44350d73c3bb8869b0963cf609be5356563dbd2713018d8695e5af70aed9
|
| 3 |
+
size 169392
|
core/__pycache__/production_guard.cpython-313.pyc
CHANGED
|
Binary files a/core/__pycache__/production_guard.cpython-313.pyc and b/core/__pycache__/production_guard.cpython-313.pyc differ
|
|
|
core/__pycache__/qualitative_engine.cpython-313.pyc
CHANGED
|
Binary files a/core/__pycache__/qualitative_engine.cpython-313.pyc and b/core/__pycache__/qualitative_engine.cpython-313.pyc differ
|
|
|
core/__pycache__/quantitative_engine.cpython-313.pyc
CHANGED
|
Binary files a/core/__pycache__/quantitative_engine.cpython-313.pyc and b/core/__pycache__/quantitative_engine.cpython-313.pyc differ
|
|
|
core/__pycache__/rag_engine.cpython-313.pyc
CHANGED
|
Binary files a/core/__pycache__/rag_engine.cpython-313.pyc and b/core/__pycache__/rag_engine.cpython-313.pyc differ
|
|
|
core/generation_runner.py
CHANGED
|
@@ -29,6 +29,9 @@ from utils.file_parser import extract_text_from_files
|
|
| 29 |
|
| 30 |
ProgressCallback = Callable[..., None]
|
| 31 |
|
|
|
|
|
|
|
|
|
|
| 32 |
|
| 33 |
def _clean_trailing_references(text, keywords=("reference", "references", "reference list", "参考文献")):
|
| 34 |
lines = str(text or "").split("\n")
|
|
@@ -75,7 +78,7 @@ class GenerationRunner:
|
|
| 75 |
self.agent = None
|
| 76 |
|
| 77 |
def _compute_request_counts(self) -> tuple[int, int, int]:
|
| 78 |
-
"""Return
|
| 79 |
agent = self.agent
|
| 80 |
if agent is None:
|
| 81 |
return 0, 0, 0
|
|
@@ -84,8 +87,8 @@ class GenerationRunner:
|
|
| 84 |
oa_metric = metrics.get("oa", {}) if isinstance(metrics, dict) else {}
|
| 85 |
return (
|
| 86 |
int(getattr(agent, "ds_calls", 0) or 0),
|
| 87 |
-
int(s2_metric.get("
|
| 88 |
-
int(oa_metric.get("
|
| 89 |
)
|
| 90 |
|
| 91 |
def _progress(self, pct: int, node: str, *, checkpoint_seq=None, checkpoint_stage=None) -> None:
|
|
@@ -231,11 +234,13 @@ class GenerationRunner:
|
|
| 231 |
refs_text = agent.fetch_references_parallel(
|
| 232 |
topic,
|
| 233 |
keywords,
|
| 234 |
-
min_required=
|
| 235 |
progress_callback=lambda: self._progress(9, "S2 + OpenAlex"),
|
| 236 |
)
|
| 237 |
-
if len(agent.citation_map) <
|
| 238 |
-
raise RuntimeError(
|
|
|
|
|
|
|
| 239 |
if user_lit_context:
|
| 240 |
core_refs = agent.parse_user_core_literature(user_lit_context)
|
| 241 |
if core_refs:
|
|
@@ -315,6 +320,7 @@ class GenerationRunner:
|
|
| 315 |
context_tail = str(state.get("context_tail") or "")
|
| 316 |
accumulated_deviation = float(state.get("accumulated_deviation") or 0.0)
|
| 317 |
all_cite_keys = list(agent.citation_map.keys())
|
|
|
|
| 318 |
valid_chunks = [
|
| 319 |
item for item in adaptive_structure
|
| 320 |
if not any(word in item["section"].lower() for word in ["abstract", "keyword", "appendix"])
|
|
@@ -322,11 +328,11 @@ class GenerationRunner:
|
|
| 322 |
allocation_map: dict[str, list[str]] = {}
|
| 323 |
current_cite_idx = 0
|
| 324 |
for idx, item in enumerate(valid_chunks):
|
| 325 |
-
count = max(1, math.ceil(len(
|
| 326 |
allocation_map[item["display_name"]] = (
|
| 327 |
-
|
| 328 |
if idx == len(valid_chunks) - 1
|
| 329 |
-
else
|
| 330 |
)
|
| 331 |
current_cite_idx += count
|
| 332 |
|
|
@@ -347,12 +353,15 @@ class GenerationRunner:
|
|
| 347 |
word in sec_title.lower()
|
| 348 |
for word in ["abstract", "keyword", "conclusion", "references", "appendix"]
|
| 349 |
)
|
| 350 |
-
|
|
|
|
| 351 |
citation_rule = (
|
| 352 |
"NO IN-TEXT CITATIONS ALLOWED IN THIS SECTION."
|
| 353 |
if is_quarantined
|
| 354 |
else "Use claim-specific citation. Each sentence may contain at most one source tag. "
|
| 355 |
-
|
|
|
|
|
|
|
| 356 |
)
|
| 357 |
memory_context = agent._build_memory_context(sec_title, 2500)
|
| 358 |
memory_block = f"\n<GLOBAL_MEMORY>\n{memory_context}\n</GLOBAL_MEMORY>\n" if memory_context else ""
|
|
@@ -431,6 +440,10 @@ Output only final academic prose. Do not report the word count."""
|
|
| 431 |
for tag, quote in quote_dict.items():
|
| 432 |
en_raw = en_raw.replace(tag, f'"{quote}"')
|
| 433 |
en_raw = agent.polish_academic_section(en_raw, display_name, abstract=False)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 434 |
prefix = ""
|
| 435 |
if sec.get("section_first"):
|
| 436 |
prefix += f"# {sec_title}\n\n"
|
|
@@ -477,8 +490,11 @@ Output only final academic prose. Do not report the word count."""
|
|
| 477 |
if ref and ref not in seen_refs:
|
| 478 |
references.append(f"- {ref}")
|
| 479 |
seen_refs.add(ref)
|
| 480 |
-
if
|
| 481 |
-
raise RuntimeError(
|
|
|
|
|
|
|
|
|
|
| 482 |
full_draft_en = full_draft_en.rstrip() + "\n\n# References\n" + "\n\n".join(references)
|
| 483 |
self._validate_target_length(full_draft_en, target_words)
|
| 484 |
state.update(
|
|
|
|
| 29 |
|
| 30 |
ProgressCallback = Callable[..., None]
|
| 31 |
|
| 32 |
+
MIN_FINAL_REFERENCES = 50
|
| 33 |
+
CITATION_COVERAGE_TARGET = 60
|
| 34 |
+
|
| 35 |
|
| 36 |
def _clean_trailing_references(text, keywords=("reference", "references", "reference list", "参考文献")):
|
| 37 |
lines = str(text or "").split("\n")
|
|
|
|
| 78 |
self.agent = None
|
| 79 |
|
| 80 |
def _compute_request_counts(self) -> tuple[int, int, int]:
|
| 81 |
+
"""Return UI metrics: DeepSeek requests and eligible literature results."""
|
| 82 |
agent = self.agent
|
| 83 |
if agent is None:
|
| 84 |
return 0, 0, 0
|
|
|
|
| 87 |
oa_metric = metrics.get("oa", {}) if isinstance(metrics, dict) else {}
|
| 88 |
return (
|
| 89 |
int(getattr(agent, "ds_calls", 0) or 0),
|
| 90 |
+
int(s2_metric.get("eligible_result_count", 0) or 0),
|
| 91 |
+
int(oa_metric.get("eligible_result_count", 0) or 0),
|
| 92 |
)
|
| 93 |
|
| 94 |
def _progress(self, pct: int, node: str, *, checkpoint_seq=None, checkpoint_stage=None) -> None:
|
|
|
|
| 234 |
refs_text = agent.fetch_references_parallel(
|
| 235 |
topic,
|
| 236 |
keywords,
|
| 237 |
+
min_required=MIN_FINAL_REFERENCES,
|
| 238 |
progress_callback=lambda: self._progress(9, "S2 + OpenAlex"),
|
| 239 |
)
|
| 240 |
+
if len(agent.citation_map) < MIN_FINAL_REFERENCES:
|
| 241 |
+
raise RuntimeError(
|
| 242 |
+
f"Insufficient eligible references: {len(agent.citation_map)}/{MIN_FINAL_REFERENCES}"
|
| 243 |
+
)
|
| 244 |
if user_lit_context:
|
| 245 |
core_refs = agent.parse_user_core_literature(user_lit_context)
|
| 246 |
if core_refs:
|
|
|
|
| 320 |
context_tail = str(state.get("context_tail") or "")
|
| 321 |
accumulated_deviation = float(state.get("accumulated_deviation") or 0.0)
|
| 322 |
all_cite_keys = list(agent.citation_map.keys())
|
| 323 |
+
coverage_cite_keys = all_cite_keys[: min(len(all_cite_keys), CITATION_COVERAGE_TARGET)]
|
| 324 |
valid_chunks = [
|
| 325 |
item for item in adaptive_structure
|
| 326 |
if not any(word in item["section"].lower() for word in ["abstract", "keyword", "appendix"])
|
|
|
|
| 328 |
allocation_map: dict[str, list[str]] = {}
|
| 329 |
current_cite_idx = 0
|
| 330 |
for idx, item in enumerate(valid_chunks):
|
| 331 |
+
count = max(1, math.ceil(len(coverage_cite_keys) / max(1, len(valid_chunks))))
|
| 332 |
allocation_map[item["display_name"]] = (
|
| 333 |
+
coverage_cite_keys[current_cite_idx:]
|
| 334 |
if idx == len(valid_chunks) - 1
|
| 335 |
+
else coverage_cite_keys[current_cite_idx:current_cite_idx + count]
|
| 336 |
)
|
| 337 |
current_cite_idx += count
|
| 338 |
|
|
|
|
| 353 |
word in sec_title.lower()
|
| 354 |
for word in ["abstract", "keyword", "conclusion", "references", "appendix"]
|
| 355 |
)
|
| 356 |
+
assigned_tag_list = list(allocation_map.get(display_name, []))
|
| 357 |
+
assigned_tags = ", ".join(assigned_tag_list) or "the single most relevant source"
|
| 358 |
citation_rule = (
|
| 359 |
"NO IN-TEXT CITATIONS ALLOWED IN THIS SECTION."
|
| 360 |
if is_quarantined
|
| 361 |
else "Use claim-specific citation. Each sentence may contain at most one source tag. "
|
| 362 |
+
"Use the assigned candidate tags across distinct supported claims whenever their evidence directly applies; "
|
| 363 |
+
"never stack sources or cite a source that does not support the sentence. "
|
| 364 |
+
f"Assigned candidate tags: {assigned_tags}. <AVAILABLE_CITATIONS>{refs_text}</AVAILABLE_CITATIONS>"
|
| 365 |
)
|
| 366 |
memory_context = agent._build_memory_context(sec_title, 2500)
|
| 367 |
memory_block = f"\n<GLOBAL_MEMORY>\n{memory_context}\n</GLOBAL_MEMORY>\n" if memory_context else ""
|
|
|
|
| 440 |
for tag, quote in quote_dict.items():
|
| 441 |
en_raw = en_raw.replace(tag, f'"{quote}"')
|
| 442 |
en_raw = agent.polish_academic_section(en_raw, display_name, abstract=False)
|
| 443 |
+
if not is_quarantined and assigned_tag_list:
|
| 444 |
+
en_raw = agent.reinforce_claim_specific_citation_coverage(
|
| 445 |
+
en_raw, assigned_tag_list, section_name=display_name
|
| 446 |
+
)
|
| 447 |
prefix = ""
|
| 448 |
if sec.get("section_first"):
|
| 449 |
prefix += f"# {sec_title}\n\n"
|
|
|
|
| 490 |
if ref and ref not in seen_refs:
|
| 491 |
references.append(f"- {ref}")
|
| 492 |
seen_refs.add(ref)
|
| 493 |
+
if len(references) < MIN_FINAL_REFERENCES:
|
| 494 |
+
raise RuntimeError(
|
| 495 |
+
"Insufficient final cited references after semantic validation: "
|
| 496 |
+
f"{len(references)}/{MIN_FINAL_REFERENCES}."
|
| 497 |
+
)
|
| 498 |
full_draft_en = full_draft_en.rstrip() + "\n\n# References\n" + "\n\n".join(references)
|
| 499 |
self._validate_target_length(full_draft_en, target_words)
|
| 500 |
state.update(
|
core/pipeline_agent.py
CHANGED
|
@@ -2456,6 +2456,81 @@ Return exactly:
|
|
| 2456 |
recovered = re.sub(r'[ \t]+\n', '\n', recovered)
|
| 2457 |
return re.sub(r'\n{3,}', '\n\n', recovered).strip()
|
| 2458 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2459 |
def audit_and_resolve_citations(self, text, section_name='', quarantine=False):
|
| 2460 |
value = str(text or '')
|
| 2461 |
if quarantine:
|
|
@@ -2675,8 +2750,9 @@ Return exactly:
|
|
| 2675 |
|
| 2676 |
@staticmethod
|
| 2677 |
def _translation_acronym_tokens(value):
|
|
|
|
| 2678 |
without_doi = re.sub(r'https?://\S+|doi:\s*\S+', ' ', str(value or ''), flags=re.I)
|
| 2679 |
-
return Counter(re.findall(r'\b[A-Z][A-Z0-9-]{1,11}\b', without_doi))
|
| 2680 |
|
| 2681 |
@staticmethod
|
| 2682 |
def _translation_citation_tokens(value):
|
|
@@ -2790,8 +2866,15 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
|
|
| 2790 |
raise DeepSeekRequestError(f'Translation changed citations in {block_id}.')
|
| 2791 |
if self._translation_hypothesis_tokens(source) != self._translation_hypothesis_tokens(translated):
|
| 2792 |
raise DeepSeekRequestError(f'Translation changed hypothesis labels in {block_id}.')
|
| 2793 |
-
|
| 2794 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2795 |
|
| 2796 |
source_compact = re.sub(r'\s+', '', str(source or ''))
|
| 2797 |
target_compact = re.sub(r'\s+', '', str(translated or ''))
|
|
@@ -2814,6 +2897,68 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
|
|
| 2814 |
if not is_chinese and not re.search(r'[А-Яа-яЁёӘәҒғҚқҢңӨөҰұҮүҺһІі]', translated):
|
| 2815 |
raise DeepSeekRequestError(f'Translation block {block_id} contains no Cyrillic target-language text.')
|
| 2816 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2817 |
def _translate_marked_batch_with_recovery(self, batch, system_prompt, target_lang):
|
| 2818 |
"""Translate one marked batch with a conservative marker-recovery path.
|
| 2819 |
|
|
@@ -2875,8 +3020,8 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
|
|
| 2875 |
raise DeepSeekRequestError(
|
| 2876 |
f'Single-block translation recovery returned empty text for SP-P{block_id}.'
|
| 2877 |
) from error
|
| 2878 |
-
self.
|
| 2879 |
-
source, translated,
|
| 2880 |
)
|
| 2881 |
logging.info(
|
| 2882 |
'Translation marker recovery accepted marker-free single block: SP-P%s',
|
|
@@ -2885,10 +3030,10 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
|
|
| 2885 |
return {block_id: translated}
|
| 2886 |
|
| 2887 |
for block_id, source in batch:
|
| 2888 |
-
translated =
|
| 2889 |
-
|
| 2890 |
-
source, translated, f'SP-P{block_id}', target_lang
|
| 2891 |
)
|
|
|
|
| 2892 |
return parsed
|
| 2893 |
|
| 2894 |
def translate_locked_manuscript(self, manuscript, target_lang):
|
|
|
|
| 2456 |
recovered = re.sub(r'[ \t]+\n', '\n', recovered)
|
| 2457 |
return re.sub(r'\n{3,}', '\n\n', recovered).strip()
|
| 2458 |
|
| 2459 |
+
def reinforce_claim_specific_citation_coverage(self, text, assigned_tags, section_name=''):
|
| 2460 |
+
"""Add assigned tags only to citation-free claims with lexical evidence overlap.
|
| 2461 |
+
|
| 2462 |
+
This pass is deterministic and makes no additional model call. It never
|
| 2463 |
+
changes claim wording: a source tag is inserted only when at least two
|
| 2464 |
+
substantive terms overlap between an existing sentence and the source
|
| 2465 |
+
evidence/reference. The ordinary semantic citation audit remains the
|
| 2466 |
+
final authority and may still reject an unsupported placement.
|
| 2467 |
+
"""
|
| 2468 |
+
value = str(text or '').strip()
|
| 2469 |
+
valid_tags = [tag for tag in dict.fromkeys(assigned_tags or []) if tag in self.citation_map]
|
| 2470 |
+
if not value or not valid_tags:
|
| 2471 |
+
return value
|
| 2472 |
+
existing = {f"[{tag.upper()}]" for tag in re.findall(r'CITE-\d+', value, flags=re.I)}
|
| 2473 |
+
missing = [tag for tag in valid_tags if tag not in existing]
|
| 2474 |
+
if not missing:
|
| 2475 |
+
return value
|
| 2476 |
+
|
| 2477 |
+
spans = self._sentence_spans(value)
|
| 2478 |
+
candidates = []
|
| 2479 |
+
for index, (start, end, sentence) in enumerate(spans):
|
| 2480 |
+
stripped = str(sentence or '').strip()
|
| 2481 |
+
if not stripped or stripped.startswith('#') or re.search(r'CITE-\d+', stripped, flags=re.I):
|
| 2482 |
+
continue
|
| 2483 |
+
terms = self._citation_terms(stripped)
|
| 2484 |
+
if len(terms) < 4:
|
| 2485 |
+
continue
|
| 2486 |
+
candidates.append((index, start, end, sentence, terms))
|
| 2487 |
+
|
| 2488 |
+
assignments = []
|
| 2489 |
+
occupied = set()
|
| 2490 |
+
for tag in missing:
|
| 2491 |
+
metadata = self.citation_map.get(tag) or {}
|
| 2492 |
+
evidence = ' '.join([str(metadata.get('ref') or ''), str(metadata.get('evidence') or '')])
|
| 2493 |
+
evidence_terms = self._citation_terms(evidence)
|
| 2494 |
+
if not evidence_terms:
|
| 2495 |
+
continue
|
| 2496 |
+
scored = []
|
| 2497 |
+
for item in candidates:
|
| 2498 |
+
index, start, end, sentence, sentence_terms = item
|
| 2499 |
+
if index in occupied:
|
| 2500 |
+
continue
|
| 2501 |
+
overlap = len(sentence_terms & evidence_terms)
|
| 2502 |
+
coverage = overlap / max(1, len(sentence_terms))
|
| 2503 |
+
scored.append((overlap, coverage, -index, item))
|
| 2504 |
+
if not scored:
|
| 2505 |
+
continue
|
| 2506 |
+
scored.sort(reverse=True, key=lambda row: (row[0], row[1], row[2]))
|
| 2507 |
+
overlap, coverage, _, item = scored[0]
|
| 2508 |
+
if overlap < 2 or coverage < 0.08:
|
| 2509 |
+
continue
|
| 2510 |
+
index, start, end, sentence, _ = item
|
| 2511 |
+
occupied.add(index)
|
| 2512 |
+
assignments.append((start, end, sentence, tag))
|
| 2513 |
+
|
| 2514 |
+
if not assignments:
|
| 2515 |
+
return value
|
| 2516 |
+
revised = value
|
| 2517 |
+
for start, end, sentence, tag in sorted(assignments, key=lambda row: row[0], reverse=True):
|
| 2518 |
+
raw = str(sentence)
|
| 2519 |
+
trailing_match = re.search(r'(\s*)$', raw)
|
| 2520 |
+
trailing = trailing_match.group(1) if trailing_match else ''
|
| 2521 |
+
core = raw[:-len(trailing)] if trailing else raw
|
| 2522 |
+
punctuation = ''
|
| 2523 |
+
if core and core[-1] in '.!?。!?':
|
| 2524 |
+
punctuation = core[-1]
|
| 2525 |
+
core = core[:-1]
|
| 2526 |
+
replacement = f"{core.rstrip()} {tag}{punctuation}{trailing}"
|
| 2527 |
+
revised = revised[:start] + replacement + revised[end:]
|
| 2528 |
+
logging.info(
|
| 2529 |
+
'Citation coverage placement: section=%s assigned=%s inserted=%s',
|
| 2530 |
+
section_name, len(valid_tags), len(assignments),
|
| 2531 |
+
)
|
| 2532 |
+
return revised
|
| 2533 |
+
|
| 2534 |
def audit_and_resolve_citations(self, text, section_name='', quarantine=False):
|
| 2535 |
value = str(text or '')
|
| 2536 |
if quarantine:
|
|
|
|
| 2750 |
|
| 2751 |
@staticmethod
|
| 2752 |
def _translation_acronym_tokens(value):
|
| 2753 |
+
"""Protect exact uppercase abbreviations, including plural suffixes."""
|
| 2754 |
without_doi = re.sub(r'https?://\S+|doi:\s*\S+', ' ', str(value or ''), flags=re.I)
|
| 2755 |
+
return Counter(re.findall(r'\b[A-Z][A-Z0-9-]{1,11}(?:s|es)?\b', without_doi))
|
| 2756 |
|
| 2757 |
@staticmethod
|
| 2758 |
def _translation_citation_tokens(value):
|
|
|
|
| 2866 |
raise DeepSeekRequestError(f'Translation changed citations in {block_id}.')
|
| 2867 |
if self._translation_hypothesis_tokens(source) != self._translation_hypothesis_tokens(translated):
|
| 2868 |
raise DeepSeekRequestError(f'Translation changed hypothesis labels in {block_id}.')
|
| 2869 |
+
source_acronyms = self._translation_acronym_tokens(source)
|
| 2870 |
+
target_acronyms = self._translation_acronym_tokens(translated)
|
| 2871 |
+
if source_acronyms != target_acronyms:
|
| 2872 |
+
missing = list((source_acronyms - target_acronyms).elements())
|
| 2873 |
+
added = list((target_acronyms - source_acronyms).elements())
|
| 2874 |
+
raise DeepSeekRequestError(
|
| 2875 |
+
f'Translation changed variable abbreviations in {block_id}: '
|
| 2876 |
+
f'missing={missing}, added={added}.'
|
| 2877 |
+
)
|
| 2878 |
|
| 2879 |
source_compact = re.sub(r'\s+', '', str(source or ''))
|
| 2880 |
target_compact = re.sub(r'\s+', '', str(translated or ''))
|
|
|
|
| 2897 |
if not is_chinese and not re.search(r'[А-Яа-яЁёӘәҒғҚқҢңӨөҰұҮүҺһІі]', translated):
|
| 2898 |
raise DeepSeekRequestError(f'Translation block {block_id} contains no Cyrillic target-language text.')
|
| 2899 |
|
| 2900 |
+
def _retry_translation_block_for_abbreviations(self, block_id, source, system_prompt, target_lang):
|
| 2901 |
+
protected_counter = self._translation_acronym_tokens(source)
|
| 2902 |
+
protected = list(protected_counter.elements())
|
| 2903 |
+
strict_prompt = (
|
| 2904 |
+
system_prompt
|
| 2905 |
+
+ '\nCRITICAL ABBREVIATION PRESERVATION: Preserve every token in this JSON list '
|
| 2906 |
+
'exactly, including capitalization and plural suffix, and preserve its exact count. '
|
| 2907 |
+
'Do not normalize plural abbreviations (for example, LLMs must not become LLM).\n'
|
| 2908 |
+
+ json.dumps(protected, ensure_ascii=False)
|
| 2909 |
+
)
|
| 2910 |
+
marked_source = f'[[SP-P{block_id}]]\n{source}'
|
| 2911 |
+
result = self.call_deepseek(
|
| 2912 |
+
strict_prompt,
|
| 2913 |
+
marked_source,
|
| 2914 |
+
max_tokens=max(1800, min(9000, self._count_tokens(marked_source) * 2 + 1200)),
|
| 2915 |
+
temp=0.0,
|
| 2916 |
+
model_role='translation',
|
| 2917 |
+
thinking=False,
|
| 2918 |
+
allow_fallback=False,
|
| 2919 |
+
)
|
| 2920 |
+
result = self._remove_ai_meta_traces(result)
|
| 2921 |
+
try:
|
| 2922 |
+
translated = self._parse_marked_translation(result, [block_id])[block_id]
|
| 2923 |
+
except DeepSeekRequestError as error:
|
| 2924 |
+
if not str(error).startswith('Translation marker mismatch:'):
|
| 2925 |
+
raise
|
| 2926 |
+
unexpected_markers = re.findall(
|
| 2927 |
+
r'(?m)^\[\[SP-P(\d{6})\]\]\s*$',
|
| 2928 |
+
str(result or ''),
|
| 2929 |
+
)
|
| 2930 |
+
if unexpected_markers:
|
| 2931 |
+
raise DeepSeekRequestError(
|
| 2932 |
+
'Abbreviation retry returned an unexpected marker: '
|
| 2933 |
+
f'expected={block_id}, got={unexpected_markers}.'
|
| 2934 |
+
) from error
|
| 2935 |
+
translated = str(result or '').strip()
|
| 2936 |
+
if not translated:
|
| 2937 |
+
raise DeepSeekRequestError(
|
| 2938 |
+
f'Abbreviation retry returned empty text for SP-P{block_id}.'
|
| 2939 |
+
) from error
|
| 2940 |
+
self._validate_translation_block(source, translated, f'SP-P{block_id}', target_lang)
|
| 2941 |
+
logging.info(
|
| 2942 |
+
'Translation abbreviation recovery accepted block: SP-P%s protected=%s',
|
| 2943 |
+
block_id, protected,
|
| 2944 |
+
)
|
| 2945 |
+
return translated
|
| 2946 |
+
|
| 2947 |
+
def _validate_translation_block_with_abbreviation_recovery(self, block_id, source, translated, system_prompt, target_lang):
|
| 2948 |
+
try:
|
| 2949 |
+
self._validate_translation_block(source, translated, f'SP-P{block_id}', target_lang)
|
| 2950 |
+
return translated
|
| 2951 |
+
except DeepSeekRequestError as error:
|
| 2952 |
+
if not str(error).startswith('Translation changed variable abbreviations in SP-P'):
|
| 2953 |
+
raise
|
| 2954 |
+
logging.warning(
|
| 2955 |
+
'Translation abbreviation recovery: block=SP-P%s error=%s',
|
| 2956 |
+
block_id, str(error),
|
| 2957 |
+
)
|
| 2958 |
+
return self._retry_translation_block_for_abbreviations(
|
| 2959 |
+
block_id, source, system_prompt, target_lang
|
| 2960 |
+
)
|
| 2961 |
+
|
| 2962 |
def _translate_marked_batch_with_recovery(self, batch, system_prompt, target_lang):
|
| 2963 |
"""Translate one marked batch with a conservative marker-recovery path.
|
| 2964 |
|
|
|
|
| 3020 |
raise DeepSeekRequestError(
|
| 3021 |
f'Single-block translation recovery returned empty text for SP-P{block_id}.'
|
| 3022 |
) from error
|
| 3023 |
+
translated = self._validate_translation_block_with_abbreviation_recovery(
|
| 3024 |
+
block_id, source, translated, system_prompt, target_lang
|
| 3025 |
)
|
| 3026 |
logging.info(
|
| 3027 |
'Translation marker recovery accepted marker-free single block: SP-P%s',
|
|
|
|
| 3030 |
return {block_id: translated}
|
| 3031 |
|
| 3032 |
for block_id, source in batch:
|
| 3033 |
+
translated = self._validate_translation_block_with_abbreviation_recovery(
|
| 3034 |
+
block_id, source, parsed[block_id], system_prompt, target_lang
|
|
|
|
| 3035 |
)
|
| 3036 |
+
parsed[block_id] = translated
|
| 3037 |
return parsed
|
| 3038 |
|
| 3039 |
def translate_locked_manuscript(self, manuscript, target_lang):
|
smartpaper_restore_bridge/__pycache__/__init__.cpython-313.pyc
CHANGED
|
Binary files a/smartpaper_restore_bridge/__pycache__/__init__.cpython-313.pyc and b/smartpaper_restore_bridge/__pycache__/__init__.cpython-313.pyc differ
|
|
|
smartpaper_restore_bridge/frontend/index.html
CHANGED
|
@@ -22,6 +22,8 @@
|
|
| 22 |
const TERMINAL_TASK_STATUSES = new Set(["completed", "failed", "cancelled"]);
|
| 23 |
let lastResponseSignature = "";
|
| 24 |
let activeTaskWatch = null;
|
|
|
|
|
|
|
| 25 |
|
| 26 |
const ACTION_PATHS = Object.freeze({
|
| 27 |
task_start: "/edge/task/start",
|
|
@@ -184,6 +186,17 @@
|
|
| 184 |
if (element) element.textContent = String(value ?? "");
|
| 185 |
}
|
| 186 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 187 |
function updateTaskMonitor(task) {
|
| 188 |
let parentDocument;
|
| 189 |
try { parentDocument = window.parent.document; }
|
|
@@ -191,16 +204,27 @@
|
|
| 191 |
const status = String(task && task.status || "");
|
| 192 |
const node = String(task && task.current_node || status || "");
|
| 193 |
const progress = Math.max(0, Math.min(100, asCount(task && task.progress_pct)));
|
|
|
|
|
|
|
| 194 |
updateText(parentDocument, "sp-live-status", status);
|
| 195 |
updateText(parentDocument, "sp-live-node", node);
|
| 196 |
-
updateText(parentDocument, "sp-live-checkpoint-seq",
|
| 197 |
-
updateText(parentDocument, "sp-live-checkpoint-stage",
|
| 198 |
updateText(parentDocument, "sp-live-ds", asCount(task && task.ds_calls));
|
| 199 |
updateText(parentDocument, "sp-live-s2", asCount(task && task.s2_calls));
|
| 200 |
updateText(parentDocument, "sp-live-oa", asCount(task && task.oa_calls));
|
| 201 |
updateText(parentDocument, "sp-live-progress-text", `${progress}% — ${node}`);
|
| 202 |
const fill = parentDocument.getElementById("sp-live-progress-fill");
|
| 203 |
if (fill) fill.style.width = `${progress}%`;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 204 |
}
|
| 205 |
|
| 206 |
function stopTaskWatch() {
|
|
@@ -217,15 +241,17 @@
|
|
| 217 |
const apiOrigin = normalizeOrigin(args.api_origin || "");
|
| 218 |
const payload = args.payload && typeof args.payload === "object" ? args.payload : {};
|
| 219 |
const taskId = typeof payload.task_id === "string" ? payload.task_id.trim() : "";
|
| 220 |
-
const requestedInterval = Number(payload.poll_interval_ms ||
|
| 221 |
-
const intervalMs = Math.max(1500, Math.min(10000, Number.isFinite(requestedInterval) ? requestedInterval :
|
|
|
|
| 222 |
if (!apiOrigin || !taskId) {
|
| 223 |
sendValue({ ok:false, action:"task_watch", command_id:commandId, error:"invalid_task_watch" });
|
| 224 |
return;
|
| 225 |
}
|
| 226 |
|
| 227 |
-
const watch = { commandId, taskId, apiOrigin, intervalMs, timer:null, controller:null, stopped:false };
|
| 228 |
activeTaskWatch = watch;
|
|
|
|
| 229 |
|
| 230 |
const schedule = () => {
|
| 231 |
if (activeTaskWatch !== watch || watch.stopped) return;
|
|
@@ -240,6 +266,7 @@
|
|
| 240 |
if (result.ok && task) {
|
| 241 |
updateTaskMonitor(task);
|
| 242 |
const status = String(task.status || "");
|
|
|
|
| 243 |
if (TERMINAL_TASK_STATUSES.has(status) || status === "resumable") {
|
| 244 |
watch.stopped = true;
|
| 245 |
activeTaskWatch = null;
|
|
@@ -247,7 +274,21 @@
|
|
| 247 |
action:"task_watch",
|
| 248 |
command_id:commandId,
|
| 249 |
terminal:TERMINAL_TASK_STATUSES.has(status),
|
| 250 |
-
boundary:true
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 251 |
}));
|
| 252 |
return;
|
| 253 |
}
|
|
|
|
| 22 |
const TERMINAL_TASK_STATUSES = new Set(["completed", "failed", "cancelled"]);
|
| 23 |
let lastResponseSignature = "";
|
| 24 |
let activeTaskWatch = null;
|
| 25 |
+
let lastMonitorNode = "";
|
| 26 |
+
let lastMonitorCheckpointSeq = 0;
|
| 27 |
|
| 28 |
const ACTION_PATHS = Object.freeze({
|
| 29 |
task_start: "/edge/task/start",
|
|
|
|
| 186 |
if (element) element.textContent = String(value ?? "");
|
| 187 |
}
|
| 188 |
|
| 189 |
+
function appendTaskActivity(parentDocument, text, complete=false) {
|
| 190 |
+
const feed = parentDocument.getElementById("sp-live-activity-feed");
|
| 191 |
+
const value = String(text || "").trim();
|
| 192 |
+
if (!feed || !value) return;
|
| 193 |
+
const row = parentDocument.createElement("div");
|
| 194 |
+
row.textContent = `${complete ? "✓" : "•"} ${value}`;
|
| 195 |
+
feed.appendChild(row);
|
| 196 |
+
while (feed.children.length > 12) feed.removeChild(feed.firstElementChild);
|
| 197 |
+
feed.scrollTop = feed.scrollHeight;
|
| 198 |
+
}
|
| 199 |
+
|
| 200 |
function updateTaskMonitor(task) {
|
| 201 |
let parentDocument;
|
| 202 |
try { parentDocument = window.parent.document; }
|
|
|
|
| 204 |
const status = String(task && task.status || "");
|
| 205 |
const node = String(task && task.current_node || status || "");
|
| 206 |
const progress = Math.max(0, Math.min(100, asCount(task && task.progress_pct)));
|
| 207 |
+
const checkpointSeq = asCount(task && task.checkpoint_seq);
|
| 208 |
+
const checkpointStage = String(task && task.checkpoint_stage || "");
|
| 209 |
updateText(parentDocument, "sp-live-status", status);
|
| 210 |
updateText(parentDocument, "sp-live-node", node);
|
| 211 |
+
updateText(parentDocument, "sp-live-checkpoint-seq", checkpointSeq);
|
| 212 |
+
updateText(parentDocument, "sp-live-checkpoint-stage", checkpointStage);
|
| 213 |
updateText(parentDocument, "sp-live-ds", asCount(task && task.ds_calls));
|
| 214 |
updateText(parentDocument, "sp-live-s2", asCount(task && task.s2_calls));
|
| 215 |
updateText(parentDocument, "sp-live-oa", asCount(task && task.oa_calls));
|
| 216 |
updateText(parentDocument, "sp-live-progress-text", `${progress}% — ${node}`);
|
| 217 |
const fill = parentDocument.getElementById("sp-live-progress-fill");
|
| 218 |
if (fill) fill.style.width = `${progress}%`;
|
| 219 |
+
|
| 220 |
+
if (node && node !== lastMonitorNode) {
|
| 221 |
+
appendTaskActivity(parentDocument, node, false);
|
| 222 |
+
lastMonitorNode = node;
|
| 223 |
+
}
|
| 224 |
+
if (checkpointSeq > lastMonitorCheckpointSeq) {
|
| 225 |
+
appendTaskActivity(parentDocument, checkpointStage || node, true);
|
| 226 |
+
lastMonitorCheckpointSeq = checkpointSeq;
|
| 227 |
+
}
|
| 228 |
}
|
| 229 |
|
| 230 |
function stopTaskWatch() {
|
|
|
|
| 241 |
const apiOrigin = normalizeOrigin(args.api_origin || "");
|
| 242 |
const payload = args.payload && typeof args.payload === "object" ? args.payload : {};
|
| 243 |
const taskId = typeof payload.task_id === "string" ? payload.task_id.trim() : "";
|
| 244 |
+
const requestedInterval = Number(payload.poll_interval_ms || 2000);
|
| 245 |
+
const intervalMs = Math.max(1500, Math.min(10000, Number.isFinite(requestedInterval) ? requestedInterval : 2000));
|
| 246 |
+
const initialCheckpointSeq = asCount(payload.initial_checkpoint_seq);
|
| 247 |
if (!apiOrigin || !taskId) {
|
| 248 |
sendValue({ ok:false, action:"task_watch", command_id:commandId, error:"invalid_task_watch" });
|
| 249 |
return;
|
| 250 |
}
|
| 251 |
|
| 252 |
+
const watch = { commandId, taskId, apiOrigin, intervalMs, timer:null, controller:null, stopped:false, lastCheckpointSeq:initialCheckpointSeq };
|
| 253 |
activeTaskWatch = watch;
|
| 254 |
+
lastMonitorCheckpointSeq = Math.max(lastMonitorCheckpointSeq, initialCheckpointSeq);
|
| 255 |
|
| 256 |
const schedule = () => {
|
| 257 |
if (activeTaskWatch !== watch || watch.stopped) return;
|
|
|
|
| 266 |
if (result.ok && task) {
|
| 267 |
updateTaskMonitor(task);
|
| 268 |
const status = String(task.status || "");
|
| 269 |
+
const checkpointSeq = asCount(task.checkpoint_seq);
|
| 270 |
if (TERMINAL_TASK_STATUSES.has(status) || status === "resumable") {
|
| 271 |
watch.stopped = true;
|
| 272 |
activeTaskWatch = null;
|
|
|
|
| 274 |
action:"task_watch",
|
| 275 |
command_id:commandId,
|
| 276 |
terminal:TERMINAL_TASK_STATUSES.has(status),
|
| 277 |
+
boundary:true,
|
| 278 |
+
boundary_type:"terminal"
|
| 279 |
+
}));
|
| 280 |
+
return;
|
| 281 |
+
}
|
| 282 |
+
if (checkpointSeq > watch.lastCheckpointSeq) {
|
| 283 |
+
watch.lastCheckpointSeq = checkpointSeq;
|
| 284 |
+
watch.stopped = true;
|
| 285 |
+
activeTaskWatch = null;
|
| 286 |
+
sendValue(Object.assign({}, result, {
|
| 287 |
+
action:"task_watch",
|
| 288 |
+
command_id:commandId,
|
| 289 |
+
terminal:false,
|
| 290 |
+
boundary:true,
|
| 291 |
+
boundary_type:"checkpoint"
|
| 292 |
}));
|
| 293 |
return;
|
| 294 |
}
|
utils/__pycache__/docx_builder.cpython-313.pyc
CHANGED
|
Binary files a/utils/__pycache__/docx_builder.cpython-313.pyc and b/utils/__pycache__/docx_builder.cpython-313.pyc differ
|
|
|
utils/__pycache__/file_parser.cpython-313.pyc
CHANGED
|
Binary files a/utils/__pycache__/file_parser.cpython-313.pyc and b/utils/__pycache__/file_parser.cpython-313.pyc differ
|
|
|