Spaces:
Running
Running
R3.2.8.6-R1.3C-KK1-R5 Multilingual Async Translation Preview Alignment and Relation-Aware Fact Integrity
#35
by GuXiaopeng - opened
- .streamlit/desktop.ini +2 -0
- app.py +1 -1
- core/desktop.ini +2 -0
- core/generation_runner.py +331 -69
- core/pipeline_agent.py +95 -9
- desktop.ini +2 -0
- smartpaper_restore_bridge/desktop.ini +2 -0
- smartpaper_restore_bridge/frontend/desktop.ini +2 -0
- smartpaper_restore_bridge/frontend/index.html +24 -0
- static/desktop.ini +2 -0
- static/previews/desktop.ini +2 -0
- utils/desktop.ini +2 -0
.streamlit/desktop.ini
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[.ShellClassInfo]
|
| 2 |
+
IconResource=C:\Program Files\Google\Drive File Stream\129.0.1.0\GoogleDriveFS.exe,27
|
app.py
CHANGED
|
@@ -844,8 +844,8 @@ def _render_checkpoint_preview(task: dict, lang_code: str) -> None:
|
|
| 844 |
st.markdown(tr(lang_code, "english_draft"))
|
| 845 |
st.markdown(
|
| 846 |
f"""
|
| 847 |
-
<div id='sp-live-manuscript-status' style='white-space:pre-wrap;font-size:0.88em;opacity:0.72;margin:0 0 8px 0;display:{english_status_display};'>{html.escape(english_status_initial)}</div>
|
| 848 |
<div id='sp-live-manuscript-shell' style='border:1px solid #e5e7eb;border-radius:8px;padding:14px 16px;min-height:220px;max-height:68vh;overflow:auto;background:#ffffff;'>
|
|
|
|
| 849 |
<div id='sp-live-manuscript-preview' data-sp-placeholder='{english_placeholder}' style='white-space:pre-wrap;overflow-wrap:anywhere;margin:0;font-family:inherit;line-height:1.72;color:#111827;'>{html.escape(english_initial)}</div>
|
| 850 |
<span id='sp-live-manuscript-cursor' aria-hidden='true' style='display:inline-block;width:7px;height:1.05em;background:#111827;margin-left:2px;vertical-align:-0.12em;animation:spBlink 1s steps(1,end) infinite;'></span>
|
| 851 |
</div>
|
|
|
|
| 844 |
st.markdown(tr(lang_code, "english_draft"))
|
| 845 |
st.markdown(
|
| 846 |
f"""
|
|
|
|
| 847 |
<div id='sp-live-manuscript-shell' style='border:1px solid #e5e7eb;border-radius:8px;padding:14px 16px;min-height:220px;max-height:68vh;overflow:auto;background:#ffffff;'>
|
| 848 |
+
<div id='sp-live-manuscript-status' style='white-space:pre-wrap;font-size:0.88em;opacity:0.72;margin:0 0 8px 0;display:{english_status_display};'>{html.escape(english_status_initial)}</div>
|
| 849 |
<div id='sp-live-manuscript-preview' data-sp-placeholder='{english_placeholder}' style='white-space:pre-wrap;overflow-wrap:anywhere;margin:0;font-family:inherit;line-height:1.72;color:#111827;'>{html.escape(english_initial)}</div>
|
| 850 |
<span id='sp-live-manuscript-cursor' aria-hidden='true' style='display:inline-block;width:7px;height:1.05em;background:#111827;margin-left:2px;vertical-align:-0.12em;animation:spBlink 1s steps(1,end) infinite;'></span>
|
| 851 |
</div>
|
core/desktop.ini
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[.ShellClassInfo]
|
| 2 |
+
IconResource=C:\Program Files\Google\Drive File Stream\129.0.1.0\GoogleDriveFS.exe,27
|
core/generation_runner.py
CHANGED
|
@@ -13,8 +13,11 @@ import json
|
|
| 13 |
import logging
|
| 14 |
import math
|
| 15 |
import os
|
|
|
|
| 16 |
import re
|
|
|
|
| 17 |
import time
|
|
|
|
| 18 |
from datetime import datetime, timezone, timedelta
|
| 19 |
from pathlib import Path
|
| 20 |
from typing import Callable, Any
|
|
@@ -34,7 +37,7 @@ ProgressCallback = Callable[..., None]
|
|
| 34 |
MIN_FINAL_REFERENCES = 50
|
| 35 |
LITERATURE_EVIDENCE_TARGET = 75
|
| 36 |
CITATION_PLAN_TARGET = 70
|
| 37 |
-
TRANSLATION_CONTRACT_VERSION = 'hierarchical-multilingual-
|
| 38 |
|
| 39 |
|
| 40 |
def _clean_trailing_references(text, keywords=("reference", "references", "reference list", "参考文献")):
|
|
@@ -56,6 +59,73 @@ def _canonical_request_hash(request_data: dict[str, Any]) -> str:
|
|
| 56 |
return hashlib.sha256(encoded).hexdigest()
|
| 57 |
|
| 58 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 59 |
class GenerationRunner:
|
| 60 |
def __init__(
|
| 61 |
self,
|
|
@@ -80,6 +150,8 @@ class GenerationRunner:
|
|
| 80 |
self.output_dir = self.task_dir / "outputs"
|
| 81 |
self.output_dir.mkdir(parents=True, exist_ok=True)
|
| 82 |
self.agent = None
|
|
|
|
|
|
|
| 83 |
preview_token = str(self.request.get("preview_token") or "").strip()
|
| 84 |
self.preview_token = preview_token if re.fullmatch(r"[A-Za-z0-9_-]{32,128}", preview_token) else ""
|
| 85 |
self.preview_path = None
|
|
@@ -90,6 +162,14 @@ class GenerationRunner:
|
|
| 90 |
self._preview_target_text = ""
|
| 91 |
self._preview_target_stage = "translation_waiting"
|
| 92 |
self._preview_target_complete = False
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 93 |
if self.preview_token:
|
| 94 |
preview_dir = Path(__file__).resolve().parents[1] / "static" / "previews"
|
| 95 |
preview_dir.mkdir(parents=True, exist_ok=True)
|
|
@@ -112,8 +192,9 @@ class GenerationRunner:
|
|
| 112 |
metrics = getattr(agent, "source_metrics", {}) or {}
|
| 113 |
s2_metric = metrics.get("s2", {}) if isinstance(metrics, dict) else {}
|
| 114 |
oa_metric = metrics.get("oa", {}) if isinstance(metrics, dict) else {}
|
|
|
|
| 115 |
return (
|
| 116 |
-
int(getattr(agent, "ds_calls", 0) or 0),
|
| 117 |
int(s2_metric.get("eligible_result_count", 0) or 0),
|
| 118 |
int(oa_metric.get("eligible_result_count", 0) or 0),
|
| 119 |
)
|
|
@@ -146,39 +227,50 @@ class GenerationRunner:
|
|
| 146 |
return text.strip()
|
| 147 |
|
| 148 |
def _write_preview_payload(self, *, force: bool = False) -> None:
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 170 |
|
| 171 |
def _publish_preview(self, text: str, stage: str, *, force: bool = False, complete: bool = False) -> None:
|
| 172 |
-
self.
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
self
|
| 180 |
-
|
| 181 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 182 |
|
| 183 |
def _stream_text(self, generator, *, preview_prefix: str | None = None, preview_stage: str = "") -> str:
|
| 184 |
chunks = []
|
|
@@ -220,6 +312,14 @@ class GenerationRunner:
|
|
| 220 |
return ''
|
| 221 |
return f"# {title}\n" + match.group(1).strip()
|
| 222 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 223 |
@staticmethod
|
| 224 |
def _extract_front_matter(manuscript: str, first_section_title: str) -> str:
|
| 225 |
value = str(manuscript or '')
|
|
@@ -276,10 +376,110 @@ class GenerationRunner:
|
|
| 276 |
parts.append(str(record['translated_text']).strip())
|
| 277 |
return '\n\n'.join(part for part in parts if part).strip()
|
| 278 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 279 |
def _translate_frozen_unit(
|
| 280 |
self, agent, *, unit_key: str, unit_name: str, source_text: str, target_lang: str,
|
| 281 |
translation_sections: dict, translation_glossary: dict, adaptive_structure, include_front=False,
|
| 282 |
-
defer_failures=True,
|
| 283 |
):
|
| 284 |
source = str(source_text or '').strip()
|
| 285 |
if not source:
|
|
@@ -298,7 +498,10 @@ class GenerationRunner:
|
|
| 298 |
):
|
| 299 |
return translation_sections, translation_glossary
|
| 300 |
partial = self._assemble_partial_translation(translation_sections, adaptive_structure, include_front=include_front)
|
| 301 |
-
self._publish_target_preview(
|
|
|
|
|
|
|
|
|
|
| 302 |
updated_sections = dict(translation_sections or {})
|
| 303 |
|
| 304 |
def publish_translation_stage(stage):
|
|
@@ -307,7 +510,10 @@ class GenerationRunner:
|
|
| 307 |
preview_stage = f'translation_qa:{unit_name}'
|
| 308 |
else:
|
| 309 |
preview_stage = f'translating:{unit_name}'
|
| 310 |
-
self._publish_target_preview(
|
|
|
|
|
|
|
|
|
|
| 311 |
|
| 312 |
try:
|
| 313 |
translate_method = agent.translate_frozen_section
|
|
@@ -349,7 +555,8 @@ class GenerationRunner:
|
|
| 349 |
unit_name, error_code, type(error).__name__,
|
| 350 |
)
|
| 351 |
self._publish_target_preview(
|
| 352 |
-
partial, f'translation_repair_pending:{unit_name}', force=True, complete=False
|
|
|
|
| 353 |
)
|
| 354 |
return updated_sections, translation_glossary
|
| 355 |
updated_glossary = dict(result.get('glossary') or translation_glossary or {})
|
|
@@ -367,7 +574,10 @@ class GenerationRunner:
|
|
| 367 |
'completed_at': datetime.now(timezone.utc).isoformat(),
|
| 368 |
}
|
| 369 |
partial = self._assemble_partial_translation(updated_sections, adaptive_structure, include_front=include_front)
|
| 370 |
-
self._publish_target_preview(
|
|
|
|
|
|
|
|
|
|
| 371 |
return updated_sections, updated_glossary
|
| 372 |
|
| 373 |
@staticmethod
|
|
@@ -485,7 +695,7 @@ class GenerationRunner:
|
|
| 485 |
output.append(str(path))
|
| 486 |
return output
|
| 487 |
|
| 488 |
-
def
|
| 489 |
req = self.request
|
| 490 |
mode = str(req.get("mode") or "").lower()
|
| 491 |
if mode not in {"amj", "asq"}:
|
|
@@ -714,6 +924,26 @@ class GenerationRunner:
|
|
| 714 |
if self._preview_target_text == '':
|
| 715 |
self._preview_target_text = str(state.get('full_draft_translated') or '')
|
| 716 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 717 |
# Backward-compatible resume path for a pre-R3.2.8.3 checkpoint that has
|
| 718 |
# an outline but no Working Abstract yet. Generate it once, checkpoint it,
|
| 719 |
# and reuse it thereafter; never regenerate an unchanged anchor on resume.
|
|
@@ -739,9 +969,9 @@ class GenerationRunner:
|
|
| 739 |
self._publish_preview(full_draft_en, "working_front_matter", force=True)
|
| 740 |
self._checkpoint("working_front_matter_ready", state, 15)
|
| 741 |
|
| 742 |
-
#
|
| 743 |
-
#
|
| 744 |
-
#
|
| 745 |
if working_abstract and not state.get("english_finalized"):
|
| 746 |
section_order_for_front = self._unique_top_level_sections(adaptive_structure)
|
| 747 |
if section_order_for_front:
|
|
@@ -752,23 +982,32 @@ class GenerationRunner:
|
|
| 752 |
# first body heading does not exist yet. Preserve legacy
|
| 753 |
# resume behavior when completed body sections are present.
|
| 754 |
working_front_source = str(full_draft_en or '').strip()
|
| 755 |
-
|
| 756 |
-
|
| 757 |
-
|
| 758 |
-
|
| 759 |
-
|
|
|
|
| 760 |
)
|
| 761 |
-
|
| 762 |
-
|
| 763 |
-
|
| 764 |
-
|
| 765 |
-
|
| 766 |
-
|
| 767 |
-
|
| 768 |
-
|
| 769 |
-
|
| 770 |
-
|
| 771 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 772 |
if not citation_plan:
|
| 773 |
self._progress(15, "citation_planning")
|
| 774 |
citation_plan = agent.build_claim_evidence_plan(
|
|
@@ -995,12 +1234,17 @@ Output only final academic prose. Do not report the word count."""
|
|
| 995 |
section_key = str(sec.get('section_id') or sec_title)
|
| 996 |
if self._section_is_complete(adaptive_structure, completed_sections, section_key):
|
| 997 |
frozen_section = self._extract_top_level_section(full_draft_en, sec_title)
|
| 998 |
-
|
| 999 |
-
|
| 1000 |
-
|
| 1001 |
-
|
| 1002 |
-
|
| 1003 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1004 |
partial_translation = self._assemble_partial_translation(translation_sections, adaptive_structure, include_front=True)
|
| 1005 |
state.update(
|
| 1006 |
{
|
|
@@ -1080,12 +1324,16 @@ Output only final academic prose. Do not report the word count."""
|
|
| 1080 |
section_order_for_front = self._unique_top_level_sections(adaptive_structure)
|
| 1081 |
if section_order_for_front:
|
| 1082 |
final_front_source = self._extract_front_matter(body_only, section_order_for_front[0][1])
|
| 1083 |
-
|
| 1084 |
-
|
| 1085 |
-
|
| 1086 |
-
|
| 1087 |
-
|
|
|
|
| 1088 |
)
|
|
|
|
|
|
|
|
|
|
| 1089 |
partial_target = self._assemble_partial_translation(
|
| 1090 |
translation_sections, adaptive_structure, include_front=True
|
| 1091 |
)
|
|
@@ -1114,6 +1362,13 @@ Output only final academic prose. Do not report the word count."""
|
|
| 1114 |
translation_glossary = dict(state.get('translation_glossary') or translation_glossary or {})
|
| 1115 |
if not state.get("translation_complete") or not full_draft_translated:
|
| 1116 |
self._progress(94, "translation")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1117 |
body_only, reference_tail = agent.validate_reference_tail(
|
| 1118 |
full_draft_en, minimum_references=MIN_FINAL_REFERENCES
|
| 1119 |
)
|
|
@@ -1126,10 +1381,11 @@ Output only final academic prose. Do not report the word count."""
|
|
| 1126 |
final_units.append((key, title, self._extract_top_level_section(body_only, title)))
|
| 1127 |
for index, (unit_key, unit_name, source_text) in enumerate(final_units, start=1):
|
| 1128 |
translation_sections, translation_glossary = self._translate_frozen_unit(
|
| 1129 |
-
|
| 1130 |
target_lang=target_lang, translation_sections=translation_sections,
|
| 1131 |
translation_glossary=translation_glossary, adaptive_structure=adaptive_structure,
|
| 1132 |
include_front=True, defer_failures=False,
|
|
|
|
| 1133 |
)
|
| 1134 |
partial = self._assemble_partial_translation(
|
| 1135 |
translation_sections, adaptive_structure, include_front=True
|
|
@@ -1207,6 +1463,12 @@ Output only final academic prose. Do not report the word count."""
|
|
| 1207 |
self._checkpoint("package_validated", state, 99)
|
| 1208 |
return {"output_filename": output_filename, "output_path": str(final)}
|
| 1209 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1210 |
def _progress_activity(self) -> None:
|
| 1211 |
# Activity remains Executor-owned. We also snapshot actual request
|
| 1212 |
# counters so the next Executor heartbeat can persist live UI metrics.
|
|
|
|
| 13 |
import logging
|
| 14 |
import math
|
| 15 |
import os
|
| 16 |
+
import queue
|
| 17 |
import re
|
| 18 |
+
import threading
|
| 19 |
import time
|
| 20 |
+
from concurrent.futures import Future
|
| 21 |
from datetime import datetime, timezone, timedelta
|
| 22 |
from pathlib import Path
|
| 23 |
from typing import Callable, Any
|
|
|
|
| 37 |
MIN_FINAL_REFERENCES = 50
|
| 38 |
LITERATURE_EVIDENCE_TARGET = 75
|
| 39 |
CITATION_PLAN_TARGET = 70
|
| 40 |
+
TRANSLATION_CONTRACT_VERSION = 'hierarchical-multilingual-v3'
|
| 41 |
|
| 42 |
|
| 43 |
def _clean_trailing_references(text, keywords=("reference", "references", "reference list", "参考文献")):
|
|
|
|
| 59 |
return hashlib.sha256(encoded).hexdigest()
|
| 60 |
|
| 61 |
|
| 62 |
+
class _OrderedTranslationWorker:
|
| 63 |
+
"""One daemon worker owned by the headless Executor process.
|
| 64 |
+
|
| 65 |
+
It intentionally has no checkpoint authority. Results are returned through
|
| 66 |
+
Future objects and are checkpointed only by GenerationRunner's main thread.
|
| 67 |
+
"""
|
| 68 |
+
|
| 69 |
+
def __init__(self, handler):
|
| 70 |
+
self._handler = handler
|
| 71 |
+
self._queue = queue.Queue()
|
| 72 |
+
self._stop = threading.Event()
|
| 73 |
+
self._thread = threading.Thread(
|
| 74 |
+
target=self._run,
|
| 75 |
+
name="smartpaper-translation-worker",
|
| 76 |
+
daemon=True,
|
| 77 |
+
)
|
| 78 |
+
self._thread.start()
|
| 79 |
+
|
| 80 |
+
@property
|
| 81 |
+
def stop_event(self):
|
| 82 |
+
return self._stop
|
| 83 |
+
|
| 84 |
+
def submit(self, job: dict[str, Any]) -> Future:
|
| 85 |
+
future = Future()
|
| 86 |
+
self._queue.put((future, dict(job)))
|
| 87 |
+
return future
|
| 88 |
+
|
| 89 |
+
def _run(self):
|
| 90 |
+
while True:
|
| 91 |
+
item = self._queue.get()
|
| 92 |
+
try:
|
| 93 |
+
if item is None:
|
| 94 |
+
return
|
| 95 |
+
future, job = item
|
| 96 |
+
if not future.set_running_or_notify_cancel():
|
| 97 |
+
continue
|
| 98 |
+
try:
|
| 99 |
+
result = self._handler(job)
|
| 100 |
+
except BaseException as error:
|
| 101 |
+
future.set_exception(error)
|
| 102 |
+
else:
|
| 103 |
+
future.set_result(result)
|
| 104 |
+
finally:
|
| 105 |
+
self._queue.task_done()
|
| 106 |
+
|
| 107 |
+
def close(self, *, wait: bool = False) -> None:
|
| 108 |
+
if self._stop.is_set():
|
| 109 |
+
if wait and self._thread.is_alive():
|
| 110 |
+
self._thread.join(timeout=2.0)
|
| 111 |
+
return
|
| 112 |
+
self._stop.set()
|
| 113 |
+
while True:
|
| 114 |
+
try:
|
| 115 |
+
item = self._queue.get_nowait()
|
| 116 |
+
except queue.Empty:
|
| 117 |
+
break
|
| 118 |
+
try:
|
| 119 |
+
if item is not None:
|
| 120 |
+
future, _job = item
|
| 121 |
+
future.cancel()
|
| 122 |
+
finally:
|
| 123 |
+
self._queue.task_done()
|
| 124 |
+
self._queue.put(None)
|
| 125 |
+
if wait and self._thread.is_alive():
|
| 126 |
+
self._thread.join(timeout=2.0)
|
| 127 |
+
|
| 128 |
+
|
| 129 |
class GenerationRunner:
|
| 130 |
def __init__(
|
| 131 |
self,
|
|
|
|
| 150 |
self.output_dir = self.task_dir / "outputs"
|
| 151 |
self.output_dir.mkdir(parents=True, exist_ok=True)
|
| 152 |
self.agent = None
|
| 153 |
+
self.translation_agent = None
|
| 154 |
+
self._preview_lock = threading.RLock()
|
| 155 |
preview_token = str(self.request.get("preview_token") or "").strip()
|
| 156 |
self.preview_token = preview_token if re.fullmatch(r"[A-Za-z0-9_-]{32,128}", preview_token) else ""
|
| 157 |
self.preview_path = None
|
|
|
|
| 162 |
self._preview_target_text = ""
|
| 163 |
self._preview_target_stage = "translation_waiting"
|
| 164 |
self._preview_target_complete = False
|
| 165 |
+
self._preview_target_english_commit_text = ""
|
| 166 |
+
self._translation_worker = None
|
| 167 |
+
self._translation_worker_sections = {}
|
| 168 |
+
self._translation_worker_glossary = {}
|
| 169 |
+
self._translation_worker_jobs = []
|
| 170 |
+
self._translation_worker_collected = 0
|
| 171 |
+
self._translation_worker_pending = set()
|
| 172 |
+
self._translation_worker_context = {}
|
| 173 |
if self.preview_token:
|
| 174 |
preview_dir = Path(__file__).resolve().parents[1] / "static" / "previews"
|
| 175 |
preview_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
| 192 |
metrics = getattr(agent, "source_metrics", {}) or {}
|
| 193 |
s2_metric = metrics.get("s2", {}) if isinstance(metrics, dict) else {}
|
| 194 |
oa_metric = metrics.get("oa", {}) if isinstance(metrics, dict) else {}
|
| 195 |
+
translator_calls = int(getattr(getattr(self, "translation_agent", None), "ds_calls", 0) or 0)
|
| 196 |
return (
|
| 197 |
+
int(getattr(agent, "ds_calls", 0) or 0) + translator_calls,
|
| 198 |
int(s2_metric.get("eligible_result_count", 0) or 0),
|
| 199 |
int(oa_metric.get("eligible_result_count", 0) or 0),
|
| 200 |
)
|
|
|
|
| 227 |
return text.strip()
|
| 228 |
|
| 229 |
def _write_preview_payload(self, *, force: bool = False) -> None:
|
| 230 |
+
with self._preview_lock:
|
| 231 |
+
if self.preview_path is None:
|
| 232 |
+
return
|
| 233 |
+
now = time.monotonic()
|
| 234 |
+
if not force and now - self._last_preview_write < 0.45:
|
| 235 |
+
return
|
| 236 |
+
payload = {
|
| 237 |
+
"task_id": str(self.request.get("task_id") or ""),
|
| 238 |
+
"request_hash": self.request_hash,
|
| 239 |
+
"execution_epoch": self.execution_epoch,
|
| 240 |
+
"stage": str(self._preview_english_stage or "")[:240],
|
| 241 |
+
"text": self._preview_visible_text(self._preview_english_text),
|
| 242 |
+
"complete": bool(self._preview_english_complete),
|
| 243 |
+
"target_stage": str(self._preview_target_stage or "")[:240],
|
| 244 |
+
"target_text": self._preview_visible_text(self._preview_target_text),
|
| 245 |
+
"target_complete": bool(self._preview_target_complete),
|
| 246 |
+
"target_english_commit_text": self._preview_visible_text(
|
| 247 |
+
self._preview_target_english_commit_text
|
| 248 |
+
),
|
| 249 |
+
"updated_at": datetime.now(timezone.utc).isoformat(),
|
| 250 |
+
}
|
| 251 |
+
temporary = self.preview_path.with_suffix(".json.tmp")
|
| 252 |
+
temporary.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
|
| 253 |
+
os.replace(temporary, self.preview_path)
|
| 254 |
+
self._last_preview_write = now
|
| 255 |
|
| 256 |
def _publish_preview(self, text: str, stage: str, *, force: bool = False, complete: bool = False) -> None:
|
| 257 |
+
with self._preview_lock:
|
| 258 |
+
self._preview_english_text = str(text or "")
|
| 259 |
+
self._preview_english_stage = str(stage or "")
|
| 260 |
+
self._preview_english_complete = bool(complete)
|
| 261 |
+
self._write_preview_payload(force=force)
|
| 262 |
+
|
| 263 |
+
def _publish_target_preview(
|
| 264 |
+
self, text: str, stage: str, *, force: bool = True, complete: bool = False,
|
| 265 |
+
english_commit_text: str | None = None,
|
| 266 |
+
) -> None:
|
| 267 |
+
with self._preview_lock:
|
| 268 |
+
self._preview_target_text = str(text or "")
|
| 269 |
+
self._preview_target_stage = str(stage or "")
|
| 270 |
+
self._preview_target_complete = bool(complete)
|
| 271 |
+
if english_commit_text is not None:
|
| 272 |
+
self._preview_target_english_commit_text = str(english_commit_text or "")
|
| 273 |
+
self._write_preview_payload(force=force)
|
| 274 |
|
| 275 |
def _stream_text(self, generator, *, preview_prefix: str | None = None, preview_stage: str = "") -> str:
|
| 276 |
chunks = []
|
|
|
|
| 312 |
return ''
|
| 313 |
return f"# {title}\n" + match.group(1).strip()
|
| 314 |
|
| 315 |
+
@staticmethod
|
| 316 |
+
def _english_prefix_through_top_level(manuscript: str, title: str) -> str:
|
| 317 |
+
value = str(manuscript or '')
|
| 318 |
+
match = re.search(
|
| 319 |
+
rf'(?ms)^#\s+{re.escape(str(title))}\s*$\n.*?(?=^#\s+[^#].*?$|\Z)', value
|
| 320 |
+
)
|
| 321 |
+
return value[:match.end()].strip() if match else value.strip()
|
| 322 |
+
|
| 323 |
@staticmethod
|
| 324 |
def _extract_front_matter(manuscript: str, first_section_title: str) -> str:
|
| 325 |
value = str(manuscript or '')
|
|
|
|
| 376 |
parts.append(str(record['translated_text']).strip())
|
| 377 |
return '\n\n'.join(part for part in parts if part).strip()
|
| 378 |
|
| 379 |
+
def _start_translation_worker(
|
| 380 |
+
self, agent, *, translation_sections, translation_glossary,
|
| 381 |
+
adaptive_structure, target_lang,
|
| 382 |
+
) -> None:
|
| 383 |
+
if self._translation_worker is not None:
|
| 384 |
+
return
|
| 385 |
+
self.translation_agent = agent
|
| 386 |
+
self._translation_worker_sections = dict(translation_sections or {})
|
| 387 |
+
self._translation_worker_glossary = dict(translation_glossary or {})
|
| 388 |
+
self._translation_worker_context = {
|
| 389 |
+
'adaptive_structure': list(adaptive_structure or []),
|
| 390 |
+
'target_lang': str(target_lang or ''),
|
| 391 |
+
}
|
| 392 |
+
self._translation_worker = _OrderedTranslationWorker(self._run_translation_job)
|
| 393 |
+
|
| 394 |
+
def _run_translation_job(self, job: dict[str, Any]) -> dict[str, Any]:
|
| 395 |
+
if self.translation_agent is None:
|
| 396 |
+
raise RuntimeError('translation_worker_agent_missing')
|
| 397 |
+
context = dict(self._translation_worker_context or {})
|
| 398 |
+
sections, glossary = self._translate_frozen_unit(
|
| 399 |
+
self.translation_agent,
|
| 400 |
+
unit_key=str(job.get('unit_key') or ''),
|
| 401 |
+
unit_name=str(job.get('unit_name') or ''),
|
| 402 |
+
source_text=str(job.get('source_text') or ''),
|
| 403 |
+
target_lang=str(context.get('target_lang') or ''),
|
| 404 |
+
translation_sections=self._translation_worker_sections,
|
| 405 |
+
translation_glossary=self._translation_worker_glossary,
|
| 406 |
+
adaptive_structure=context.get('adaptive_structure') or [],
|
| 407 |
+
include_front=True,
|
| 408 |
+
defer_failures=True,
|
| 409 |
+
english_commit_text=str(job.get('english_commit_text') or ''),
|
| 410 |
+
)
|
| 411 |
+
self._translation_worker_sections = dict(sections or {})
|
| 412 |
+
self._translation_worker_glossary = dict(glossary or {})
|
| 413 |
+
return {
|
| 414 |
+
'unit_key': str(job.get('unit_key') or ''),
|
| 415 |
+
'source_hash': str(job.get('source_hash') or ''),
|
| 416 |
+
'translation_sections': dict(self._translation_worker_sections),
|
| 417 |
+
'translation_glossary': dict(self._translation_worker_glossary),
|
| 418 |
+
}
|
| 419 |
+
|
| 420 |
+
def _enqueue_translation_unit(
|
| 421 |
+
self, agent, *, unit_key: str, unit_name: str, source_text: str,
|
| 422 |
+
translation_sections: dict, english_commit_text: str,
|
| 423 |
+
) -> bool:
|
| 424 |
+
source = str(source_text or '').strip()
|
| 425 |
+
if not source or self._translation_worker is None:
|
| 426 |
+
return False
|
| 427 |
+
source_hash = self._text_hash(source)
|
| 428 |
+
fact_ledger_hash = self._translation_ledger_hash(agent, source)
|
| 429 |
+
existing = dict((translation_sections or {}).get(unit_key) or {})
|
| 430 |
+
if (
|
| 431 |
+
existing.get('qa_status') in {'passed', 'passed_with_warning'}
|
| 432 |
+
and existing.get('source_hash') == source_hash
|
| 433 |
+
and existing.get('target_lang') == str(self._translation_worker_context.get('target_lang') or '')
|
| 434 |
+
and existing.get('translation_contract_version') == TRANSLATION_CONTRACT_VERSION
|
| 435 |
+
and existing.get('fact_ledger_hash') == fact_ledger_hash
|
| 436 |
+
and existing.get('translated_text')
|
| 437 |
+
):
|
| 438 |
+
return False
|
| 439 |
+
pending_key = (str(unit_key), source_hash, fact_ledger_hash)
|
| 440 |
+
if pending_key in self._translation_worker_pending:
|
| 441 |
+
return False
|
| 442 |
+
future = self._translation_worker.submit({
|
| 443 |
+
'unit_key': str(unit_key),
|
| 444 |
+
'unit_name': str(unit_name),
|
| 445 |
+
'source_text': source,
|
| 446 |
+
'source_hash': source_hash,
|
| 447 |
+
'fact_ledger_hash': fact_ledger_hash,
|
| 448 |
+
'english_commit_text': str(english_commit_text or ''),
|
| 449 |
+
})
|
| 450 |
+
self._translation_worker_pending.add(pending_key)
|
| 451 |
+
self._translation_worker_jobs.append({
|
| 452 |
+
'pending_key': pending_key,
|
| 453 |
+
'future': future,
|
| 454 |
+
})
|
| 455 |
+
return True
|
| 456 |
+
|
| 457 |
+
def _collect_translation_results(self, translation_sections, translation_glossary, *, block=False):
|
| 458 |
+
sections = dict(translation_sections or {})
|
| 459 |
+
glossary = dict(translation_glossary or {})
|
| 460 |
+
while self._translation_worker_collected < len(self._translation_worker_jobs):
|
| 461 |
+
item = self._translation_worker_jobs[self._translation_worker_collected]
|
| 462 |
+
future = item['future']
|
| 463 |
+
if not block and not future.done():
|
| 464 |
+
break
|
| 465 |
+
result = future.result()
|
| 466 |
+
sections = dict(result.get('translation_sections') or sections)
|
| 467 |
+
glossary = dict(result.get('translation_glossary') or glossary)
|
| 468 |
+
self._translation_worker_pending.discard(item['pending_key'])
|
| 469 |
+
self._translation_worker_collected += 1
|
| 470 |
+
return sections, glossary
|
| 471 |
+
|
| 472 |
+
def _shutdown_translation_worker(self, *, wait=False) -> None:
|
| 473 |
+
worker = self._translation_worker
|
| 474 |
+
if worker is None:
|
| 475 |
+
return
|
| 476 |
+
worker.close(wait=bool(wait))
|
| 477 |
+
self._translation_worker = None
|
| 478 |
+
|
| 479 |
def _translate_frozen_unit(
|
| 480 |
self, agent, *, unit_key: str, unit_name: str, source_text: str, target_lang: str,
|
| 481 |
translation_sections: dict, translation_glossary: dict, adaptive_structure, include_front=False,
|
| 482 |
+
defer_failures=True, english_commit_text: str | None = None,
|
| 483 |
):
|
| 484 |
source = str(source_text or '').strip()
|
| 485 |
if not source:
|
|
|
|
| 498 |
):
|
| 499 |
return translation_sections, translation_glossary
|
| 500 |
partial = self._assemble_partial_translation(translation_sections, adaptive_structure, include_front=include_front)
|
| 501 |
+
self._publish_target_preview(
|
| 502 |
+
partial, f'translating:{unit_name}', force=True, complete=False,
|
| 503 |
+
english_commit_text=english_commit_text,
|
| 504 |
+
)
|
| 505 |
updated_sections = dict(translation_sections or {})
|
| 506 |
|
| 507 |
def publish_translation_stage(stage):
|
|
|
|
| 510 |
preview_stage = f'translation_qa:{unit_name}'
|
| 511 |
else:
|
| 512 |
preview_stage = f'translating:{unit_name}'
|
| 513 |
+
self._publish_target_preview(
|
| 514 |
+
partial, preview_stage, force=True, complete=False,
|
| 515 |
+
english_commit_text=english_commit_text,
|
| 516 |
+
)
|
| 517 |
|
| 518 |
try:
|
| 519 |
translate_method = agent.translate_frozen_section
|
|
|
|
| 555 |
unit_name, error_code, type(error).__name__,
|
| 556 |
)
|
| 557 |
self._publish_target_preview(
|
| 558 |
+
partial, f'translation_repair_pending:{unit_name}', force=True, complete=False,
|
| 559 |
+
english_commit_text=english_commit_text,
|
| 560 |
)
|
| 561 |
return updated_sections, translation_glossary
|
| 562 |
updated_glossary = dict(result.get('glossary') or translation_glossary or {})
|
|
|
|
| 574 |
'completed_at': datetime.now(timezone.utc).isoformat(),
|
| 575 |
}
|
| 576 |
partial = self._assemble_partial_translation(updated_sections, adaptive_structure, include_front=include_front)
|
| 577 |
+
self._publish_target_preview(
|
| 578 |
+
partial, f'translation_ready:{unit_name}', force=True, complete=False,
|
| 579 |
+
english_commit_text=english_commit_text,
|
| 580 |
+
)
|
| 581 |
return updated_sections, updated_glossary
|
| 582 |
|
| 583 |
@staticmethod
|
|
|
|
| 695 |
output.append(str(path))
|
| 696 |
return output
|
| 697 |
|
| 698 |
+
def _run_impl(self) -> dict[str, Any]:
|
| 699 |
req = self.request
|
| 700 |
mode = str(req.get("mode") or "").lower()
|
| 701 |
if mode not in {"amj", "asq"}:
|
|
|
|
| 924 |
if self._preview_target_text == '':
|
| 925 |
self._preview_target_text = str(state.get('full_draft_translated') or '')
|
| 926 |
|
| 927 |
+
# Target-language work remains Executor-owned but no longer blocks the
|
| 928 |
+
# English writer. A dedicated single translation agent/worker keeps
|
| 929 |
+
# glossary order deterministic and avoids sharing mutable writer state.
|
| 930 |
+
translation_agent = SSciPipelineAgent(ds_key, s2_key)
|
| 931 |
+
translation_agent.set_execution_progress_hook(lambda: self._progress_activity())
|
| 932 |
+
self._start_translation_worker(
|
| 933 |
+
translation_agent,
|
| 934 |
+
translation_sections=translation_sections,
|
| 935 |
+
translation_glossary=translation_glossary,
|
| 936 |
+
adaptive_structure=adaptive_structure,
|
| 937 |
+
target_lang=target_lang,
|
| 938 |
+
)
|
| 939 |
+
# Capture the worker stop Event itself. `_shutdown_translation_worker()`
|
| 940 |
+
# clears `self._translation_worker`; a dynamic self lookup here would
|
| 941 |
+
# otherwise let an already-running target call miss task shutdown.
|
| 942 |
+
translation_stop_event = self._translation_worker.stop_event
|
| 943 |
+
translation_agent.set_cancel_checker(
|
| 944 |
+
lambda: self.cancel_checker() or translation_stop_event.is_set()
|
| 945 |
+
)
|
| 946 |
+
|
| 947 |
# Backward-compatible resume path for a pre-R3.2.8.3 checkpoint that has
|
| 948 |
# an outline but no Working Abstract yet. Generate it once, checkpoint it,
|
| 949 |
# and reuse it thereafter; never regenerate an unchanged anchor on resume.
|
|
|
|
| 969 |
self._publish_preview(full_draft_en, "working_front_matter", force=True)
|
| 970 |
self._checkpoint("working_front_matter_ready", state, 15)
|
| 971 |
|
| 972 |
+
# Queue the QA-frozen Working front matter. Translation is allowed to
|
| 973 |
+
# overlap later English work, but only frozen English units may enter
|
| 974 |
+
# this Executor-owned ordered worker.
|
| 975 |
if working_abstract and not state.get("english_finalized"):
|
| 976 |
section_order_for_front = self._unique_top_level_sections(adaptive_structure)
|
| 977 |
if section_order_for_front:
|
|
|
|
| 982 |
# first body heading does not exist yet. Preserve legacy
|
| 983 |
# resume behavior when completed body sections are present.
|
| 984 |
working_front_source = str(full_draft_en or '').strip()
|
| 985 |
+
self._enqueue_translation_unit(
|
| 986 |
+
translation_agent,
|
| 987 |
+
unit_key='__front_matter__', unit_name='Working Front Matter',
|
| 988 |
+
source_text=working_front_source,
|
| 989 |
+
translation_sections=translation_sections,
|
| 990 |
+
english_commit_text=working_front_source,
|
| 991 |
)
|
| 992 |
+
|
| 993 |
+
# Resume/idempotency: completed English top-level sections whose current
|
| 994 |
+
# source-hash translation is absent/stale are queued exactly once.
|
| 995 |
+
for section_key, section_title in self._unique_top_level_sections(adaptive_structure):
|
| 996 |
+
if not self._section_is_complete(adaptive_structure, completed_sections, section_key):
|
| 997 |
+
continue
|
| 998 |
+
frozen_section = self._extract_top_level_section(full_draft_en, section_title)
|
| 999 |
+
self._enqueue_translation_unit(
|
| 1000 |
+
translation_agent,
|
| 1001 |
+
unit_key=section_key,
|
| 1002 |
+
unit_name=section_title,
|
| 1003 |
+
source_text=frozen_section,
|
| 1004 |
+
translation_sections=translation_sections,
|
| 1005 |
+
english_commit_text=self._english_prefix_through_top_level(full_draft_en, section_title),
|
| 1006 |
+
)
|
| 1007 |
+
|
| 1008 |
+
translation_sections, translation_glossary = self._collect_translation_results(
|
| 1009 |
+
translation_sections, translation_glossary, block=False
|
| 1010 |
+
)
|
| 1011 |
if not citation_plan:
|
| 1012 |
self._progress(15, "citation_planning")
|
| 1013 |
citation_plan = agent.build_claim_evidence_plan(
|
|
|
|
| 1234 |
section_key = str(sec.get('section_id') or sec_title)
|
| 1235 |
if self._section_is_complete(adaptive_structure, completed_sections, section_key):
|
| 1236 |
frozen_section = self._extract_top_level_section(full_draft_en, sec_title)
|
| 1237 |
+
self._enqueue_translation_unit(
|
| 1238 |
+
translation_agent,
|
| 1239 |
+
unit_key=section_key, unit_name=sec_title, source_text=frozen_section,
|
| 1240 |
+
translation_sections=translation_sections,
|
| 1241 |
+
english_commit_text=full_draft_en,
|
| 1242 |
)
|
| 1243 |
+
# Collect only already-finished target jobs. This call never waits,
|
| 1244 |
+
# so English Section N+1 can start while target Section N runs.
|
| 1245 |
+
translation_sections, translation_glossary = self._collect_translation_results(
|
| 1246 |
+
translation_sections, translation_glossary, block=False
|
| 1247 |
+
)
|
| 1248 |
partial_translation = self._assemble_partial_translation(translation_sections, adaptive_structure, include_front=True)
|
| 1249 |
state.update(
|
| 1250 |
{
|
|
|
|
| 1324 |
section_order_for_front = self._unique_top_level_sections(adaptive_structure)
|
| 1325 |
if section_order_for_front:
|
| 1326 |
final_front_source = self._extract_front_matter(body_only, section_order_for_front[0][1])
|
| 1327 |
+
self._enqueue_translation_unit(
|
| 1328 |
+
translation_agent,
|
| 1329 |
+
unit_key='__front_matter__', unit_name='Final Front Matter',
|
| 1330 |
+
source_text=final_front_source,
|
| 1331 |
+
translation_sections=translation_sections,
|
| 1332 |
+
english_commit_text=full_draft_en,
|
| 1333 |
)
|
| 1334 |
+
translation_sections, translation_glossary = self._collect_translation_results(
|
| 1335 |
+
translation_sections, translation_glossary, block=False
|
| 1336 |
+
)
|
| 1337 |
partial_target = self._assemble_partial_translation(
|
| 1338 |
translation_sections, adaptive_structure, include_front=True
|
| 1339 |
)
|
|
|
|
| 1362 |
translation_glossary = dict(state.get('translation_glossary') or translation_glossary or {})
|
| 1363 |
if not state.get("translation_complete") or not full_draft_translated:
|
| 1364 |
self._progress(94, "translation")
|
| 1365 |
+
# English is fully frozen. Wait for all previously queued section
|
| 1366 |
+
# translations before the strict final gate; checkpoint ownership
|
| 1367 |
+
# remains on this main execution thread.
|
| 1368 |
+
translation_sections, translation_glossary = self._collect_translation_results(
|
| 1369 |
+
translation_sections, translation_glossary, block=True
|
| 1370 |
+
)
|
| 1371 |
+
self._shutdown_translation_worker(wait=True)
|
| 1372 |
body_only, reference_tail = agent.validate_reference_tail(
|
| 1373 |
full_draft_en, minimum_references=MIN_FINAL_REFERENCES
|
| 1374 |
)
|
|
|
|
| 1381 |
final_units.append((key, title, self._extract_top_level_section(body_only, title)))
|
| 1382 |
for index, (unit_key, unit_name, source_text) in enumerate(final_units, start=1):
|
| 1383 |
translation_sections, translation_glossary = self._translate_frozen_unit(
|
| 1384 |
+
translation_agent, unit_key=unit_key, unit_name=unit_name, source_text=source_text,
|
| 1385 |
target_lang=target_lang, translation_sections=translation_sections,
|
| 1386 |
translation_glossary=translation_glossary, adaptive_structure=adaptive_structure,
|
| 1387 |
include_front=True, defer_failures=False,
|
| 1388 |
+
english_commit_text=full_draft_en,
|
| 1389 |
)
|
| 1390 |
partial = self._assemble_partial_translation(
|
| 1391 |
translation_sections, adaptive_structure, include_front=True
|
|
|
|
| 1463 |
self._checkpoint("package_validated", state, 99)
|
| 1464 |
return {"output_filename": output_filename, "output_path": str(final)}
|
| 1465 |
|
| 1466 |
+
def run(self) -> dict[str, Any]:
|
| 1467 |
+
try:
|
| 1468 |
+
return self._run_impl()
|
| 1469 |
+
finally:
|
| 1470 |
+
self._shutdown_translation_worker(wait=False)
|
| 1471 |
+
|
| 1472 |
def _progress_activity(self) -> None:
|
| 1473 |
# Activity remains Executor-owned. We also snapshot actual request
|
| 1474 |
# counters so the next Executor heartbeat can persist live UI metrics.
|
core/pipeline_agent.py
CHANGED
|
@@ -61,7 +61,7 @@ class CitationAuditError(DeepSeekRequestError):
|
|
| 61 |
|
| 62 |
|
| 63 |
class SSciPipelineAgent:
|
| 64 |
-
TRANSLATION_CONTRACT_VERSION = 'hierarchical-multilingual-
|
| 65 |
|
| 66 |
def __init__(self, ds_key, s2_key=""):
|
| 67 |
self.ds_key = str(ds_key or "").strip()
|
|
@@ -3621,7 +3621,13 @@ Output only the repaired section."""
|
|
| 3621 |
@classmethod
|
| 3622 |
def translation_error_code(cls, error):
|
| 3623 |
message = str(error or '')
|
| 3624 |
-
if
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3625 |
return 'FACT_INTEGRITY'
|
| 3626 |
if 'claim marker mismatch' in message or 'marker mismatch' in message or 'heading-tree mismatch' in message:
|
| 3627 |
return 'STRUCTURE_INTEGRITY'
|
|
@@ -4184,6 +4190,55 @@ Output only the repaired section."""
|
|
| 4184 |
unowned['doi_url'] = urls
|
| 4185 |
return {key: values for key, values in unowned.items() if values}
|
| 4186 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4187 |
@classmethod
|
| 4188 |
def _validate_translation_skeleton(cls, value, mapping, block_id='', claim_sources=None):
|
| 4189 |
text = str(value or '')
|
|
@@ -4215,19 +4270,20 @@ Output only the repaired section."""
|
|
| 4215 |
f'Translation fact moved across claim boundary in {block_id}: {manifest.get("fact_id") or placeholder}.'
|
| 4216 |
)
|
| 4217 |
|
| 4218 |
-
ordered_types = {'NUMBER','QUANTITY','STAT','HYPOTHESIS','DOI_URL','FORMULA','QUOTE_ID'}
|
| 4219 |
for claim_id in claim_order:
|
| 4220 |
target_claim = str(target_claims.get(claim_id) or '')
|
| 4221 |
-
|
| 4222 |
placeholder for placeholder, raw in (mapping or {}).items()
|
| 4223 |
if str((raw if isinstance(raw, dict) else {}).get('claim_id') or 'C000001') == claim_id
|
| 4224 |
-
and str((raw if isinstance(raw, dict) else {}).get('type') or '') in ordered_types
|
| 4225 |
]
|
| 4226 |
-
|
| 4227 |
-
|
| 4228 |
-
|
|
|
|
|
|
|
|
|
|
| 4229 |
raise DeepSeekRequestError(
|
| 4230 |
-
f'Translation
|
| 4231 |
)
|
| 4232 |
|
| 4233 |
unowned = cls._translation_unowned_objects(target_claim)
|
|
@@ -4731,6 +4787,34 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
|
|
| 4731 |
raise failed[first_id]
|
| 4732 |
source_by_id = {block_id: source for block_id, source in batch}
|
| 4733 |
for block_id, error in failed.items():
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4734 |
logging.warning(
|
| 4735 |
'Translation protected block recovery: block=SP-P%s error=%s; strict isolated retry.',
|
| 4736 |
block_id, self._safe_source_error(error),
|
|
@@ -4974,6 +5058,8 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
|
|
| 4974 |
if (
|
| 4975 |
message.startswith('Translation claim-local QA failed')
|
| 4976 |
or message.startswith('Translation introduced unowned factual content')
|
|
|
|
|
|
|
| 4977 |
or message.startswith('Translation alignment')
|
| 4978 |
or message.startswith('Translation changed ')
|
| 4979 |
or message.startswith('Translation protected ')
|
|
|
|
| 61 |
|
| 62 |
|
| 63 |
class SSciPipelineAgent:
|
| 64 |
+
TRANSLATION_CONTRACT_VERSION = 'hierarchical-multilingual-v3'
|
| 65 |
|
| 66 |
def __init__(self, ds_key, s2_key=""):
|
| 67 |
self.ds_key = str(ds_key or "").strip()
|
|
|
|
| 3621 |
@classmethod
|
| 3622 |
def translation_error_code(cls, error):
|
| 3623 |
message = str(error or '')
|
| 3624 |
+
if (
|
| 3625 |
+
'unowned factual content' in message
|
| 3626 |
+
or 'protected placeholder mismatch' in message
|
| 3627 |
+
or 'fact moved across claim boundary' in message
|
| 3628 |
+
or 'fact order changed' in message
|
| 3629 |
+
or 'order-sensitive relation changed' in message
|
| 3630 |
+
):
|
| 3631 |
return 'FACT_INTEGRITY'
|
| 3632 |
if 'claim marker mismatch' in message or 'marker mismatch' in message or 'heading-tree mismatch' in message:
|
| 3633 |
return 'STRUCTURE_INTEGRITY'
|
|
|
|
| 4190 |
unowned['doi_url'] = urls
|
| 4191 |
return {key: values for key, values in unowned.items() if values}
|
| 4192 |
|
| 4193 |
+
@staticmethod
|
| 4194 |
+
def _translation_order_sensitive_placeholder_groups(protected_source):
|
| 4195 |
+
"""Return only high-confidence placeholder relations whose order is semantic.
|
| 4196 |
+
|
| 4197 |
+
Ordinary facts inside one claim are intentionally absent from this list:
|
| 4198 |
+
target-language grammar may reorder them. We retain order only for
|
| 4199 |
+
explicit directional/range/comparison surfaces where reversing the two
|
| 4200 |
+
protected endpoints would change the source relation.
|
| 4201 |
+
"""
|
| 4202 |
+
text = str(protected_source or '')
|
| 4203 |
+
p = r'(\[\[SP-T\d{6}\]\])'
|
| 4204 |
+
patterns = (
|
| 4205 |
+
rf'(?i:\bfrom\s+{p}\s+(?:up\s+)?to\s+{p})',
|
| 4206 |
+
rf'{p}\s*(?:->|→|⇒|⟶)\s*{p}',
|
| 4207 |
+
rf'{p}\s*(?:<|>|≤|≥)\s*{p}',
|
| 4208 |
+
rf'{p}\s*(?:–|—)\s*{p}',
|
| 4209 |
+
)
|
| 4210 |
+
groups = []
|
| 4211 |
+
for pattern in patterns:
|
| 4212 |
+
for match in re.finditer(pattern, text):
|
| 4213 |
+
pair = tuple(group for group in match.groups() if group)
|
| 4214 |
+
if len(pair) == 2 and pair not in groups:
|
| 4215 |
+
groups.append(pair)
|
| 4216 |
+
return groups
|
| 4217 |
+
|
| 4218 |
+
@classmethod
|
| 4219 |
+
def _translation_order_sensitive_relations_for_claim(cls, source_claim, claim_placeholders):
|
| 4220 |
+
"""Map local source relation placeholders back to the claim's global IDs."""
|
| 4221 |
+
protected_local, local_mapping = cls._protect_translation_tokens(source_claim, claim_id='C000001')
|
| 4222 |
+
local_ids = list(local_mapping)
|
| 4223 |
+
global_ids = list(claim_placeholders or [])
|
| 4224 |
+
if len(local_ids) != len(global_ids):
|
| 4225 |
+
return []
|
| 4226 |
+
id_map = dict(zip(local_ids, global_ids))
|
| 4227 |
+
relations = []
|
| 4228 |
+
for group in cls._translation_order_sensitive_placeholder_groups(protected_local):
|
| 4229 |
+
mapped = tuple(id_map.get(item, '') for item in group)
|
| 4230 |
+
if all(mapped) and mapped not in relations:
|
| 4231 |
+
relations.append(mapped)
|
| 4232 |
+
return relations
|
| 4233 |
+
|
| 4234 |
+
@staticmethod
|
| 4235 |
+
def _is_translation_relation_order_error(error):
|
| 4236 |
+
message = str(error or '')
|
| 4237 |
+
return (
|
| 4238 |
+
message.startswith('Translation order-sensitive relation changed')
|
| 4239 |
+
or message.startswith('Translation fact order changed')
|
| 4240 |
+
)
|
| 4241 |
+
|
| 4242 |
@classmethod
|
| 4243 |
def _validate_translation_skeleton(cls, value, mapping, block_id='', claim_sources=None):
|
| 4244 |
text = str(value or '')
|
|
|
|
| 4270 |
f'Translation fact moved across claim boundary in {block_id}: {manifest.get("fact_id") or placeholder}.'
|
| 4271 |
)
|
| 4272 |
|
|
|
|
| 4273 |
for claim_id in claim_order:
|
| 4274 |
target_claim = str(target_claims.get(claim_id) or '')
|
| 4275 |
+
claim_placeholders = [
|
| 4276 |
placeholder for placeholder, raw in (mapping or {}).items()
|
| 4277 |
if str((raw if isinstance(raw, dict) else {}).get('claim_id') or 'C000001') == claim_id
|
|
|
|
| 4278 |
]
|
| 4279 |
+
source_claim = str(private_claims.get(claim_id) or '')
|
| 4280 |
+
for relation in cls._translation_order_sensitive_relations_for_claim(
|
| 4281 |
+
source_claim, claim_placeholders
|
| 4282 |
+
):
|
| 4283 |
+
actual_order = sorted(relation, key=lambda placeholder: target_claim.find(placeholder))
|
| 4284 |
+
if tuple(actual_order) != tuple(relation):
|
| 4285 |
raise DeepSeekRequestError(
|
| 4286 |
+
f'Translation order-sensitive relation changed within owner claim in {block_id}: claim={claim_id}.'
|
| 4287 |
)
|
| 4288 |
|
| 4289 |
unowned = cls._translation_unowned_objects(target_claim)
|
|
|
|
| 4787 |
raise failed[first_id]
|
| 4788 |
source_by_id = {block_id: source for block_id, source in batch}
|
| 4789 |
for block_id, error in failed.items():
|
| 4790 |
+
if self._is_translation_relation_order_error(error):
|
| 4791 |
+
claim_match = re.search(r'claim=(C\d{6})', str(error))
|
| 4792 |
+
source_claims = dict(claim_sources_by_block.get(block_id) or {})
|
| 4793 |
+
claim_id = claim_match.group(1) if claim_match else ''
|
| 4794 |
+
if claim_id and claim_id in source_claims:
|
| 4795 |
+
logging.warning(
|
| 4796 |
+
'Translation relation-local recovery: block=SP-P%s claim=%s error=%s',
|
| 4797 |
+
block_id, claim_id, self._safe_source_error(error),
|
| 4798 |
+
)
|
| 4799 |
+
claim_order = list(source_claims)
|
| 4800 |
+
_order, segments = self._parse_translation_claim_segments(
|
| 4801 |
+
parsed[block_id], claim_order
|
| 4802 |
+
)
|
| 4803 |
+
segments[claim_id] = self._translate_single_claim_with_recovery(
|
| 4804 |
+
block_id, claim_id, source_claims[claim_id],
|
| 4805 |
+
system_prompt, target_lang, ['FACT_RELATION_ORDER'],
|
| 4806 |
+
)
|
| 4807 |
+
repaired = self._compose_translation_claim_segments(claim_order, segments)
|
| 4808 |
+
translated_with_claims = self._restore_translation_tokens(
|
| 4809 |
+
repaired, protected_maps.get(block_id, {}), f'SP-P{block_id}', target_lang,
|
| 4810 |
+
claim_sources=source_claims, keep_claim_markers=True,
|
| 4811 |
+
)
|
| 4812 |
+
clean_translation = self._strip_translation_claim_markers(translated_with_claims)
|
| 4813 |
+
self._validate_translation_block_protected(
|
| 4814 |
+
block_id, source_by_id[block_id], clean_translation, target_lang
|
| 4815 |
+
)
|
| 4816 |
+
restored[block_id] = translated_with_claims
|
| 4817 |
+
continue
|
| 4818 |
logging.warning(
|
| 4819 |
'Translation protected block recovery: block=SP-P%s error=%s; strict isolated retry.',
|
| 4820 |
block_id, self._safe_source_error(error),
|
|
|
|
| 5058 |
if (
|
| 5059 |
message.startswith('Translation claim-local QA failed')
|
| 5060 |
or message.startswith('Translation introduced unowned factual content')
|
| 5061 |
+
or message.startswith('Translation order-sensitive relation changed')
|
| 5062 |
+
or message.startswith('Translation fact order changed')
|
| 5063 |
or message.startswith('Translation alignment')
|
| 5064 |
or message.startswith('Translation changed ')
|
| 5065 |
or message.startswith('Translation protected ')
|
desktop.ini
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[.ShellClassInfo]
|
| 2 |
+
IconResource=C:\Program Files\Google\Drive File Stream\129.0.1.0\GoogleDriveFS.exe,27
|
smartpaper_restore_bridge/desktop.ini
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[.ShellClassInfo]
|
| 2 |
+
IconResource=C:\Program Files\Google\Drive File Stream\129.0.1.0\GoogleDriveFS.exe,27
|
smartpaper_restore_bridge/frontend/desktop.ini
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[.ShellClassInfo]
|
| 2 |
+
IconResource=C:\Program Files\Google\Drive File Stream\129.0.1.0\GoogleDriveFS.exe,27
|
smartpaper_restore_bridge/frontend/index.html
CHANGED
|
@@ -510,6 +510,28 @@
|
|
| 510 |
pumpRenderQueue(parentDocument, target, shell, channel);
|
| 511 |
}
|
| 512 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 513 |
function previewShouldFollow(parentWindow, shell, hasText) {
|
| 514 |
if (!shell) return false;
|
| 515 |
if (!hasText) return true;
|
|
@@ -543,6 +565,7 @@
|
|
| 543 |
catch (_) { return; }
|
| 544 |
const englishText = String(payload && payload.text || "");
|
| 545 |
const targetText = String(payload && payload.target_text || "");
|
|
|
|
| 546 |
if (englishText && englishText !== lastPreviewText) {
|
| 547 |
updateOnePreview(parentDocument, parentWindow, "sp-live-manuscript-preview", "sp-live-manuscript-shell", englishText, renderChannels.english);
|
| 548 |
const englishStatus = parentDocument.getElementById("sp-live-manuscript-status");
|
|
@@ -553,6 +576,7 @@
|
|
| 553 |
lastPreviewText = englishText;
|
| 554 |
}
|
| 555 |
if (targetText && targetText !== lastTargetPreviewText) {
|
|
|
|
| 556 |
updateOnePreview(parentDocument, parentWindow, "sp-live-translation-preview", "sp-live-translation-shell", targetText, renderChannels.target);
|
| 557 |
lastTargetPreviewText = targetText;
|
| 558 |
}
|
|
|
|
| 510 |
pumpRenderQueue(parentDocument, target, shell, channel);
|
| 511 |
}
|
| 512 |
|
| 513 |
+
function ensureEnglishCommitVisible(parentDocument, commitText) {
|
| 514 |
+
const commit = normalizePreviewDisplay(commitText);
|
| 515 |
+
if (!commit) return;
|
| 516 |
+
const channel = renderChannels.english;
|
| 517 |
+
const target = parentDocument.getElementById("sp-live-manuscript-preview");
|
| 518 |
+
const shell = parentDocument.getElementById("sp-live-manuscript-shell");
|
| 519 |
+
if (!target || !shell) return;
|
| 520 |
+
if (channel.displayed.startsWith(commit)) return;
|
| 521 |
+
// The server only publishes this barrier for a fully frozen English source
|
| 522 |
+
// unit. Never synthesize text that is not already authoritative.
|
| 523 |
+
if (!channel.authoritative.startsWith(commit)) return;
|
| 524 |
+
const stream = ensureStreamNode(parentDocument, target, channel);
|
| 525 |
+
stopRenderTimer(channel);
|
| 526 |
+
stream.textContent = commit;
|
| 527 |
+
channel.displayed = commit;
|
| 528 |
+
channel.queue = [];
|
| 529 |
+
const remaining = channel.authoritative.slice(commit.length);
|
| 530 |
+
if (remaining) channel.queue.push(...wordLikeUnits(remaining));
|
| 531 |
+
if (shell.dataset.spAutoFollow !== "0") shell.scrollTop = shell.scrollHeight;
|
| 532 |
+
pumpRenderQueue(parentDocument, target, shell, channel);
|
| 533 |
+
}
|
| 534 |
+
|
| 535 |
function previewShouldFollow(parentWindow, shell, hasText) {
|
| 536 |
if (!shell) return false;
|
| 537 |
if (!hasText) return true;
|
|
|
|
| 565 |
catch (_) { return; }
|
| 566 |
const englishText = String(payload && payload.text || "");
|
| 567 |
const targetText = String(payload && payload.target_text || "");
|
| 568 |
+
const targetEnglishCommitText = String(payload && payload.target_english_commit_text || "");
|
| 569 |
if (englishText && englishText !== lastPreviewText) {
|
| 570 |
updateOnePreview(parentDocument, parentWindow, "sp-live-manuscript-preview", "sp-live-manuscript-shell", englishText, renderChannels.english);
|
| 571 |
const englishStatus = parentDocument.getElementById("sp-live-manuscript-status");
|
|
|
|
| 576 |
lastPreviewText = englishText;
|
| 577 |
}
|
| 578 |
if (targetText && targetText !== lastTargetPreviewText) {
|
| 579 |
+
ensureEnglishCommitVisible(parentDocument, targetEnglishCommitText);
|
| 580 |
updateOnePreview(parentDocument, parentWindow, "sp-live-translation-preview", "sp-live-translation-shell", targetText, renderChannels.target);
|
| 581 |
lastTargetPreviewText = targetText;
|
| 582 |
}
|
static/desktop.ini
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[.ShellClassInfo]
|
| 2 |
+
IconResource=C:\Program Files\Google\Drive File Stream\129.0.1.0\GoogleDriveFS.exe,27
|
static/previews/desktop.ini
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[.ShellClassInfo]
|
| 2 |
+
IconResource=C:\Program Files\Google\Drive File Stream\129.0.1.0\GoogleDriveFS.exe,27
|
utils/desktop.ini
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[.ShellClassInfo]
|
| 2 |
+
IconResource=C:\Program Files\Google\Drive File Stream\129.0.1.0\GoogleDriveFS.exe,27
|