Spaces:
Running
Running
Step 4A-6-C-R3.2.8.4 Translation Fact Ledger and 60ms Streaming Fix
#22
by GuXiaopeng - opened
- .gitattributes +0 -1
- core/generation_runner.py +15 -0
- core/pipeline_agent.py +215 -64
- smartpaper_restore_bridge/frontend/index.html +2 -2
.gitattributes
CHANGED
|
@@ -33,4 +33,3 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
-
core/__pycache__/pipeline_agent.cpython-313.pyc filter=lfs diff=lfs merge=lfs -text
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
core/generation_runner.py
CHANGED
|
@@ -33,6 +33,7 @@ ProgressCallback = Callable[..., None]
|
|
| 33 |
MIN_FINAL_REFERENCES = 50
|
| 34 |
LITERATURE_EVIDENCE_TARGET = 75
|
| 35 |
CITATION_PLAN_TARGET = 70
|
|
|
|
| 36 |
|
| 37 |
|
| 38 |
def _clean_trailing_references(text, keywords=("reference", "references", "reference list", "参考文献")):
|
|
@@ -252,6 +253,14 @@ class GenerationRunner:
|
|
| 252 |
encoded = json.dumps(dict(glossary or {}), ensure_ascii=False, sort_keys=True).encode('utf-8')
|
| 253 |
return hashlib.sha256(encoded).hexdigest()[:16]
|
| 254 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 255 |
@staticmethod
|
| 256 |
def _assemble_partial_translation(translation_sections, adaptive_structure, *, include_front=False) -> str:
|
| 257 |
mapping = translation_sections if isinstance(translation_sections, dict) else {}
|
|
@@ -274,11 +283,15 @@ class GenerationRunner:
|
|
| 274 |
if not source:
|
| 275 |
return translation_sections, translation_glossary
|
| 276 |
source_hash = self._text_hash(source)
|
|
|
|
|
|
|
| 277 |
existing = dict((translation_sections or {}).get(unit_key) or {})
|
| 278 |
if (
|
| 279 |
existing.get('qa_status') == 'passed'
|
| 280 |
and existing.get('source_hash') == source_hash
|
| 281 |
and existing.get('target_lang') == target_lang
|
|
|
|
|
|
|
| 282 |
and existing.get('translated_text')
|
| 283 |
):
|
| 284 |
return translation_sections, translation_glossary
|
|
@@ -293,6 +306,8 @@ class GenerationRunner:
|
|
| 293 |
'source_hash': source_hash,
|
| 294 |
'target_lang': target_lang,
|
| 295 |
'glossary_version': self._glossary_version(updated_glossary),
|
|
|
|
|
|
|
| 296 |
'translated_text': str(result.get('text') or '').strip(),
|
| 297 |
'qa_status': 'passed',
|
| 298 |
'attempts': int(result.get('attempts') or 1),
|
|
|
|
| 33 |
MIN_FINAL_REFERENCES = 50
|
| 34 |
LITERATURE_EVIDENCE_TARGET = 75
|
| 35 |
CITATION_PLAN_TARGET = 70
|
| 36 |
+
TRANSLATION_CONTRACT_VERSION = 'fact-ledger-v1'
|
| 37 |
|
| 38 |
|
| 39 |
def _clean_trailing_references(text, keywords=("reference", "references", "reference list", "参考文献")):
|
|
|
|
| 253 |
encoded = json.dumps(dict(glossary or {}), ensure_ascii=False, sort_keys=True).encode('utf-8')
|
| 254 |
return hashlib.sha256(encoded).hexdigest()[:16]
|
| 255 |
|
| 256 |
+
@staticmethod
|
| 257 |
+
def _translation_ledger_hash(agent, source: str) -> str:
|
| 258 |
+
method = getattr(agent, 'translation_fact_ledger_hash', None)
|
| 259 |
+
if callable(method):
|
| 260 |
+
return str(method(source) or '')
|
| 261 |
+
# Test doubles / legacy adapters still get deterministic invalidation.
|
| 262 |
+
return hashlib.sha256(str(source or '').encode('utf-8')).hexdigest()
|
| 263 |
+
|
| 264 |
@staticmethod
|
| 265 |
def _assemble_partial_translation(translation_sections, adaptive_structure, *, include_front=False) -> str:
|
| 266 |
mapping = translation_sections if isinstance(translation_sections, dict) else {}
|
|
|
|
| 283 |
if not source:
|
| 284 |
return translation_sections, translation_glossary
|
| 285 |
source_hash = self._text_hash(source)
|
| 286 |
+
contract_version = TRANSLATION_CONTRACT_VERSION
|
| 287 |
+
fact_ledger_hash = self._translation_ledger_hash(agent, source)
|
| 288 |
existing = dict((translation_sections or {}).get(unit_key) or {})
|
| 289 |
if (
|
| 290 |
existing.get('qa_status') == 'passed'
|
| 291 |
and existing.get('source_hash') == source_hash
|
| 292 |
and existing.get('target_lang') == target_lang
|
| 293 |
+
and existing.get('translation_contract_version') == contract_version
|
| 294 |
+
and existing.get('fact_ledger_hash') == fact_ledger_hash
|
| 295 |
and existing.get('translated_text')
|
| 296 |
):
|
| 297 |
return translation_sections, translation_glossary
|
|
|
|
| 306 |
'source_hash': source_hash,
|
| 307 |
'target_lang': target_lang,
|
| 308 |
'glossary_version': self._glossary_version(updated_glossary),
|
| 309 |
+
'translation_contract_version': str(result.get('translation_contract_version') or contract_version),
|
| 310 |
+
'fact_ledger_hash': str(result.get('fact_ledger_hash') or fact_ledger_hash),
|
| 311 |
'translated_text': str(result.get('text') or '').strip(),
|
| 312 |
'qa_status': 'passed',
|
| 313 |
'attempts': int(result.get('attempts') or 1),
|
core/pipeline_agent.py
CHANGED
|
@@ -10,6 +10,7 @@ import random
|
|
| 10 |
import tiktoken
|
| 11 |
import sys
|
| 12 |
import logging
|
|
|
|
| 13 |
import concurrent.futures
|
| 14 |
from collections import Counter
|
| 15 |
from datetime import datetime, timezone
|
|
@@ -55,6 +56,8 @@ class CitationAuditError(DeepSeekRequestError):
|
|
| 55 |
|
| 56 |
|
| 57 |
class SSciPipelineAgent:
|
|
|
|
|
|
|
| 58 |
def __init__(self, ds_key, s2_key=""):
|
| 59 |
self.ds_key = str(ds_key or "").strip()
|
| 60 |
if not self.ds_key:
|
|
@@ -3448,14 +3451,78 @@ Output only the revised Discussion body, without the '# Discussion' heading."""
|
|
| 3448 |
text = re.sub(r'(?<!\d)(?:19|20)\d{2}\s+жылғы\s+\d{1,2}\s+(?:қаңтар|ақпан|наурыз|сәуір|мамыр|маусым|шілде|тамыз|қыркүйек|қазан|қараша|желтоқсан)', ' ', text, flags=re.I)
|
| 3449 |
return cls._numeric_tokens(text)
|
| 3450 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3451 |
@classmethod
|
| 3452 |
def _translation_anchor_type(cls, token):
|
| 3453 |
value = str(token or '')
|
| 3454 |
if re.fullmatch(cls._translation_date_regex(), value, flags=re.I): return 'DATE'
|
|
|
|
|
|
|
| 3455 |
if re.fullmatch(r'\([^()\n]*?(?:19|20)\d{2}[a-z]?[^()\n]*?\)', value, flags=re.I): return 'CITATION'
|
| 3456 |
if re.fullmatch(r'(?i:https?://[^\s)>]+|doi:\s*[^\s)>]+|10\.\d{4,9}/[-._;()/:A-Z0-9]+)', value): return 'DOI_URL'
|
| 3457 |
if re.fullmatch(r'(?i:H\d+[A-Za-z]?)', value): return 'HYPOTHESIS'
|
| 3458 |
-
if re.search(r'(?:p|t|f|z|r|β|χ²|R²|ΔR²)\s*(?:<=|>=|=|<|>|≤|≥)', value, flags=re.I): return 'STAT'
|
| 3459 |
if re.fullmatch(r'(?:fsQCA|SmartPLS|Mplus|OpenAlex|DeepSeek|ChatGPT|RStudio|SPSS|AMOS|Stata)', value): return 'NAME'
|
| 3460 |
if re.fullmatch(cls._translation_acronym_regex(), value): return 'ACRONYM'
|
| 3461 |
if re.fullmatch(r'(?:QUOTE|CASE|RESP|P)-\d+', value, flags=re.I): return 'QUOTE_ID'
|
|
@@ -3487,19 +3554,21 @@ Output only the revised Discussion body, without the '# Discussion' heading."""
|
|
| 3487 |
|
| 3488 |
@classmethod
|
| 3489 |
def _translation_protected_pattern(cls):
|
| 3490 |
-
# One-pass alternation prevents a placeholder created for a
|
| 3491 |
-
#
|
| 3492 |
-
# conservative: protect immutable research objects, not prose.
|
| 3493 |
acronym = cls._translation_acronym_regex()
|
| 3494 |
numeric = cls._translation_numeric_regex()
|
| 3495 |
date_pattern = cls._translation_date_regex()
|
|
|
|
|
|
|
| 3496 |
return re.compile(
|
| 3497 |
rf'(?:{date_pattern})'
|
| 3498 |
-
|
| 3499 |
-
|
|
|
|
| 3500 |
r'|(?i:\bdoi:\s*[^\s)>]+)'
|
| 3501 |
r'|(?i:\b10\.\d{4,9}/[-._;()/:A-Z0-9]+)'
|
| 3502 |
-
r'|(?i:\b(?:p|t|f|z|r)\s*(?:<=|>=|=|<|>|≤|≥)\s*[-+]?(?:\d+(?:\.\d+)?|\.\d+))'
|
| 3503 |
r'|(?:β|χ²|R²|ΔR²)\s*(?:<=|>=|=|<|>|≤|≥)\s*[-+]?(?:\d+(?:\.\d+)?|\.\d+)'
|
| 3504 |
r'|\([^()\n]*?(?:19|20)\d{2}[a-z]?[^()\n]*?\)'
|
| 3505 |
r'|`[^`\n]+`'
|
|
@@ -3515,6 +3584,7 @@ Output only the revised Discussion body, without the '# Discussion' heading."""
|
|
| 3515 |
|
| 3516 |
@classmethod
|
| 3517 |
def _protect_translation_tokens(cls, value):
|
|
|
|
| 3518 |
source = str(value or '')
|
| 3519 |
mapping = {}
|
| 3520 |
counter = 0
|
|
@@ -3526,13 +3596,13 @@ Output only the revised Discussion body, without the '# Discussion' heading."""
|
|
| 3526 |
placeholder = f'[[SP-T{counter:06d}]]'
|
| 3527 |
original = match.group(0)
|
| 3528 |
anchor_type = cls._translation_anchor_type(original)
|
| 3529 |
-
|
| 3530 |
-
|
| 3531 |
-
|
| 3532 |
-
|
| 3533 |
-
|
| 3534 |
-
|
| 3535 |
-
|
| 3536 |
return placeholder
|
| 3537 |
|
| 3538 |
protected = pattern.sub(replace, source)
|
|
@@ -3542,32 +3612,129 @@ Output only the revised Discussion body, without the '# Discussion' heading."""
|
|
| 3542 |
return protected, mapping
|
| 3543 |
|
| 3544 |
@classmethod
|
| 3545 |
-
def
|
| 3546 |
-
|
| 3547 |
-
|
| 3548 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3549 |
if found != expected:
|
| 3550 |
missing = list((expected - found).elements())[:8]
|
| 3551 |
added = list((found - expected).elements())[:8]
|
| 3552 |
raise DeepSeekRequestError(
|
| 3553 |
f'Translation protected placeholder mismatch in {block_id}: missing={missing}, added={added}.'
|
| 3554 |
)
|
| 3555 |
-
|
| 3556 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3557 |
source_sentence = int(manifest.get('sentence_index') or 0)
|
| 3558 |
-
target_sentence = cls._translation_sentence_index(
|
| 3559 |
-
if
|
| 3560 |
raise DeepSeekRequestError(
|
| 3561 |
-
f'Translation
|
| 3562 |
)
|
| 3563 |
-
|
| 3564 |
-
|
| 3565 |
-
|
| 3566 |
-
|
| 3567 |
-
|
| 3568 |
-
|
| 3569 |
-
|
| 3570 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3571 |
if re.search(r'\[\[SP-T\d{6}\]\]', text):
|
| 3572 |
raise DeepSeekRequestError(f'Translation left unresolved protected placeholders in {block_id}.')
|
| 3573 |
return text
|
|
@@ -3827,23 +3994,11 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
|
|
| 3827 |
return parsed
|
| 3828 |
|
| 3829 |
def _validate_translation_block(self, source, translated, block_id, target_lang):
|
| 3830 |
-
|
| 3831 |
-
|
| 3832 |
-
|
| 3833 |
-
|
| 3834 |
-
|
| 3835 |
-
raise DeepSeekRequestError(f'Translation changed citations in {block_id}.')
|
| 3836 |
-
if self._translation_hypothesis_tokens(source) != self._translation_hypothesis_tokens(translated):
|
| 3837 |
-
raise DeepSeekRequestError(f'Translation changed hypothesis labels in {block_id}.')
|
| 3838 |
-
source_acronyms = self._translation_acronym_tokens(source)
|
| 3839 |
-
target_acronyms = self._translation_acronym_tokens(translated)
|
| 3840 |
-
if source_acronyms != target_acronyms:
|
| 3841 |
-
missing = list((source_acronyms - target_acronyms).elements())
|
| 3842 |
-
added = list((target_acronyms - source_acronyms).elements())
|
| 3843 |
-
raise DeepSeekRequestError(
|
| 3844 |
-
f'Translation changed variable abbreviations in {block_id}: '
|
| 3845 |
-
f'missing={missing}, added={added}.'
|
| 3846 |
-
)
|
| 3847 |
for _long_form, acronym in self._translation_longform_acronym_pairs(source):
|
| 3848 |
if not re.search(rf'[((]\s*{re.escape(acronym)}\s*[))]', str(translated or '')):
|
| 3849 |
raise DeepSeekRequestError(
|
|
@@ -3871,8 +4026,8 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
|
|
| 3871 |
if not is_chinese and not re.search(r'[А-Яа-яЁёӘәҒғҚқҢңӨөҰұҮүҺһІі]', translated):
|
| 3872 |
raise DeepSeekRequestError(f'Translation block {block_id} contains no Cyrillic target-language text.')
|
| 3873 |
|
| 3874 |
-
def _validate_translation_block_protected(self, block_id, source, translated, target_lang):
|
| 3875 |
-
"""Validate one restored block
|
| 3876 |
self._validate_translation_block(source, translated, f'SP-P{block_id}', target_lang)
|
| 3877 |
return translated
|
| 3878 |
|
|
@@ -3896,18 +4051,16 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
|
|
| 3896 |
for block_id, _, protected_source in protected_batch
|
| 3897 |
)
|
| 3898 |
semantic_manifest = {
|
| 3899 |
-
block_id:
|
| 3900 |
-
{'id': placeholder, **manifest}
|
| 3901 |
-
for placeholder, manifest in protected_maps.get(block_id, {}).items()
|
| 3902 |
-
]
|
| 3903 |
for block_id, _source, _protected in protected_batch
|
| 3904 |
}
|
| 3905 |
effective_prompt = system_prompt + (
|
| 3906 |
'\n<SEMANTIC_ANCHOR_MANIFEST>\n'
|
| 3907 |
+ json.dumps(semantic_manifest, ensure_ascii=False)
|
| 3908 |
+ '\n</SEMANTIC_ANCHOR_MANIFEST>\n'
|
| 3909 |
-
'
|
| 3910 |
-
'
|
|
|
|
| 3911 |
)
|
| 3912 |
if _strict:
|
| 3913 |
effective_prompt += (
|
|
@@ -4113,6 +4266,8 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
|
|
| 4113 |
'glossary': frozen_glossary,
|
| 4114 |
'attempts': attempt,
|
| 4115 |
'qa_status': 'passed',
|
|
|
|
|
|
|
| 4116 |
}
|
| 4117 |
except PipelineCancelledError:
|
| 4118 |
raise
|
|
@@ -4207,14 +4362,10 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
|
|
| 4207 |
raise DeepSeekRequestError(
|
| 4208 |
f'Translation heading-tree mismatch: English={en_tree}, target={tr_tree}.'
|
| 4209 |
)
|
| 4210 |
-
|
| 4211 |
-
|
| 4212 |
-
|
| 4213 |
-
|
| 4214 |
-
if SSciPipelineAgent._translation_hypothesis_tokens(source) != SSciPipelineAgent._translation_hypothesis_tokens(target):
|
| 4215 |
-
raise DeepSeekRequestError('Translation hypothesis-label alignment failed.')
|
| 4216 |
-
if SSciPipelineAgent._translation_acronym_tokens(source) != SSciPipelineAgent._translation_acronym_tokens(target):
|
| 4217 |
-
raise DeepSeekRequestError('Translation abbreviation alignment failed.')
|
| 4218 |
source_body = re.split(r'(?m)^#\s+References\s*$', source, maxsplit=1)[0]
|
| 4219 |
target_body = re.split(r'(?m)^#\s+(?:参考文献|Литература|Пайдаланылған әдебиеттер)\s*$', target, maxsplit=1)[0]
|
| 4220 |
source_chars = len(re.sub(r'\s+', '', source_body))
|
|
|
|
| 10 |
import tiktoken
|
| 11 |
import sys
|
| 12 |
import logging
|
| 13 |
+
import hashlib
|
| 14 |
import concurrent.futures
|
| 15 |
from collections import Counter
|
| 16 |
from datetime import datetime, timezone
|
|
|
|
| 56 |
|
| 57 |
|
| 58 |
class SSciPipelineAgent:
|
| 59 |
+
TRANSLATION_CONTRACT_VERSION = 'fact-ledger-v1'
|
| 60 |
+
|
| 61 |
def __init__(self, ds_key, s2_key=""):
|
| 62 |
self.ds_key = str(ds_key or "").strip()
|
| 63 |
if not self.ds_key:
|
|
|
|
| 3451 |
text = re.sub(r'(?<!\d)(?:19|20)\d{2}\s+жылғы\s+\d{1,2}\s+(?:қаңтар|ақпан|наурыз|сәуір|мамыр|маусым|шілде|тамыз|қыркүйек|қазан|қараша|желтоқсан)', ' ', text, flags=re.I)
|
| 3452 |
return cls._numeric_tokens(text)
|
| 3453 |
|
| 3454 |
+
@staticmethod
|
| 3455 |
+
def _translation_number_word_regex():
|
| 3456 |
+
# Conservative explicit English number-word quantities. Restrict to
|
| 3457 |
+
# phrases containing a magnitude word so ordinary prose such as "one of"
|
| 3458 |
+
# is not rewritten as a numeric fact.
|
| 3459 |
+
units = r'one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve|thirteen|fourteen|fifteen|sixteen|seventeen|eighteen|nineteen'
|
| 3460 |
+
tens = r'twenty|thirty|forty|fifty|sixty|seventy|eighty|ninety'
|
| 3461 |
+
magnitudes = r'hundred|thousand|million|billion'
|
| 3462 |
+
word = rf'(?:{units}|{tens}|{magnitudes}|and)'
|
| 3463 |
+
return rf'(?<![A-Za-z])(?=[A-Za-z -]*(?:{magnitudes})\b){word}(?:[ -]+{word})+(?![A-Za-z])'
|
| 3464 |
+
|
| 3465 |
+
@classmethod
|
| 3466 |
+
def _canonicalize_translation_quantity(cls, value):
|
| 3467 |
+
text = re.sub(r'[-]+', ' ', str(value or '').strip().casefold())
|
| 3468 |
+
tokens = [t for t in text.split() if t != 'and']
|
| 3469 |
+
small = {
|
| 3470 |
+
'one':1,'two':2,'three':3,'four':4,'five':5,'six':6,'seven':7,'eight':8,'nine':9,
|
| 3471 |
+
'ten':10,'eleven':11,'twelve':12,'thirteen':13,'fourteen':14,'fifteen':15,'sixteen':16,
|
| 3472 |
+
'seventeen':17,'eighteen':18,'nineteen':19,'twenty':20,'thirty':30,'forty':40,'fifty':50,
|
| 3473 |
+
'sixty':60,'seventy':70,'eighty':80,'ninety':90,
|
| 3474 |
+
}
|
| 3475 |
+
total = 0
|
| 3476 |
+
current = 0
|
| 3477 |
+
for token in tokens:
|
| 3478 |
+
if token in small:
|
| 3479 |
+
current += small[token]
|
| 3480 |
+
elif token == 'hundred':
|
| 3481 |
+
current = max(1, current) * 100
|
| 3482 |
+
elif token in {'thousand','million','billion'}:
|
| 3483 |
+
scale = {'thousand':1000,'million':1000000,'billion':1000000000}[token]
|
| 3484 |
+
total += max(1, current) * scale
|
| 3485 |
+
current = 0
|
| 3486 |
+
else:
|
| 3487 |
+
return ''
|
| 3488 |
+
return str(total + current) if (total or current) else ''
|
| 3489 |
+
|
| 3490 |
+
@staticmethod
|
| 3491 |
+
def _canonicalize_numeric_fact(value):
|
| 3492 |
+
text = str(value or '').strip()
|
| 3493 |
+
if not text:
|
| 3494 |
+
return ''
|
| 3495 |
+
return text.replace(',', '')
|
| 3496 |
+
|
| 3497 |
+
@classmethod
|
| 3498 |
+
def _translation_fact_render_policy(cls, anchor_type):
|
| 3499 |
+
if anchor_type == 'DATE':
|
| 3500 |
+
return 'localized_date'
|
| 3501 |
+
if anchor_type == 'QUANTITY':
|
| 3502 |
+
return 'canonical_numeric'
|
| 3503 |
+
return 'exact'
|
| 3504 |
+
|
| 3505 |
+
@classmethod
|
| 3506 |
+
def _translation_fact_canonical(cls, token, anchor_type):
|
| 3507 |
+
value = str(token or '')
|
| 3508 |
+
if anchor_type == 'DATE':
|
| 3509 |
+
return cls._canonicalize_translation_date(value)
|
| 3510 |
+
if anchor_type == 'QUANTITY':
|
| 3511 |
+
return cls._canonicalize_translation_quantity(value)
|
| 3512 |
+
if anchor_type in {'NUMBER','STAT'}:
|
| 3513 |
+
return cls._canonicalize_numeric_fact(value)
|
| 3514 |
+
return value
|
| 3515 |
+
|
| 3516 |
@classmethod
|
| 3517 |
def _translation_anchor_type(cls, token):
|
| 3518 |
value = str(token or '')
|
| 3519 |
if re.fullmatch(cls._translation_date_regex(), value, flags=re.I): return 'DATE'
|
| 3520 |
+
if re.fullmatch(cls._translation_number_word_regex(), value, flags=re.I): return 'QUANTITY'
|
| 3521 |
+
if re.fullmatch(r'(?i:(?:\d+(?:\.\d+)?%)?\s*CI\s*\[\s*[-+]?\.?\d+(?:\.\d+)?\s*,\s*[-+]?\.?\d+(?:\.\d+)?\s*\])', value): return 'STAT'
|
| 3522 |
if re.fullmatch(r'\([^()\n]*?(?:19|20)\d{2}[a-z]?[^()\n]*?\)', value, flags=re.I): return 'CITATION'
|
| 3523 |
if re.fullmatch(r'(?i:https?://[^\s)>]+|doi:\s*[^\s)>]+|10\.\d{4,9}/[-._;()/:A-Z0-9]+)', value): return 'DOI_URL'
|
| 3524 |
if re.fullmatch(r'(?i:H\d+[A-Za-z]?)', value): return 'HYPOTHESIS'
|
| 3525 |
+
if re.search(r'(?:p|t|f|z|r|β|χ²|R²|ΔR²|N|n)\s*(?:<=|>=|=|<|>|≤|≥)', value, flags=re.I): return 'STAT'
|
| 3526 |
if re.fullmatch(r'(?:fsQCA|SmartPLS|Mplus|OpenAlex|DeepSeek|ChatGPT|RStudio|SPSS|AMOS|Stata)', value): return 'NAME'
|
| 3527 |
if re.fullmatch(cls._translation_acronym_regex(), value): return 'ACRONYM'
|
| 3528 |
if re.fullmatch(r'(?:QUOTE|CASE|RESP|P)-\d+', value, flags=re.I): return 'QUOTE_ID'
|
|
|
|
| 3554 |
|
| 3555 |
@classmethod
|
| 3556 |
def _translation_protected_pattern(cls):
|
| 3557 |
+
# One-pass alternation prevents a placeholder created for a compound
|
| 3558 |
+
# statistical/date object from being split into unrelated numbers.
|
|
|
|
| 3559 |
acronym = cls._translation_acronym_regex()
|
| 3560 |
numeric = cls._translation_numeric_regex()
|
| 3561 |
date_pattern = cls._translation_date_regex()
|
| 3562 |
+
quantity_words = cls._translation_number_word_regex()
|
| 3563 |
+
compound_ci = r'(?i:(?:\d+(?:\.\d+)?%)?\s*CI\s*\[\s*[-+]?\.?\d+(?:\.\d+)?\s*,\s*[-+]?\.?\d+(?:\.\d+)?\s*\])'
|
| 3564 |
return re.compile(
|
| 3565 |
rf'(?:{date_pattern})'
|
| 3566 |
+
rf'|(?:{compound_ci})'
|
| 3567 |
+
rf'|(?:{quantity_words})'
|
| 3568 |
+
r'|(?i:https?://[^\s)>]+)'
|
| 3569 |
r'|(?i:\bdoi:\s*[^\s)>]+)'
|
| 3570 |
r'|(?i:\b10\.\d{4,9}/[-._;()/:A-Z0-9]+)'
|
| 3571 |
+
r'|(?i:\b(?:p|t|f|z|r|n)\s*(?:<=|>=|=|<|>|≤|≥)\s*[-+]?(?:\d+(?:\.\d+)?|\.\d+))'
|
| 3572 |
r'|(?:β|χ²|R²|ΔR²)\s*(?:<=|>=|=|<|>|≤|≥)\s*[-+]?(?:\d+(?:\.\d+)?|\.\d+)'
|
| 3573 |
r'|\([^()\n]*?(?:19|20)\d{2}[a-z]?[^()\n]*?\)'
|
| 3574 |
r'|`[^`\n]+`'
|
|
|
|
| 3584 |
|
| 3585 |
@classmethod
|
| 3586 |
def _protect_translation_tokens(cls, value):
|
| 3587 |
+
"""Build a private fact ledger and replace model-inaccessible values."""
|
| 3588 |
source = str(value or '')
|
| 3589 |
mapping = {}
|
| 3590 |
counter = 0
|
|
|
|
| 3596 |
placeholder = f'[[SP-T{counter:06d}]]'
|
| 3597 |
original = match.group(0)
|
| 3598 |
anchor_type = cls._translation_anchor_type(original)
|
| 3599 |
+
mapping[placeholder] = {
|
| 3600 |
+
'fact_id': f'F{counter:06d}',
|
| 3601 |
+
'source': original, # private server ledger only
|
| 3602 |
+
'type': anchor_type,
|
| 3603 |
+
'canonical': cls._translation_fact_canonical(original, anchor_type),
|
| 3604 |
+
'render_policy': cls._translation_fact_render_policy(anchor_type),
|
| 3605 |
+
}
|
| 3606 |
return placeholder
|
| 3607 |
|
| 3608 |
protected = pattern.sub(replace, source)
|
|
|
|
| 3612 |
return protected, mapping
|
| 3613 |
|
| 3614 |
@classmethod
|
| 3615 |
+
def _model_safe_translation_manifest(cls, mapping):
|
| 3616 |
+
"""Expose only placement metadata; factual values remain server-private."""
|
| 3617 |
+
safe = []
|
| 3618 |
+
for placeholder, raw in (mapping or {}).items():
|
| 3619 |
+
item = raw if isinstance(raw, dict) else {'type':'IMMUTABLE'}
|
| 3620 |
+
anchor_type = str(item.get('type') or 'IMMUTABLE')
|
| 3621 |
+
safe.append({
|
| 3622 |
+
'id': placeholder,
|
| 3623 |
+
'fact_id': str(item.get('fact_id') or placeholder),
|
| 3624 |
+
'type': anchor_type,
|
| 3625 |
+
'sentence_index': int(item.get('sentence_index') or 0),
|
| 3626 |
+
'mobility': 'same_sentence' if anchor_type in {'DATE','CITATION'} else 'same_owner',
|
| 3627 |
+
})
|
| 3628 |
+
return safe
|
| 3629 |
+
|
| 3630 |
+
@classmethod
|
| 3631 |
+
def _fact_ledger_hash(cls, ledger):
|
| 3632 |
+
records = []
|
| 3633 |
+
for block_id, mapping in sorted((ledger or {}).items()):
|
| 3634 |
+
for placeholder, raw in sorted((mapping or {}).items()):
|
| 3635 |
+
item = raw if isinstance(raw, dict) else {'source':str(raw),'type':'IMMUTABLE'}
|
| 3636 |
+
records.append({
|
| 3637 |
+
'block_id': str(block_id),
|
| 3638 |
+
'fact_id': str(item.get('fact_id') or placeholder),
|
| 3639 |
+
'type': str(item.get('type') or 'IMMUTABLE'),
|
| 3640 |
+
'canonical': str(item.get('canonical') or item.get('source') or ''),
|
| 3641 |
+
'sentence_index': int(item.get('sentence_index') or 0),
|
| 3642 |
+
'render_policy': str(item.get('render_policy') or 'exact'),
|
| 3643 |
+
})
|
| 3644 |
+
payload = json.dumps(records, ensure_ascii=False, sort_keys=True, separators=(',', ':')).encode('utf-8')
|
| 3645 |
+
return hashlib.sha256(payload).hexdigest()
|
| 3646 |
+
|
| 3647 |
+
@classmethod
|
| 3648 |
+
def translation_fact_ledger_hash(cls, section_text):
|
| 3649 |
+
"""Deterministic source ledger hash used by translation checkpoint reuse."""
|
| 3650 |
+
units = cls._translation_body_units(section_text)
|
| 3651 |
+
ledger = {}
|
| 3652 |
+
body_index = 0
|
| 3653 |
+
for unit in units:
|
| 3654 |
+
if unit.get('kind') != 'body':
|
| 3655 |
+
continue
|
| 3656 |
+
body_index += 1
|
| 3657 |
+
_protected, mapping = cls._protect_translation_tokens(unit.get('text') or '')
|
| 3658 |
+
ledger[f'{body_index:06d}'] = mapping
|
| 3659 |
+
return cls._fact_ledger_hash(ledger)
|
| 3660 |
+
|
| 3661 |
+
@classmethod
|
| 3662 |
+
def _render_translation_fact(cls, manifest, target_lang):
|
| 3663 |
+
item = manifest if isinstance(manifest, dict) else {'source':str(manifest), 'type':'IMMUTABLE'}
|
| 3664 |
+
policy = str(item.get('render_policy') or 'exact')
|
| 3665 |
+
if policy == 'localized_date' and item.get('canonical'):
|
| 3666 |
+
return cls._render_translation_date(item.get('canonical'), target_lang)
|
| 3667 |
+
if policy == 'canonical_numeric' and item.get('canonical'):
|
| 3668 |
+
return str(item.get('canonical'))
|
| 3669 |
+
return str(item.get('source') or '')
|
| 3670 |
+
|
| 3671 |
+
@classmethod
|
| 3672 |
+
def _translation_unowned_objects(cls, protected_target):
|
| 3673 |
+
"""Find factual objects invented by the model outside SP-T anchors."""
|
| 3674 |
+
text = re.sub(r'\[\[SP-T\d{6}\]\]', ' ', str(protected_target or ''))
|
| 3675 |
+
text = re.sub(r'\[\[SP-P\d{6}\]\]', ' ', text)
|
| 3676 |
+
unowned = {
|
| 3677 |
+
'numeric': list(cls._numeric_tokens(text).elements()),
|
| 3678 |
+
'citation': list(cls._translation_citation_tokens(text).elements()),
|
| 3679 |
+
'hypothesis': list(cls._translation_hypothesis_tokens(text).elements()),
|
| 3680 |
+
'acronym': list(cls._translation_acronym_tokens(text).elements()),
|
| 3681 |
+
}
|
| 3682 |
+
names = re.findall(r'\b(?:fsQCA|SmartPLS|Mplus|OpenAlex|DeepSeek|ChatGPT|RStudio|SPSS|AMOS|Stata)\b', text)
|
| 3683 |
+
if names:
|
| 3684 |
+
unowned['name'] = names
|
| 3685 |
+
urls = re.findall(r'(?i:https?://[^\s)>]+|\bdoi:\s*[^\s)>]+|\b10\.\d{4,9}/[-._;()/:A-Z0-9]+)', text)
|
| 3686 |
+
if urls:
|
| 3687 |
+
unowned['doi_url'] = urls
|
| 3688 |
+
return {key: values for key, values in unowned.items() if values}
|
| 3689 |
+
|
| 3690 |
+
@classmethod
|
| 3691 |
+
def _validate_translation_skeleton(cls, value, mapping, block_id=''):
|
| 3692 |
+
found = Counter(re.findall(r'\[\[SP-T\d{6}\]\]', str(value or '')))
|
| 3693 |
+
expected = Counter((mapping or {}).keys())
|
| 3694 |
if found != expected:
|
| 3695 |
missing = list((expected - found).elements())[:8]
|
| 3696 |
added = list((found - expected).elements())[:8]
|
| 3697 |
raise DeepSeekRequestError(
|
| 3698 |
f'Translation protected placeholder mismatch in {block_id}: missing={missing}, added={added}.'
|
| 3699 |
)
|
| 3700 |
+
unowned = cls._translation_unowned_objects(value)
|
| 3701 |
+
if unowned:
|
| 3702 |
+
safe_summary = {key: len(values) for key, values in unowned.items()}
|
| 3703 |
+
logging.warning('Translation fact integrity failed: block=%s unowned=%s', block_id, safe_summary)
|
| 3704 |
+
raise DeepSeekRequestError(f'Translation introduced unowned factual content in {block_id}: {safe_summary}.')
|
| 3705 |
+
target_text = str(value or '')
|
| 3706 |
+
for placeholder, raw_manifest in (mapping or {}).items():
|
| 3707 |
+
manifest = raw_manifest if isinstance(raw_manifest, dict) else {'type':'IMMUTABLE'}
|
| 3708 |
source_sentence = int(manifest.get('sentence_index') or 0)
|
| 3709 |
+
target_sentence = cls._translation_sentence_index(target_text, target_text.find(placeholder))
|
| 3710 |
+
if target_sentence != source_sentence:
|
| 3711 |
raise DeepSeekRequestError(
|
| 3712 |
+
f'Translation fact moved across sentence boundary in {block_id}: {manifest.get("fact_id") or placeholder}.'
|
| 3713 |
)
|
| 3714 |
+
# Preserve the relative order of hard quantitative/identifier facts
|
| 3715 |
+
# inside each claim. DATE/CITATION/NAME may legitimately move for target
|
| 3716 |
+
# grammar, but two sample/statistical values must never trade owners.
|
| 3717 |
+
ordered_types = {'NUMBER','QUANTITY','STAT','HYPOTHESIS','DOI_URL','FORMULA','QUOTE_ID'}
|
| 3718 |
+
for sentence_index in sorted({int((m if isinstance(m, dict) else {}).get('sentence_index') or 0) for m in (mapping or {}).values()}):
|
| 3719 |
+
expected_order = [
|
| 3720 |
+
ph for ph, raw in (mapping or {}).items()
|
| 3721 |
+
if int((raw if isinstance(raw, dict) else {}).get('sentence_index') or 0) == sentence_index
|
| 3722 |
+
and str((raw if isinstance(raw, dict) else {}).get('type') or '') in ordered_types
|
| 3723 |
+
]
|
| 3724 |
+
if len(expected_order) > 1:
|
| 3725 |
+
actual_order = sorted(expected_order, key=lambda ph: target_text.find(ph))
|
| 3726 |
+
if actual_order != expected_order:
|
| 3727 |
+
raise DeepSeekRequestError(
|
| 3728 |
+
f'Translation fact order changed within owner sentence in {block_id}: sentence={sentence_index}.'
|
| 3729 |
+
)
|
| 3730 |
+
return True
|
| 3731 |
+
|
| 3732 |
+
@classmethod
|
| 3733 |
+
def _restore_translation_tokens(cls, value, mapping, block_id='', target_lang=''):
|
| 3734 |
+
text = str(value or '')
|
| 3735 |
+
cls._validate_translation_skeleton(text, mapping, block_id)
|
| 3736 |
+
for placeholder, manifest in (mapping or {}).items():
|
| 3737 |
+
text = text.replace(placeholder, cls._render_translation_fact(manifest, target_lang))
|
| 3738 |
if re.search(r'\[\[SP-T\d{6}\]\]', text):
|
| 3739 |
raise DeepSeekRequestError(f'Translation left unresolved protected placeholders in {block_id}.')
|
| 3740 |
return text
|
|
|
|
| 3994 |
return parsed
|
| 3995 |
|
| 3996 |
def _validate_translation_block(self, source, translated, block_id, target_lang):
|
| 3997 |
+
# R3.2.8.4: factual equality is enforced on the protected skeleton /
|
| 3998 |
+
# private Fact Ledger. This routine now owns language/coverage and the
|
| 3999 |
+
# long-form/acronym adjacency check only; raw numeric Counter equality is
|
| 4000 |
+
# deliberately non-authoritative because it creates multilingual false
|
| 4001 |
+
# failures and can miss value swaps.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4002 |
for _long_form, acronym in self._translation_longform_acronym_pairs(source):
|
| 4003 |
if not re.search(rf'[((]\s*{re.escape(acronym)}\s*[))]', str(translated or '')):
|
| 4004 |
raise DeepSeekRequestError(
|
|
|
|
| 4026 |
if not is_chinese and not re.search(r'[А-Яа-яЁёӘәҒғҚқҢңӨөҰұҮүҺһІі]', translated):
|
| 4027 |
raise DeepSeekRequestError(f'Translation block {block_id} contains no Cyrillic target-language text.')
|
| 4028 |
|
| 4029 |
+
def _validate_translation_block_protected(self, block_id, source, translated, target_lang, mapping=None):
|
| 4030 |
+
"""Validate one restored block after Fact Ledger integrity passed."""
|
| 4031 |
self._validate_translation_block(source, translated, f'SP-P{block_id}', target_lang)
|
| 4032 |
return translated
|
| 4033 |
|
|
|
|
| 4051 |
for block_id, _, protected_source in protected_batch
|
| 4052 |
)
|
| 4053 |
semantic_manifest = {
|
| 4054 |
+
block_id: self._model_safe_translation_manifest(protected_maps.get(block_id, {}))
|
|
|
|
|
|
|
|
|
|
| 4055 |
for block_id, _source, _protected in protected_batch
|
| 4056 |
}
|
| 4057 |
effective_prompt = system_prompt + (
|
| 4058 |
'\n<SEMANTIC_ANCHOR_MANIFEST>\n'
|
| 4059 |
+ json.dumps(semantic_manifest, ensure_ascii=False)
|
| 4060 |
+ '\n</SEMANTIC_ANCHOR_MANIFEST>\n'
|
| 4061 |
+
'This manifest intentionally contains no factual values. Use only fact type and sentence ownership to place each placeholder. '
|
| 4062 |
+
'Never infer, spell out, calculate, or recreate a hidden number, date, citation, acronym, name, formula, or identifier. '
|
| 4063 |
+
'Do not print the manifest. DATE and CITATION anchors may move only within their original sentence when grammar requires it.'
|
| 4064 |
)
|
| 4065 |
if _strict:
|
| 4066 |
effective_prompt += (
|
|
|
|
| 4266 |
'glossary': frozen_glossary,
|
| 4267 |
'attempts': attempt,
|
| 4268 |
'qa_status': 'passed',
|
| 4269 |
+
'translation_contract_version': self.TRANSLATION_CONTRACT_VERSION,
|
| 4270 |
+
'fact_ledger_hash': self.translation_fact_ledger_hash(value),
|
| 4271 |
}
|
| 4272 |
except PipelineCancelledError:
|
| 4273 |
raise
|
|
|
|
| 4362 |
raise DeepSeekRequestError(
|
| 4363 |
f'Translation heading-tree mismatch: English={en_tree}, target={tr_tree}.'
|
| 4364 |
)
|
| 4365 |
+
# Fact integrity is authoritative at the protected-block ledger gate.
|
| 4366 |
+
# Whole-unit raw Counter equality is intentionally not used here: valid
|
| 4367 |
+
# localized dates/number words can change surface digits, while swapped
|
| 4368 |
+
# values can preserve the same Counter.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4369 |
source_body = re.split(r'(?m)^#\s+References\s*$', source, maxsplit=1)[0]
|
| 4370 |
target_body = re.split(r'(?m)^#\s+(?:参考文献|Литература|Пайдаланылған әдебиеттер)\s*$', target, maxsplit=1)[0]
|
| 4371 |
source_chars = len(re.sub(r'\s+', '', source_body))
|
smartpaper_restore_bridge/frontend/index.html
CHANGED
|
@@ -311,7 +311,7 @@
|
|
| 311 |
if (!channel.queue.length) return;
|
| 312 |
const stream = ensureStreamNode(parentDocument, target, channel);
|
| 313 |
const backlog = channel.queue.length;
|
| 314 |
-
const perTick = backlog >
|
| 315 |
let chunk = "";
|
| 316 |
for (let i=0; i<perTick && channel.queue.length; i++) chunk += channel.queue.shift();
|
| 317 |
if (chunk) {
|
|
@@ -319,7 +319,7 @@
|
|
| 319 |
channel.displayed += chunk;
|
| 320 |
if (shell && shell.dataset.spAutoFollow !== "0") shell.scrollTop = shell.scrollHeight;
|
| 321 |
}
|
| 322 |
-
channel.timer = window.setTimeout(() => pumpRenderQueue(parentDocument, target, shell, channel),
|
| 323 |
}
|
| 324 |
|
| 325 |
function renderBufferedPreview(parentDocument, target, shell, text, channel) {
|
|
|
|
| 311 |
if (!channel.queue.length) return;
|
| 312 |
const stream = ensureStreamNode(parentDocument, target, channel);
|
| 313 |
const backlog = channel.queue.length;
|
| 314 |
+
const perTick = backlog > 220 ? 3 : backlog > 90 ? 2 : 1;
|
| 315 |
let chunk = "";
|
| 316 |
for (let i=0; i<perTick && channel.queue.length; i++) chunk += channel.queue.shift();
|
| 317 |
if (chunk) {
|
|
|
|
| 319 |
channel.displayed += chunk;
|
| 320 |
if (shell && shell.dataset.spAutoFollow !== "0") shell.scrollTop = shell.scrollHeight;
|
| 321 |
}
|
| 322 |
+
channel.timer = window.setTimeout(() => pumpRenderQueue(parentDocument, target, shell, channel), 60);
|
| 323 |
}
|
| 324 |
|
| 325 |
function renderBufferedPreview(parentDocument, target, shell, text, channel) {
|