Step 4A-6-C-R3.2.8.4 Translation Fact Ledger and 60ms Streaming Fix

#22
.gitattributes CHANGED
@@ -33,4 +33,3 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
- core/__pycache__/pipeline_agent.cpython-313.pyc filter=lfs diff=lfs merge=lfs -text
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
core/generation_runner.py CHANGED
@@ -33,6 +33,7 @@ ProgressCallback = Callable[..., None]
33
  MIN_FINAL_REFERENCES = 50
34
  LITERATURE_EVIDENCE_TARGET = 75
35
  CITATION_PLAN_TARGET = 70
 
36
 
37
 
38
  def _clean_trailing_references(text, keywords=("reference", "references", "reference list", "参考文献")):
@@ -252,6 +253,14 @@ class GenerationRunner:
252
  encoded = json.dumps(dict(glossary or {}), ensure_ascii=False, sort_keys=True).encode('utf-8')
253
  return hashlib.sha256(encoded).hexdigest()[:16]
254
 
 
 
 
 
 
 
 
 
255
  @staticmethod
256
  def _assemble_partial_translation(translation_sections, adaptive_structure, *, include_front=False) -> str:
257
  mapping = translation_sections if isinstance(translation_sections, dict) else {}
@@ -274,11 +283,15 @@ class GenerationRunner:
274
  if not source:
275
  return translation_sections, translation_glossary
276
  source_hash = self._text_hash(source)
 
 
277
  existing = dict((translation_sections or {}).get(unit_key) or {})
278
  if (
279
  existing.get('qa_status') == 'passed'
280
  and existing.get('source_hash') == source_hash
281
  and existing.get('target_lang') == target_lang
 
 
282
  and existing.get('translated_text')
283
  ):
284
  return translation_sections, translation_glossary
@@ -293,6 +306,8 @@ class GenerationRunner:
293
  'source_hash': source_hash,
294
  'target_lang': target_lang,
295
  'glossary_version': self._glossary_version(updated_glossary),
 
 
296
  'translated_text': str(result.get('text') or '').strip(),
297
  'qa_status': 'passed',
298
  'attempts': int(result.get('attempts') or 1),
 
33
  MIN_FINAL_REFERENCES = 50
34
  LITERATURE_EVIDENCE_TARGET = 75
35
  CITATION_PLAN_TARGET = 70
36
+ TRANSLATION_CONTRACT_VERSION = 'fact-ledger-v1'
37
 
38
 
39
  def _clean_trailing_references(text, keywords=("reference", "references", "reference list", "参考文献")):
 
253
  encoded = json.dumps(dict(glossary or {}), ensure_ascii=False, sort_keys=True).encode('utf-8')
254
  return hashlib.sha256(encoded).hexdigest()[:16]
255
 
256
+ @staticmethod
257
+ def _translation_ledger_hash(agent, source: str) -> str:
258
+ method = getattr(agent, 'translation_fact_ledger_hash', None)
259
+ if callable(method):
260
+ return str(method(source) or '')
261
+ # Test doubles / legacy adapters still get deterministic invalidation.
262
+ return hashlib.sha256(str(source or '').encode('utf-8')).hexdigest()
263
+
264
  @staticmethod
265
  def _assemble_partial_translation(translation_sections, adaptive_structure, *, include_front=False) -> str:
266
  mapping = translation_sections if isinstance(translation_sections, dict) else {}
 
283
  if not source:
284
  return translation_sections, translation_glossary
285
  source_hash = self._text_hash(source)
286
+ contract_version = TRANSLATION_CONTRACT_VERSION
287
+ fact_ledger_hash = self._translation_ledger_hash(agent, source)
288
  existing = dict((translation_sections or {}).get(unit_key) or {})
289
  if (
290
  existing.get('qa_status') == 'passed'
291
  and existing.get('source_hash') == source_hash
292
  and existing.get('target_lang') == target_lang
293
+ and existing.get('translation_contract_version') == contract_version
294
+ and existing.get('fact_ledger_hash') == fact_ledger_hash
295
  and existing.get('translated_text')
296
  ):
297
  return translation_sections, translation_glossary
 
306
  'source_hash': source_hash,
307
  'target_lang': target_lang,
308
  'glossary_version': self._glossary_version(updated_glossary),
309
+ 'translation_contract_version': str(result.get('translation_contract_version') or contract_version),
310
+ 'fact_ledger_hash': str(result.get('fact_ledger_hash') or fact_ledger_hash),
311
  'translated_text': str(result.get('text') or '').strip(),
312
  'qa_status': 'passed',
313
  'attempts': int(result.get('attempts') or 1),
core/pipeline_agent.py CHANGED
@@ -10,6 +10,7 @@ import random
10
  import tiktoken
11
  import sys
12
  import logging
 
13
  import concurrent.futures
14
  from collections import Counter
15
  from datetime import datetime, timezone
@@ -55,6 +56,8 @@ class CitationAuditError(DeepSeekRequestError):
55
 
56
 
57
  class SSciPipelineAgent:
 
 
58
  def __init__(self, ds_key, s2_key=""):
59
  self.ds_key = str(ds_key or "").strip()
60
  if not self.ds_key:
@@ -3448,14 +3451,78 @@ Output only the revised Discussion body, without the '# Discussion' heading."""
3448
  text = re.sub(r'(?<!\d)(?:19|20)\d{2}\s+жылғы\s+\d{1,2}\s+(?:қаңтар|ақпан|наурыз|сәуір|мамыр|маусым|шілде|тамыз|қыркүйек|қазан|қараша|желтоқсан)', ' ', text, flags=re.I)
3449
  return cls._numeric_tokens(text)
3450
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3451
  @classmethod
3452
  def _translation_anchor_type(cls, token):
3453
  value = str(token or '')
3454
  if re.fullmatch(cls._translation_date_regex(), value, flags=re.I): return 'DATE'
 
 
3455
  if re.fullmatch(r'\([^()\n]*?(?:19|20)\d{2}[a-z]?[^()\n]*?\)', value, flags=re.I): return 'CITATION'
3456
  if re.fullmatch(r'(?i:https?://[^\s)>]+|doi:\s*[^\s)>]+|10\.\d{4,9}/[-._;()/:A-Z0-9]+)', value): return 'DOI_URL'
3457
  if re.fullmatch(r'(?i:H\d+[A-Za-z]?)', value): return 'HYPOTHESIS'
3458
- if re.search(r'(?:p|t|f|z|r|β|χ²|R²|ΔR²)\s*(?:<=|>=|=|<|>|≤|≥)', value, flags=re.I): return 'STAT'
3459
  if re.fullmatch(r'(?:fsQCA|SmartPLS|Mplus|OpenAlex|DeepSeek|ChatGPT|RStudio|SPSS|AMOS|Stata)', value): return 'NAME'
3460
  if re.fullmatch(cls._translation_acronym_regex(), value): return 'ACRONYM'
3461
  if re.fullmatch(r'(?:QUOTE|CASE|RESP|P)-\d+', value, flags=re.I): return 'QUOTE_ID'
@@ -3487,19 +3554,21 @@ Output only the revised Discussion body, without the '# Discussion' heading."""
3487
 
3488
  @classmethod
3489
  def _translation_protected_pattern(cls):
3490
- # One-pass alternation prevents a placeholder created for a citation or
3491
- # URL from being matched again as an acronym/number. Keep this list
3492
- # conservative: protect immutable research objects, not prose.
3493
  acronym = cls._translation_acronym_regex()
3494
  numeric = cls._translation_numeric_regex()
3495
  date_pattern = cls._translation_date_regex()
 
 
3496
  return re.compile(
3497
  rf'(?:{date_pattern})'
3498
- r'|'
3499
- r'(?i:https?://[^\s)>]+)'
 
3500
  r'|(?i:\bdoi:\s*[^\s)>]+)'
3501
  r'|(?i:\b10\.\d{4,9}/[-._;()/:A-Z0-9]+)'
3502
- r'|(?i:\b(?:p|t|f|z|r)\s*(?:<=|>=|=|<|>|≤|≥)\s*[-+]?(?:\d+(?:\.\d+)?|\.\d+))'
3503
  r'|(?:β|χ²|R²|ΔR²)\s*(?:<=|>=|=|<|>|≤|≥)\s*[-+]?(?:\d+(?:\.\d+)?|\.\d+)'
3504
  r'|\([^()\n]*?(?:19|20)\d{2}[a-z]?[^()\n]*?\)'
3505
  r'|`[^`\n]+`'
@@ -3515,6 +3584,7 @@ Output only the revised Discussion body, without the '# Discussion' heading."""
3515
 
3516
  @classmethod
3517
  def _protect_translation_tokens(cls, value):
 
3518
  source = str(value or '')
3519
  mapping = {}
3520
  counter = 0
@@ -3526,13 +3596,13 @@ Output only the revised Discussion body, without the '# Discussion' heading."""
3526
  placeholder = f'[[SP-T{counter:06d}]]'
3527
  original = match.group(0)
3528
  anchor_type = cls._translation_anchor_type(original)
3529
- manifest = {'source': original, 'type': anchor_type}
3530
- if anchor_type == 'DATE':
3531
- manifest['canonical'] = cls._canonicalize_translation_date(original)
3532
- left = source[max(0, match.start() - 80):match.start()]
3533
- right = source[match.end():min(len(source), match.end() + 60)]
3534
- manifest['owner_hint'] = (left + '{ANCHOR}' + right).replace('\n', ' ')
3535
- mapping[placeholder] = manifest
3536
  return placeholder
3537
 
3538
  protected = pattern.sub(replace, source)
@@ -3542,32 +3612,129 @@ Output only the revised Discussion body, without the '# Discussion' heading."""
3542
  return protected, mapping
3543
 
3544
  @classmethod
3545
- def _restore_translation_tokens(cls, value, mapping, block_id='', target_lang=''):
3546
- text = str(value or '')
3547
- found = Counter(re.findall(r'\[\[SP-T\d{6}\]\]', text))
3548
- expected = Counter(mapping.keys())
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3549
  if found != expected:
3550
  missing = list((expected - found).elements())[:8]
3551
  added = list((found - expected).elements())[:8]
3552
  raise DeepSeekRequestError(
3553
  f'Translation protected placeholder mismatch in {block_id}: missing={missing}, added={added}.'
3554
  )
3555
- for placeholder, raw_manifest in mapping.items():
3556
- manifest = raw_manifest if isinstance(raw_manifest, dict) else {'source': str(raw_manifest), 'type': 'IMMUTABLE'}
 
 
 
 
 
 
3557
  source_sentence = int(manifest.get('sentence_index') or 0)
3558
- target_sentence = cls._translation_sentence_index(text, text.find(placeholder))
3559
- if manifest.get('type') in {'DATE','CITATION','STAT','HYPOTHESIS','ACRONYM','NAME'} and target_sentence != source_sentence:
3560
  raise DeepSeekRequestError(
3561
- f'Translation semantic anchor moved across sentence boundary in {block_id}: {placeholder}.'
3562
  )
3563
- for placeholder, raw_manifest in mapping.items():
3564
- manifest = raw_manifest if isinstance(raw_manifest, dict) else {'source': str(raw_manifest), 'type': 'IMMUTABLE'}
3565
- original = str(manifest.get('source') or '')
3566
- if target_lang and manifest.get('type') == 'DATE' and manifest.get('canonical'):
3567
- replacement = cls._render_translation_date(manifest.get('canonical'), target_lang)
3568
- else:
3569
- replacement = original
3570
- text = text.replace(placeholder, replacement)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3571
  if re.search(r'\[\[SP-T\d{6}\]\]', text):
3572
  raise DeepSeekRequestError(f'Translation left unresolved protected placeholders in {block_id}.')
3573
  return text
@@ -3827,23 +3994,11 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
3827
  return parsed
3828
 
3829
  def _validate_translation_block(self, source, translated, block_id, target_lang):
3830
- if self._translation_date_canonicals(source) != self._translation_date_canonicals(translated):
3831
- raise DeepSeekRequestError(f'Translation changed date content in {block_id}.')
3832
- if self._numeric_tokens_without_dates(source) != self._numeric_tokens_without_dates(translated):
3833
- raise DeepSeekRequestError(f'Translation changed numerical content in {block_id}.')
3834
- if self._translation_citation_tokens(source) != self._translation_citation_tokens(translated):
3835
- raise DeepSeekRequestError(f'Translation changed citations in {block_id}.')
3836
- if self._translation_hypothesis_tokens(source) != self._translation_hypothesis_tokens(translated):
3837
- raise DeepSeekRequestError(f'Translation changed hypothesis labels in {block_id}.')
3838
- source_acronyms = self._translation_acronym_tokens(source)
3839
- target_acronyms = self._translation_acronym_tokens(translated)
3840
- if source_acronyms != target_acronyms:
3841
- missing = list((source_acronyms - target_acronyms).elements())
3842
- added = list((target_acronyms - source_acronyms).elements())
3843
- raise DeepSeekRequestError(
3844
- f'Translation changed variable abbreviations in {block_id}: '
3845
- f'missing={missing}, added={added}.'
3846
- )
3847
  for _long_form, acronym in self._translation_longform_acronym_pairs(source):
3848
  if not re.search(rf'[((]\s*{re.escape(acronym)}\s*[))]', str(translated or '')):
3849
  raise DeepSeekRequestError(
@@ -3871,8 +4026,8 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
3871
  if not is_chinese and not re.search(r'[А-Яа-яЁёӘәҒғҚқҢңӨөҰұҮүҺһІі]', translated):
3872
  raise DeepSeekRequestError(f'Translation block {block_id} contains no Cyrillic target-language text.')
3873
 
3874
- def _validate_translation_block_protected(self, block_id, source, translated, target_lang):
3875
- """Validate one restored block; recovery is owned by the protected gateway."""
3876
  self._validate_translation_block(source, translated, f'SP-P{block_id}', target_lang)
3877
  return translated
3878
 
@@ -3896,18 +4051,16 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
3896
  for block_id, _, protected_source in protected_batch
3897
  )
3898
  semantic_manifest = {
3899
- block_id: [
3900
- {'id': placeholder, **manifest}
3901
- for placeholder, manifest in protected_maps.get(block_id, {}).items()
3902
- ]
3903
  for block_id, _source, _protected in protected_batch
3904
  }
3905
  effective_prompt = system_prompt + (
3906
  '\n<SEMANTIC_ANCHOR_MANIFEST>\n'
3907
  + json.dumps(semantic_manifest, ensure_ascii=False)
3908
  + '\n</SEMANTIC_ANCHOR_MANIFEST>\n'
3909
- 'Use anchor type, canonical value, sentence_index, and owner_hint only to keep each placeholder attached to its original semantic owner. '
3910
- 'Do not print the manifest. DATE anchors may change local word order inside their sentence; CITATION anchors must remain with their original claim.'
 
3911
  )
3912
  if _strict:
3913
  effective_prompt += (
@@ -4113,6 +4266,8 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
4113
  'glossary': frozen_glossary,
4114
  'attempts': attempt,
4115
  'qa_status': 'passed',
 
 
4116
  }
4117
  except PipelineCancelledError:
4118
  raise
@@ -4207,14 +4362,10 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
4207
  raise DeepSeekRequestError(
4208
  f'Translation heading-tree mismatch: English={en_tree}, target={tr_tree}.'
4209
  )
4210
- if SSciPipelineAgent._numeric_tokens(source) != SSciPipelineAgent._numeric_tokens(target):
4211
- raise DeepSeekRequestError('Translation numerical alignment failed.')
4212
- if SSciPipelineAgent._translation_citation_tokens(source) != SSciPipelineAgent._translation_citation_tokens(target):
4213
- raise DeepSeekRequestError('Translation citation alignment failed.')
4214
- if SSciPipelineAgent._translation_hypothesis_tokens(source) != SSciPipelineAgent._translation_hypothesis_tokens(target):
4215
- raise DeepSeekRequestError('Translation hypothesis-label alignment failed.')
4216
- if SSciPipelineAgent._translation_acronym_tokens(source) != SSciPipelineAgent._translation_acronym_tokens(target):
4217
- raise DeepSeekRequestError('Translation abbreviation alignment failed.')
4218
  source_body = re.split(r'(?m)^#\s+References\s*$', source, maxsplit=1)[0]
4219
  target_body = re.split(r'(?m)^#\s+(?:参考文献|Литература|Пайдаланылған әдебиеттер)\s*$', target, maxsplit=1)[0]
4220
  source_chars = len(re.sub(r'\s+', '', source_body))
 
10
  import tiktoken
11
  import sys
12
  import logging
13
+ import hashlib
14
  import concurrent.futures
15
  from collections import Counter
16
  from datetime import datetime, timezone
 
56
 
57
 
58
  class SSciPipelineAgent:
59
+ TRANSLATION_CONTRACT_VERSION = 'fact-ledger-v1'
60
+
61
  def __init__(self, ds_key, s2_key=""):
62
  self.ds_key = str(ds_key or "").strip()
63
  if not self.ds_key:
 
3451
  text = re.sub(r'(?<!\d)(?:19|20)\d{2}\s+жылғы\s+\d{1,2}\s+(?:қаңтар|ақпан|наурыз|сәуір|мамыр|маусым|шілде|тамыз|қыркүйек|қазан|қараша|желтоқсан)', ' ', text, flags=re.I)
3452
  return cls._numeric_tokens(text)
3453
 
3454
+ @staticmethod
3455
+ def _translation_number_word_regex():
3456
+ # Conservative explicit English number-word quantities. Restrict to
3457
+ # phrases containing a magnitude word so ordinary prose such as "one of"
3458
+ # is not rewritten as a numeric fact.
3459
+ units = r'one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve|thirteen|fourteen|fifteen|sixteen|seventeen|eighteen|nineteen'
3460
+ tens = r'twenty|thirty|forty|fifty|sixty|seventy|eighty|ninety'
3461
+ magnitudes = r'hundred|thousand|million|billion'
3462
+ word = rf'(?:{units}|{tens}|{magnitudes}|and)'
3463
+ return rf'(?<![A-Za-z])(?=[A-Za-z -]*(?:{magnitudes})\b){word}(?:[ -]+{word})+(?![A-Za-z])'
3464
+
3465
+ @classmethod
3466
+ def _canonicalize_translation_quantity(cls, value):
3467
+ text = re.sub(r'[-]+', ' ', str(value or '').strip().casefold())
3468
+ tokens = [t for t in text.split() if t != 'and']
3469
+ small = {
3470
+ 'one':1,'two':2,'three':3,'four':4,'five':5,'six':6,'seven':7,'eight':8,'nine':9,
3471
+ 'ten':10,'eleven':11,'twelve':12,'thirteen':13,'fourteen':14,'fifteen':15,'sixteen':16,
3472
+ 'seventeen':17,'eighteen':18,'nineteen':19,'twenty':20,'thirty':30,'forty':40,'fifty':50,
3473
+ 'sixty':60,'seventy':70,'eighty':80,'ninety':90,
3474
+ }
3475
+ total = 0
3476
+ current = 0
3477
+ for token in tokens:
3478
+ if token in small:
3479
+ current += small[token]
3480
+ elif token == 'hundred':
3481
+ current = max(1, current) * 100
3482
+ elif token in {'thousand','million','billion'}:
3483
+ scale = {'thousand':1000,'million':1000000,'billion':1000000000}[token]
3484
+ total += max(1, current) * scale
3485
+ current = 0
3486
+ else:
3487
+ return ''
3488
+ return str(total + current) if (total or current) else ''
3489
+
3490
+ @staticmethod
3491
+ def _canonicalize_numeric_fact(value):
3492
+ text = str(value or '').strip()
3493
+ if not text:
3494
+ return ''
3495
+ return text.replace(',', '')
3496
+
3497
+ @classmethod
3498
+ def _translation_fact_render_policy(cls, anchor_type):
3499
+ if anchor_type == 'DATE':
3500
+ return 'localized_date'
3501
+ if anchor_type == 'QUANTITY':
3502
+ return 'canonical_numeric'
3503
+ return 'exact'
3504
+
3505
+ @classmethod
3506
+ def _translation_fact_canonical(cls, token, anchor_type):
3507
+ value = str(token or '')
3508
+ if anchor_type == 'DATE':
3509
+ return cls._canonicalize_translation_date(value)
3510
+ if anchor_type == 'QUANTITY':
3511
+ return cls._canonicalize_translation_quantity(value)
3512
+ if anchor_type in {'NUMBER','STAT'}:
3513
+ return cls._canonicalize_numeric_fact(value)
3514
+ return value
3515
+
3516
  @classmethod
3517
  def _translation_anchor_type(cls, token):
3518
  value = str(token or '')
3519
  if re.fullmatch(cls._translation_date_regex(), value, flags=re.I): return 'DATE'
3520
+ if re.fullmatch(cls._translation_number_word_regex(), value, flags=re.I): return 'QUANTITY'
3521
+ if re.fullmatch(r'(?i:(?:\d+(?:\.\d+)?%)?\s*CI\s*\[\s*[-+]?\.?\d+(?:\.\d+)?\s*,\s*[-+]?\.?\d+(?:\.\d+)?\s*\])', value): return 'STAT'
3522
  if re.fullmatch(r'\([^()\n]*?(?:19|20)\d{2}[a-z]?[^()\n]*?\)', value, flags=re.I): return 'CITATION'
3523
  if re.fullmatch(r'(?i:https?://[^\s)>]+|doi:\s*[^\s)>]+|10\.\d{4,9}/[-._;()/:A-Z0-9]+)', value): return 'DOI_URL'
3524
  if re.fullmatch(r'(?i:H\d+[A-Za-z]?)', value): return 'HYPOTHESIS'
3525
+ if re.search(r'(?:p|t|f|z|r|β|χ²|R²|ΔR²|N|n)\s*(?:<=|>=|=|<|>|≤|≥)', value, flags=re.I): return 'STAT'
3526
  if re.fullmatch(r'(?:fsQCA|SmartPLS|Mplus|OpenAlex|DeepSeek|ChatGPT|RStudio|SPSS|AMOS|Stata)', value): return 'NAME'
3527
  if re.fullmatch(cls._translation_acronym_regex(), value): return 'ACRONYM'
3528
  if re.fullmatch(r'(?:QUOTE|CASE|RESP|P)-\d+', value, flags=re.I): return 'QUOTE_ID'
 
3554
 
3555
  @classmethod
3556
  def _translation_protected_pattern(cls):
3557
+ # One-pass alternation prevents a placeholder created for a compound
3558
+ # statistical/date object from being split into unrelated numbers.
 
3559
  acronym = cls._translation_acronym_regex()
3560
  numeric = cls._translation_numeric_regex()
3561
  date_pattern = cls._translation_date_regex()
3562
+ quantity_words = cls._translation_number_word_regex()
3563
+ compound_ci = r'(?i:(?:\d+(?:\.\d+)?%)?\s*CI\s*\[\s*[-+]?\.?\d+(?:\.\d+)?\s*,\s*[-+]?\.?\d+(?:\.\d+)?\s*\])'
3564
  return re.compile(
3565
  rf'(?:{date_pattern})'
3566
+ rf'|(?:{compound_ci})'
3567
+ rf'|(?:{quantity_words})'
3568
+ r'|(?i:https?://[^\s)>]+)'
3569
  r'|(?i:\bdoi:\s*[^\s)>]+)'
3570
  r'|(?i:\b10\.\d{4,9}/[-._;()/:A-Z0-9]+)'
3571
+ r'|(?i:\b(?:p|t|f|z|r|n)\s*(?:<=|>=|=|<|>|≤|≥)\s*[-+]?(?:\d+(?:\.\d+)?|\.\d+))'
3572
  r'|(?:β|χ²|R²|ΔR²)\s*(?:<=|>=|=|<|>|≤|≥)\s*[-+]?(?:\d+(?:\.\d+)?|\.\d+)'
3573
  r'|\([^()\n]*?(?:19|20)\d{2}[a-z]?[^()\n]*?\)'
3574
  r'|`[^`\n]+`'
 
3584
 
3585
  @classmethod
3586
  def _protect_translation_tokens(cls, value):
3587
+ """Build a private fact ledger and replace model-inaccessible values."""
3588
  source = str(value or '')
3589
  mapping = {}
3590
  counter = 0
 
3596
  placeholder = f'[[SP-T{counter:06d}]]'
3597
  original = match.group(0)
3598
  anchor_type = cls._translation_anchor_type(original)
3599
+ mapping[placeholder] = {
3600
+ 'fact_id': f'F{counter:06d}',
3601
+ 'source': original, # private server ledger only
3602
+ 'type': anchor_type,
3603
+ 'canonical': cls._translation_fact_canonical(original, anchor_type),
3604
+ 'render_policy': cls._translation_fact_render_policy(anchor_type),
3605
+ }
3606
  return placeholder
3607
 
3608
  protected = pattern.sub(replace, source)
 
3612
  return protected, mapping
3613
 
3614
  @classmethod
3615
+ def _model_safe_translation_manifest(cls, mapping):
3616
+ """Expose only placement metadata; factual values remain server-private."""
3617
+ safe = []
3618
+ for placeholder, raw in (mapping or {}).items():
3619
+ item = raw if isinstance(raw, dict) else {'type':'IMMUTABLE'}
3620
+ anchor_type = str(item.get('type') or 'IMMUTABLE')
3621
+ safe.append({
3622
+ 'id': placeholder,
3623
+ 'fact_id': str(item.get('fact_id') or placeholder),
3624
+ 'type': anchor_type,
3625
+ 'sentence_index': int(item.get('sentence_index') or 0),
3626
+ 'mobility': 'same_sentence' if anchor_type in {'DATE','CITATION'} else 'same_owner',
3627
+ })
3628
+ return safe
3629
+
3630
+ @classmethod
3631
+ def _fact_ledger_hash(cls, ledger):
3632
+ records = []
3633
+ for block_id, mapping in sorted((ledger or {}).items()):
3634
+ for placeholder, raw in sorted((mapping or {}).items()):
3635
+ item = raw if isinstance(raw, dict) else {'source':str(raw),'type':'IMMUTABLE'}
3636
+ records.append({
3637
+ 'block_id': str(block_id),
3638
+ 'fact_id': str(item.get('fact_id') or placeholder),
3639
+ 'type': str(item.get('type') or 'IMMUTABLE'),
3640
+ 'canonical': str(item.get('canonical') or item.get('source') or ''),
3641
+ 'sentence_index': int(item.get('sentence_index') or 0),
3642
+ 'render_policy': str(item.get('render_policy') or 'exact'),
3643
+ })
3644
+ payload = json.dumps(records, ensure_ascii=False, sort_keys=True, separators=(',', ':')).encode('utf-8')
3645
+ return hashlib.sha256(payload).hexdigest()
3646
+
3647
+ @classmethod
3648
+ def translation_fact_ledger_hash(cls, section_text):
3649
+ """Deterministic source ledger hash used by translation checkpoint reuse."""
3650
+ units = cls._translation_body_units(section_text)
3651
+ ledger = {}
3652
+ body_index = 0
3653
+ for unit in units:
3654
+ if unit.get('kind') != 'body':
3655
+ continue
3656
+ body_index += 1
3657
+ _protected, mapping = cls._protect_translation_tokens(unit.get('text') or '')
3658
+ ledger[f'{body_index:06d}'] = mapping
3659
+ return cls._fact_ledger_hash(ledger)
3660
+
3661
+ @classmethod
3662
+ def _render_translation_fact(cls, manifest, target_lang):
3663
+ item = manifest if isinstance(manifest, dict) else {'source':str(manifest), 'type':'IMMUTABLE'}
3664
+ policy = str(item.get('render_policy') or 'exact')
3665
+ if policy == 'localized_date' and item.get('canonical'):
3666
+ return cls._render_translation_date(item.get('canonical'), target_lang)
3667
+ if policy == 'canonical_numeric' and item.get('canonical'):
3668
+ return str(item.get('canonical'))
3669
+ return str(item.get('source') or '')
3670
+
3671
+ @classmethod
3672
+ def _translation_unowned_objects(cls, protected_target):
3673
+ """Find factual objects invented by the model outside SP-T anchors."""
3674
+ text = re.sub(r'\[\[SP-T\d{6}\]\]', ' ', str(protected_target or ''))
3675
+ text = re.sub(r'\[\[SP-P\d{6}\]\]', ' ', text)
3676
+ unowned = {
3677
+ 'numeric': list(cls._numeric_tokens(text).elements()),
3678
+ 'citation': list(cls._translation_citation_tokens(text).elements()),
3679
+ 'hypothesis': list(cls._translation_hypothesis_tokens(text).elements()),
3680
+ 'acronym': list(cls._translation_acronym_tokens(text).elements()),
3681
+ }
3682
+ names = re.findall(r'\b(?:fsQCA|SmartPLS|Mplus|OpenAlex|DeepSeek|ChatGPT|RStudio|SPSS|AMOS|Stata)\b', text)
3683
+ if names:
3684
+ unowned['name'] = names
3685
+ urls = re.findall(r'(?i:https?://[^\s)>]+|\bdoi:\s*[^\s)>]+|\b10\.\d{4,9}/[-._;()/:A-Z0-9]+)', text)
3686
+ if urls:
3687
+ unowned['doi_url'] = urls
3688
+ return {key: values for key, values in unowned.items() if values}
3689
+
3690
+ @classmethod
3691
+ def _validate_translation_skeleton(cls, value, mapping, block_id=''):
3692
+ found = Counter(re.findall(r'\[\[SP-T\d{6}\]\]', str(value or '')))
3693
+ expected = Counter((mapping or {}).keys())
3694
  if found != expected:
3695
  missing = list((expected - found).elements())[:8]
3696
  added = list((found - expected).elements())[:8]
3697
  raise DeepSeekRequestError(
3698
  f'Translation protected placeholder mismatch in {block_id}: missing={missing}, added={added}.'
3699
  )
3700
+ unowned = cls._translation_unowned_objects(value)
3701
+ if unowned:
3702
+ safe_summary = {key: len(values) for key, values in unowned.items()}
3703
+ logging.warning('Translation fact integrity failed: block=%s unowned=%s', block_id, safe_summary)
3704
+ raise DeepSeekRequestError(f'Translation introduced unowned factual content in {block_id}: {safe_summary}.')
3705
+ target_text = str(value or '')
3706
+ for placeholder, raw_manifest in (mapping or {}).items():
3707
+ manifest = raw_manifest if isinstance(raw_manifest, dict) else {'type':'IMMUTABLE'}
3708
  source_sentence = int(manifest.get('sentence_index') or 0)
3709
+ target_sentence = cls._translation_sentence_index(target_text, target_text.find(placeholder))
3710
+ if target_sentence != source_sentence:
3711
  raise DeepSeekRequestError(
3712
+ f'Translation fact moved across sentence boundary in {block_id}: {manifest.get("fact_id") or placeholder}.'
3713
  )
3714
+ # Preserve the relative order of hard quantitative/identifier facts
3715
+ # inside each claim. DATE/CITATION/NAME may legitimately move for target
3716
+ # grammar, but two sample/statistical values must never trade owners.
3717
+ ordered_types = {'NUMBER','QUANTITY','STAT','HYPOTHESIS','DOI_URL','FORMULA','QUOTE_ID'}
3718
+ for sentence_index in sorted({int((m if isinstance(m, dict) else {}).get('sentence_index') or 0) for m in (mapping or {}).values()}):
3719
+ expected_order = [
3720
+ ph for ph, raw in (mapping or {}).items()
3721
+ if int((raw if isinstance(raw, dict) else {}).get('sentence_index') or 0) == sentence_index
3722
+ and str((raw if isinstance(raw, dict) else {}).get('type') or '') in ordered_types
3723
+ ]
3724
+ if len(expected_order) > 1:
3725
+ actual_order = sorted(expected_order, key=lambda ph: target_text.find(ph))
3726
+ if actual_order != expected_order:
3727
+ raise DeepSeekRequestError(
3728
+ f'Translation fact order changed within owner sentence in {block_id}: sentence={sentence_index}.'
3729
+ )
3730
+ return True
3731
+
3732
+ @classmethod
3733
+ def _restore_translation_tokens(cls, value, mapping, block_id='', target_lang=''):
3734
+ text = str(value or '')
3735
+ cls._validate_translation_skeleton(text, mapping, block_id)
3736
+ for placeholder, manifest in (mapping or {}).items():
3737
+ text = text.replace(placeholder, cls._render_translation_fact(manifest, target_lang))
3738
  if re.search(r'\[\[SP-T\d{6}\]\]', text):
3739
  raise DeepSeekRequestError(f'Translation left unresolved protected placeholders in {block_id}.')
3740
  return text
 
3994
  return parsed
3995
 
3996
  def _validate_translation_block(self, source, translated, block_id, target_lang):
3997
+ # R3.2.8.4: factual equality is enforced on the protected skeleton /
3998
+ # private Fact Ledger. This routine now owns language/coverage and the
3999
+ # long-form/acronym adjacency check only; raw numeric Counter equality is
4000
+ # deliberately non-authoritative because it creates multilingual false
4001
+ # failures and can miss value swaps.
 
 
 
 
 
 
 
 
 
 
 
 
4002
  for _long_form, acronym in self._translation_longform_acronym_pairs(source):
4003
  if not re.search(rf'[((]\s*{re.escape(acronym)}\s*[))]', str(translated or '')):
4004
  raise DeepSeekRequestError(
 
4026
  if not is_chinese and not re.search(r'[А-Яа-яЁёӘәҒғҚқҢңӨөҰұҮүҺһІі]', translated):
4027
  raise DeepSeekRequestError(f'Translation block {block_id} contains no Cyrillic target-language text.')
4028
 
4029
+ def _validate_translation_block_protected(self, block_id, source, translated, target_lang, mapping=None):
4030
+ """Validate one restored block after Fact Ledger integrity passed."""
4031
  self._validate_translation_block(source, translated, f'SP-P{block_id}', target_lang)
4032
  return translated
4033
 
 
4051
  for block_id, _, protected_source in protected_batch
4052
  )
4053
  semantic_manifest = {
4054
+ block_id: self._model_safe_translation_manifest(protected_maps.get(block_id, {}))
 
 
 
4055
  for block_id, _source, _protected in protected_batch
4056
  }
4057
  effective_prompt = system_prompt + (
4058
  '\n<SEMANTIC_ANCHOR_MANIFEST>\n'
4059
  + json.dumps(semantic_manifest, ensure_ascii=False)
4060
  + '\n</SEMANTIC_ANCHOR_MANIFEST>\n'
4061
+ 'This manifest intentionally contains no factual values. Use only fact type and sentence ownership to place each placeholder. '
4062
+ 'Never infer, spell out, calculate, or recreate a hidden number, date, citation, acronym, name, formula, or identifier. '
4063
+ 'Do not print the manifest. DATE and CITATION anchors may move only within their original sentence when grammar requires it.'
4064
  )
4065
  if _strict:
4066
  effective_prompt += (
 
4266
  'glossary': frozen_glossary,
4267
  'attempts': attempt,
4268
  'qa_status': 'passed',
4269
+ 'translation_contract_version': self.TRANSLATION_CONTRACT_VERSION,
4270
+ 'fact_ledger_hash': self.translation_fact_ledger_hash(value),
4271
  }
4272
  except PipelineCancelledError:
4273
  raise
 
4362
  raise DeepSeekRequestError(
4363
  f'Translation heading-tree mismatch: English={en_tree}, target={tr_tree}.'
4364
  )
4365
+ # Fact integrity is authoritative at the protected-block ledger gate.
4366
+ # Whole-unit raw Counter equality is intentionally not used here: valid
4367
+ # localized dates/number words can change surface digits, while swapped
4368
+ # values can preserve the same Counter.
 
 
 
 
4369
  source_body = re.split(r'(?m)^#\s+References\s*$', source, maxsplit=1)[0]
4370
  target_body = re.split(r'(?m)^#\s+(?:参考文献|Литература|Пайдаланылған әдебиеттер)\s*$', target, maxsplit=1)[0]
4371
  source_chars = len(re.sub(r'\s+', '', source_body))
smartpaper_restore_bridge/frontend/index.html CHANGED
@@ -311,7 +311,7 @@
311
  if (!channel.queue.length) return;
312
  const stream = ensureStreamNode(parentDocument, target, channel);
313
  const backlog = channel.queue.length;
314
- const perTick = backlog > 260 ? 5 : backlog > 140 ? 3 : backlog > 70 ? 2 : 1;
315
  let chunk = "";
316
  for (let i=0; i<perTick && channel.queue.length; i++) chunk += channel.queue.shift();
317
  if (chunk) {
@@ -319,7 +319,7 @@
319
  channel.displayed += chunk;
320
  if (shell && shell.dataset.spAutoFollow !== "0") shell.scrollTop = shell.scrollHeight;
321
  }
322
- channel.timer = window.setTimeout(() => pumpRenderQueue(parentDocument, target, shell, channel), 34);
323
  }
324
 
325
  function renderBufferedPreview(parentDocument, target, shell, text, channel) {
 
311
  if (!channel.queue.length) return;
312
  const stream = ensureStreamNode(parentDocument, target, channel);
313
  const backlog = channel.queue.length;
314
+ const perTick = backlog > 220 ? 3 : backlog > 90 ? 2 : 1;
315
  let chunk = "";
316
  for (let i=0; i<perTick && channel.queue.length; i++) chunk += channel.queue.shift();
317
  if (chunk) {
 
319
  channel.displayed += chunk;
320
  if (shell && shell.dataset.spAutoFollow !== "0") shell.scrollTop = shell.scrollHeight;
321
  }
322
+ channel.timer = window.setTimeout(() => pumpRenderQueue(parentDocument, target, shell, channel), 60);
323
  }
324
 
325
  function renderBufferedPreview(parentDocument, target, shell, text, channel) {