R3.2.8.6 Hierarchical Multilingual Translation Assurance Compiler

#26
Files changed (2) hide show
  1. core/generation_runner.py +46 -11
  2. core/pipeline_agent.py +221 -80
core/generation_runner.py CHANGED
@@ -21,7 +21,7 @@ from typing import Callable, Any
21
  from config import APP_CONFIG, AMJ_PROMPTS, ASQ_PROMPTS
22
  from core.checkpoint_store import TaskCheckpointStore
23
  from core.meta_extractor import extract_golden_metadata
24
- from core.pipeline_agent import SSciPipelineAgent
25
  from core.production_guard import build_validated_package, replace_abstract
26
  from core.qualitative_engine import get_qualitative_engine
27
  from core.quantitative_engine import get_quantitative_engine
@@ -33,7 +33,7 @@ ProgressCallback = Callable[..., None]
33
  MIN_FINAL_REFERENCES = 50
34
  LITERATURE_EVIDENCE_TARGET = 75
35
  CITATION_PLAN_TARGET = 70
36
- TRANSLATION_CONTRACT_VERSION = 'claim-bound-v2'
37
 
38
 
39
  def _clean_trailing_references(text, keywords=("reference", "references", "reference list", "参考文献")):
@@ -267,17 +267,18 @@ class GenerationRunner:
267
  parts = []
268
  if include_front:
269
  record = mapping.get('__front_matter__') or {}
270
- if record.get('qa_status') == 'passed' and record.get('translated_text'):
271
  parts.append(str(record['translated_text']).strip())
272
  for key, _title in GenerationRunner._unique_top_level_sections(adaptive_structure):
273
  record = mapping.get(key) or {}
274
- if record.get('qa_status') == 'passed' and record.get('translated_text'):
275
  parts.append(str(record['translated_text']).strip())
276
  return '\n\n'.join(part for part in parts if part).strip()
277
 
278
  def _translate_frozen_unit(
279
  self, agent, *, unit_key: str, unit_name: str, source_text: str, target_lang: str,
280
  translation_sections: dict, translation_glossary: dict, adaptive_structure, include_front=False,
 
281
  ):
282
  source = str(source_text or '').strip()
283
  if not source:
@@ -287,7 +288,7 @@ class GenerationRunner:
287
  fact_ledger_hash = self._translation_ledger_hash(agent, source)
288
  existing = dict((translation_sections or {}).get(unit_key) or {})
289
  if (
290
- existing.get('qa_status') == 'passed'
291
  and existing.get('source_hash') == source_hash
292
  and existing.get('target_lang') == target_lang
293
  and existing.get('translation_contract_version') == contract_version
@@ -297,11 +298,36 @@ class GenerationRunner:
297
  return translation_sections, translation_glossary
298
  partial = self._assemble_partial_translation(translation_sections, adaptive_structure, include_front=include_front)
299
  self._publish_target_preview(partial, f'translating:{unit_name}', force=True, complete=False)
300
- result = agent.translate_frozen_section(
301
- source, target_lang, glossary=translation_glossary, section_name=unit_name, max_attempts=2
302
- )
303
- updated_glossary = dict(result.get('glossary') or translation_glossary or {})
304
  updated_sections = dict(translation_sections or {})
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
305
  updated_sections[unit_key] = {
306
  'source_hash': source_hash,
307
  'target_lang': target_lang,
@@ -309,7 +335,9 @@ class GenerationRunner:
309
  'translation_contract_version': str(result.get('translation_contract_version') or contract_version),
310
  'fact_ledger_hash': str(result.get('fact_ledger_hash') or fact_ledger_hash),
311
  'translated_text': str(result.get('text') or '').strip(),
312
- 'qa_status': 'passed',
 
 
313
  'attempts': int(result.get('attempts') or 1),
314
  'completed_at': datetime.now(timezone.utc).isoformat(),
315
  }
@@ -1070,7 +1098,7 @@ Output only final academic prose. Do not report the word count."""
1070
  agent, unit_key=unit_key, unit_name=unit_name, source_text=source_text,
1071
  target_lang=target_lang, translation_sections=translation_sections,
1072
  translation_glossary=translation_glossary, adaptive_structure=adaptive_structure,
1073
- include_front=True,
1074
  )
1075
  partial = self._assemble_partial_translation(
1076
  translation_sections, adaptive_structure, include_front=True
@@ -1085,6 +1113,13 @@ Output only final academic prose. Do not report the word count."""
1085
  pct = min(96, 94 + int(index * 2 / max(1, len(final_units))))
1086
  self._checkpoint(f'translation_{unit_key}', state, pct)
1087
 
 
 
 
 
 
 
 
1088
  full_draft_translated = self._assemble_partial_translation(
1089
  translation_sections, adaptive_structure, include_front=True
1090
  ).rstrip()
 
21
  from config import APP_CONFIG, AMJ_PROMPTS, ASQ_PROMPTS
22
  from core.checkpoint_store import TaskCheckpointStore
23
  from core.meta_extractor import extract_golden_metadata
24
+ from core.pipeline_agent import SSciPipelineAgent, PipelineCancelledError
25
  from core.production_guard import build_validated_package, replace_abstract
26
  from core.qualitative_engine import get_qualitative_engine
27
  from core.quantitative_engine import get_quantitative_engine
 
33
  MIN_FINAL_REFERENCES = 50
34
  LITERATURE_EVIDENCE_TARGET = 75
35
  CITATION_PLAN_TARGET = 70
36
+ TRANSLATION_CONTRACT_VERSION = 'hierarchical-multilingual-v1'
37
 
38
 
39
  def _clean_trailing_references(text, keywords=("reference", "references", "reference list", "参考文献")):
 
267
  parts = []
268
  if include_front:
269
  record = mapping.get('__front_matter__') or {}
270
+ if record.get('qa_status') in {'passed', 'passed_with_warning'} and record.get('translated_text'):
271
  parts.append(str(record['translated_text']).strip())
272
  for key, _title in GenerationRunner._unique_top_level_sections(adaptive_structure):
273
  record = mapping.get(key) or {}
274
+ if record.get('qa_status') in {'passed', 'passed_with_warning'} and record.get('translated_text'):
275
  parts.append(str(record['translated_text']).strip())
276
  return '\n\n'.join(part for part in parts if part).strip()
277
 
278
  def _translate_frozen_unit(
279
  self, agent, *, unit_key: str, unit_name: str, source_text: str, target_lang: str,
280
  translation_sections: dict, translation_glossary: dict, adaptive_structure, include_front=False,
281
+ defer_failures=True,
282
  ):
283
  source = str(source_text or '').strip()
284
  if not source:
 
288
  fact_ledger_hash = self._translation_ledger_hash(agent, source)
289
  existing = dict((translation_sections or {}).get(unit_key) or {})
290
  if (
291
+ existing.get('qa_status') in {'passed', 'passed_with_warning'}
292
  and existing.get('source_hash') == source_hash
293
  and existing.get('target_lang') == target_lang
294
  and existing.get('translation_contract_version') == contract_version
 
298
  return translation_sections, translation_glossary
299
  partial = self._assemble_partial_translation(translation_sections, adaptive_structure, include_front=include_front)
300
  self._publish_target_preview(partial, f'translating:{unit_name}', force=True, complete=False)
 
 
 
 
301
  updated_sections = dict(translation_sections or {})
302
+ try:
303
+ result = agent.translate_frozen_section(
304
+ source, target_lang, glossary=translation_glossary, section_name=unit_name, max_attempts=2
305
+ )
306
+ except PipelineCancelledError:
307
+ raise
308
+ except Exception as error:
309
+ if not defer_failures:
310
+ raise
311
+ code_method = getattr(agent, 'translation_error_code', None)
312
+ error_code = str(code_method(error) if callable(code_method) else 'TRANSLATION_PENDING')
313
+ updated_sections[unit_key] = {
314
+ 'source_hash': source_hash,
315
+ 'target_lang': target_lang,
316
+ 'glossary_version': self._glossary_version(translation_glossary),
317
+ 'translation_contract_version': contract_version,
318
+ 'fact_ledger_hash': fact_ledger_hash,
319
+ 'translated_text': '',
320
+ 'qa_status': 'repair_pending',
321
+ 'error_code': error_code,
322
+ 'attempts': int(existing.get('attempts') or 0) + 1,
323
+ 'deferred_at': datetime.now(timezone.utc).isoformat(),
324
+ }
325
+ logging.warning(
326
+ 'Translation deferred until final gate | unit=%s code=%s error=%s',
327
+ unit_name, error_code, type(error).__name__,
328
+ )
329
+ return updated_sections, translation_glossary
330
+ updated_glossary = dict(result.get('glossary') or translation_glossary or {})
331
  updated_sections[unit_key] = {
332
  'source_hash': source_hash,
333
  'target_lang': target_lang,
 
335
  'translation_contract_version': str(result.get('translation_contract_version') or contract_version),
336
  'fact_ledger_hash': str(result.get('fact_ledger_hash') or fact_ledger_hash),
337
  'translated_text': str(result.get('text') or '').strip(),
338
+ 'qa_status': str(result.get('qa_status') or 'passed'),
339
+ 'language_adapter': str(result.get('language_adapter') or ''),
340
+ 'diagnostics': list(result.get('diagnostics') or [])[-20:],
341
  'attempts': int(result.get('attempts') or 1),
342
  'completed_at': datetime.now(timezone.utc).isoformat(),
343
  }
 
1098
  agent, unit_key=unit_key, unit_name=unit_name, source_text=source_text,
1099
  target_lang=target_lang, translation_sections=translation_sections,
1100
  translation_glossary=translation_glossary, adaptive_structure=adaptive_structure,
1101
+ include_front=True, defer_failures=False,
1102
  )
1103
  partial = self._assemble_partial_translation(
1104
  translation_sections, adaptive_structure, include_front=True
 
1113
  pct = min(96, 94 + int(index * 2 / max(1, len(final_units))))
1114
  self._checkpoint(f'translation_{unit_key}', state, pct)
1115
 
1116
+ unresolved_units = [
1117
+ key for key, _name, _source in final_units
1118
+ if (translation_sections.get(key) or {}).get('qa_status') not in {'passed', 'passed_with_warning'}
1119
+ or not (translation_sections.get(key) or {}).get('translated_text')
1120
+ ]
1121
+ if unresolved_units:
1122
+ raise RuntimeError('translation_final_gate_unresolved:' + ','.join(unresolved_units[:8]))
1123
  full_draft_translated = self._assemble_partial_translation(
1124
  translation_sections, adaptive_structure, include_front=True
1125
  ).rstrip()
core/pipeline_agent.py CHANGED
@@ -56,7 +56,7 @@ class CitationAuditError(DeepSeekRequestError):
56
 
57
 
58
  class SSciPipelineAgent:
59
- TRANSLATION_CONTRACT_VERSION = 'claim-bound-v2'
60
 
61
  def __init__(self, ds_key, s2_key=""):
62
  self.ds_key = str(ds_key or "").strip()
@@ -120,6 +120,9 @@ class SSciPipelineAgent:
120
  self.citation_map = {}
121
  self.used_citations = set()
122
  self.citation_audit_log = []
 
 
 
123
  self.lock = threading.RLock()
124
 
125
  self.lit_target_papers = APP_CONFIG["lit_target_papers"]
@@ -273,6 +276,7 @@ class SSciPipelineAgent:
273
  "citation_map": self.citation_map,
274
  "used_citations": list(self.used_citations),
275
  "citation_audit_log": list(self.citation_audit_log),
 
276
  "global_memory": self.global_memory,
277
  "s2_access_mode": self._s2_access_mode,
278
  "s2_logical_searches": int(self._s2_logical_searches),
@@ -294,6 +298,7 @@ class SSciPipelineAgent:
294
  self.citation_map = state_dict.get("citation_map", {})
295
  self.used_citations = set(state_dict.get("used_citations", []))
296
  self.citation_audit_log = list(state_dict.get("citation_audit_log", []))
 
297
  self.global_memory = state_dict.get("global_memory", self.global_memory)
298
  loaded_mode = str(state_dict.get("s2_access_mode") or "").strip().lower()
299
  expected_mode = "key" if self.s2_key else "anonymous"
@@ -3277,12 +3282,104 @@ Output only the repaired section."""
3277
  )['abstract']
3278
 
3279
  @staticmethod
3280
- def _translation_language_rule(target_lang):
3281
- if 'Russian' in target_lang:
3282
- return 'Use idiomatic academic Russian with correct case government, gender-number agreement, aspect, punctuation, and terminology. Reorganize English syntax where necessary and avoid literal English calques.'
3283
- if 'Kazakh' in target_lang:
3284
- return 'Use idiomatic academic Kazakh with correct case and possessive suffixes, person/plural endings, vowel harmony, consonant assimilation, natural word order, and correct Kazakh-specific letters. Avoid Russian syntactic calques.'
3285
- return 'Use formal academic Simplified Chinese with explicit concepts, natural Chinese syntax, clear hierarchy, and logically connected claims. Avoid carrying English nominalization and clause order into Chinese mechanically.'
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3286
 
3287
  @staticmethod
3288
  def _translation_prompt(target_lang):
@@ -3955,26 +4052,40 @@ Output only the repaired section."""
3955
  mapping[acronym] = acronym
3956
  return mapping
3957
 
3958
- @staticmethod
3959
- def _validate_translation_language_quality(translated, target_lang):
3960
  text = str(translated or '')
3961
  compact = re.sub(r'\s+', '', text)
3962
  if len(compact) < 80:
3963
  return True
3964
- if 'Chinese' in target_lang:
3965
- if len(re.findall(r'[\u4e00-\u9fff]', text)) < max(12, int(len(compact) * 0.08)):
3966
- raise DeepSeekRequestError('Chinese translation quality validation failed: insufficient Chinese text.')
 
 
 
 
 
 
 
3967
  return True
 
3968
  cyrillic = re.findall(r'[А-Яа-яЁёӘәҒғҚқҢңӨөҰұҮүҺһІі]', text)
3969
  if not cyrillic:
3970
- raise DeepSeekRequestError('Target-language translation contains no Cyrillic text.')
3971
  kazakh_specific = re.findall(r'[ӘәҒғҚқҢңӨөҰұҮүҺһІі]', text)
3972
- if 'Kazakh' in target_lang and len(cyrillic) >= 200:
3973
  minimum = max(2, int(len(cyrillic) * 0.002))
3974
  if len(kazakh_specific) < minimum:
3975
- raise DeepSeekRequestError('Kazakh translation quality validation failed: insufficient Kazakh-specific letters.')
3976
- if 'Russian' in target_lang and cyrillic and len(kazakh_specific) / len(cyrillic) > 0.02:
3977
- raise DeepSeekRequestError('Russian translation quality validation failed: excessive Kazakh-specific letters.')
 
 
 
 
 
 
3978
  return True
3979
 
3980
  def validate_translation_semantic_equivalence(self, source, translated, target_lang, section_name=''):
@@ -4055,8 +4166,11 @@ SECTION: {section_name}
4055
  'allowed_issue_codes': sorted(allowed_codes),
4056
  'items': [{'id': row['id'], 'english': row['english'], 'target': row['target']} for row in rows],
4057
  }
 
4058
  system_prompt = f"""You are a bilingual academic translation QA classifier for {target_lang}.
4059
  Review EACH claim independently. Do not rewrite and do not provide free-form explanations. Check semantic equivalence and language quality: omissions/additions, negation, uncertainty, association-versus-causation strength, terminology, grammar, language contamination, citation attachment, and acronym attachment.
 
 
4060
  Return valid JSON only in this exact form: {{"items":[{{"id":"SP-P000001-C000001","ok":true,"codes":[]}}]}}.
4061
  If ok=false, codes MUST contain only values from allowed_issue_codes. Never return numbered prose, examples, quotes, or corrective wording. Return every supplied id exactly once."""
4062
  response = self.call_deepseek(
@@ -4184,37 +4298,61 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
4184
  return parsed
4185
 
4186
  def _validate_translation_block(self, source, translated, block_id, target_lang):
4187
- # R3.2.8.4: factual equality is enforced on the protected skeleton /
4188
- # private Fact Ledger. This routine now owns language/coverage and the
4189
- # long-form/acronym adjacency check only; raw numeric Counter equality is
4190
- # deliberately non-authoritative because it creates multilingual false
4191
- # failures and can miss value swaps.
4192
- for _long_form, acronym in self._translation_longform_acronym_pairs(source):
4193
- if not re.search(rf'[((]\s*{re.escape(acronym)}\s*[))]', str(translated or '')):
4194
- raise DeepSeekRequestError(
4195
- f'Translation detached long-form/acronym pair in {block_id}: {acronym}.'
4196
- )
4197
 
 
 
 
 
4198
  source_compact = re.sub(r'\s+', '', str(source or ''))
4199
  target_compact = re.sub(r'\s+', '', str(translated or ''))
 
 
 
 
 
 
4200
  if len(source_compact) >= 120:
4201
- is_chinese = 'Chinese' in target_lang
4202
- minimum = float(APP_CONFIG.get(
4203
- 'translation_min_block_ratio_zh' if is_chinese else 'translation_min_block_ratio_cyrillic',
4204
- 0.28 if is_chinese else 0.65,
4205
- ))
4206
- maximum = float(APP_CONFIG.get('translation_max_block_ratio', 2.20))
4207
  ratio = len(target_compact) / max(1, len(source_compact))
4208
- if ratio < minimum or ratio > maximum:
4209
- raise DeepSeekRequestError(
4210
- f'Translation coverage ratio is implausible in {block_id}: {ratio:.3f}.'
4211
- )
4212
- if source_compact.casefold() == target_compact.casefold():
4213
- raise DeepSeekRequestError(f'Translation block {block_id} was returned unchanged.')
4214
- if is_chinese and not re.search(r'[\u4e00-\u9fff]', translated):
4215
- raise DeepSeekRequestError(f'Translation block {block_id} contains no Chinese text.')
4216
- if not is_chinese and not re.search(r'[А-Яа-яЁёӘәҒғҚқҢңӨөҰұҮүҺһІі]', translated):
4217
- raise DeepSeekRequestError(f'Translation block {block_id} contains no Cyrillic target-language text.')
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4218
 
4219
  def _validate_translation_block_protected(self, block_id, source, translated, target_lang, mapping=None):
4220
  """Validate one restored block after Fact Ledger integrity passed."""
@@ -4228,10 +4366,10 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
4228
  protected_maps = {}
4229
  claim_sources_by_block = {}
4230
  for block_id, source in batch:
4231
- protected_source, token_map, claim_sources = self._protect_translation_claims(source)
4232
- protected_batch.append((block_id, source, protected_source))
4233
- protected_maps[block_id] = token_map
4234
- claim_sources_by_block[block_id] = claim_sources
4235
  marked_source = '\n\n'.join(
4236
  f'[[SP-P{block_id}]]\n{protected_source}'
4237
  for block_id, _, protected_source in protected_batch
@@ -4399,12 +4537,21 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
4399
  except Exception as error:
4400
  last_error = error
4401
  if attempt >= attempts:
4402
- raise
 
 
 
 
 
 
 
 
 
4403
  logging.warning(
4404
  'Translation QA-only retry: section=%s attempt=%s/%s error=%s',
4405
  section_name or 'section', attempt, attempts, self._safe_source_error(error),
4406
  )
4407
- raise last_error or DeepSeekRequestError('Translation QA transport failed.')
4408
 
4409
  def translate_frozen_section(self, section_text, target_lang, glossary=None, section_name='', max_attempts=2):
4410
  """Translate one frozen English unit and publish only after all QA passes."""
@@ -4413,6 +4560,8 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
4413
  raise DeepSeekRequestError('Frozen English translation unit is empty.')
4414
  if re.search(r'(?m)^#\s+References\s*$', value):
4415
  raise DeepSeekRequestError('References must not be sent to the translation model.')
 
 
4416
  frozen_glossary = self.extend_translation_glossary(value, target_lang, glossary)
4417
  units = self._translation_body_units(value)
4418
  body_units = [unit for unit in units if unit['kind'] == 'body']
@@ -4510,13 +4659,16 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
4510
  output = '\n\n'.join(output_units).strip()
4511
  self.validate_translation_alignment(value, output)
4512
  self._validate_translation_language_quality(output, target_lang)
 
4513
  return {
4514
  'text': output,
4515
  'glossary': frozen_glossary,
4516
  'attempts': attempt,
4517
- 'qa_status': 'passed',
4518
  'translation_contract_version': self.TRANSLATION_CONTRACT_VERSION,
4519
  'fact_ledger_hash': self.translation_fact_ledger_hash(value),
 
 
4520
  }
4521
  except PipelineCancelledError:
4522
  raise
@@ -4529,7 +4681,7 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
4529
  or message.startswith('Translation alignment')
4530
  or message.startswith('Translation changed ')
4531
  or message.startswith('Translation protected ')
4532
- or message.startswith('Translation coverage ratio')
4533
  or message.startswith('Translation block ')
4534
  or message.startswith('Chinese translation quality validation failed')
4535
  or message.startswith('Kazakh translation quality validation failed')
@@ -4600,54 +4752,43 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
4600
  self.validate_translation_alignment(value, output)
4601
  return output.strip()
4602
 
4603
- @staticmethod
4604
- def validate_translation_alignment(english_text, translated_text):
4605
  source = str(english_text or '').strip()
4606
  target = str(translated_text or '').strip()
4607
  if not target:
4608
  raise DeepSeekRequestError('Target-language manuscript is empty.')
4609
- en_tree = [level for level, _ in SSciPipelineAgent._markdown_heading_tree(source)]
4610
- tr_tree = [level for level, _ in SSciPipelineAgent._markdown_heading_tree(target)]
4611
  if en_tree != tr_tree:
4612
  raise DeepSeekRequestError(
4613
  f'Translation heading-tree mismatch: English={en_tree}, target={tr_tree}.'
4614
  )
4615
- # Fact integrity is authoritative at the protected-block ledger gate.
4616
- # Whole-unit raw Counter equality is intentionally not used here: valid
4617
- # localized dates/number words can change surface digits, while swapped
4618
- # values can preserve the same Counter.
4619
  source_body = re.split(r'(?m)^#\s+References\s*$', source, maxsplit=1)[0]
4620
  target_body = re.split(r'(?m)^#\s+(?:参考文献|Литература|Пайдаланылған әдебиеттер)\s*$', target, maxsplit=1)[0]
4621
  source_chars = len(re.sub(r'\s+', '', source_body))
4622
  target_chars = len(re.sub(r'\s+', '', target_body))
4623
- if re.search(r'(?m)^#\s+参考文献\s*$', target):
4624
- minimum_ratio = float(APP_CONFIG.get('translation_min_block_ratio_zh', 0.28))
4625
- elif re.search(r'(?m)^#\s+(?:Литература|Пайдаланылған әдебиеттер)\s*$', target):
4626
- minimum_ratio = float(APP_CONFIG.get('translation_min_block_ratio_cyrillic', 0.65))
4627
- else:
4628
- minimum_ratio = float(APP_CONFIG.get('translation_min_coverage_ratio', 0.25))
4629
  ratio = target_chars / source_chars if source_chars else 0.0
4630
- if source_chars and ratio < minimum_ratio:
4631
- raise DeepSeekRequestError(
4632
- f'Translation content coverage is too low: {ratio:.3f}.'
4633
- )
4634
 
4635
- # Russian and Kazakh both use Cyrillic, so a generic Cyrillic check is
4636
- # insufficient. Long Kazakh manuscripts must contain a meaningful
4637
- # number of Kazakh-specific letters; Russian manuscripts must not be
4638
- # dominated by those letters. This is deterministic and adds no API cost.
4639
  kazakh_specific = re.findall(r'[ӘәҒғҚқҢңӨөҰұҮүҺһІі]', target_body)
4640
  cyrillic = re.findall(r'[А-Яа-яЁёӘәҒғҚқҢңӨөҰұҮүҺһІі]', target_body)
4641
- if re.search(r'(?m)^#\s+Пайдаланылған әдебиеттер\s*$', target):
4642
  minimum_kazakh = max(8, int(len(cyrillic) * 0.003))
4643
  if len(kazakh_specific) < minimum_kazakh:
4644
- raise DeepSeekRequestError(
4645
- 'Kazakh language validation failed: insufficient Kazakh-specific letters.'
 
4646
  )
4647
- elif re.search(r'(?m)^#\s+Литература\s*$', target):
4648
- if cyrillic and len(kazakh_specific) / len(cyrillic) > 0.02:
4649
- raise DeepSeekRequestError(
4650
- 'Russian language validation failed: excessive Kazakh-specific letters.'
 
4651
  )
4652
  return True
4653
 
 
56
 
57
 
58
  class SSciPipelineAgent:
59
+ TRANSLATION_CONTRACT_VERSION = 'hierarchical-multilingual-v1'
60
 
61
  def __init__(self, ds_key, s2_key=""):
62
  self.ds_key = str(ds_key or "").strip()
 
120
  self.citation_map = {}
121
  self.used_citations = set()
122
  self.citation_audit_log = []
123
+ # R3.2.8.6: translation validators emit hierarchical assurance evidence.
124
+ # Low-level diagnostics cannot independently terminate a manuscript.
125
+ self.translation_assurance_log = []
126
  self.lock = threading.RLock()
127
 
128
  self.lit_target_papers = APP_CONFIG["lit_target_papers"]
 
276
  "citation_map": self.citation_map,
277
  "used_citations": list(self.used_citations),
278
  "citation_audit_log": list(self.citation_audit_log),
279
+ "translation_assurance_log": list(self.translation_assurance_log),
280
  "global_memory": self.global_memory,
281
  "s2_access_mode": self._s2_access_mode,
282
  "s2_logical_searches": int(self._s2_logical_searches),
 
298
  self.citation_map = state_dict.get("citation_map", {})
299
  self.used_citations = set(state_dict.get("used_citations", []))
300
  self.citation_audit_log = list(state_dict.get("citation_audit_log", []))
301
+ self.translation_assurance_log = list(state_dict.get("translation_assurance_log", []))
302
  self.global_memory = state_dict.get("global_memory", self.global_memory)
303
  loaded_mode = str(state_dict.get("s2_access_mode") or "").strip().lower()
304
  expected_mode = "key" if self.s2_key else "anonymous"
 
3282
  )['abstract']
3283
 
3284
  @staticmethod
3285
+ def _translation_language_profile(target_lang):
3286
+ """Language-specific rendering policy over one language-neutral claim/fact IR.
3287
+
3288
+ Surface length and script statistics are diagnostic evidence only.
3289
+ Fact/claim integrity remains language-neutral and authoritative.
3290
+ """
3291
+ name = str(target_lang or '')
3292
+ if 'Russian' in name:
3293
+ return {
3294
+ 'adapter_id': 'ru-academic-v1', 'language_code': 'ru',
3295
+ 'morphology': 'inflectional', 'script': 'cyrillic',
3296
+ 'surface_min_key': 'translation_min_block_ratio_cyrillic',
3297
+ 'surface_min_default': 0.65, 'surface_max_default': 2.20,
3298
+ 'term_policy': 'Allow grammatically required Russian case, gender, number, and inflection around translated terms while preserving concept identity and protected acronyms.',
3299
+ 'contamination_policy': 'Treat sustained non-Russian language contamination as a repair signal; Kazakh-specific letters are diagnostic evidence, not a standalone failure rule.',
3300
+ }
3301
+ if 'Kazakh' in name:
3302
+ return {
3303
+ 'adapter_id': 'kk-academic-v1', 'language_code': 'kk',
3304
+ 'morphology': 'agglutinative', 'script': 'cyrillic',
3305
+ 'surface_min_key': 'translation_min_block_ratio_cyrillic',
3306
+ 'surface_min_default': 0.65, 'surface_max_default': 2.20,
3307
+ 'term_policy': 'Translate directly from English into academic Kazakh. Allow case, possessive, person, plural and other productive suffixes to attach naturally to translated term stems while preserving concept identity and protected acronyms.',
3308
+ 'contamination_policy': 'Treat sustained Russian contamination as a repair signal. Absence of Kazakh-specific letters in a short claim is never sufficient by itself to reject the claim.',
3309
+ }
3310
+ return {
3311
+ 'adapter_id': 'zh-academic-v1', 'language_code': 'zh',
3312
+ 'morphology': 'analytic', 'script': 'han',
3313
+ 'surface_min_key': 'translation_min_block_ratio_zh',
3314
+ 'surface_min_default': 0.28, 'surface_max_default': 2.20,
3315
+ 'term_policy': 'Use stable academic Chinese terminology while preserving concept identity and protected acronyms. Natural Chinese reordering and sentence merging/splitting are allowed inside the same claim.',
3316
+ 'contamination_policy': 'Treat sustained untranslated English prose as a repair signal; normal acronyms, model names and protected terms are allowed.',
3317
+ }
3318
+
3319
+ @classmethod
3320
+ def _translation_language_rule(cls, target_lang):
3321
+ profile = cls._translation_language_profile(target_lang)
3322
+ if profile['language_code'] == 'ru':
3323
+ core = 'Use idiomatic academic Russian with correct case government, gender-number agreement, aspect, punctuation, and terminology. Reorganize English syntax where necessary and avoid literal English calques.'
3324
+ elif profile['language_code'] == 'kk':
3325
+ core = 'Use idiomatic academic Kazakh with correct case and possessive suffixes, person/plural endings, vowel harmony, consonant assimilation, natural word order, and correct Kazakh-specific letters. Translate directly from English and avoid Russian syntactic calques.'
3326
+ else:
3327
+ core = 'Use formal academic Simplified Chinese with explicit concepts, natural Chinese syntax, clear hierarchy, and logically connected claims. Avoid carrying English nominalization and clause order into Chinese mechanically.'
3328
+ return core + ' ' + profile['term_policy'] + ' ' + profile['contamination_policy']
3329
+
3330
+ @classmethod
3331
+ def _compile_translation_ir(cls, source):
3332
+ """Compile a private, language-neutral translation intermediate representation."""
3333
+ protected_source, fact_ledger, claim_sources = cls._protect_translation_claims(source)
3334
+ return {
3335
+ 'ir_version': 'translation-ir-v1',
3336
+ 'protected_source': protected_source,
3337
+ 'fact_ledger': fact_ledger,
3338
+ 'claim_sources': claim_sources,
3339
+ 'claim_ids': list(claim_sources),
3340
+ }
3341
+
3342
+ def _record_translation_assurance(self, code, *, target_lang='', block_id='', severity='diagnostic', decision='PASS_WITH_WARNING', detail=None):
3343
+ row = {
3344
+ 'code': str(code or 'TRANSLATION_DIAGNOSTIC'),
3345
+ 'language': self._translation_language_profile(target_lang).get('language_code'),
3346
+ 'block_id': str(block_id or ''),
3347
+ 'severity': str(severity or 'diagnostic'),
3348
+ 'decision': str(decision or 'PASS_WITH_WARNING'),
3349
+ }
3350
+ if isinstance(detail, dict):
3351
+ row['detail'] = {str(k): v for k, v in detail.items() if isinstance(v, (str, int, float, bool))}
3352
+ self.translation_assurance_log.append(row)
3353
+ logging.warning(
3354
+ 'Translation assurance | code=%s language=%s block=%s severity=%s decision=%s detail=%s',
3355
+ row['code'], row['language'], row['block_id'], row['severity'], row['decision'], row.get('detail', {}),
3356
+ )
3357
+ return row
3358
+
3359
+ @staticmethod
3360
+ def _translation_assurance_decision(evidence):
3361
+ """Central decision manager: validators provide evidence; this method owns consequence."""
3362
+ items = list(evidence or [])
3363
+ if any(str(item.get('severity')) == 'critical' for item in items):
3364
+ return 'HOLD_FINAL'
3365
+ if any(str(item.get('severity')) == 'repair' for item in items):
3366
+ return 'REPAIR_BLOCK'
3367
+ if items:
3368
+ return 'PASS_WITH_WARNING'
3369
+ return 'PASS'
3370
+
3371
+ @classmethod
3372
+ def translation_error_code(cls, error):
3373
+ message = str(error or '')
3374
+ if 'unowned factual content' in message or 'protected placeholder mismatch' in message or 'fact moved across claim boundary' in message or 'fact order changed' in message:
3375
+ return 'FACT_INTEGRITY'
3376
+ if 'claim marker mismatch' in message or 'marker mismatch' in message or 'heading-tree mismatch' in message:
3377
+ return 'STRUCTURE_INTEGRITY'
3378
+ if 'claim-local QA failed' in message or 'semantic' in message.lower() or 'caus' in message.lower():
3379
+ return 'SEMANTIC_INTEGRITY'
3380
+ if 'target-language signal' in message.lower() or 'returned unchanged' in message.lower():
3381
+ return 'LANGUAGE_QUALITY'
3382
+ return 'TRANSLATION_TECHNICAL'
3383
 
3384
  @staticmethod
3385
  def _translation_prompt(target_lang):
 
4052
  mapping[acronym] = acronym
4053
  return mapping
4054
 
4055
+ def _validate_translation_language_quality(self, translated, target_lang):
4056
+ """Validate strong target-language identity; keep weak script heuristics diagnostic-only."""
4057
  text = str(translated or '')
4058
  compact = re.sub(r'\s+', '', text)
4059
  if len(compact) < 80:
4060
  return True
4061
+ profile = self._translation_language_profile(target_lang)
4062
+ if profile['language_code'] == 'zh':
4063
+ han = re.findall(r'[\u4e00-\u9fff]', text)
4064
+ if not han:
4065
+ raise DeepSeekRequestError('Translation target-language signal missing: Chinese text contains no Han characters.')
4066
+ if len(han) < max(12, int(len(compact) * 0.08)):
4067
+ self._record_translation_assurance(
4068
+ 'ZH_LANGUAGE_SIGNAL_LOW', target_lang=target_lang, severity='diagnostic',
4069
+ decision='PASS_WITH_WARNING', detail={'han_chars': len(han), 'compact_chars': len(compact)},
4070
+ )
4071
  return True
4072
+
4073
  cyrillic = re.findall(r'[А-Яа-яЁёӘәҒғҚқҢңӨөҰұҮүҺһІі]', text)
4074
  if not cyrillic:
4075
+ raise DeepSeekRequestError('Translation target-language signal missing: target contains no Cyrillic text.')
4076
  kazakh_specific = re.findall(r'[ӘәҒғҚқҢңӨөҰұҮүҺһІі]', text)
4077
+ if profile['language_code'] == 'kk' and len(cyrillic) >= 200:
4078
  minimum = max(2, int(len(cyrillic) * 0.002))
4079
  if len(kazakh_specific) < minimum:
4080
+ self._record_translation_assurance(
4081
+ 'KK_SPECIFIC_LETTER_SIGNAL_LOW', target_lang=target_lang, severity='diagnostic',
4082
+ decision='PASS_WITH_WARNING', detail={'kazakh_specific': len(kazakh_specific), 'cyrillic': len(cyrillic)},
4083
+ )
4084
+ if profile['language_code'] == 'ru' and cyrillic and len(kazakh_specific) / len(cyrillic) > 0.02:
4085
+ self._record_translation_assurance(
4086
+ 'RU_KAZAKH_LETTER_SIGNAL_HIGH', target_lang=target_lang, severity='diagnostic',
4087
+ decision='PASS_WITH_WARNING', detail={'kazakh_specific': len(kazakh_specific), 'cyrillic': len(cyrillic)},
4088
+ )
4089
  return True
4090
 
4091
  def validate_translation_semantic_equivalence(self, source, translated, target_lang, section_name=''):
 
4166
  'allowed_issue_codes': sorted(allowed_codes),
4167
  'items': [{'id': row['id'], 'english': row['english'], 'target': row['target']} for row in rows],
4168
  }
4169
+ profile = self._translation_language_profile(target_lang)
4170
  system_prompt = f"""You are a bilingual academic translation QA classifier for {target_lang}.
4171
  Review EACH claim independently. Do not rewrite and do not provide free-form explanations. Check semantic equivalence and language quality: omissions/additions, negation, uncertainty, association-versus-causation strength, terminology, grammar, language contamination, citation attachment, and acronym attachment.
4172
+ LANGUAGE ADAPTER: {profile['adapter_id']}; morphology={profile['morphology']}. {profile['term_policy']} {profile['contamination_policy']}
4173
+ For Russian, do not flag a terminology error merely because a translated term is inflected for grammatical case/number/gender. For Kazakh, do not flag a terminology error merely because productive suffixes attach to the correct term stem. For Kazakh language identity, absence of Kazakh-specific letters in one short claim is not sufficient evidence of Russian contamination.
4174
  Return valid JSON only in this exact form: {{"items":[{{"id":"SP-P000001-C000001","ok":true,"codes":[]}}]}}.
4175
  If ok=false, codes MUST contain only values from allowed_issue_codes. Never return numbered prose, examples, quotes, or corrective wording. Return every supplied id exactly once."""
4176
  response = self.call_deepseek(
 
4298
  return parsed
4299
 
4300
  def _validate_translation_block(self, source, translated, block_id, target_lang):
4301
+ """Hierarchical multilingual block assurance.
 
 
 
 
 
 
 
 
 
4302
 
4303
+ Fact/claim invariants are enforced earlier by the private ledger. Surface
4304
+ length and weak language statistics are diagnostic only; strong wrong-
4305
+ language/identity signals request bounded local repair.
4306
+ """
4307
  source_compact = re.sub(r'\s+', '', str(source or ''))
4308
  target_compact = re.sub(r'\s+', '', str(translated or ''))
4309
+ profile = self._translation_language_profile(target_lang)
4310
+ evidence = []
4311
+
4312
+ if source_compact and source_compact.casefold() == target_compact.casefold():
4313
+ evidence.append({'code': 'UNCHANGED_SOURCE', 'severity': 'repair'})
4314
+
4315
  if len(source_compact) >= 120:
4316
+ minimum = float(APP_CONFIG.get(profile['surface_min_key'], profile['surface_min_default']))
4317
+ maximum = float(APP_CONFIG.get('translation_max_block_ratio', profile['surface_max_default']))
 
 
 
 
4318
  ratio = len(target_compact) / max(1, len(source_compact))
4319
+ if ratio < minimum:
4320
+ evidence.append({'code': 'SURFACE_RATIO_LOW', 'severity': 'diagnostic', 'ratio': round(ratio, 4), 'threshold': minimum})
4321
+ elif ratio > maximum:
4322
+ evidence.append({'code': 'SURFACE_RATIO_HIGH', 'severity': 'diagnostic', 'ratio': round(ratio, 4), 'threshold': maximum})
4323
+
4324
+ if len(target_compact) >= 40:
4325
+ if profile['language_code'] == 'zh':
4326
+ han = len(re.findall(r'[\u4e00-\u9fff]', str(translated or '')))
4327
+ if han == 0:
4328
+ evidence.append({'code': 'TARGET_SCRIPT_MISSING', 'severity': 'repair'})
4329
+ else:
4330
+ cyrillic = re.findall(r'[А-Яа-яЁёӘәҒғҚқҢңӨөҰұҮүҺһІі]', str(translated or ''))
4331
+ if not cyrillic:
4332
+ evidence.append({'code': 'TARGET_SCRIPT_MISSING', 'severity': 'repair'})
4333
+ else:
4334
+ kazakh_specific = re.findall(r'[ӘәҒғҚқҢңӨөҰұҮүҺһІі]', str(translated or ''))
4335
+ if profile['language_code'] == 'kk' and len(cyrillic) >= 200:
4336
+ minimum_kk = max(2, int(len(cyrillic) * 0.002))
4337
+ if len(kazakh_specific) < minimum_kk:
4338
+ evidence.append({'code': 'KK_SPECIFIC_LETTER_SIGNAL_LOW', 'severity': 'diagnostic', 'kazakh_specific': len(kazakh_specific), 'cyrillic': len(cyrillic)})
4339
+ if profile['language_code'] == 'ru' and len(kazakh_specific) / max(1, len(cyrillic)) > 0.02:
4340
+ evidence.append({'code': 'RU_KAZAKH_LETTER_SIGNAL_HIGH', 'severity': 'diagnostic', 'kazakh_specific': len(kazakh_specific), 'cyrillic': len(cyrillic)})
4341
+
4342
+ decision = self._translation_assurance_decision(evidence)
4343
+ for item in evidence:
4344
+ self._record_translation_assurance(
4345
+ item.get('code'), target_lang=target_lang, block_id=block_id,
4346
+ severity=item.get('severity'),
4347
+ decision='REPAIR_BLOCK' if item.get('severity') == 'repair' else 'PASS_WITH_WARNING',
4348
+ detail={key: value for key, value in item.items() if key not in {'code', 'severity'}},
4349
+ )
4350
+ if decision == 'REPAIR_BLOCK':
4351
+ codes = [item.get('code') for item in evidence if item.get('severity') == 'repair']
4352
+ raise DeepSeekRequestError(
4353
+ f'Translation assurance requires local block repair in {block_id}: {codes[:5]}.'
4354
+ )
4355
+ return True
4356
 
4357
  def _validate_translation_block_protected(self, block_id, source, translated, target_lang, mapping=None):
4358
  """Validate one restored block after Fact Ledger integrity passed."""
 
4366
  protected_maps = {}
4367
  claim_sources_by_block = {}
4368
  for block_id, source in batch:
4369
+ ir = self._compile_translation_ir(source)
4370
+ protected_batch.append((block_id, source, ir['protected_source']))
4371
+ protected_maps[block_id] = ir['fact_ledger']
4372
+ claim_sources_by_block[block_id] = ir['claim_sources']
4373
  marked_source = '\n\n'.join(
4374
  f'[[SP-P{block_id}]]\n{protected_source}'
4375
  for block_id, _, protected_source in protected_batch
 
4537
  except Exception as error:
4538
  last_error = error
4539
  if attempt >= attempts:
4540
+ self._record_translation_assurance(
4541
+ 'SEMANTIC_QA_UNCERTAIN', target_lang=target_lang, block_id=section_name or 'section',
4542
+ severity='diagnostic', decision='PASS_WITH_WARNING',
4543
+ detail={'attempts': attempts, 'error_type': type(error).__name__},
4544
+ )
4545
+ logging.warning(
4546
+ 'Translation semantic QA technical fallback: section=%s attempts=%s error=%s',
4547
+ section_name or 'section', attempts, self._safe_source_error(error),
4548
+ )
4549
+ return {}
4550
  logging.warning(
4551
  'Translation QA-only retry: section=%s attempt=%s/%s error=%s',
4552
  section_name or 'section', attempt, attempts, self._safe_source_error(error),
4553
  )
4554
+ return {}
4555
 
4556
  def translate_frozen_section(self, section_text, target_lang, glossary=None, section_name='', max_attempts=2):
4557
  """Translate one frozen English unit and publish only after all QA passes."""
 
4560
  raise DeepSeekRequestError('Frozen English translation unit is empty.')
4561
  if re.search(r'(?m)^#\s+References\s*$', value):
4562
  raise DeepSeekRequestError('References must not be sent to the translation model.')
4563
+ assurance_start = len(self.translation_assurance_log)
4564
+ profile = self._translation_language_profile(target_lang)
4565
  frozen_glossary = self.extend_translation_glossary(value, target_lang, glossary)
4566
  units = self._translation_body_units(value)
4567
  body_units = [unit for unit in units if unit['kind'] == 'body']
 
4659
  output = '\n\n'.join(output_units).strip()
4660
  self.validate_translation_alignment(value, output)
4661
  self._validate_translation_language_quality(output, target_lang)
4662
+ diagnostics = list(self.translation_assurance_log[assurance_start:])
4663
  return {
4664
  'text': output,
4665
  'glossary': frozen_glossary,
4666
  'attempts': attempt,
4667
+ 'qa_status': 'passed_with_warning' if diagnostics else 'passed',
4668
  'translation_contract_version': self.TRANSLATION_CONTRACT_VERSION,
4669
  'fact_ledger_hash': self.translation_fact_ledger_hash(value),
4670
+ 'language_adapter': profile['adapter_id'],
4671
+ 'diagnostics': diagnostics[-20:],
4672
  }
4673
  except PipelineCancelledError:
4674
  raise
 
4681
  or message.startswith('Translation alignment')
4682
  or message.startswith('Translation changed ')
4683
  or message.startswith('Translation protected ')
4684
+ or message.startswith('Translation assurance requires local block repair')
4685
  or message.startswith('Translation block ')
4686
  or message.startswith('Chinese translation quality validation failed')
4687
  or message.startswith('Kazakh translation quality validation failed')
 
4752
  self.validate_translation_alignment(value, output)
4753
  return output.strip()
4754
 
4755
+ @classmethod
4756
+ def validate_translation_alignment(cls, english_text, translated_text):
4757
  source = str(english_text or '').strip()
4758
  target = str(translated_text or '').strip()
4759
  if not target:
4760
  raise DeepSeekRequestError('Target-language manuscript is empty.')
4761
+ en_tree = [level for level, _ in cls._markdown_heading_tree(source)]
4762
+ tr_tree = [level for level, _ in cls._markdown_heading_tree(target)]
4763
  if en_tree != tr_tree:
4764
  raise DeepSeekRequestError(
4765
  f'Translation heading-tree mismatch: English={en_tree}, target={tr_tree}.'
4766
  )
4767
+ # R3.2.8.6: whole-unit character ratios and Cyrillic subtype signals are
4768
+ # diagnostics only. Claim markers + private Fact Ledger + semantic claim
4769
+ # QA are authoritative for completeness and fidelity.
 
4770
  source_body = re.split(r'(?m)^#\s+References\s*$', source, maxsplit=1)[0]
4771
  target_body = re.split(r'(?m)^#\s+(?:参考文献|Литература|Пайдаланылған әдебиеттер)\s*$', target, maxsplit=1)[0]
4772
  source_chars = len(re.sub(r'\s+', '', source_body))
4773
  target_chars = len(re.sub(r'\s+', '', target_body))
 
 
 
 
 
 
4774
  ratio = target_chars / source_chars if source_chars else 0.0
4775
+ if source_chars and (ratio < 0.20 or ratio > 2.80):
4776
+ logging.warning('Translation whole-unit surface diagnostic | ratio=%.3f | nonfatal=true', ratio)
 
 
4777
 
 
 
 
 
4778
  kazakh_specific = re.findall(r'[ӘәҒғҚқҢңӨөҰұҮүҺһІі]', target_body)
4779
  cyrillic = re.findall(r'[А-Яа-яЁёӘәҒғҚқҢңӨөҰұҮүҺһІі]', target_body)
4780
+ if re.search(r'(?m)^#\s+Пайдаланылған әдебиеттер\s*$', target) and len(cyrillic) >= 200:
4781
  minimum_kazakh = max(8, int(len(cyrillic) * 0.003))
4782
  if len(kazakh_specific) < minimum_kazakh:
4783
+ logging.warning(
4784
+ 'Kazakh whole-unit language signal diagnostic | specific=%s cyrillic=%s | nonfatal=true',
4785
+ len(kazakh_specific), len(cyrillic),
4786
  )
4787
+ elif re.search(r'(?m)^#\s+Литература\s*$', target) and cyrillic:
4788
+ if len(kazakh_specific) / len(cyrillic) > 0.02:
4789
+ logging.warning(
4790
+ 'Russian whole-unit language signal diagnostic | kazakh_specific=%s cyrillic=%s | nonfatal=true',
4791
+ len(kazakh_specific), len(cyrillic),
4792
  )
4793
  return True
4794