R6A-R4-R14 four-language lexical purity and canonical construct ledger

#48
Files changed (2) hide show
  1. core/generation_runner.py +116 -1
  2. core/pipeline_agent.py +238 -21
core/generation_runner.py CHANGED
@@ -37,7 +37,7 @@ ProgressCallback = Callable[..., None]
37
  MIN_FINAL_REFERENCES = 50
38
  LITERATURE_EVIDENCE_TARGET = 75
39
  CITATION_PLAN_TARGET = 70
40
- TRANSLATION_CONTRACT_VERSION = 'hierarchical-multilingual-v4'
41
 
42
 
43
  def _clean_trailing_references(text, keywords=("reference", "references", "reference list", "参考文献")):
@@ -858,6 +858,99 @@ class GenerationRunner:
858
  encoded = json.dumps(dict(glossary or {}), ensure_ascii=False, sort_keys=True).encode('utf-8')
859
  return hashlib.sha256(encoded).hexdigest()[:16]
860
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
861
  @staticmethod
862
  def _translation_ledger_hash(agent, source: str) -> str:
863
  method = getattr(agent, 'translation_fact_ledger_hash', None)
@@ -1571,6 +1664,10 @@ class GenerationRunner:
1571
  citation_plan = agent.build_claim_evidence_plan(
1572
  adaptive_structure, topic, synthesis_result, mode=mode, target_unique=CITATION_PLAN_TARGET
1573
  )
 
 
 
 
1574
  planned_unique = {
1575
  item.get("tag")
1576
  for items in citation_plan.values()
@@ -1596,6 +1693,7 @@ class GenerationRunner:
1596
  "full_draft_translated": "",
1597
  "translation_sections": {},
1598
  "translation_glossary": {},
 
1599
  "completed_sections": [],
1600
  "context_tail": "",
1601
  "accumulated_deviation": 0.0,
@@ -1619,6 +1717,13 @@ class GenerationRunner:
1619
  citation_plan = state.get("citation_plan") if isinstance(state.get("citation_plan"), dict) else {}
1620
  translation_sections = dict(state.get('translation_sections') or {})
1621
  translation_glossary = dict(state.get('translation_glossary') or {})
 
 
 
 
 
 
 
1622
  working_abstract = str(state.get("working_abstract") or "").strip()
1623
  working_keywords = list(state.get("working_keywords") or [])
1624
  if working_abstract:
@@ -1631,6 +1736,15 @@ class GenerationRunner:
1631
  # glossary order deterministic and avoids sharing mutable writer state.
1632
  translation_agent = SSciPipelineAgent(ds_key, s2_key)
1633
  translation_agent.set_execution_progress_hook(lambda: self._progress_activity())
 
 
 
 
 
 
 
 
 
1634
  self._start_translation_worker(
1635
  translation_agent,
1636
  translation_sections=translation_sections,
@@ -1885,6 +1999,7 @@ Planning length: approximately {int(adjusted_target * min_ratio)}-{int(adjusted_
1885
  {str(sec.get('required_content') or '')}
1886
  {qualitative_rule}
1887
  {anti_repeat}
 
1888
  <PROVISIONAL_RESEARCH_LOGIC>
1889
  {research_logic_contract}
1890
  </PROVISIONAL_RESEARCH_LOGIC>
 
37
  MIN_FINAL_REFERENCES = 50
38
  LITERATURE_EVIDENCE_TARGET = 75
39
  CITATION_PLAN_TARGET = 70
40
+ TRANSLATION_CONTRACT_VERSION = 'hierarchical-multilingual-v5'
41
 
42
 
43
  def _clean_trailing_references(text, keywords=("reference", "references", "reference list", "参考文献")):
 
858
  encoded = json.dumps(dict(glossary or {}), ensure_ascii=False, sort_keys=True).encode('utf-8')
859
  return hashlib.sha256(encoded).hexdigest()[:16]
860
 
861
+ @staticmethod
862
+ def _build_canonical_concept_ledger(
863
+ mode: str, data_metadata: dict, *, keywords='', adaptive_structure=None, research_logic_contract=''
864
+ ) -> dict:
865
+ """Freeze task-level research concept identity before body translation.
866
+
867
+ This ledger is intentionally conservative: quantitative construct names
868
+ come directly from frozen metadata; qualitative concept/theme labels come
869
+ from the Evidence Ledger/outline. Lower-case hyphenated concepts used in
870
+ the frozen research logic are added because the legacy Title-Case glossary
871
+ discovery cannot see them. No semantic synonym merging is performed.
872
+ """
873
+ items = []
874
+ seen = set()
875
+
876
+ def add(value, kind):
877
+ term = re.sub(r'\s+', ' ', str(value or '')).strip(' \t\r\n,;:.')
878
+ if not term or len(term) < 3 or len(term) > 100:
879
+ return
880
+ words = re.findall(r"[A-Za-zÀ-ÖØ-öø-ÿА-Яа-яЁёӘәҒғҚқҢңӨөҰұҮүҺһІі][\w’'\-]*", term, flags=re.UNICODE)
881
+ if not words or len(words) > 10:
882
+ return
883
+ key = re.sub(r'[–—-]+', '-', term).casefold()
884
+ if key in seen:
885
+ return
886
+ seen.add(key)
887
+ items.append({'source': term, 'kind': str(kind or 'concept')})
888
+
889
+ meta = dict(data_metadata or {})
890
+ if str(mode or '') == 'amj':
891
+ constructs = meta.get('constructs') if isinstance(meta.get('constructs'), dict) else {}
892
+ for role in ('predictor', 'mediator', 'outcome'):
893
+ add(constructs.get(role), f'quantitative_{role}')
894
+ else:
895
+ for theme in (meta.get('themes') or []):
896
+ if not isinstance(theme, dict):
897
+ continue
898
+ add(theme.get('title'), 'qualitative_theme')
899
+ for concept in (theme.get('concepts') or []):
900
+ add(concept, 'qualitative_concept')
901
+ for row in (meta.get('evidence') or []):
902
+ if isinstance(row, dict):
903
+ add(row.get('concept'), 'qualitative_concept')
904
+
905
+ for task in (adaptive_structure or []):
906
+ if not isinstance(task, dict):
907
+ continue
908
+ for concept in (task.get('concepts') or []):
909
+ add(concept, 'outline_concept')
910
+
911
+ # User keywords remain planning seeds, not canonical identities. Use
912
+ # them only as a fallback when frozen empirical/qualitative metadata and
913
+ # outline concepts provide no concept labels at all.
914
+ if not items:
915
+ for seed in re.split(r'[,;\n|]+', str(keywords or '')):
916
+ seed = re.sub(r'\s+', ' ', seed).strip()
917
+ if 1 <= len(seed.split()) <= 6:
918
+ add(seed, 'keyword_fallback')
919
+
920
+ # Capture explicit lower-case hyphenated academic expressions only when
921
+ # they recur or carry a research-concept cue. This closes the
922
+ # context-awareness/context-aware blind spot without freezing generic
923
+ # methodological adjectives such as cross-sectional by default.
924
+ hyphenated = re.findall(r'(?<![A-Za-z])(?:[a-z][a-z]{2,}(?:-[a-z][a-z]{2,})+)(?![A-Za-z])', str(research_logic_contract or ''))
925
+ concept_cues = {
926
+ 'awareness','trust','governance','literacy','fidelity','performance','context','capability',
927
+ 'calibration','memory','acceptance','effectiveness','efficiency','alignment','recommendation',
928
+ }
929
+ counts = {}
930
+ for term in hyphenated:
931
+ counts[term.casefold()] = counts.get(term.casefold(), 0) + 1
932
+ for term in hyphenated:
933
+ parts = set(term.casefold().split('-'))
934
+ if counts.get(term.casefold(), 0) >= 2 or parts.intersection(concept_cues):
935
+ add(term, 'logic_hyphenated_concept')
936
+
937
+ return {'version': 'concept-ledger-v1', 'items': items}
938
+
939
+ @staticmethod
940
+ def _canonical_concept_writer_directive(ledger: dict) -> str:
941
+ terms = [str(item.get('source') or '').strip() for item in (dict(ledger or {}).get('items') or []) if isinstance(item, dict)]
942
+ terms = [term for term in terms if term]
943
+ if not terms:
944
+ return ''
945
+ rendered = '; '.join(terms[:40])
946
+ return (
947
+ 'CANONICAL CONCEPT IDENTITY: The following research labels are frozen task-level concepts: '
948
+ + rendered
949
+ + '. When referring to the same construct or concept, preserve its canonical English label; '
950
+ 'do not silently replace it with a near-synonym that changes concept identity. Ordinary grammatical '
951
+ 'articles, possessives, and surrounding syntax remain free. Distinct listed labels are not synonyms unless the evidence explicitly says so.'
952
+ )
953
+
954
  @staticmethod
955
  def _translation_ledger_hash(agent, source: str) -> str:
956
  method = getattr(agent, 'translation_fact_ledger_hash', None)
 
1664
  citation_plan = agent.build_claim_evidence_plan(
1665
  adaptive_structure, topic, synthesis_result, mode=mode, target_unique=CITATION_PLAN_TARGET
1666
  )
1667
+ canonical_concept_ledger = self._build_canonical_concept_ledger(
1668
+ mode, data_metadata, keywords=keywords, adaptive_structure=adaptive_structure,
1669
+ research_logic_contract=research_logic_contract,
1670
+ )
1671
  planned_unique = {
1672
  item.get("tag")
1673
  for items in citation_plan.values()
 
1693
  "full_draft_translated": "",
1694
  "translation_sections": {},
1695
  "translation_glossary": {},
1696
+ "canonical_concept_ledger": canonical_concept_ledger,
1697
  "completed_sections": [],
1698
  "context_tail": "",
1699
  "accumulated_deviation": 0.0,
 
1717
  citation_plan = state.get("citation_plan") if isinstance(state.get("citation_plan"), dict) else {}
1718
  translation_sections = dict(state.get('translation_sections') or {})
1719
  translation_glossary = dict(state.get('translation_glossary') or {})
1720
+ canonical_concept_ledger = state.get('canonical_concept_ledger')
1721
+ if not isinstance(canonical_concept_ledger, dict) or not canonical_concept_ledger.get('items'):
1722
+ canonical_concept_ledger = self._build_canonical_concept_ledger(
1723
+ mode, data_metadata, keywords=keywords, adaptive_structure=adaptive_structure,
1724
+ research_logic_contract=research_logic_contract,
1725
+ )
1726
+ state['canonical_concept_ledger'] = canonical_concept_ledger
1727
  working_abstract = str(state.get("working_abstract") or "").strip()
1728
  working_keywords = list(state.get("working_keywords") or [])
1729
  if working_abstract:
 
1736
  # glossary order deterministic and avoids sharing mutable writer state.
1737
  translation_agent = SSciPipelineAgent(ds_key, s2_key)
1738
  translation_agent.set_execution_progress_hook(lambda: self._progress_activity())
1739
+ translation_agent.set_cancel_checker(lambda: self.cancel_checker())
1740
+ seeded_glossary = translation_agent.seed_canonical_concept_glossary(
1741
+ canonical_concept_ledger, target_lang, translation_glossary
1742
+ )
1743
+ if seeded_glossary != translation_glossary:
1744
+ translation_glossary = dict(seeded_glossary)
1745
+ state['translation_glossary'] = translation_glossary
1746
+ state['canonical_concept_ledger'] = canonical_concept_ledger
1747
+ self._checkpoint('canonical_concept_ledger_ready', state, 15)
1748
  self._start_translation_worker(
1749
  translation_agent,
1750
  translation_sections=translation_sections,
 
1999
  {str(sec.get('required_content') or '')}
2000
  {qualitative_rule}
2001
  {anti_repeat}
2002
+ {self._canonical_concept_writer_directive(canonical_concept_ledger)}
2003
  <PROVISIONAL_RESEARCH_LOGIC>
2004
  {research_logic_contract}
2005
  </PROVISIONAL_RESEARCH_LOGIC>
core/pipeline_agent.py CHANGED
@@ -61,7 +61,7 @@ class CitationAuditError(DeepSeekRequestError):
61
 
62
 
63
  class SSciPipelineAgent:
64
- TRANSLATION_CONTRACT_VERSION = 'hierarchical-multilingual-v4'
65
 
66
  def __init__(self, ds_key, s2_key=""):
67
  self.ds_key = str(ds_key or "").strip()
@@ -3878,6 +3878,143 @@ Output only the repaired section."""
3878
  'source_technical_tokens': int(len(technical)),
3879
  }
3880
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3881
  @classmethod
3882
  def _compile_translation_ir(cls, source):
3883
  """Compile a private, language-neutral translation intermediate representation."""
@@ -4625,6 +4762,55 @@ Output only the repaired section."""
4625
  raise DeepSeekRequestError(f'Translation left unresolved protected placeholders in {block_id}.')
4626
  return text.strip() if keep_claim_markers else cls._strip_translation_claim_markers(text)
4627
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4628
  def extend_translation_glossary(self, body, target_lang, existing=None):
4629
  """Extend a task-level glossary without remapping an existing term."""
4630
  mapping = dict(existing or {})
@@ -4840,7 +5026,7 @@ SECTION: {section_name}
4840
  relevant_glossary = {
4841
  str(source_term): str(target_term)
4842
  for source_term, target_term in glossary_value.items()
4843
- if source_term and str(source_term) in joined_source
4844
  }
4845
  payload = {
4846
  'section': str(section_name or 'section'),
@@ -4853,7 +5039,7 @@ SECTION: {section_name}
4853
  system_prompt = f"""You are a bilingual academic translation QA classifier for {target_lang}.
4854
  Review EACH claim independently. Do not rewrite and do not provide free-form explanations. Check semantic equivalence and language quality: omissions/additions, negation, uncertainty, association-versus-causation strength, terminology, grammar, language contamination, citation attachment, and acronym attachment.
4855
  LANGUAGE ADAPTER: {profile['adapter_id']}; morphology={profile['morphology']}. {profile['term_policy']} {profile['contamination_policy']}
4856
- For Russian, do not flag a terminology error merely because a translated term is inflected for grammatical case/number/gender. For Russian LANGUAGE_CONTAMINATION, require sustained unprotected non-Russian prose such as a multi-word English clause or sentence. Never use LANGUAGE_CONTAMINATION merely for protected acronyms, AI/LLM/SEM, model or tool names, variable names, citations, glossary terms, source-matched proper names/titles, or isolated Latin terminology. For Kazakh, do not flag a terminology error merely because productive suffixes attach to the correct term stem. For Kazakh language identity, absence of Kazakh-specific letters in one short claim is not sufficient evidence of Russian contamination.
4857
  Return valid JSON only in this exact form: {{"items":[{{"id":"SP-P000001-C000001","ok":true,"codes":[]}}]}}.
4858
  If ok=false, codes MUST contain only values from allowed_issue_codes. Never return numbered prose, examples, quotes, or corrective wording. Return every supplied id exactly once."""
4859
  response = self.call_deepseek(
@@ -4877,30 +5063,54 @@ If ok=false, codes MUST contain only values from allowed_issue_codes. Never retu
4877
  raw_codes = [str(code or '').strip().upper() for code in (returned_row.get('codes') or [])]
4878
  codes = [code for code in raw_codes if code in allowed_codes]
4879
  model_contamination = 'LANGUAGE_CONTAMINATION' in codes
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4880
  contamination = None
 
 
 
 
 
 
 
 
 
 
4881
  if language_code == 'ru':
4882
  contamination = self._russian_contamination_evidence(
4883
  meta['english'], meta['target'], relevant_glossary
4884
  )
4885
- if contamination.get('confirmed'):
4886
- if 'LANGUAGE_CONTAMINATION' not in codes:
4887
- codes.append('LANGUAGE_CONTAMINATION')
4888
- elif model_contamination:
4889
  codes = [code for code in codes if code != 'LANGUAGE_CONTAMINATION']
4890
  self._record_translation_assurance(
4891
  'RU_LANGUAGE_CONTAMINATION_UNCONFIRMED', target_lang=target_lang, block_id=row_id,
4892
  severity='diagnostic', decision='PASS_WITH_WARNING', detail=contamination,
4893
  )
4894
 
4895
- # A deterministic confirmed Russian contamination signal can
4896
- # override an LLM false negative. Conversely, a sole Russian
4897
- # contamination code without deterministic evidence is suppressed.
4898
- if returned_row.get('ok') is True and not (contamination and contamination.get('confirmed')):
4899
  continue
4900
  if not codes:
4901
  only_suppressed_contamination = (
4902
- language_code == 'ru' and model_contamination
4903
- and not (contamination or {}).get('confirmed')
4904
  and bool(raw_codes) and all(code == 'LANGUAGE_CONTAMINATION' for code in raw_codes)
4905
  )
4906
  if only_suppressed_contamination:
@@ -5334,13 +5544,20 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
5334
  if claim_id not in source_claims or claim_id not in target_claims:
5335
  raise DeepSeekRequestError(f'Translation QA adjudication claim identity mismatch: SP-P{block_id}-{claim_id}.')
5336
  for code in codes:
5337
- # Russian sustained Latin prose already has deterministic
5338
- # evidence authority inside audit_translation_blocks.
5339
- if code == 'LANGUAGE_CONTAMINATION' and profile.get('language_code') == 'ru':
5340
- evidence = self._russian_contamination_evidence(
5341
- source_claims[claim_id], target_claims[claim_id], glossary
 
5342
  )
5343
- if evidence.get('confirmed'):
 
 
 
 
 
 
5344
  rows.append({
5345
  'id': f'SP-P{block_id}-{claim_id}', 'block_id': block_id,
5346
  'claim_id': claim_id, 'code': code, 'english': source_claims[claim_id],
@@ -5370,7 +5587,7 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
5370
  allowed_evidence = {
5371
  'CLAIM_MISSING','CLAIM_ADDED','POLARITY_CHANGED','CAUSALITY_CHANGED',
5372
  'UNCERTAINTY_CHANGED','MEANING_CHANGED','CITATION_MOVED','ACRONYM_MISATTACHED',
5373
- 'GRAMMATICAL_VIOLATION','GLOSSARY_CONCEPT_MISMATCH','UNTRANSLATED_PROSE',
5374
  'ACCEPTABLE_VARIANT','MORPHOLOGICAL_VARIANT','STYLISTIC_PREFERENCE',
5375
  'QA_DISAGREEMENT','NO_BLOCKING_EVIDENCE',
5376
  }
@@ -5387,7 +5604,7 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
5387
  }
5388
  system_prompt = f"""You are an independent bilingual translation QA adjudicator for {target_lang}.
5389
  This is verification-only: do not rewrite, suggest wording, or emit free-form explanations. Each supplied issue already survived two bounded claim-local repair rounds. Confirm a BLOCKING defect only when the English/target pair itself provides clear evidence. Do not treat stylistic preference or a legitimate grammatical/morphological variant as blocking.
5390
- For Russian, allow required case/number/gender inflection. For Kazakh, allow productive case, possessive, person/plural and vowel-harmony suffixes on the correct term stem. For Chinese, allow natural reordering and sentence reshaping inside the same claim while preserving meaning.
5391
  Return valid JSON only: {{"items":[{{"id":"SP-P000001-C000001","issue_code":"GRAMMAR","decision":"CONFIRMED_BLOCKING","evidence_code":"GRAMMATICAL_VIOLATION"}}]}}.
5392
  Decision must be CONFIRMED_BLOCKING or NOT_CONFIRMED. evidence_code must be one of: {sorted(allowed_evidence)}. Return every supplied id+issue_code pair exactly once."""
5393
  try:
 
61
 
62
 
63
  class SSciPipelineAgent:
64
+ TRANSLATION_CONTRACT_VERSION = 'hierarchical-multilingual-v5'
65
 
66
  def __init__(self, ds_key, s2_key=""):
67
  self.ds_key = str(ds_key or "").strip()
 
3878
  'source_technical_tokens': int(len(technical)),
3879
  }
3880
 
3881
+ @classmethod
3882
+ def _translation_lexical_purity_evidence(cls, source, target, target_lang, glossary=None):
3883
+ """Detect isolated ordinary English lexemes left in ZH/RU/KK prose.
3884
+
3885
+ This is deliberately narrower than general language identification. It
3886
+ protects research objects, source-matched proper names, and legitimate
3887
+ Latin glossary targets, then treats any remaining ordinary Latin word as
3888
+ untranslated lexical residue. The same gate is shared by Chinese, Russian
3889
+ and Kazakh. Kazakh-vs-Russian discrimination remains evidence-grounded QA
3890
+ because the languages share Cyrillic script and overlapping vocabulary.
3891
+ """
3892
+ profile = cls._translation_language_profile(target_lang)
3893
+ language_code = profile.get('language_code')
3894
+ if language_code not in {'zh', 'ru', 'kk'}:
3895
+ return {'confirmed': False, 'tokens': [], 'language_code': language_code or ''}
3896
+
3897
+ source_value = str(source or '')
3898
+ text = cls._strip_translation_claim_markers(str(target or ''))
3899
+ # Do not exempt source-matched English quotations here: participant
3900
+ # quotations are target-language content and must not remain ordinary
3901
+ # English merely because the same literal exists in the frozen source.
3902
+ text = cls._translation_protected_pattern().sub(' ', text)
3903
+ text = re.sub(r'\[\[SP-(?:P\d{6}|C\d{6}|T\d{6})\]\]', ' ', text)
3904
+
3905
+ english_function_words = {'a','an','and','as','at','by','for','from','in','of','on','or','the','to','with'}
3906
+ legal_latin_terms = set()
3907
+ for value in dict(glossary or {}).values():
3908
+ term = str(value or '').strip()
3909
+ words = re.findall(r'[A-Za-z][A-Za-z0-9+.#-]*', term)
3910
+ title_like = bool(words) and len(words) <= 5 and all(
3911
+ word.casefold() in english_function_words or word[:1].isupper() for word in words
3912
+ )
3913
+ technical_shape = bool(words) and any(
3914
+ any(ch.isdigit() for ch in word) or word.isupper()
3915
+ or (len(word) > 2 and any(ch.isupper() for ch in word[1:])) for word in words
3916
+ )
3917
+ if term and re.search(r'[A-Za-z]', term) and (title_like or technical_shape):
3918
+ legal_latin_terms.add(term)
3919
+ for term in sorted(legal_latin_terms, key=len, reverse=True):
3920
+ text = re.sub(re.escape(term), ' ', text, flags=re.I)
3921
+
3922
+ source_tokens = list(re.finditer(r'\b[A-Za-z][A-Za-z0-9+.#-]*\b', source_value))
3923
+ technical = set()
3924
+ proper = set()
3925
+ ordinary_sentence_starters = {
3926
+ 'the','this','these','those','prior','previous','current','our','we','results','findings',
3927
+ 'participants','evidence','analysis','study','research','however','therefore','because',
3928
+ }
3929
+ for match in source_tokens:
3930
+ token = match.group(0)
3931
+ if any(ch.isdigit() for ch in token) or token.isupper() or (len(token) > 2 and any(ch.isupper() for ch in token[1:])):
3932
+ technical.add(token)
3933
+ continue
3934
+ if token[:1].isupper() and token.casefold() not in ordinary_sentence_starters:
3935
+ proper.add(token)
3936
+ for token in sorted(technical | proper, key=len, reverse=True):
3937
+ text = re.sub(rf'(?<![A-Za-z0-9_]){re.escape(token)}(?![A-Za-z0-9_])', ' ', text)
3938
+
3939
+ # Source-matched bounded proper titles/institutions remain legal even if
3940
+ # they contain English function words, e.g. University of California.
3941
+ latin_run_pattern = re.compile(r"(?:[A-Za-z][A-Za-z'’-]*(?:[ \t]+|$)){2,8}")
3942
+ for match in list(latin_run_pattern.finditer(text)):
3943
+ candidate = re.sub(r'\s+', ' ', match.group(0)).strip()
3944
+ if not candidate:
3945
+ continue
3946
+ words = candidate.split()
3947
+ title_like = all(word.casefold() in english_function_words or word[:1].isupper() for word in words)
3948
+ if title_like and candidate.casefold() in re.sub(r'\s+', ' ', source_value).casefold():
3949
+ text = text[:match.start()] + (' ' * (match.end() - match.start())) + text[match.end():]
3950
+
3951
+ allow_words = {'et', 'al', 'vs'}
3952
+ residual = []
3953
+ for match in re.finditer(r"[A-Za-z][A-Za-z'’-]*", text):
3954
+ token = match.group(0).strip("-'’–—")
3955
+ folded = token.casefold()
3956
+ if folded in allow_words or len(token) < 4:
3957
+ continue
3958
+ residual.append(token)
3959
+ unique = list(dict.fromkeys(residual))
3960
+ return {
3961
+ 'confirmed': bool(unique),
3962
+ 'tokens': unique[:8],
3963
+ 'token_count': len(residual),
3964
+ 'language_code': language_code,
3965
+ }
3966
+
3967
+ @classmethod
3968
+ def _canonical_glossary_violations(cls, source, target, target_lang, glossary=None, canonical_sources=None):
3969
+ """Return canonical concept terms whose frozen target identity disappeared.
3970
+
3971
+ Chinese uses compact exact identity. Russian/Kazakh use conservative
3972
+ lexical stems so grammatical inflection and productive suffixation remain
3973
+ legal while a switch to a different concept word is still detectable.
3974
+ """
3975
+ source_value = str(source or '')
3976
+ target_value = cls._strip_translation_claim_markers(str(target or ''))
3977
+ profile = cls._translation_language_profile(target_lang)
3978
+ language_code = profile.get('language_code')
3979
+ violations = []
3980
+ canonical = {str(item or '').strip().casefold() for item in (canonical_sources or []) if str(item or '').strip()}
3981
+ for source_term, target_term in dict(glossary or {}).items():
3982
+ source_term = str(source_term or '').strip()
3983
+ target_term = str(target_term or '').strip()
3984
+ if canonical and source_term.casefold() not in canonical:
3985
+ continue
3986
+ if not source_term or not target_term or source_term.casefold() not in source_value.casefold():
3987
+ continue
3988
+ # Pure technical/acronym identities are protected elsewhere.
3989
+ compact = re.sub(r'[^A-Za-z0-9]', '', target_term)
3990
+ if compact and ' ' not in target_term and (
3991
+ compact.isupper() or any(ch.isdigit() for ch in compact)
3992
+ or (len(compact) > 2 and any(ch.isupper() for ch in compact[1:]))
3993
+ ):
3994
+ continue
3995
+ if language_code == 'zh':
3996
+ expected = re.sub(r'\s+', '', target_term)
3997
+ actual = re.sub(r'\s+', '', target_value)
3998
+ if expected and expected not in actual:
3999
+ violations.append(source_term)
4000
+ continue
4001
+ if language_code in {'ru', 'kk'}:
4002
+ expected_words = re.findall(r'[А-Яа-яЁёӘәҒғҚқҢңӨөҰұҮүҺһІі]{3,}', target_term)
4003
+ actual_words = [word.casefold() for word in re.findall(r'[А-Яа-яЁёӘәҒғҚқҢңӨөҰұҮүҺһІі]{3,}', target_value)]
4004
+ if not expected_words:
4005
+ continue
4006
+ missing = False
4007
+ for word in expected_words:
4008
+ folded = word.casefold()
4009
+ stem_len = min(len(folded), 5 if len(folded) >= 5 else len(folded))
4010
+ stem = folded[:stem_len]
4011
+ if not any(actual.startswith(stem) for actual in actual_words):
4012
+ missing = True
4013
+ break
4014
+ if missing:
4015
+ violations.append(source_term)
4016
+ return list(dict.fromkeys(violations))[:8]
4017
+
4018
  @classmethod
4019
  def _compile_translation_ir(cls, source):
4020
  """Compile a private, language-neutral translation intermediate representation."""
 
4762
  raise DeepSeekRequestError(f'Translation left unresolved protected placeholders in {block_id}.')
4763
  return text.strip() if keep_claim_markers else cls._strip_translation_claim_markers(text)
4764
 
4765
+ def seed_canonical_concept_glossary(self, concept_ledger, target_lang, existing=None):
4766
+ """Translate/freeze canonical task concepts once before section work."""
4767
+ mapping = dict(existing or {})
4768
+ rows = (dict(concept_ledger or {}).get('items') or []) if isinstance(concept_ledger, dict) else []
4769
+ sources = []
4770
+ for row in rows:
4771
+ if not isinstance(row, dict):
4772
+ continue
4773
+ term = re.sub(r'\s+', ' ', str(row.get('source') or '')).strip()
4774
+ if term and term not in sources:
4775
+ sources.append(term)
4776
+ self._canonical_concept_sources = set(sources)
4777
+ if not sources:
4778
+ return mapping
4779
+
4780
+ missing = [term for term in sources if not str(mapping.get(term) or '').strip()]
4781
+ # Acronyms/technical identifiers preserve literal identity; conceptual
4782
+ # phrases must be translated instead of becoming legal English residue.
4783
+ translate_terms = []
4784
+ for term in missing:
4785
+ compact = re.sub(r'[^A-Za-z0-9]', '', term)
4786
+ technical = bool(compact) and (compact.isupper() or any(ch.isdigit() for ch in compact) or (len(compact) > 2 and any(ch.isupper() for ch in compact[1:]))) and ' ' not in term
4787
+ if technical:
4788
+ mapping[term] = term
4789
+ else:
4790
+ translate_terms.append(term)
4791
+ if translate_terms:
4792
+ rule = self._translation_language_rule(target_lang)
4793
+ response = self.call_deepseek(
4794
+ f"Translate the supplied frozen research concepts into {target_lang}. {rule} Each source is a canonical concept identity, not a stylistic phrase. Give one stable canonical target term for each source; do not merge distinct sources or replace one concept with a near-synonym. Return JSON only as {{\"terms\":[{{\"source\":\"...\",\"target\":\"...\"}}]}} and return every source exactly once.",
4795
+ json.dumps({'terms': translate_terms}, ensure_ascii=False),
4796
+ max_tokens=max(1200, min(5000, 120 + len(translate_terms) * 90)),
4797
+ temp=0.0, model_role='translation', thinking=False, allow_fallback=False,
4798
+ )
4799
+ parsed = self._extract_json_from_text(response)
4800
+ items = parsed.get('terms', []) if isinstance(parsed, dict) else []
4801
+ returned = {
4802
+ re.sub(r'\s+', ' ', str(item.get('source') or '')).strip(): re.sub(r'\s+', ' ', str(item.get('target') or '')).strip()
4803
+ for item in items if isinstance(item, dict)
4804
+ }
4805
+ if set(returned) != set(translate_terms) or any(not returned.get(term) for term in translate_terms):
4806
+ raise DeepSeekRequestError('Canonical concept glossary identity mismatch or omitted term.')
4807
+ for term in translate_terms:
4808
+ target = returned[term]
4809
+ if target.casefold() == term.casefold() and re.search(r'[a-z]{4,}', term):
4810
+ raise DeepSeekRequestError(f'Canonical concept remained untranslated: {term[:80]}')
4811
+ mapping[term] = target
4812
+ return mapping
4813
+
4814
  def extend_translation_glossary(self, body, target_lang, existing=None):
4815
  """Extend a task-level glossary without remapping an existing term."""
4816
  mapping = dict(existing or {})
 
5026
  relevant_glossary = {
5027
  str(source_term): str(target_term)
5028
  for source_term, target_term in glossary_value.items()
5029
+ if source_term and str(source_term).casefold() in joined_source.casefold()
5030
  }
5031
  payload = {
5032
  'section': str(section_name or 'section'),
 
5039
  system_prompt = f"""You are a bilingual academic translation QA classifier for {target_lang}.
5040
  Review EACH claim independently. Do not rewrite and do not provide free-form explanations. Check semantic equivalence and language quality: omissions/additions, negation, uncertainty, association-versus-causation strength, terminology, grammar, language contamination, citation attachment, and acronym attachment.
5041
  LANGUAGE ADAPTER: {profile['adapter_id']}; morphology={profile['morphology']}. {profile['term_policy']} {profile['contamination_policy']}
5042
+ For Russian, do not flag a terminology error merely because a translated term is inflected for grammatical case/number/gender. Russian LANGUAGE_CONTAMINATION includes sustained unprotected English prose and ordinary untranslated English lexemes; never use it for protected acronyms, AI/LLM/SEM, model or tool names, variable names, citations, canonical glossary terms, or source-matched proper names/titles. For Chinese, ordinary untranslated English lexical words are contamination while protected technical objects remain legal. For Kazakh, do not flag a terminology error merely because productive suffixes attach to the correct term stem. Kazakh LANGUAGE_CONTAMINATION includes ordinary untranslated English lexemes and clearly Russian lexical/syntactic intrusions not justified by shared international terminology; Cyrillic script or absence of Kazakh-specific letters alone is never sufficient evidence.
5043
  Return valid JSON only in this exact form: {{"items":[{{"id":"SP-P000001-C000001","ok":true,"codes":[]}}]}}.
5044
  If ok=false, codes MUST contain only values from allowed_issue_codes. Never return numbered prose, examples, quotes, or corrective wording. Return every supplied id exactly once."""
5045
  response = self.call_deepseek(
 
5063
  raw_codes = [str(code or '').strip().upper() for code in (returned_row.get('codes') or [])]
5064
  codes = [code for code in raw_codes if code in allowed_codes]
5065
  model_contamination = 'LANGUAGE_CONTAMINATION' in codes
5066
+ lexical = self._translation_lexical_purity_evidence(
5067
+ meta['english'], meta['target'], target_lang, relevant_glossary
5068
+ )
5069
+ canonical_violations = self._canonical_glossary_violations(
5070
+ meta['english'], meta['target'], target_lang, relevant_glossary,
5071
+ canonical_sources=getattr(self, '_canonical_concept_sources', set()),
5072
+ )
5073
+ if canonical_violations:
5074
+ if 'TERMINOLOGY' not in codes:
5075
+ codes.append('TERMINOLOGY')
5076
+ self._record_translation_assurance(
5077
+ 'CANONICAL_CONCEPT_MISMATCH', target_lang=target_lang, block_id=row_id,
5078
+ severity='diagnostic', decision='REPAIR_LOCAL',
5079
+ detail={'count': len(canonical_violations), 'sources': '|'.join(canonical_violations)[:160]},
5080
+ )
5081
  contamination = None
5082
+ deterministic_contamination = bool(lexical.get('confirmed'))
5083
+ deterministic_issue = bool(canonical_violations)
5084
+ if deterministic_contamination:
5085
+ if 'LANGUAGE_CONTAMINATION' not in codes:
5086
+ codes.append('LANGUAGE_CONTAMINATION')
5087
+ self._record_translation_assurance(
5088
+ 'LEXICAL_PURITY_CONFIRMED', target_lang=target_lang, block_id=row_id,
5089
+ severity='repair', decision='REPAIR_LOCAL',
5090
+ detail={'token_count': int(lexical.get('token_count') or 0), 'sample': ','.join(lexical.get('tokens') or [])[:120]},
5091
+ )
5092
  if language_code == 'ru':
5093
  contamination = self._russian_contamination_evidence(
5094
  meta['english'], meta['target'], relevant_glossary
5095
  )
5096
+ deterministic_contamination = deterministic_contamination or bool(contamination.get('confirmed'))
5097
+ if contamination.get('confirmed') and 'LANGUAGE_CONTAMINATION' not in codes:
5098
+ codes.append('LANGUAGE_CONTAMINATION')
5099
+ elif model_contamination and not deterministic_contamination:
5100
  codes = [code for code in codes if code != 'LANGUAGE_CONTAMINATION']
5101
  self._record_translation_assurance(
5102
  'RU_LANGUAGE_CONTAMINATION_UNCONFIRMED', target_lang=target_lang, block_id=row_id,
5103
  severity='diagnostic', decision='PASS_WITH_WARNING', detail=contamination,
5104
  )
5105
 
5106
+ # Deterministic lexical residue can override an LLM false negative
5107
+ # in ZH/RU/KK. Russian LLM-only contamination remains suppressed
5108
+ # when neither sustained-prose nor lexical evidence confirms it.
5109
+ if returned_row.get('ok') is True and not deterministic_contamination and not deterministic_issue:
5110
  continue
5111
  if not codes:
5112
  only_suppressed_contamination = (
5113
+ language_code == 'ru' and model_contamination and not deterministic_contamination
 
5114
  and bool(raw_codes) and all(code == 'LANGUAGE_CONTAMINATION' for code in raw_codes)
5115
  )
5116
  if only_suppressed_contamination:
 
5544
  if claim_id not in source_claims or claim_id not in target_claims:
5545
  raise DeepSeekRequestError(f'Translation QA adjudication claim identity mismatch: SP-P{block_id}-{claim_id}.')
5546
  for code in codes:
5547
+ # Lexical purity has deterministic authority in all
5548
+ # supported target languages. Russian sustained Latin prose
5549
+ # retains its existing independent deterministic authority.
5550
+ if code == 'LANGUAGE_CONTAMINATION':
5551
+ lexical = self._translation_lexical_purity_evidence(
5552
+ source_claims[claim_id], target_claims[claim_id], target_lang, glossary
5553
  )
5554
+ deterministic = bool(lexical.get('confirmed'))
5555
+ if profile.get('language_code') == 'ru' and not deterministic:
5556
+ evidence = self._russian_contamination_evidence(
5557
+ source_claims[claim_id], target_claims[claim_id], glossary
5558
+ )
5559
+ deterministic = bool(evidence.get('confirmed'))
5560
+ if deterministic:
5561
  rows.append({
5562
  'id': f'SP-P{block_id}-{claim_id}', 'block_id': block_id,
5563
  'claim_id': claim_id, 'code': code, 'english': source_claims[claim_id],
 
5587
  allowed_evidence = {
5588
  'CLAIM_MISSING','CLAIM_ADDED','POLARITY_CHANGED','CAUSALITY_CHANGED',
5589
  'UNCERTAINTY_CHANGED','MEANING_CHANGED','CITATION_MOVED','ACRONYM_MISATTACHED',
5590
+ 'GRAMMATICAL_VIOLATION','GLOSSARY_CONCEPT_MISMATCH','UNTRANSLATED_PROSE','UNTRANSLATED_LEXEME',
5591
  'ACCEPTABLE_VARIANT','MORPHOLOGICAL_VARIANT','STYLISTIC_PREFERENCE',
5592
  'QA_DISAGREEMENT','NO_BLOCKING_EVIDENCE',
5593
  }
 
5604
  }
5605
  system_prompt = f"""You are an independent bilingual translation QA adjudicator for {target_lang}.
5606
  This is verification-only: do not rewrite, suggest wording, or emit free-form explanations. Each supplied issue already survived two bounded claim-local repair rounds. Confirm a BLOCKING defect only when the English/target pair itself provides clear evidence. Do not treat stylistic preference or a legitimate grammatical/morphological variant as blocking.
5607
+ For Russian, allow required case/number/gender inflection. For Kazakh, allow productive case, possessive, person/plural and vowel-harmony suffixes on the correct term stem, but treat clearly Russian lexical/syntactic substitution as blocking only when the pair itself supports that conclusion. For Chinese, allow natural reordering and sentence reshaping inside the same claim while preserving meaning. For all three languages, an ordinary untranslated English lexical word is blocking; protected acronyms, model/tool names, variables, citations, canonical glossary terms, and source-matched proper names are not.
5608
  Return valid JSON only: {{"items":[{{"id":"SP-P000001-C000001","issue_code":"GRAMMAR","decision":"CONFIRMED_BLOCKING","evidence_code":"GRAMMATICAL_VIOLATION"}}]}}.
5609
  Decision must be CONFIRMED_BLOCKING or NOT_CONFIRMED. evidence_code must be one of: {sorted(allowed_evidence)}. Return every supplied id+issue_code pair exactly once."""
5610
  try: