R14-TQ1-KK-A2: typed contamination authority and glossary-aware fact ownership

#55
Files changed (1) hide show
  1. core/pipeline_agent.py +172 -17
core/pipeline_agent.py CHANGED
@@ -4631,16 +4631,44 @@ Output only the repaired section."""
4631
  return ' '.join(parts).strip()
4632
 
4633
  @classmethod
4634
- def _translation_unowned_objects(cls, protected_target):
4635
- """Find non-anchor factual surfaces; numeric candidates are adjudicated per claim."""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4636
  text = re.sub(r'\[\[SP-T\d{6}\]\]', ' ', str(protected_target or ''))
4637
  text = re.sub(r'\[\[SP-P\d{6}\]\]', ' ', text)
4638
  text = re.sub(r'\[\[SP-C\d{6}\]\]', ' ', text)
 
 
 
 
 
 
 
 
4639
  unowned = {
4640
  'numeric': list(cls._numeric_tokens(text).elements()),
4641
  'citation': list(cls._translation_citation_tokens(text).elements()),
4642
  'hypothesis': list(cls._translation_hypothesis_tokens(text).elements()),
4643
- 'acronym': list(cls._translation_acronym_tokens(text).elements()),
4644
  }
4645
  names = re.findall(r'\b(?:fsQCA|SmartPLS|Mplus|OpenAlex|DeepSeek|ChatGPT|RStudio|SPSS|AMOS|Stata)\b', text)
4646
  if names:
@@ -4700,7 +4728,7 @@ Output only the repaired section."""
4700
  )
4701
 
4702
  @classmethod
4703
- def _validate_translation_skeleton(cls, value, mapping, block_id='', claim_sources=None):
4704
  text = str(value or '')
4705
  found = Counter(re.findall(r'\[\[SP-T\d{6}\]\]', text))
4706
  expected = Counter((mapping or {}).keys())
@@ -4746,7 +4774,9 @@ Output only the repaired section."""
4746
  f'Translation order-sensitive relation changed within owner claim in {block_id}: claim={claim_id}.'
4747
  )
4748
 
4749
- unowned = cls._translation_unowned_objects(target_claim)
 
 
4750
  numeric = list(unowned.pop('numeric', []))
4751
  if unowned:
4752
  safe_summary = {key: len(values) for key, values in unowned.items()}
@@ -4775,9 +4805,14 @@ Output only the repaired section."""
4775
  return True
4776
 
4777
  @classmethod
4778
- def _restore_translation_tokens(cls, value, mapping, block_id='', target_lang='', claim_sources=None, keep_claim_markers=False):
 
 
 
4779
  text = str(value or '')
4780
- cls._validate_translation_skeleton(text, mapping, block_id, claim_sources=claim_sources)
 
 
4781
  for placeholder, manifest in (mapping or {}).items():
4782
  text = text.replace(placeholder, cls._render_translation_fact(manifest, target_lang))
4783
  if re.search(r'\[\[SP-T\d{6}\]\]', text):
@@ -5047,6 +5082,10 @@ SECTION: {section_name}
5047
  'CASE_GOVERNMENT','AGREEMENT','MORPHOLOGY','VERB_FORM','WORD_ORDER',
5048
  'SYNTAX_STRUCTURE','PUNCTUATION','SYNTACTIC_COMPLETENESS','GRAMMAR_UNSPECIFIED',
5049
  }
 
 
 
 
5050
  if not isinstance(getattr(self, '_translation_claim_qa_evidence', None), dict):
5051
  self._translation_claim_qa_evidence = {}
5052
  claim_rows = []
@@ -5089,7 +5128,7 @@ Review EACH claim independently. Do not rewrite and do not provide free-form exp
5089
  LANGUAGE ADAPTER: {profile['adapter_id']}; morphology={profile['morphology']}. {profile['term_policy']} {lexical_contract} {profile['contamination_policy']}
5090
  For Russian, do not flag a terminology error merely because a translated term is inflected for grammatical case/number/gender; sustained non-Russian prose remains an additional contamination signal. For Kazakh, do not flag a terminology error merely because productive suffixes attach to the correct term stem; clearly Russian lexical/syntactic intrusion remains an additional language-specific signal when the pair supports that conclusion. Cyrillic script or absence of Kazakh-specific letters alone is never sufficient evidence.
5091
  Return valid JSON only in this exact form: {{"items":[{{"id":"SP-P000001-C000001","ok":true,"codes":[],"evidence_codes":[]}}]}}.
5092
- If ok=false, codes MUST contain only values from allowed_issue_codes. For SEMANTIC_DRIFT, NEGATION_DRIFT, CAUSALITY_STRENGTH_DRIFT, UNCERTAINTY_DRIFT, OMITTED_CLAIM, or ADDED_CLAIM, evidence_codes should identify the affected semantic dimension using only: {sorted(allowed_semantic_evidence)}. For GRAMMAR, evidence_codes should identify the narrow grammatical defect using only: {sorted(allowed_grammar_evidence)}. Use the narrowest supported evidence; use MEANING_CHANGED or GRAMMAR_UNSPECIFIED only when the narrower dimension cannot be identified from the pair. Do not invent an error dimension. Never return numbered prose, examples, quotes, or corrective wording. Never return replacement text. Return every supplied id exactly once."""
5093
  response = self.call_deepseek(
5094
  system_prompt, json.dumps(payload, ensure_ascii=False),
5095
  max_tokens=max(1200, min(3600, 650 + 260 * len(rows))),
@@ -5116,6 +5155,9 @@ If ok=false, codes MUST contain only values from allowed_issue_codes. For SEMANT
5116
  ]
5117
  evidence_codes = [code for code in raw_evidence_codes if code in allowed_semantic_evidence]
5118
  grammar_evidence_codes = [code for code in raw_evidence_codes if code in allowed_grammar_evidence]
 
 
 
5119
  semantic_code_set = {
5120
  'SEMANTIC_DRIFT','NEGATION_DRIFT','CAUSALITY_STRENGTH_DRIFT',
5121
  'UNCERTAINTY_DRIFT','OMITTED_CLAIM','ADDED_CLAIM',
@@ -5145,6 +5187,11 @@ If ok=false, codes MUST contain only values from allowed_issue_codes. For SEMANT
5145
  (meta['block_id'], meta['claim_id']), {}
5146
  )
5147
  evidence_row['grammar_evidence_codes'] = grammar_evidence_codes[:4]
 
 
 
 
 
5148
  model_contamination = 'LANGUAGE_CONTAMINATION' in codes
5149
  lexical = self._translation_lexical_purity_evidence(
5150
  meta['english'], meta['target'], target_lang, relevant_glossary
@@ -5185,15 +5232,41 @@ If ok=false, codes MUST contain only values from allowed_issue_codes. For SEMANT
5185
  'RU_LANGUAGE_CONTAMINATION_UNCONFIRMED', target_lang=target_lang, block_id=row_id,
5186
  severity='diagnostic', decision='PASS_WITH_WARNING', detail=contamination,
5187
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5188
 
5189
  # Deterministic lexical residue can override an LLM false negative
5190
- # in ZH/RU/KK. Russian LLM-only contamination remains suppressed
5191
- # when neither sustained-prose nor lexical evidence confirms it.
 
5192
  if returned_row.get('ok') is True and not deterministic_contamination and not deterministic_issue:
5193
  continue
5194
  if not codes:
5195
  only_suppressed_contamination = (
5196
- language_code == 'ru' and model_contamination and not deterministic_contamination
 
5197
  and bool(raw_codes) and all(code == 'LANGUAGE_CONTAMINATION' for code in raw_codes)
5198
  )
5199
  if only_suppressed_contamination:
@@ -5485,6 +5558,7 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
5485
  translated_with_claims = self._restore_translation_tokens(
5486
  parsed[block_id], protected_maps.get(block_id, {}), f'SP-P{block_id}', target_lang,
5487
  claim_sources=claim_sources_by_block.get(block_id, {}), keep_claim_markers=True,
 
5488
  )
5489
  clean_translation = self._strip_translation_claim_markers(translated_with_claims)
5490
  self._validate_translation_block_protected(block_id, source, clean_translation, target_lang)
@@ -5519,6 +5593,7 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
5519
  translated_with_claims = self._restore_translation_tokens(
5520
  repaired, protected_maps.get(block_id, {}), f'SP-P{block_id}', target_lang,
5521
  claim_sources=source_claims, keep_claim_markers=True,
 
5522
  )
5523
  clean_translation = self._strip_translation_claim_markers(translated_with_claims)
5524
  self._validate_translation_block_protected(
@@ -5544,6 +5619,7 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
5544
  'issue_codes': sorted({str(code or '').strip().upper() for code in (issue_codes or []) if str(code or '').strip()}),
5545
  'semantic_delta_codes': sorted({str(code or '').strip().upper() for code in (evidence.get('semantic_delta_codes') or []) if str(code or '').strip()}),
5546
  'grammar_evidence_codes': sorted({str(code or '').strip().upper() for code in (evidence.get('grammar_evidence_codes') or []) if str(code or '').strip()}),
 
5547
  'detected_unprotected_english_tokens': sorted({str(token or '').strip().casefold() for token in (evidence.get('detected_unprotected_english_tokens') or []) if str(token or '').strip()}),
5548
  }
5549
  payload = json.dumps(bounded, ensure_ascii=False, sort_keys=True, separators=(',', ':'))
@@ -5626,6 +5702,12 @@ Each code must be one of: {sorted(allowed)}. Return one or more specific dimensi
5626
  ]
5627
  if grammar_evidence_codes:
5628
  evidence_payload['grammar_evidence_codes'] = grammar_evidence_codes[:4]
 
 
 
 
 
 
5629
  lexical_contract = self._translation_lexical_retention_contract(target_lang)
5630
  retry_mode = (
5631
  '\nCLAIM SOURCE REGENERATION. The previous target wording has failed bounded repair and MUST NOT be used as drafting context. Generate a fresh translation of only this one claim directly from the frozen marked English source. '
@@ -5640,6 +5722,7 @@ Each code must be one of: {sorted(allowed)}. Return one or more specific dimensi
5640
  'If detected_unprotected_english_tokens is non-empty, every listed token is deterministic lexical-purity evidence from a prior target and MUST NOT survive as ordinary untranslated target-language prose unless represented by a protected SP-T anchor. '
5641
  'If semantic_delta_codes is non-empty, enforce ONLY those evidenced semantic dimensions against the frozen English source and preserve all unaffected actor, relation, object, polarity, modality, condition, comparison, temporal scope, and evidentiary strength. UNRESOLVED_SEMANTIC_DELTA is a signal to regenerate faithfully from source, not an instruction to guess a specific correction. '
5642
  'If grammar_evidence_codes is non-empty, repair or avoid only those bounded grammatical defect classes; GRAMMAR_UNSPECIFIED requires a fresh natural target-language rendering rather than blind micro-editing. '
 
5643
  'Preserve every SP-P, SP-C, and SP-T marker byte-for-byte. Do not add claims or factual content. Return only the marked translated claim.'
5644
  + '\n<CLAIM_REPAIR_EVIDENCE>\n'
5645
  + json.dumps(evidence_payload, ensure_ascii=False)
@@ -5658,6 +5741,7 @@ Each code must be one of: {sorted(allowed)}. Return one or more specific dimensi
5658
  restored = self._restore_translation_tokens(
5659
  parsed[block_id], mapping, f'SP-P{block_id}', target_lang,
5660
  claim_sources={claim_id: source_claim}, keep_claim_markers=True,
 
5661
  )
5662
  except DeepSeekRequestError as error:
5663
  if str(error or '').startswith('Translation protected placeholder mismatch'):
@@ -5682,6 +5766,7 @@ Each code must be one of: {sorted(allowed)}. Return one or more specific dimensi
5682
  restored = self._restore_translation_tokens(
5683
  strict_parsed[block_id], mapping, f'SP-P{block_id}', target_lang,
5684
  claim_sources={claim_id: source_claim}, keep_claim_markers=True,
 
5685
  )
5686
  elif self._is_translation_unowned_numeric_error(error):
5687
  logging.warning(
@@ -5706,6 +5791,7 @@ Each code must be one of: {sorted(allowed)}. Return one or more specific dimensi
5706
  restored = self._restore_translation_tokens(
5707
  retry_parsed[block_id], mapping, f'SP-P{block_id}', target_lang,
5708
  claim_sources={claim_id: source_claim}, keep_claim_markers=True,
 
5709
  )
5710
  else:
5711
  raise
@@ -5820,15 +5906,20 @@ Each code must be one of: {sorted(allowed)}. Return one or more specific dimensi
5820
  # supported target languages. Russian sustained Latin prose
5821
  # retains its existing independent deterministic authority.
5822
  if code == 'LANGUAGE_CONTAMINATION':
 
 
 
 
 
5823
  lexical = self._translation_lexical_purity_evidence(
5824
- source_claims[claim_id], target_claims[claim_id], target_lang, glossary
5825
  )
5826
  lexical_confirmed = bool(lexical.get('confirmed'))
5827
  deterministic = lexical_confirmed
5828
  deterministic_evidence_code = 'UNTRANSLATED_LEXEME' if lexical_confirmed else ''
5829
  if profile.get('language_code') == 'ru' and not deterministic:
5830
  evidence = self._russian_contamination_evidence(
5831
- source_claims[claim_id], target_claims[claim_id], glossary
5832
  )
5833
  deterministic = bool(evidence.get('confirmed'))
5834
  if deterministic:
@@ -5839,8 +5930,37 @@ Each code must be one of: {sorted(allowed)}. Return one or more specific dimensi
5839
  'claim_id': claim_id, 'code': code, 'english': source_claims[claim_id],
5840
  'target': target_claims[claim_id], 'deterministic_confirmed': True,
5841
  'deterministic_evidence_code': deterministic_evidence_code,
 
5842
  })
5843
  continue
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5844
  rows.append({
5845
  'id': f'SP-P{block_id}-{claim_id}', 'block_id': block_id,
5846
  'claim_id': claim_id, 'code': code, 'english': source_claims[claim_id],
@@ -5868,6 +5988,7 @@ Each code must be one of: {sorted(allowed)}. Return one or more specific dimensi
5868
  'CLAIM_MISSING','CLAIM_ADDED','POLARITY_CHANGED','CAUSALITY_CHANGED',
5869
  'UNCERTAINTY_CHANGED','MEANING_CHANGED','CITATION_MOVED','ACRONYM_MISATTACHED',
5870
  'GRAMMATICAL_VIOLATION','GLOSSARY_CONCEPT_MISMATCH','UNTRANSLATED_PROSE','UNTRANSLATED_LEXEME',
 
5871
  'ACCEPTABLE_VARIANT','MORPHOLOGICAL_VARIANT','STYLISTIC_PREFERENCE',
5872
  'QA_DISAGREEMENT','NO_BLOCKING_EVIDENCE',
5873
  }
@@ -5877,16 +5998,23 @@ Each code must be one of: {sorted(allowed)}. Return one or more specific dimensi
5877
  'language_adapter': profile.get('adapter_id'),
5878
  'morphology': profile.get('morphology'),
5879
  'term_policy': profile.get('term_policy'),
5880
- 'items': [
5881
- {'id': row['id'], 'issue_code': row['code'], 'english': row['english'], 'target': row['target']}
5882
- for row in model_rows
5883
- ],
 
 
 
 
 
 
5884
  }
5885
  lexical_contract = self._translation_lexical_retention_contract(target_lang)
5886
  system_prompt = f"""You are an independent bilingual translation QA adjudicator for {target_lang}.
5887
  This is verification-only: do not rewrite, suggest wording, or emit free-form explanations. Each supplied issue already survived two bounded claim-local repair rounds. Confirm a BLOCKING defect only when the English/target pair itself provides clear evidence. Do not treat stylistic preference or a legitimate grammatical/morphological variant as blocking.
5888
  {lexical_contract}
5889
  For Russian, allow required case/number/gender inflection and retain sustained non-Russian prose as an additional contamination signal. For Kazakh, allow productive case, possessive, person/plural and vowel-harmony suffixes on the correct term stem, but treat clearly Russian lexical/syntactic substitution as blocking only when the pair itself supports that conclusion. For Chinese, allow natural reordering and sentence reshaping inside the same claim while preserving meaning.
 
5890
  Return valid JSON only: {{"items":[{{"id":"SP-P000001-C000001","issue_code":"GRAMMAR","decision":"CONFIRMED_BLOCKING","evidence_code":"GRAMMATICAL_VIOLATION"}}]}}.
5891
  Decision must be CONFIRMED_BLOCKING or NOT_CONFIRMED. evidence_code must be one of: {sorted(allowed_evidence)}. Return every supplied id+issue_code pair exactly once."""
5892
  try:
@@ -5914,6 +6042,15 @@ Decision must be CONFIRMED_BLOCKING or NOT_CONFIRMED. evidence_code must be one
5914
  evidence_code = str(item.get('evidence_code') or '').strip().upper()
5915
  if decision not in {'CONFIRMED_BLOCKING', 'NOT_CONFIRMED'} or evidence_code not in allowed_evidence:
5916
  raise DeepSeekRequestError(f'Translation QA adjudication returned invalid bounded decision for {row["id"]}.')
 
 
 
 
 
 
 
 
 
5917
  if decision == 'CONFIRMED_BLOCKING':
5918
  confirmed.setdefault(row['block_id'], {}).setdefault(row['claim_id'], []).append(row['code'])
5919
  self._record_translation_assurance(
@@ -5989,6 +6126,7 @@ Decision must be CONFIRMED_BLOCKING or NOT_CONFIRMED. evidence_code must be one
5989
  assurance_start = len(self.translation_assurance_log)
5990
  profile = self._translation_language_profile(target_lang)
5991
  frozen_glossary = self.extend_translation_glossary(value, target_lang, glossary)
 
5992
  units = self._translation_body_units(value)
5993
  body_units = [unit for unit in units if unit['kind'] == 'body']
5994
  if not body_units:
@@ -6163,6 +6301,7 @@ Decision must be CONFIRMED_BLOCKING or NOT_CONFIRMED. evidence_code must be one
6163
  )
6164
  if lexical.get('confirmed'):
6165
  repair_evidence['detected_unprotected_english_tokens'] = list(lexical.get('tokens') or [])[:8]
 
6166
  self._record_translation_assurance(
6167
  'LEXICAL_PURITY_REPAIR_EVIDENCE', target_lang=target_lang,
6168
  block_id=f'SP-P{block_id}-{claim_id}', severity='repair', decision='REPAIR_LOCAL',
@@ -6171,6 +6310,21 @@ Decision must be CONFIRMED_BLOCKING or NOT_CONFIRMED. evidence_code must be one
6171
  'sample': ','.join(lexical.get('tokens') or [])[:120],
6172
  },
6173
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6174
  self._translation_claim_qa_evidence[evidence_key] = dict(repair_evidence)
6175
  signature = self._translation_repair_evidence_signature(codes, repair_evidence)
6176
  meta = self._translation_claim_repair_meta.setdefault(evidence_key, {
@@ -6371,6 +6525,7 @@ Decision must be CONFIRMED_BLOCKING or NOT_CONFIRMED. evidence_code must be one
6371
  raise DeepSeekRequestError('English manuscript contains no translatable content.')
6372
 
6373
  glossary = self._build_translation_glossary(body, target_lang)
 
6374
  body_units = [unit for unit in units if unit['kind'] == 'body']
6375
  translations = {}
6376
  token_budget = max(800, int(APP_CONFIG.get('translation_batch_tokens', 2500)))
 
4631
  return ' '.join(parts).strip()
4632
 
4633
  @classmethod
4634
+ def _translation_glossary_authorized_acronyms(cls, source_claim, glossary=None):
4635
+ """Return claim-local target acronyms authorized by a frozen glossary."""
4636
+ source_value = str(source_claim or '')
4637
+ allowed = Counter()
4638
+ for source_term, target_term in dict(glossary or {}).items():
4639
+ source_key = str(source_term or '').strip()
4640
+ target_value = str(target_term or '').strip()
4641
+ if not source_key or not target_value:
4642
+ continue
4643
+ occurrences = len(re.findall(
4644
+ rf'(?<![A-Za-z0-9_]){re.escape(source_key)}(?![A-Za-z0-9_])',
4645
+ source_value, flags=re.I,
4646
+ ))
4647
+ if occurrences <= 0:
4648
+ continue
4649
+ for acronym, count in cls._translation_acronym_tokens(target_value).items():
4650
+ allowed[acronym] += int(count) * int(occurrences)
4651
+ return allowed
4652
+
4653
+ @classmethod
4654
+ def _translation_unowned_objects(cls, protected_target, source_claim='', glossary=None):
4655
+ """Find non-anchor factual surfaces with claim-local glossary authorization."""
4656
  text = re.sub(r'\[\[SP-T\d{6}\]\]', ' ', str(protected_target or ''))
4657
  text = re.sub(r'\[\[SP-P\d{6}\]\]', ' ', text)
4658
  text = re.sub(r'\[\[SP-C\d{6}\]\]', ' ', text)
4659
+ acronym_tokens = cls._translation_acronym_tokens(text)
4660
+ authorized_acronyms = cls._translation_glossary_authorized_acronyms(source_claim, glossary)
4661
+ for acronym, count in authorized_acronyms.items():
4662
+ if acronym_tokens.get(acronym, 0) <= 0:
4663
+ continue
4664
+ acronym_tokens[acronym] -= min(int(count), int(acronym_tokens[acronym]))
4665
+ if acronym_tokens[acronym] <= 0:
4666
+ del acronym_tokens[acronym]
4667
  unowned = {
4668
  'numeric': list(cls._numeric_tokens(text).elements()),
4669
  'citation': list(cls._translation_citation_tokens(text).elements()),
4670
  'hypothesis': list(cls._translation_hypothesis_tokens(text).elements()),
4671
+ 'acronym': list(acronym_tokens.elements()),
4672
  }
4673
  names = re.findall(r'\b(?:fsQCA|SmartPLS|Mplus|OpenAlex|DeepSeek|ChatGPT|RStudio|SPSS|AMOS|Stata)\b', text)
4674
  if names:
 
4728
  )
4729
 
4730
  @classmethod
4731
+ def _validate_translation_skeleton(cls, value, mapping, block_id='', claim_sources=None, glossary=None):
4732
  text = str(value or '')
4733
  found = Counter(re.findall(r'\[\[SP-T\d{6}\]\]', text))
4734
  expected = Counter((mapping or {}).keys())
 
4774
  f'Translation order-sensitive relation changed within owner claim in {block_id}: claim={claim_id}.'
4775
  )
4776
 
4777
+ unowned = cls._translation_unowned_objects(
4778
+ target_claim, source_claim=source_claim, glossary=glossary
4779
+ )
4780
  numeric = list(unowned.pop('numeric', []))
4781
  if unowned:
4782
  safe_summary = {key: len(values) for key, values in unowned.items()}
 
4805
  return True
4806
 
4807
  @classmethod
4808
+ def _restore_translation_tokens(
4809
+ cls, value, mapping, block_id='', target_lang='', claim_sources=None,
4810
+ keep_claim_markers=False, glossary=None,
4811
+ ):
4812
  text = str(value or '')
4813
+ cls._validate_translation_skeleton(
4814
+ text, mapping, block_id, claim_sources=claim_sources, glossary=glossary
4815
+ )
4816
  for placeholder, manifest in (mapping or {}).items():
4817
  text = text.replace(placeholder, cls._render_translation_fact(manifest, target_lang))
4818
  if re.search(r'\[\[SP-T\d{6}\]\]', text):
 
5082
  'CASE_GOVERNMENT','AGREEMENT','MORPHOLOGY','VERB_FORM','WORD_ORDER',
5083
  'SYNTAX_STRUCTURE','PUNCTUATION','SYNTACTIC_COMPLETENESS','GRAMMAR_UNSPECIFIED',
5084
  }
5085
+ allowed_contamination_evidence = {
5086
+ 'UNTRANSLATED_LEXEME','UNTRANSLATED_PROSE',
5087
+ 'KK_RUSSIAN_LEXICAL_INTRUSION','KK_RUSSIAN_SYNTACTIC_INTRUSION',
5088
+ }
5089
  if not isinstance(getattr(self, '_translation_claim_qa_evidence', None), dict):
5090
  self._translation_claim_qa_evidence = {}
5091
  claim_rows = []
 
5128
  LANGUAGE ADAPTER: {profile['adapter_id']}; morphology={profile['morphology']}. {profile['term_policy']} {lexical_contract} {profile['contamination_policy']}
5129
  For Russian, do not flag a terminology error merely because a translated term is inflected for grammatical case/number/gender; sustained non-Russian prose remains an additional contamination signal. For Kazakh, do not flag a terminology error merely because productive suffixes attach to the correct term stem; clearly Russian lexical/syntactic intrusion remains an additional language-specific signal when the pair supports that conclusion. Cyrillic script or absence of Kazakh-specific letters alone is never sufficient evidence.
5130
  Return valid JSON only in this exact form: {{"items":[{{"id":"SP-P000001-C000001","ok":true,"codes":[],"evidence_codes":[]}}]}}.
5131
+ If ok=false, codes MUST contain only values from allowed_issue_codes. For SEMANTIC_DRIFT, NEGATION_DRIFT, CAUSALITY_STRENGTH_DRIFT, UNCERTAINTY_DRIFT, OMITTED_CLAIM, or ADDED_CLAIM, evidence_codes should identify the affected semantic dimension using only: {sorted(allowed_semantic_evidence)}. For GRAMMAR, evidence_codes should identify the narrow grammatical defect using only: {sorted(allowed_grammar_evidence)}. For LANGUAGE_CONTAMINATION, evidence_codes MUST identify the supported subtype using only: {sorted(allowed_contamination_evidence)}. Use UNTRANSLATED_LEXEME/UNTRANSLATED_PROSE only for Latin/English residue. For Kazakh, use KK_RUSSIAN_LEXICAL_INTRUSION or KK_RUSSIAN_SYNTACTIC_INTRUSION only when the English/target pair itself clearly supports Russian substitution or a Russian syntactic calque; Cyrillic script or absence of Kazakh-specific letters is never sufficient. Use the narrowest supported evidence; use MEANING_CHANGED or GRAMMAR_UNSPECIFIED only when the narrower dimension cannot be identified from the pair. Do not invent an error dimension. Never return numbered prose, examples, quotes, or corrective wording. Never return replacement text. Return every supplied id exactly once."""
5132
  response = self.call_deepseek(
5133
  system_prompt, json.dumps(payload, ensure_ascii=False),
5134
  max_tokens=max(1200, min(3600, 650 + 260 * len(rows))),
 
5155
  ]
5156
  evidence_codes = [code for code in raw_evidence_codes if code in allowed_semantic_evidence]
5157
  grammar_evidence_codes = [code for code in raw_evidence_codes if code in allowed_grammar_evidence]
5158
+ contamination_evidence_codes = [
5159
+ code for code in raw_evidence_codes if code in allowed_contamination_evidence
5160
+ ]
5161
  semantic_code_set = {
5162
  'SEMANTIC_DRIFT','NEGATION_DRIFT','CAUSALITY_STRENGTH_DRIFT',
5163
  'UNCERTAINTY_DRIFT','OMITTED_CLAIM','ADDED_CLAIM',
 
5187
  (meta['block_id'], meta['claim_id']), {}
5188
  )
5189
  evidence_row['grammar_evidence_codes'] = grammar_evidence_codes[:4]
5190
+ if 'LANGUAGE_CONTAMINATION' in codes and contamination_evidence_codes:
5191
+ evidence_row = self._translation_claim_qa_evidence.setdefault(
5192
+ (meta['block_id'], meta['claim_id']), {}
5193
+ )
5194
+ evidence_row['contamination_evidence_codes'] = contamination_evidence_codes[:4]
5195
  model_contamination = 'LANGUAGE_CONTAMINATION' in codes
5196
  lexical = self._translation_lexical_purity_evidence(
5197
  meta['english'], meta['target'], target_lang, relevant_glossary
 
5232
  'RU_LANGUAGE_CONTAMINATION_UNCONFIRMED', target_lang=target_lang, block_id=row_id,
5233
  severity='diagnostic', decision='PASS_WITH_WARNING', detail=contamination,
5234
  )
5235
+ elif language_code == 'zh' and model_contamination and not deterministic_contamination:
5236
+ codes = [code for code in codes if code != 'LANGUAGE_CONTAMINATION']
5237
+ self._record_translation_assurance(
5238
+ 'ZH_LANGUAGE_CONTAMINATION_UNCONFIRMED', target_lang=target_lang, block_id=row_id,
5239
+ severity='diagnostic', decision='PASS_WITH_WARNING',
5240
+ detail={'deterministic_token_count': int(lexical.get('token_count') or 0)},
5241
+ )
5242
+ elif language_code == 'kk' and model_contamination and not deterministic_contamination:
5243
+ russian_intrusion = [
5244
+ code for code in contamination_evidence_codes
5245
+ if code in {'KK_RUSSIAN_LEXICAL_INTRUSION','KK_RUSSIAN_SYNTACTIC_INTRUSION'}
5246
+ ]
5247
+ if russian_intrusion:
5248
+ evidence_row = self._translation_claim_qa_evidence.setdefault(
5249
+ (meta['block_id'], meta['claim_id']), {}
5250
+ )
5251
+ evidence_row['contamination_evidence_codes'] = russian_intrusion[:2]
5252
+ else:
5253
+ codes = [code for code in codes if code != 'LANGUAGE_CONTAMINATION']
5254
+ self._record_translation_assurance(
5255
+ 'KK_LANGUAGE_CONTAMINATION_UNCONFIRMED', target_lang=target_lang, block_id=row_id,
5256
+ severity='diagnostic', decision='PASS_WITH_WARNING',
5257
+ detail={'deterministic_token_count': int(lexical.get('token_count') or 0)},
5258
+ )
5259
 
5260
  # Deterministic lexical residue can override an LLM false negative
5261
+ # in ZH/RU/KK. LLM-only English/Latin contamination is suppressed
5262
+ # when deterministic evidence is absent; Kazakh Russian intrusion
5263
+ # survives only with its dedicated typed evidence.
5264
  if returned_row.get('ok') is True and not deterministic_contamination and not deterministic_issue:
5265
  continue
5266
  if not codes:
5267
  only_suppressed_contamination = (
5268
+ language_code in {'zh','ru','kk'} and model_contamination and not deterministic_contamination
5269
+ and 'LANGUAGE_CONTAMINATION' not in codes
5270
  and bool(raw_codes) and all(code == 'LANGUAGE_CONTAMINATION' for code in raw_codes)
5271
  )
5272
  if only_suppressed_contamination:
 
5558
  translated_with_claims = self._restore_translation_tokens(
5559
  parsed[block_id], protected_maps.get(block_id, {}), f'SP-P{block_id}', target_lang,
5560
  claim_sources=claim_sources_by_block.get(block_id, {}), keep_claim_markers=True,
5561
+ glossary=getattr(self, '_active_translation_glossary', {}),
5562
  )
5563
  clean_translation = self._strip_translation_claim_markers(translated_with_claims)
5564
  self._validate_translation_block_protected(block_id, source, clean_translation, target_lang)
 
5593
  translated_with_claims = self._restore_translation_tokens(
5594
  repaired, protected_maps.get(block_id, {}), f'SP-P{block_id}', target_lang,
5595
  claim_sources=source_claims, keep_claim_markers=True,
5596
+ glossary=getattr(self, '_active_translation_glossary', {}),
5597
  )
5598
  clean_translation = self._strip_translation_claim_markers(translated_with_claims)
5599
  self._validate_translation_block_protected(
 
5619
  'issue_codes': sorted({str(code or '').strip().upper() for code in (issue_codes or []) if str(code or '').strip()}),
5620
  'semantic_delta_codes': sorted({str(code or '').strip().upper() for code in (evidence.get('semantic_delta_codes') or []) if str(code or '').strip()}),
5621
  'grammar_evidence_codes': sorted({str(code or '').strip().upper() for code in (evidence.get('grammar_evidence_codes') or []) if str(code or '').strip()}),
5622
+ 'contamination_evidence_codes': sorted({str(code or '').strip().upper() for code in (evidence.get('contamination_evidence_codes') or []) if str(code or '').strip()}),
5623
  'detected_unprotected_english_tokens': sorted({str(token or '').strip().casefold() for token in (evidence.get('detected_unprotected_english_tokens') or []) if str(token or '').strip()}),
5624
  }
5625
  payload = json.dumps(bounded, ensure_ascii=False, sort_keys=True, separators=(',', ':'))
 
5702
  ]
5703
  if grammar_evidence_codes:
5704
  evidence_payload['grammar_evidence_codes'] = grammar_evidence_codes[:4]
5705
+ contamination_evidence_codes = [
5706
+ str(code or '').strip().upper() for code in (supplied_evidence.get('contamination_evidence_codes') or [])
5707
+ if str(code or '').strip()
5708
+ ]
5709
+ if contamination_evidence_codes:
5710
+ evidence_payload['contamination_evidence_codes'] = contamination_evidence_codes[:4]
5711
  lexical_contract = self._translation_lexical_retention_contract(target_lang)
5712
  retry_mode = (
5713
  '\nCLAIM SOURCE REGENERATION. The previous target wording has failed bounded repair and MUST NOT be used as drafting context. Generate a fresh translation of only this one claim directly from the frozen marked English source. '
 
5722
  'If detected_unprotected_english_tokens is non-empty, every listed token is deterministic lexical-purity evidence from a prior target and MUST NOT survive as ordinary untranslated target-language prose unless represented by a protected SP-T anchor. '
5723
  'If semantic_delta_codes is non-empty, enforce ONLY those evidenced semantic dimensions against the frozen English source and preserve all unaffected actor, relation, object, polarity, modality, condition, comparison, temporal scope, and evidentiary strength. UNRESOLVED_SEMANTIC_DELTA is a signal to regenerate faithfully from source, not an instruction to guess a specific correction. '
5724
  'If grammar_evidence_codes is non-empty, repair or avoid only those bounded grammatical defect classes; GRAMMAR_UNSPECIFIED requires a fresh natural target-language rendering rather than blind micro-editing. '
5725
+ 'If contamination_evidence_codes identifies KK_RUSSIAN_LEXICAL_INTRUSION or KK_RUSSIAN_SYNTACTIC_INTRUSION, remove only the evidenced Russian substitution/calque while preserving the frozen English meaning; do not invent an English lexical defect when deterministic English-token evidence is absent. '
5726
  'Preserve every SP-P, SP-C, and SP-T marker byte-for-byte. Do not add claims or factual content. Return only the marked translated claim.'
5727
  + '\n<CLAIM_REPAIR_EVIDENCE>\n'
5728
  + json.dumps(evidence_payload, ensure_ascii=False)
 
5741
  restored = self._restore_translation_tokens(
5742
  parsed[block_id], mapping, f'SP-P{block_id}', target_lang,
5743
  claim_sources={claim_id: source_claim}, keep_claim_markers=True,
5744
+ glossary=getattr(self, '_active_translation_glossary', {}),
5745
  )
5746
  except DeepSeekRequestError as error:
5747
  if str(error or '').startswith('Translation protected placeholder mismatch'):
 
5766
  restored = self._restore_translation_tokens(
5767
  strict_parsed[block_id], mapping, f'SP-P{block_id}', target_lang,
5768
  claim_sources={claim_id: source_claim}, keep_claim_markers=True,
5769
+ glossary=getattr(self, '_active_translation_glossary', {}),
5770
  )
5771
  elif self._is_translation_unowned_numeric_error(error):
5772
  logging.warning(
 
5791
  restored = self._restore_translation_tokens(
5792
  retry_parsed[block_id], mapping, f'SP-P{block_id}', target_lang,
5793
  claim_sources={claim_id: source_claim}, keep_claim_markers=True,
5794
+ glossary=getattr(self, '_active_translation_glossary', {}),
5795
  )
5796
  else:
5797
  raise
 
5906
  # supported target languages. Russian sustained Latin prose
5907
  # retains its existing independent deterministic authority.
5908
  if code == 'LANGUAGE_CONTAMINATION':
5909
+ relevant_glossary = {
5910
+ str(source_term): str(target_term)
5911
+ for source_term, target_term in dict(glossary or {}).items()
5912
+ if source_term and str(source_term).casefold() in source_claims[claim_id].casefold()
5913
+ }
5914
  lexical = self._translation_lexical_purity_evidence(
5915
+ source_claims[claim_id], target_claims[claim_id], target_lang, relevant_glossary
5916
  )
5917
  lexical_confirmed = bool(lexical.get('confirmed'))
5918
  deterministic = lexical_confirmed
5919
  deterministic_evidence_code = 'UNTRANSLATED_LEXEME' if lexical_confirmed else ''
5920
  if profile.get('language_code') == 'ru' and not deterministic:
5921
  evidence = self._russian_contamination_evidence(
5922
+ source_claims[claim_id], target_claims[claim_id], relevant_glossary
5923
  )
5924
  deterministic = bool(evidence.get('confirmed'))
5925
  if deterministic:
 
5930
  'claim_id': claim_id, 'code': code, 'english': source_claims[claim_id],
5931
  'target': target_claims[claim_id], 'deterministic_confirmed': True,
5932
  'deterministic_evidence_code': deterministic_evidence_code,
5933
+ 'lexical_evidence': lexical, 'glossary': relevant_glossary,
5934
  })
5935
  continue
5936
+ repair_evidence = dict(
5937
+ getattr(self, '_translation_claim_qa_evidence', {}).get((block_id, claim_id), {}) or {}
5938
+ )
5939
+ typed = [
5940
+ str(item or '').strip().upper()
5941
+ for item in (repair_evidence.get('contamination_evidence_codes') or [])
5942
+ if str(item or '').strip()
5943
+ ]
5944
+ kk_typed = [
5945
+ item for item in typed
5946
+ if item in {'KK_RUSSIAN_LEXICAL_INTRUSION','KK_RUSSIAN_SYNTACTIC_INTRUSION'}
5947
+ ]
5948
+ if profile.get('language_code') in {'zh','kk'} and not kk_typed:
5949
+ self._record_translation_assurance(
5950
+ f"{profile.get('language_code','').upper()}_QA_ADJUDICATION_LEXICAL_UNCONFIRMED",
5951
+ target_lang=target_lang, block_id=f'SP-P{block_id}-{claim_id}',
5952
+ severity='diagnostic', decision='PASS_WITH_WARNING',
5953
+ detail={'deterministic_token_count': int(lexical.get('token_count') or 0)},
5954
+ )
5955
+ continue
5956
+ rows.append({
5957
+ 'id': f'SP-P{block_id}-{claim_id}', 'block_id': block_id,
5958
+ 'claim_id': claim_id, 'code': code, 'english': source_claims[claim_id],
5959
+ 'target': target_claims[claim_id], 'deterministic_confirmed': False,
5960
+ 'lexical_evidence': lexical, 'glossary': relevant_glossary,
5961
+ 'contamination_evidence_codes': kk_typed[:2],
5962
+ })
5963
+ continue
5964
  rows.append({
5965
  'id': f'SP-P{block_id}-{claim_id}', 'block_id': block_id,
5966
  'claim_id': claim_id, 'code': code, 'english': source_claims[claim_id],
 
5988
  'CLAIM_MISSING','CLAIM_ADDED','POLARITY_CHANGED','CAUSALITY_CHANGED',
5989
  'UNCERTAINTY_CHANGED','MEANING_CHANGED','CITATION_MOVED','ACRONYM_MISATTACHED',
5990
  'GRAMMATICAL_VIOLATION','GLOSSARY_CONCEPT_MISMATCH','UNTRANSLATED_PROSE','UNTRANSLATED_LEXEME',
5991
+ 'KK_RUSSIAN_LEXICAL_INTRUSION','KK_RUSSIAN_SYNTACTIC_INTRUSION',
5992
  'ACCEPTABLE_VARIANT','MORPHOLOGICAL_VARIANT','STYLISTIC_PREFERENCE',
5993
  'QA_DISAGREEMENT','NO_BLOCKING_EVIDENCE',
5994
  }
 
5998
  'language_adapter': profile.get('adapter_id'),
5999
  'morphology': profile.get('morphology'),
6000
  'term_policy': profile.get('term_policy'),
6001
+ 'items': [{
6002
+ 'id': row['id'], 'issue_code': row['code'], 'english': row['english'], 'target': row['target'],
6003
+ 'glossary': dict(row.get('glossary') or {}),
6004
+ 'deterministic_lexical': {
6005
+ 'confirmed': bool((row.get('lexical_evidence') or {}).get('confirmed')),
6006
+ 'token_count': int((row.get('lexical_evidence') or {}).get('token_count') or 0),
6007
+ 'tokens': list((row.get('lexical_evidence') or {}).get('tokens') or [])[:8],
6008
+ },
6009
+ 'contamination_evidence_codes': list(row.get('contamination_evidence_codes') or [])[:2],
6010
+ } for row in model_rows],
6011
  }
6012
  lexical_contract = self._translation_lexical_retention_contract(target_lang)
6013
  system_prompt = f"""You are an independent bilingual translation QA adjudicator for {target_lang}.
6014
  This is verification-only: do not rewrite, suggest wording, or emit free-form explanations. Each supplied issue already survived two bounded claim-local repair rounds. Confirm a BLOCKING defect only when the English/target pair itself provides clear evidence. Do not treat stylistic preference or a legitimate grammatical/morphological variant as blocking.
6015
  {lexical_contract}
6016
  For Russian, allow required case/number/gender inflection and retain sustained non-Russian prose as an additional contamination signal. For Kazakh, allow productive case, possessive, person/plural and vowel-harmony suffixes on the correct term stem, but treat clearly Russian lexical/syntactic substitution as blocking only when the pair itself supports that conclusion. For Chinese, allow natural reordering and sentence reshaping inside the same claim while preserving meaning.
6017
+ For LANGUAGE_CONTAMINATION, deterministic_lexical is authoritative for ordinary English/Latin residue: when confirmed=false and token_count=0, you MUST NOT confirm UNTRANSLATED_LEXEME or UNTRANSLATED_PROSE. For Kazakh, a non-Latin contamination blocker may be confirmed only when contamination_evidence_codes contains KK_RUSSIAN_LEXICAL_INTRUSION or KK_RUSSIAN_SYNTACTIC_INTRUSION, and the evidence_code must match one of those supplied typed codes. The per-item glossary lists the frozen target terms that are legal for that source claim.
6018
  Return valid JSON only: {{"items":[{{"id":"SP-P000001-C000001","issue_code":"GRAMMAR","decision":"CONFIRMED_BLOCKING","evidence_code":"GRAMMATICAL_VIOLATION"}}]}}.
6019
  Decision must be CONFIRMED_BLOCKING or NOT_CONFIRMED. evidence_code must be one of: {sorted(allowed_evidence)}. Return every supplied id+issue_code pair exactly once."""
6020
  try:
 
6042
  evidence_code = str(item.get('evidence_code') or '').strip().upper()
6043
  if decision not in {'CONFIRMED_BLOCKING', 'NOT_CONFIRMED'} or evidence_code not in allowed_evidence:
6044
  raise DeepSeekRequestError(f'Translation QA adjudication returned invalid bounded decision for {row["id"]}.')
6045
+ if row['code'] == 'LANGUAGE_CONTAMINATION':
6046
+ lexical = dict(row.get('lexical_evidence') or {})
6047
+ typed = set(row.get('contamination_evidence_codes') or [])
6048
+ if not bool(lexical.get('confirmed')) and evidence_code in {'UNTRANSLATED_LEXEME','UNTRANSLATED_PROSE'}:
6049
+ decision = 'NOT_CONFIRMED'
6050
+ evidence_code = 'QA_DISAGREEMENT'
6051
+ if typed and evidence_code not in typed and decision == 'CONFIRMED_BLOCKING':
6052
+ decision = 'NOT_CONFIRMED'
6053
+ evidence_code = 'QA_DISAGREEMENT'
6054
  if decision == 'CONFIRMED_BLOCKING':
6055
  confirmed.setdefault(row['block_id'], {}).setdefault(row['claim_id'], []).append(row['code'])
6056
  self._record_translation_assurance(
 
6126
  assurance_start = len(self.translation_assurance_log)
6127
  profile = self._translation_language_profile(target_lang)
6128
  frozen_glossary = self.extend_translation_glossary(value, target_lang, glossary)
6129
+ self._active_translation_glossary = dict(frozen_glossary)
6130
  units = self._translation_body_units(value)
6131
  body_units = [unit for unit in units if unit['kind'] == 'body']
6132
  if not body_units:
 
6301
  )
6302
  if lexical.get('confirmed'):
6303
  repair_evidence['detected_unprotected_english_tokens'] = list(lexical.get('tokens') or [])[:8]
6304
+ repair_evidence['contamination_evidence_codes'] = ['UNTRANSLATED_LEXEME']
6305
  self._record_translation_assurance(
6306
  'LEXICAL_PURITY_REPAIR_EVIDENCE', target_lang=target_lang,
6307
  block_id=f'SP-P{block_id}-{claim_id}', severity='repair', decision='REPAIR_LOCAL',
 
6310
  'sample': ','.join(lexical.get('tokens') or [])[:120],
6311
  },
6312
  )
6313
+ elif self._translation_language_profile(target_lang).get('language_code') == 'kk':
6314
+ kk_intrusion = [
6315
+ str(code or '').strip().upper()
6316
+ for code in (repair_evidence.get('contamination_evidence_codes') or [])
6317
+ if str(code or '').strip().upper() in {
6318
+ 'KK_RUSSIAN_LEXICAL_INTRUSION','KK_RUSSIAN_SYNTACTIC_INTRUSION'
6319
+ }
6320
+ ]
6321
+ if kk_intrusion:
6322
+ repair_evidence['contamination_evidence_codes'] = kk_intrusion[:2]
6323
+ self._record_translation_assurance(
6324
+ 'KK_RUSSIAN_INTRUSION_REPAIR_EVIDENCE', target_lang=target_lang,
6325
+ block_id=f'SP-P{block_id}-{claim_id}', severity='repair', decision='REPAIR_LOCAL',
6326
+ detail={'classes': '|'.join(kk_intrusion)[:120]},
6327
+ )
6328
  self._translation_claim_qa_evidence[evidence_key] = dict(repair_evidence)
6329
  signature = self._translation_repair_evidence_signature(codes, repair_evidence)
6330
  meta = self._translation_claim_repair_meta.setdefault(evidence_key, {
 
6525
  raise DeepSeekRequestError('English manuscript contains no translatable content.')
6526
 
6527
  glossary = self._build_translation_glossary(body, target_lang)
6528
+ self._active_translation_glossary = dict(glossary)
6529
  body_units = [unit for unit in units if unit['kind'] == 'body']
6530
  translations = {}
6531
  token_budget = max(800, int(APP_CONFIG.get('translation_batch_tokens', 2500)))