Spaces:
Running
Running
R14-TQ1-KK-A2: typed contamination authority and glossary-aware fact ownership
#55
by GuXiaopeng - opened
- core/pipeline_agent.py +172 -17
core/pipeline_agent.py
CHANGED
|
@@ -4631,16 +4631,44 @@ Output only the repaired section."""
|
|
| 4631 |
return ' '.join(parts).strip()
|
| 4632 |
|
| 4633 |
@classmethod
|
| 4634 |
-
def
|
| 4635 |
-
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4636 |
text = re.sub(r'\[\[SP-T\d{6}\]\]', ' ', str(protected_target or ''))
|
| 4637 |
text = re.sub(r'\[\[SP-P\d{6}\]\]', ' ', text)
|
| 4638 |
text = re.sub(r'\[\[SP-C\d{6}\]\]', ' ', text)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4639 |
unowned = {
|
| 4640 |
'numeric': list(cls._numeric_tokens(text).elements()),
|
| 4641 |
'citation': list(cls._translation_citation_tokens(text).elements()),
|
| 4642 |
'hypothesis': list(cls._translation_hypothesis_tokens(text).elements()),
|
| 4643 |
-
'acronym': list(
|
| 4644 |
}
|
| 4645 |
names = re.findall(r'\b(?:fsQCA|SmartPLS|Mplus|OpenAlex|DeepSeek|ChatGPT|RStudio|SPSS|AMOS|Stata)\b', text)
|
| 4646 |
if names:
|
|
@@ -4700,7 +4728,7 @@ Output only the repaired section."""
|
|
| 4700 |
)
|
| 4701 |
|
| 4702 |
@classmethod
|
| 4703 |
-
def _validate_translation_skeleton(cls, value, mapping, block_id='', claim_sources=None):
|
| 4704 |
text = str(value or '')
|
| 4705 |
found = Counter(re.findall(r'\[\[SP-T\d{6}\]\]', text))
|
| 4706 |
expected = Counter((mapping or {}).keys())
|
|
@@ -4746,7 +4774,9 @@ Output only the repaired section."""
|
|
| 4746 |
f'Translation order-sensitive relation changed within owner claim in {block_id}: claim={claim_id}.'
|
| 4747 |
)
|
| 4748 |
|
| 4749 |
-
unowned = cls._translation_unowned_objects(
|
|
|
|
|
|
|
| 4750 |
numeric = list(unowned.pop('numeric', []))
|
| 4751 |
if unowned:
|
| 4752 |
safe_summary = {key: len(values) for key, values in unowned.items()}
|
|
@@ -4775,9 +4805,14 @@ Output only the repaired section."""
|
|
| 4775 |
return True
|
| 4776 |
|
| 4777 |
@classmethod
|
| 4778 |
-
def _restore_translation_tokens(
|
|
|
|
|
|
|
|
|
|
| 4779 |
text = str(value or '')
|
| 4780 |
-
cls._validate_translation_skeleton(
|
|
|
|
|
|
|
| 4781 |
for placeholder, manifest in (mapping or {}).items():
|
| 4782 |
text = text.replace(placeholder, cls._render_translation_fact(manifest, target_lang))
|
| 4783 |
if re.search(r'\[\[SP-T\d{6}\]\]', text):
|
|
@@ -5047,6 +5082,10 @@ SECTION: {section_name}
|
|
| 5047 |
'CASE_GOVERNMENT','AGREEMENT','MORPHOLOGY','VERB_FORM','WORD_ORDER',
|
| 5048 |
'SYNTAX_STRUCTURE','PUNCTUATION','SYNTACTIC_COMPLETENESS','GRAMMAR_UNSPECIFIED',
|
| 5049 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5050 |
if not isinstance(getattr(self, '_translation_claim_qa_evidence', None), dict):
|
| 5051 |
self._translation_claim_qa_evidence = {}
|
| 5052 |
claim_rows = []
|
|
@@ -5089,7 +5128,7 @@ Review EACH claim independently. Do not rewrite and do not provide free-form exp
|
|
| 5089 |
LANGUAGE ADAPTER: {profile['adapter_id']}; morphology={profile['morphology']}. {profile['term_policy']} {lexical_contract} {profile['contamination_policy']}
|
| 5090 |
For Russian, do not flag a terminology error merely because a translated term is inflected for grammatical case/number/gender; sustained non-Russian prose remains an additional contamination signal. For Kazakh, do not flag a terminology error merely because productive suffixes attach to the correct term stem; clearly Russian lexical/syntactic intrusion remains an additional language-specific signal when the pair supports that conclusion. Cyrillic script or absence of Kazakh-specific letters alone is never sufficient evidence.
|
| 5091 |
Return valid JSON only in this exact form: {{"items":[{{"id":"SP-P000001-C000001","ok":true,"codes":[],"evidence_codes":[]}}]}}.
|
| 5092 |
-
If ok=false, codes MUST contain only values from allowed_issue_codes. For SEMANTIC_DRIFT, NEGATION_DRIFT, CAUSALITY_STRENGTH_DRIFT, UNCERTAINTY_DRIFT, OMITTED_CLAIM, or ADDED_CLAIM, evidence_codes should identify the affected semantic dimension using only: {sorted(allowed_semantic_evidence)}. For GRAMMAR, evidence_codes should identify the narrow grammatical defect using only: {sorted(allowed_grammar_evidence)}. Use the narrowest supported evidence; use MEANING_CHANGED or GRAMMAR_UNSPECIFIED only when the narrower dimension cannot be identified from the pair. Do not invent an error dimension. Never return numbered prose, examples, quotes, or corrective wording. Never return replacement text. Return every supplied id exactly once."""
|
| 5093 |
response = self.call_deepseek(
|
| 5094 |
system_prompt, json.dumps(payload, ensure_ascii=False),
|
| 5095 |
max_tokens=max(1200, min(3600, 650 + 260 * len(rows))),
|
|
@@ -5116,6 +5155,9 @@ If ok=false, codes MUST contain only values from allowed_issue_codes. For SEMANT
|
|
| 5116 |
]
|
| 5117 |
evidence_codes = [code for code in raw_evidence_codes if code in allowed_semantic_evidence]
|
| 5118 |
grammar_evidence_codes = [code for code in raw_evidence_codes if code in allowed_grammar_evidence]
|
|
|
|
|
|
|
|
|
|
| 5119 |
semantic_code_set = {
|
| 5120 |
'SEMANTIC_DRIFT','NEGATION_DRIFT','CAUSALITY_STRENGTH_DRIFT',
|
| 5121 |
'UNCERTAINTY_DRIFT','OMITTED_CLAIM','ADDED_CLAIM',
|
|
@@ -5145,6 +5187,11 @@ If ok=false, codes MUST contain only values from allowed_issue_codes. For SEMANT
|
|
| 5145 |
(meta['block_id'], meta['claim_id']), {}
|
| 5146 |
)
|
| 5147 |
evidence_row['grammar_evidence_codes'] = grammar_evidence_codes[:4]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5148 |
model_contamination = 'LANGUAGE_CONTAMINATION' in codes
|
| 5149 |
lexical = self._translation_lexical_purity_evidence(
|
| 5150 |
meta['english'], meta['target'], target_lang, relevant_glossary
|
|
@@ -5185,15 +5232,41 @@ If ok=false, codes MUST contain only values from allowed_issue_codes. For SEMANT
|
|
| 5185 |
'RU_LANGUAGE_CONTAMINATION_UNCONFIRMED', target_lang=target_lang, block_id=row_id,
|
| 5186 |
severity='diagnostic', decision='PASS_WITH_WARNING', detail=contamination,
|
| 5187 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5188 |
|
| 5189 |
# Deterministic lexical residue can override an LLM false negative
|
| 5190 |
-
# in ZH/RU/KK.
|
| 5191 |
-
# when
|
|
|
|
| 5192 |
if returned_row.get('ok') is True and not deterministic_contamination and not deterministic_issue:
|
| 5193 |
continue
|
| 5194 |
if not codes:
|
| 5195 |
only_suppressed_contamination = (
|
| 5196 |
-
language_code
|
|
|
|
| 5197 |
and bool(raw_codes) and all(code == 'LANGUAGE_CONTAMINATION' for code in raw_codes)
|
| 5198 |
)
|
| 5199 |
if only_suppressed_contamination:
|
|
@@ -5485,6 +5558,7 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
|
|
| 5485 |
translated_with_claims = self._restore_translation_tokens(
|
| 5486 |
parsed[block_id], protected_maps.get(block_id, {}), f'SP-P{block_id}', target_lang,
|
| 5487 |
claim_sources=claim_sources_by_block.get(block_id, {}), keep_claim_markers=True,
|
|
|
|
| 5488 |
)
|
| 5489 |
clean_translation = self._strip_translation_claim_markers(translated_with_claims)
|
| 5490 |
self._validate_translation_block_protected(block_id, source, clean_translation, target_lang)
|
|
@@ -5519,6 +5593,7 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
|
|
| 5519 |
translated_with_claims = self._restore_translation_tokens(
|
| 5520 |
repaired, protected_maps.get(block_id, {}), f'SP-P{block_id}', target_lang,
|
| 5521 |
claim_sources=source_claims, keep_claim_markers=True,
|
|
|
|
| 5522 |
)
|
| 5523 |
clean_translation = self._strip_translation_claim_markers(translated_with_claims)
|
| 5524 |
self._validate_translation_block_protected(
|
|
@@ -5544,6 +5619,7 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
|
|
| 5544 |
'issue_codes': sorted({str(code or '').strip().upper() for code in (issue_codes or []) if str(code or '').strip()}),
|
| 5545 |
'semantic_delta_codes': sorted({str(code or '').strip().upper() for code in (evidence.get('semantic_delta_codes') or []) if str(code or '').strip()}),
|
| 5546 |
'grammar_evidence_codes': sorted({str(code or '').strip().upper() for code in (evidence.get('grammar_evidence_codes') or []) if str(code or '').strip()}),
|
|
|
|
| 5547 |
'detected_unprotected_english_tokens': sorted({str(token or '').strip().casefold() for token in (evidence.get('detected_unprotected_english_tokens') or []) if str(token or '').strip()}),
|
| 5548 |
}
|
| 5549 |
payload = json.dumps(bounded, ensure_ascii=False, sort_keys=True, separators=(',', ':'))
|
|
@@ -5626,6 +5702,12 @@ Each code must be one of: {sorted(allowed)}. Return one or more specific dimensi
|
|
| 5626 |
]
|
| 5627 |
if grammar_evidence_codes:
|
| 5628 |
evidence_payload['grammar_evidence_codes'] = grammar_evidence_codes[:4]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5629 |
lexical_contract = self._translation_lexical_retention_contract(target_lang)
|
| 5630 |
retry_mode = (
|
| 5631 |
'\nCLAIM SOURCE REGENERATION. The previous target wording has failed bounded repair and MUST NOT be used as drafting context. Generate a fresh translation of only this one claim directly from the frozen marked English source. '
|
|
@@ -5640,6 +5722,7 @@ Each code must be one of: {sorted(allowed)}. Return one or more specific dimensi
|
|
| 5640 |
'If detected_unprotected_english_tokens is non-empty, every listed token is deterministic lexical-purity evidence from a prior target and MUST NOT survive as ordinary untranslated target-language prose unless represented by a protected SP-T anchor. '
|
| 5641 |
'If semantic_delta_codes is non-empty, enforce ONLY those evidenced semantic dimensions against the frozen English source and preserve all unaffected actor, relation, object, polarity, modality, condition, comparison, temporal scope, and evidentiary strength. UNRESOLVED_SEMANTIC_DELTA is a signal to regenerate faithfully from source, not an instruction to guess a specific correction. '
|
| 5642 |
'If grammar_evidence_codes is non-empty, repair or avoid only those bounded grammatical defect classes; GRAMMAR_UNSPECIFIED requires a fresh natural target-language rendering rather than blind micro-editing. '
|
|
|
|
| 5643 |
'Preserve every SP-P, SP-C, and SP-T marker byte-for-byte. Do not add claims or factual content. Return only the marked translated claim.'
|
| 5644 |
+ '\n<CLAIM_REPAIR_EVIDENCE>\n'
|
| 5645 |
+ json.dumps(evidence_payload, ensure_ascii=False)
|
|
@@ -5658,6 +5741,7 @@ Each code must be one of: {sorted(allowed)}. Return one or more specific dimensi
|
|
| 5658 |
restored = self._restore_translation_tokens(
|
| 5659 |
parsed[block_id], mapping, f'SP-P{block_id}', target_lang,
|
| 5660 |
claim_sources={claim_id: source_claim}, keep_claim_markers=True,
|
|
|
|
| 5661 |
)
|
| 5662 |
except DeepSeekRequestError as error:
|
| 5663 |
if str(error or '').startswith('Translation protected placeholder mismatch'):
|
|
@@ -5682,6 +5766,7 @@ Each code must be one of: {sorted(allowed)}. Return one or more specific dimensi
|
|
| 5682 |
restored = self._restore_translation_tokens(
|
| 5683 |
strict_parsed[block_id], mapping, f'SP-P{block_id}', target_lang,
|
| 5684 |
claim_sources={claim_id: source_claim}, keep_claim_markers=True,
|
|
|
|
| 5685 |
)
|
| 5686 |
elif self._is_translation_unowned_numeric_error(error):
|
| 5687 |
logging.warning(
|
|
@@ -5706,6 +5791,7 @@ Each code must be one of: {sorted(allowed)}. Return one or more specific dimensi
|
|
| 5706 |
restored = self._restore_translation_tokens(
|
| 5707 |
retry_parsed[block_id], mapping, f'SP-P{block_id}', target_lang,
|
| 5708 |
claim_sources={claim_id: source_claim}, keep_claim_markers=True,
|
|
|
|
| 5709 |
)
|
| 5710 |
else:
|
| 5711 |
raise
|
|
@@ -5820,15 +5906,20 @@ Each code must be one of: {sorted(allowed)}. Return one or more specific dimensi
|
|
| 5820 |
# supported target languages. Russian sustained Latin prose
|
| 5821 |
# retains its existing independent deterministic authority.
|
| 5822 |
if code == 'LANGUAGE_CONTAMINATION':
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5823 |
lexical = self._translation_lexical_purity_evidence(
|
| 5824 |
-
source_claims[claim_id], target_claims[claim_id], target_lang,
|
| 5825 |
)
|
| 5826 |
lexical_confirmed = bool(lexical.get('confirmed'))
|
| 5827 |
deterministic = lexical_confirmed
|
| 5828 |
deterministic_evidence_code = 'UNTRANSLATED_LEXEME' if lexical_confirmed else ''
|
| 5829 |
if profile.get('language_code') == 'ru' and not deterministic:
|
| 5830 |
evidence = self._russian_contamination_evidence(
|
| 5831 |
-
source_claims[claim_id], target_claims[claim_id],
|
| 5832 |
)
|
| 5833 |
deterministic = bool(evidence.get('confirmed'))
|
| 5834 |
if deterministic:
|
|
@@ -5839,8 +5930,37 @@ Each code must be one of: {sorted(allowed)}. Return one or more specific dimensi
|
|
| 5839 |
'claim_id': claim_id, 'code': code, 'english': source_claims[claim_id],
|
| 5840 |
'target': target_claims[claim_id], 'deterministic_confirmed': True,
|
| 5841 |
'deterministic_evidence_code': deterministic_evidence_code,
|
|
|
|
| 5842 |
})
|
| 5843 |
continue
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5844 |
rows.append({
|
| 5845 |
'id': f'SP-P{block_id}-{claim_id}', 'block_id': block_id,
|
| 5846 |
'claim_id': claim_id, 'code': code, 'english': source_claims[claim_id],
|
|
@@ -5868,6 +5988,7 @@ Each code must be one of: {sorted(allowed)}. Return one or more specific dimensi
|
|
| 5868 |
'CLAIM_MISSING','CLAIM_ADDED','POLARITY_CHANGED','CAUSALITY_CHANGED',
|
| 5869 |
'UNCERTAINTY_CHANGED','MEANING_CHANGED','CITATION_MOVED','ACRONYM_MISATTACHED',
|
| 5870 |
'GRAMMATICAL_VIOLATION','GLOSSARY_CONCEPT_MISMATCH','UNTRANSLATED_PROSE','UNTRANSLATED_LEXEME',
|
|
|
|
| 5871 |
'ACCEPTABLE_VARIANT','MORPHOLOGICAL_VARIANT','STYLISTIC_PREFERENCE',
|
| 5872 |
'QA_DISAGREEMENT','NO_BLOCKING_EVIDENCE',
|
| 5873 |
}
|
|
@@ -5877,16 +5998,23 @@ Each code must be one of: {sorted(allowed)}. Return one or more specific dimensi
|
|
| 5877 |
'language_adapter': profile.get('adapter_id'),
|
| 5878 |
'morphology': profile.get('morphology'),
|
| 5879 |
'term_policy': profile.get('term_policy'),
|
| 5880 |
-
'items': [
|
| 5881 |
-
|
| 5882 |
-
|
| 5883 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5884 |
}
|
| 5885 |
lexical_contract = self._translation_lexical_retention_contract(target_lang)
|
| 5886 |
system_prompt = f"""You are an independent bilingual translation QA adjudicator for {target_lang}.
|
| 5887 |
This is verification-only: do not rewrite, suggest wording, or emit free-form explanations. Each supplied issue already survived two bounded claim-local repair rounds. Confirm a BLOCKING defect only when the English/target pair itself provides clear evidence. Do not treat stylistic preference or a legitimate grammatical/morphological variant as blocking.
|
| 5888 |
{lexical_contract}
|
| 5889 |
For Russian, allow required case/number/gender inflection and retain sustained non-Russian prose as an additional contamination signal. For Kazakh, allow productive case, possessive, person/plural and vowel-harmony suffixes on the correct term stem, but treat clearly Russian lexical/syntactic substitution as blocking only when the pair itself supports that conclusion. For Chinese, allow natural reordering and sentence reshaping inside the same claim while preserving meaning.
|
|
|
|
| 5890 |
Return valid JSON only: {{"items":[{{"id":"SP-P000001-C000001","issue_code":"GRAMMAR","decision":"CONFIRMED_BLOCKING","evidence_code":"GRAMMATICAL_VIOLATION"}}]}}.
|
| 5891 |
Decision must be CONFIRMED_BLOCKING or NOT_CONFIRMED. evidence_code must be one of: {sorted(allowed_evidence)}. Return every supplied id+issue_code pair exactly once."""
|
| 5892 |
try:
|
|
@@ -5914,6 +6042,15 @@ Decision must be CONFIRMED_BLOCKING or NOT_CONFIRMED. evidence_code must be one
|
|
| 5914 |
evidence_code = str(item.get('evidence_code') or '').strip().upper()
|
| 5915 |
if decision not in {'CONFIRMED_BLOCKING', 'NOT_CONFIRMED'} or evidence_code not in allowed_evidence:
|
| 5916 |
raise DeepSeekRequestError(f'Translation QA adjudication returned invalid bounded decision for {row["id"]}.')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5917 |
if decision == 'CONFIRMED_BLOCKING':
|
| 5918 |
confirmed.setdefault(row['block_id'], {}).setdefault(row['claim_id'], []).append(row['code'])
|
| 5919 |
self._record_translation_assurance(
|
|
@@ -5989,6 +6126,7 @@ Decision must be CONFIRMED_BLOCKING or NOT_CONFIRMED. evidence_code must be one
|
|
| 5989 |
assurance_start = len(self.translation_assurance_log)
|
| 5990 |
profile = self._translation_language_profile(target_lang)
|
| 5991 |
frozen_glossary = self.extend_translation_glossary(value, target_lang, glossary)
|
|
|
|
| 5992 |
units = self._translation_body_units(value)
|
| 5993 |
body_units = [unit for unit in units if unit['kind'] == 'body']
|
| 5994 |
if not body_units:
|
|
@@ -6163,6 +6301,7 @@ Decision must be CONFIRMED_BLOCKING or NOT_CONFIRMED. evidence_code must be one
|
|
| 6163 |
)
|
| 6164 |
if lexical.get('confirmed'):
|
| 6165 |
repair_evidence['detected_unprotected_english_tokens'] = list(lexical.get('tokens') or [])[:8]
|
|
|
|
| 6166 |
self._record_translation_assurance(
|
| 6167 |
'LEXICAL_PURITY_REPAIR_EVIDENCE', target_lang=target_lang,
|
| 6168 |
block_id=f'SP-P{block_id}-{claim_id}', severity='repair', decision='REPAIR_LOCAL',
|
|
@@ -6171,6 +6310,21 @@ Decision must be CONFIRMED_BLOCKING or NOT_CONFIRMED. evidence_code must be one
|
|
| 6171 |
'sample': ','.join(lexical.get('tokens') or [])[:120],
|
| 6172 |
},
|
| 6173 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6174 |
self._translation_claim_qa_evidence[evidence_key] = dict(repair_evidence)
|
| 6175 |
signature = self._translation_repair_evidence_signature(codes, repair_evidence)
|
| 6176 |
meta = self._translation_claim_repair_meta.setdefault(evidence_key, {
|
|
@@ -6371,6 +6525,7 @@ Decision must be CONFIRMED_BLOCKING or NOT_CONFIRMED. evidence_code must be one
|
|
| 6371 |
raise DeepSeekRequestError('English manuscript contains no translatable content.')
|
| 6372 |
|
| 6373 |
glossary = self._build_translation_glossary(body, target_lang)
|
|
|
|
| 6374 |
body_units = [unit for unit in units if unit['kind'] == 'body']
|
| 6375 |
translations = {}
|
| 6376 |
token_budget = max(800, int(APP_CONFIG.get('translation_batch_tokens', 2500)))
|
|
|
|
| 4631 |
return ' '.join(parts).strip()
|
| 4632 |
|
| 4633 |
@classmethod
|
| 4634 |
+
def _translation_glossary_authorized_acronyms(cls, source_claim, glossary=None):
|
| 4635 |
+
"""Return claim-local target acronyms authorized by a frozen glossary."""
|
| 4636 |
+
source_value = str(source_claim or '')
|
| 4637 |
+
allowed = Counter()
|
| 4638 |
+
for source_term, target_term in dict(glossary or {}).items():
|
| 4639 |
+
source_key = str(source_term or '').strip()
|
| 4640 |
+
target_value = str(target_term or '').strip()
|
| 4641 |
+
if not source_key or not target_value:
|
| 4642 |
+
continue
|
| 4643 |
+
occurrences = len(re.findall(
|
| 4644 |
+
rf'(?<![A-Za-z0-9_]){re.escape(source_key)}(?![A-Za-z0-9_])',
|
| 4645 |
+
source_value, flags=re.I,
|
| 4646 |
+
))
|
| 4647 |
+
if occurrences <= 0:
|
| 4648 |
+
continue
|
| 4649 |
+
for acronym, count in cls._translation_acronym_tokens(target_value).items():
|
| 4650 |
+
allowed[acronym] += int(count) * int(occurrences)
|
| 4651 |
+
return allowed
|
| 4652 |
+
|
| 4653 |
+
@classmethod
|
| 4654 |
+
def _translation_unowned_objects(cls, protected_target, source_claim='', glossary=None):
|
| 4655 |
+
"""Find non-anchor factual surfaces with claim-local glossary authorization."""
|
| 4656 |
text = re.sub(r'\[\[SP-T\d{6}\]\]', ' ', str(protected_target or ''))
|
| 4657 |
text = re.sub(r'\[\[SP-P\d{6}\]\]', ' ', text)
|
| 4658 |
text = re.sub(r'\[\[SP-C\d{6}\]\]', ' ', text)
|
| 4659 |
+
acronym_tokens = cls._translation_acronym_tokens(text)
|
| 4660 |
+
authorized_acronyms = cls._translation_glossary_authorized_acronyms(source_claim, glossary)
|
| 4661 |
+
for acronym, count in authorized_acronyms.items():
|
| 4662 |
+
if acronym_tokens.get(acronym, 0) <= 0:
|
| 4663 |
+
continue
|
| 4664 |
+
acronym_tokens[acronym] -= min(int(count), int(acronym_tokens[acronym]))
|
| 4665 |
+
if acronym_tokens[acronym] <= 0:
|
| 4666 |
+
del acronym_tokens[acronym]
|
| 4667 |
unowned = {
|
| 4668 |
'numeric': list(cls._numeric_tokens(text).elements()),
|
| 4669 |
'citation': list(cls._translation_citation_tokens(text).elements()),
|
| 4670 |
'hypothesis': list(cls._translation_hypothesis_tokens(text).elements()),
|
| 4671 |
+
'acronym': list(acronym_tokens.elements()),
|
| 4672 |
}
|
| 4673 |
names = re.findall(r'\b(?:fsQCA|SmartPLS|Mplus|OpenAlex|DeepSeek|ChatGPT|RStudio|SPSS|AMOS|Stata)\b', text)
|
| 4674 |
if names:
|
|
|
|
| 4728 |
)
|
| 4729 |
|
| 4730 |
@classmethod
|
| 4731 |
+
def _validate_translation_skeleton(cls, value, mapping, block_id='', claim_sources=None, glossary=None):
|
| 4732 |
text = str(value or '')
|
| 4733 |
found = Counter(re.findall(r'\[\[SP-T\d{6}\]\]', text))
|
| 4734 |
expected = Counter((mapping or {}).keys())
|
|
|
|
| 4774 |
f'Translation order-sensitive relation changed within owner claim in {block_id}: claim={claim_id}.'
|
| 4775 |
)
|
| 4776 |
|
| 4777 |
+
unowned = cls._translation_unowned_objects(
|
| 4778 |
+
target_claim, source_claim=source_claim, glossary=glossary
|
| 4779 |
+
)
|
| 4780 |
numeric = list(unowned.pop('numeric', []))
|
| 4781 |
if unowned:
|
| 4782 |
safe_summary = {key: len(values) for key, values in unowned.items()}
|
|
|
|
| 4805 |
return True
|
| 4806 |
|
| 4807 |
@classmethod
|
| 4808 |
+
def _restore_translation_tokens(
|
| 4809 |
+
cls, value, mapping, block_id='', target_lang='', claim_sources=None,
|
| 4810 |
+
keep_claim_markers=False, glossary=None,
|
| 4811 |
+
):
|
| 4812 |
text = str(value or '')
|
| 4813 |
+
cls._validate_translation_skeleton(
|
| 4814 |
+
text, mapping, block_id, claim_sources=claim_sources, glossary=glossary
|
| 4815 |
+
)
|
| 4816 |
for placeholder, manifest in (mapping or {}).items():
|
| 4817 |
text = text.replace(placeholder, cls._render_translation_fact(manifest, target_lang))
|
| 4818 |
if re.search(r'\[\[SP-T\d{6}\]\]', text):
|
|
|
|
| 5082 |
'CASE_GOVERNMENT','AGREEMENT','MORPHOLOGY','VERB_FORM','WORD_ORDER',
|
| 5083 |
'SYNTAX_STRUCTURE','PUNCTUATION','SYNTACTIC_COMPLETENESS','GRAMMAR_UNSPECIFIED',
|
| 5084 |
}
|
| 5085 |
+
allowed_contamination_evidence = {
|
| 5086 |
+
'UNTRANSLATED_LEXEME','UNTRANSLATED_PROSE',
|
| 5087 |
+
'KK_RUSSIAN_LEXICAL_INTRUSION','KK_RUSSIAN_SYNTACTIC_INTRUSION',
|
| 5088 |
+
}
|
| 5089 |
if not isinstance(getattr(self, '_translation_claim_qa_evidence', None), dict):
|
| 5090 |
self._translation_claim_qa_evidence = {}
|
| 5091 |
claim_rows = []
|
|
|
|
| 5128 |
LANGUAGE ADAPTER: {profile['adapter_id']}; morphology={profile['morphology']}. {profile['term_policy']} {lexical_contract} {profile['contamination_policy']}
|
| 5129 |
For Russian, do not flag a terminology error merely because a translated term is inflected for grammatical case/number/gender; sustained non-Russian prose remains an additional contamination signal. For Kazakh, do not flag a terminology error merely because productive suffixes attach to the correct term stem; clearly Russian lexical/syntactic intrusion remains an additional language-specific signal when the pair supports that conclusion. Cyrillic script or absence of Kazakh-specific letters alone is never sufficient evidence.
|
| 5130 |
Return valid JSON only in this exact form: {{"items":[{{"id":"SP-P000001-C000001","ok":true,"codes":[],"evidence_codes":[]}}]}}.
|
| 5131 |
+
If ok=false, codes MUST contain only values from allowed_issue_codes. For SEMANTIC_DRIFT, NEGATION_DRIFT, CAUSALITY_STRENGTH_DRIFT, UNCERTAINTY_DRIFT, OMITTED_CLAIM, or ADDED_CLAIM, evidence_codes should identify the affected semantic dimension using only: {sorted(allowed_semantic_evidence)}. For GRAMMAR, evidence_codes should identify the narrow grammatical defect using only: {sorted(allowed_grammar_evidence)}. For LANGUAGE_CONTAMINATION, evidence_codes MUST identify the supported subtype using only: {sorted(allowed_contamination_evidence)}. Use UNTRANSLATED_LEXEME/UNTRANSLATED_PROSE only for Latin/English residue. For Kazakh, use KK_RUSSIAN_LEXICAL_INTRUSION or KK_RUSSIAN_SYNTACTIC_INTRUSION only when the English/target pair itself clearly supports Russian substitution or a Russian syntactic calque; Cyrillic script or absence of Kazakh-specific letters is never sufficient. Use the narrowest supported evidence; use MEANING_CHANGED or GRAMMAR_UNSPECIFIED only when the narrower dimension cannot be identified from the pair. Do not invent an error dimension. Never return numbered prose, examples, quotes, or corrective wording. Never return replacement text. Return every supplied id exactly once."""
|
| 5132 |
response = self.call_deepseek(
|
| 5133 |
system_prompt, json.dumps(payload, ensure_ascii=False),
|
| 5134 |
max_tokens=max(1200, min(3600, 650 + 260 * len(rows))),
|
|
|
|
| 5155 |
]
|
| 5156 |
evidence_codes = [code for code in raw_evidence_codes if code in allowed_semantic_evidence]
|
| 5157 |
grammar_evidence_codes = [code for code in raw_evidence_codes if code in allowed_grammar_evidence]
|
| 5158 |
+
contamination_evidence_codes = [
|
| 5159 |
+
code for code in raw_evidence_codes if code in allowed_contamination_evidence
|
| 5160 |
+
]
|
| 5161 |
semantic_code_set = {
|
| 5162 |
'SEMANTIC_DRIFT','NEGATION_DRIFT','CAUSALITY_STRENGTH_DRIFT',
|
| 5163 |
'UNCERTAINTY_DRIFT','OMITTED_CLAIM','ADDED_CLAIM',
|
|
|
|
| 5187 |
(meta['block_id'], meta['claim_id']), {}
|
| 5188 |
)
|
| 5189 |
evidence_row['grammar_evidence_codes'] = grammar_evidence_codes[:4]
|
| 5190 |
+
if 'LANGUAGE_CONTAMINATION' in codes and contamination_evidence_codes:
|
| 5191 |
+
evidence_row = self._translation_claim_qa_evidence.setdefault(
|
| 5192 |
+
(meta['block_id'], meta['claim_id']), {}
|
| 5193 |
+
)
|
| 5194 |
+
evidence_row['contamination_evidence_codes'] = contamination_evidence_codes[:4]
|
| 5195 |
model_contamination = 'LANGUAGE_CONTAMINATION' in codes
|
| 5196 |
lexical = self._translation_lexical_purity_evidence(
|
| 5197 |
meta['english'], meta['target'], target_lang, relevant_glossary
|
|
|
|
| 5232 |
'RU_LANGUAGE_CONTAMINATION_UNCONFIRMED', target_lang=target_lang, block_id=row_id,
|
| 5233 |
severity='diagnostic', decision='PASS_WITH_WARNING', detail=contamination,
|
| 5234 |
)
|
| 5235 |
+
elif language_code == 'zh' and model_contamination and not deterministic_contamination:
|
| 5236 |
+
codes = [code for code in codes if code != 'LANGUAGE_CONTAMINATION']
|
| 5237 |
+
self._record_translation_assurance(
|
| 5238 |
+
'ZH_LANGUAGE_CONTAMINATION_UNCONFIRMED', target_lang=target_lang, block_id=row_id,
|
| 5239 |
+
severity='diagnostic', decision='PASS_WITH_WARNING',
|
| 5240 |
+
detail={'deterministic_token_count': int(lexical.get('token_count') or 0)},
|
| 5241 |
+
)
|
| 5242 |
+
elif language_code == 'kk' and model_contamination and not deterministic_contamination:
|
| 5243 |
+
russian_intrusion = [
|
| 5244 |
+
code for code in contamination_evidence_codes
|
| 5245 |
+
if code in {'KK_RUSSIAN_LEXICAL_INTRUSION','KK_RUSSIAN_SYNTACTIC_INTRUSION'}
|
| 5246 |
+
]
|
| 5247 |
+
if russian_intrusion:
|
| 5248 |
+
evidence_row = self._translation_claim_qa_evidence.setdefault(
|
| 5249 |
+
(meta['block_id'], meta['claim_id']), {}
|
| 5250 |
+
)
|
| 5251 |
+
evidence_row['contamination_evidence_codes'] = russian_intrusion[:2]
|
| 5252 |
+
else:
|
| 5253 |
+
codes = [code for code in codes if code != 'LANGUAGE_CONTAMINATION']
|
| 5254 |
+
self._record_translation_assurance(
|
| 5255 |
+
'KK_LANGUAGE_CONTAMINATION_UNCONFIRMED', target_lang=target_lang, block_id=row_id,
|
| 5256 |
+
severity='diagnostic', decision='PASS_WITH_WARNING',
|
| 5257 |
+
detail={'deterministic_token_count': int(lexical.get('token_count') or 0)},
|
| 5258 |
+
)
|
| 5259 |
|
| 5260 |
# Deterministic lexical residue can override an LLM false negative
|
| 5261 |
+
# in ZH/RU/KK. LLM-only English/Latin contamination is suppressed
|
| 5262 |
+
# when deterministic evidence is absent; Kazakh Russian intrusion
|
| 5263 |
+
# survives only with its dedicated typed evidence.
|
| 5264 |
if returned_row.get('ok') is True and not deterministic_contamination and not deterministic_issue:
|
| 5265 |
continue
|
| 5266 |
if not codes:
|
| 5267 |
only_suppressed_contamination = (
|
| 5268 |
+
language_code in {'zh','ru','kk'} and model_contamination and not deterministic_contamination
|
| 5269 |
+
and 'LANGUAGE_CONTAMINATION' not in codes
|
| 5270 |
and bool(raw_codes) and all(code == 'LANGUAGE_CONTAMINATION' for code in raw_codes)
|
| 5271 |
)
|
| 5272 |
if only_suppressed_contamination:
|
|
|
|
| 5558 |
translated_with_claims = self._restore_translation_tokens(
|
| 5559 |
parsed[block_id], protected_maps.get(block_id, {}), f'SP-P{block_id}', target_lang,
|
| 5560 |
claim_sources=claim_sources_by_block.get(block_id, {}), keep_claim_markers=True,
|
| 5561 |
+
glossary=getattr(self, '_active_translation_glossary', {}),
|
| 5562 |
)
|
| 5563 |
clean_translation = self._strip_translation_claim_markers(translated_with_claims)
|
| 5564 |
self._validate_translation_block_protected(block_id, source, clean_translation, target_lang)
|
|
|
|
| 5593 |
translated_with_claims = self._restore_translation_tokens(
|
| 5594 |
repaired, protected_maps.get(block_id, {}), f'SP-P{block_id}', target_lang,
|
| 5595 |
claim_sources=source_claims, keep_claim_markers=True,
|
| 5596 |
+
glossary=getattr(self, '_active_translation_glossary', {}),
|
| 5597 |
)
|
| 5598 |
clean_translation = self._strip_translation_claim_markers(translated_with_claims)
|
| 5599 |
self._validate_translation_block_protected(
|
|
|
|
| 5619 |
'issue_codes': sorted({str(code or '').strip().upper() for code in (issue_codes or []) if str(code or '').strip()}),
|
| 5620 |
'semantic_delta_codes': sorted({str(code or '').strip().upper() for code in (evidence.get('semantic_delta_codes') or []) if str(code or '').strip()}),
|
| 5621 |
'grammar_evidence_codes': sorted({str(code or '').strip().upper() for code in (evidence.get('grammar_evidence_codes') or []) if str(code or '').strip()}),
|
| 5622 |
+
'contamination_evidence_codes': sorted({str(code or '').strip().upper() for code in (evidence.get('contamination_evidence_codes') or []) if str(code or '').strip()}),
|
| 5623 |
'detected_unprotected_english_tokens': sorted({str(token or '').strip().casefold() for token in (evidence.get('detected_unprotected_english_tokens') or []) if str(token or '').strip()}),
|
| 5624 |
}
|
| 5625 |
payload = json.dumps(bounded, ensure_ascii=False, sort_keys=True, separators=(',', ':'))
|
|
|
|
| 5702 |
]
|
| 5703 |
if grammar_evidence_codes:
|
| 5704 |
evidence_payload['grammar_evidence_codes'] = grammar_evidence_codes[:4]
|
| 5705 |
+
contamination_evidence_codes = [
|
| 5706 |
+
str(code or '').strip().upper() for code in (supplied_evidence.get('contamination_evidence_codes') or [])
|
| 5707 |
+
if str(code or '').strip()
|
| 5708 |
+
]
|
| 5709 |
+
if contamination_evidence_codes:
|
| 5710 |
+
evidence_payload['contamination_evidence_codes'] = contamination_evidence_codes[:4]
|
| 5711 |
lexical_contract = self._translation_lexical_retention_contract(target_lang)
|
| 5712 |
retry_mode = (
|
| 5713 |
'\nCLAIM SOURCE REGENERATION. The previous target wording has failed bounded repair and MUST NOT be used as drafting context. Generate a fresh translation of only this one claim directly from the frozen marked English source. '
|
|
|
|
| 5722 |
'If detected_unprotected_english_tokens is non-empty, every listed token is deterministic lexical-purity evidence from a prior target and MUST NOT survive as ordinary untranslated target-language prose unless represented by a protected SP-T anchor. '
|
| 5723 |
'If semantic_delta_codes is non-empty, enforce ONLY those evidenced semantic dimensions against the frozen English source and preserve all unaffected actor, relation, object, polarity, modality, condition, comparison, temporal scope, and evidentiary strength. UNRESOLVED_SEMANTIC_DELTA is a signal to regenerate faithfully from source, not an instruction to guess a specific correction. '
|
| 5724 |
'If grammar_evidence_codes is non-empty, repair or avoid only those bounded grammatical defect classes; GRAMMAR_UNSPECIFIED requires a fresh natural target-language rendering rather than blind micro-editing. '
|
| 5725 |
+
'If contamination_evidence_codes identifies KK_RUSSIAN_LEXICAL_INTRUSION or KK_RUSSIAN_SYNTACTIC_INTRUSION, remove only the evidenced Russian substitution/calque while preserving the frozen English meaning; do not invent an English lexical defect when deterministic English-token evidence is absent. '
|
| 5726 |
'Preserve every SP-P, SP-C, and SP-T marker byte-for-byte. Do not add claims or factual content. Return only the marked translated claim.'
|
| 5727 |
+ '\n<CLAIM_REPAIR_EVIDENCE>\n'
|
| 5728 |
+ json.dumps(evidence_payload, ensure_ascii=False)
|
|
|
|
| 5741 |
restored = self._restore_translation_tokens(
|
| 5742 |
parsed[block_id], mapping, f'SP-P{block_id}', target_lang,
|
| 5743 |
claim_sources={claim_id: source_claim}, keep_claim_markers=True,
|
| 5744 |
+
glossary=getattr(self, '_active_translation_glossary', {}),
|
| 5745 |
)
|
| 5746 |
except DeepSeekRequestError as error:
|
| 5747 |
if str(error or '').startswith('Translation protected placeholder mismatch'):
|
|
|
|
| 5766 |
restored = self._restore_translation_tokens(
|
| 5767 |
strict_parsed[block_id], mapping, f'SP-P{block_id}', target_lang,
|
| 5768 |
claim_sources={claim_id: source_claim}, keep_claim_markers=True,
|
| 5769 |
+
glossary=getattr(self, '_active_translation_glossary', {}),
|
| 5770 |
)
|
| 5771 |
elif self._is_translation_unowned_numeric_error(error):
|
| 5772 |
logging.warning(
|
|
|
|
| 5791 |
restored = self._restore_translation_tokens(
|
| 5792 |
retry_parsed[block_id], mapping, f'SP-P{block_id}', target_lang,
|
| 5793 |
claim_sources={claim_id: source_claim}, keep_claim_markers=True,
|
| 5794 |
+
glossary=getattr(self, '_active_translation_glossary', {}),
|
| 5795 |
)
|
| 5796 |
else:
|
| 5797 |
raise
|
|
|
|
| 5906 |
# supported target languages. Russian sustained Latin prose
|
| 5907 |
# retains its existing independent deterministic authority.
|
| 5908 |
if code == 'LANGUAGE_CONTAMINATION':
|
| 5909 |
+
relevant_glossary = {
|
| 5910 |
+
str(source_term): str(target_term)
|
| 5911 |
+
for source_term, target_term in dict(glossary or {}).items()
|
| 5912 |
+
if source_term and str(source_term).casefold() in source_claims[claim_id].casefold()
|
| 5913 |
+
}
|
| 5914 |
lexical = self._translation_lexical_purity_evidence(
|
| 5915 |
+
source_claims[claim_id], target_claims[claim_id], target_lang, relevant_glossary
|
| 5916 |
)
|
| 5917 |
lexical_confirmed = bool(lexical.get('confirmed'))
|
| 5918 |
deterministic = lexical_confirmed
|
| 5919 |
deterministic_evidence_code = 'UNTRANSLATED_LEXEME' if lexical_confirmed else ''
|
| 5920 |
if profile.get('language_code') == 'ru' and not deterministic:
|
| 5921 |
evidence = self._russian_contamination_evidence(
|
| 5922 |
+
source_claims[claim_id], target_claims[claim_id], relevant_glossary
|
| 5923 |
)
|
| 5924 |
deterministic = bool(evidence.get('confirmed'))
|
| 5925 |
if deterministic:
|
|
|
|
| 5930 |
'claim_id': claim_id, 'code': code, 'english': source_claims[claim_id],
|
| 5931 |
'target': target_claims[claim_id], 'deterministic_confirmed': True,
|
| 5932 |
'deterministic_evidence_code': deterministic_evidence_code,
|
| 5933 |
+
'lexical_evidence': lexical, 'glossary': relevant_glossary,
|
| 5934 |
})
|
| 5935 |
continue
|
| 5936 |
+
repair_evidence = dict(
|
| 5937 |
+
getattr(self, '_translation_claim_qa_evidence', {}).get((block_id, claim_id), {}) or {}
|
| 5938 |
+
)
|
| 5939 |
+
typed = [
|
| 5940 |
+
str(item or '').strip().upper()
|
| 5941 |
+
for item in (repair_evidence.get('contamination_evidence_codes') or [])
|
| 5942 |
+
if str(item or '').strip()
|
| 5943 |
+
]
|
| 5944 |
+
kk_typed = [
|
| 5945 |
+
item for item in typed
|
| 5946 |
+
if item in {'KK_RUSSIAN_LEXICAL_INTRUSION','KK_RUSSIAN_SYNTACTIC_INTRUSION'}
|
| 5947 |
+
]
|
| 5948 |
+
if profile.get('language_code') in {'zh','kk'} and not kk_typed:
|
| 5949 |
+
self._record_translation_assurance(
|
| 5950 |
+
f"{profile.get('language_code','').upper()}_QA_ADJUDICATION_LEXICAL_UNCONFIRMED",
|
| 5951 |
+
target_lang=target_lang, block_id=f'SP-P{block_id}-{claim_id}',
|
| 5952 |
+
severity='diagnostic', decision='PASS_WITH_WARNING',
|
| 5953 |
+
detail={'deterministic_token_count': int(lexical.get('token_count') or 0)},
|
| 5954 |
+
)
|
| 5955 |
+
continue
|
| 5956 |
+
rows.append({
|
| 5957 |
+
'id': f'SP-P{block_id}-{claim_id}', 'block_id': block_id,
|
| 5958 |
+
'claim_id': claim_id, 'code': code, 'english': source_claims[claim_id],
|
| 5959 |
+
'target': target_claims[claim_id], 'deterministic_confirmed': False,
|
| 5960 |
+
'lexical_evidence': lexical, 'glossary': relevant_glossary,
|
| 5961 |
+
'contamination_evidence_codes': kk_typed[:2],
|
| 5962 |
+
})
|
| 5963 |
+
continue
|
| 5964 |
rows.append({
|
| 5965 |
'id': f'SP-P{block_id}-{claim_id}', 'block_id': block_id,
|
| 5966 |
'claim_id': claim_id, 'code': code, 'english': source_claims[claim_id],
|
|
|
|
| 5988 |
'CLAIM_MISSING','CLAIM_ADDED','POLARITY_CHANGED','CAUSALITY_CHANGED',
|
| 5989 |
'UNCERTAINTY_CHANGED','MEANING_CHANGED','CITATION_MOVED','ACRONYM_MISATTACHED',
|
| 5990 |
'GRAMMATICAL_VIOLATION','GLOSSARY_CONCEPT_MISMATCH','UNTRANSLATED_PROSE','UNTRANSLATED_LEXEME',
|
| 5991 |
+
'KK_RUSSIAN_LEXICAL_INTRUSION','KK_RUSSIAN_SYNTACTIC_INTRUSION',
|
| 5992 |
'ACCEPTABLE_VARIANT','MORPHOLOGICAL_VARIANT','STYLISTIC_PREFERENCE',
|
| 5993 |
'QA_DISAGREEMENT','NO_BLOCKING_EVIDENCE',
|
| 5994 |
}
|
|
|
|
| 5998 |
'language_adapter': profile.get('adapter_id'),
|
| 5999 |
'morphology': profile.get('morphology'),
|
| 6000 |
'term_policy': profile.get('term_policy'),
|
| 6001 |
+
'items': [{
|
| 6002 |
+
'id': row['id'], 'issue_code': row['code'], 'english': row['english'], 'target': row['target'],
|
| 6003 |
+
'glossary': dict(row.get('glossary') or {}),
|
| 6004 |
+
'deterministic_lexical': {
|
| 6005 |
+
'confirmed': bool((row.get('lexical_evidence') or {}).get('confirmed')),
|
| 6006 |
+
'token_count': int((row.get('lexical_evidence') or {}).get('token_count') or 0),
|
| 6007 |
+
'tokens': list((row.get('lexical_evidence') or {}).get('tokens') or [])[:8],
|
| 6008 |
+
},
|
| 6009 |
+
'contamination_evidence_codes': list(row.get('contamination_evidence_codes') or [])[:2],
|
| 6010 |
+
} for row in model_rows],
|
| 6011 |
}
|
| 6012 |
lexical_contract = self._translation_lexical_retention_contract(target_lang)
|
| 6013 |
system_prompt = f"""You are an independent bilingual translation QA adjudicator for {target_lang}.
|
| 6014 |
This is verification-only: do not rewrite, suggest wording, or emit free-form explanations. Each supplied issue already survived two bounded claim-local repair rounds. Confirm a BLOCKING defect only when the English/target pair itself provides clear evidence. Do not treat stylistic preference or a legitimate grammatical/morphological variant as blocking.
|
| 6015 |
{lexical_contract}
|
| 6016 |
For Russian, allow required case/number/gender inflection and retain sustained non-Russian prose as an additional contamination signal. For Kazakh, allow productive case, possessive, person/plural and vowel-harmony suffixes on the correct term stem, but treat clearly Russian lexical/syntactic substitution as blocking only when the pair itself supports that conclusion. For Chinese, allow natural reordering and sentence reshaping inside the same claim while preserving meaning.
|
| 6017 |
+
For LANGUAGE_CONTAMINATION, deterministic_lexical is authoritative for ordinary English/Latin residue: when confirmed=false and token_count=0, you MUST NOT confirm UNTRANSLATED_LEXEME or UNTRANSLATED_PROSE. For Kazakh, a non-Latin contamination blocker may be confirmed only when contamination_evidence_codes contains KK_RUSSIAN_LEXICAL_INTRUSION or KK_RUSSIAN_SYNTACTIC_INTRUSION, and the evidence_code must match one of those supplied typed codes. The per-item glossary lists the frozen target terms that are legal for that source claim.
|
| 6018 |
Return valid JSON only: {{"items":[{{"id":"SP-P000001-C000001","issue_code":"GRAMMAR","decision":"CONFIRMED_BLOCKING","evidence_code":"GRAMMATICAL_VIOLATION"}}]}}.
|
| 6019 |
Decision must be CONFIRMED_BLOCKING or NOT_CONFIRMED. evidence_code must be one of: {sorted(allowed_evidence)}. Return every supplied id+issue_code pair exactly once."""
|
| 6020 |
try:
|
|
|
|
| 6042 |
evidence_code = str(item.get('evidence_code') or '').strip().upper()
|
| 6043 |
if decision not in {'CONFIRMED_BLOCKING', 'NOT_CONFIRMED'} or evidence_code not in allowed_evidence:
|
| 6044 |
raise DeepSeekRequestError(f'Translation QA adjudication returned invalid bounded decision for {row["id"]}.')
|
| 6045 |
+
if row['code'] == 'LANGUAGE_CONTAMINATION':
|
| 6046 |
+
lexical = dict(row.get('lexical_evidence') or {})
|
| 6047 |
+
typed = set(row.get('contamination_evidence_codes') or [])
|
| 6048 |
+
if not bool(lexical.get('confirmed')) and evidence_code in {'UNTRANSLATED_LEXEME','UNTRANSLATED_PROSE'}:
|
| 6049 |
+
decision = 'NOT_CONFIRMED'
|
| 6050 |
+
evidence_code = 'QA_DISAGREEMENT'
|
| 6051 |
+
if typed and evidence_code not in typed and decision == 'CONFIRMED_BLOCKING':
|
| 6052 |
+
decision = 'NOT_CONFIRMED'
|
| 6053 |
+
evidence_code = 'QA_DISAGREEMENT'
|
| 6054 |
if decision == 'CONFIRMED_BLOCKING':
|
| 6055 |
confirmed.setdefault(row['block_id'], {}).setdefault(row['claim_id'], []).append(row['code'])
|
| 6056 |
self._record_translation_assurance(
|
|
|
|
| 6126 |
assurance_start = len(self.translation_assurance_log)
|
| 6127 |
profile = self._translation_language_profile(target_lang)
|
| 6128 |
frozen_glossary = self.extend_translation_glossary(value, target_lang, glossary)
|
| 6129 |
+
self._active_translation_glossary = dict(frozen_glossary)
|
| 6130 |
units = self._translation_body_units(value)
|
| 6131 |
body_units = [unit for unit in units if unit['kind'] == 'body']
|
| 6132 |
if not body_units:
|
|
|
|
| 6301 |
)
|
| 6302 |
if lexical.get('confirmed'):
|
| 6303 |
repair_evidence['detected_unprotected_english_tokens'] = list(lexical.get('tokens') or [])[:8]
|
| 6304 |
+
repair_evidence['contamination_evidence_codes'] = ['UNTRANSLATED_LEXEME']
|
| 6305 |
self._record_translation_assurance(
|
| 6306 |
'LEXICAL_PURITY_REPAIR_EVIDENCE', target_lang=target_lang,
|
| 6307 |
block_id=f'SP-P{block_id}-{claim_id}', severity='repair', decision='REPAIR_LOCAL',
|
|
|
|
| 6310 |
'sample': ','.join(lexical.get('tokens') or [])[:120],
|
| 6311 |
},
|
| 6312 |
)
|
| 6313 |
+
elif self._translation_language_profile(target_lang).get('language_code') == 'kk':
|
| 6314 |
+
kk_intrusion = [
|
| 6315 |
+
str(code or '').strip().upper()
|
| 6316 |
+
for code in (repair_evidence.get('contamination_evidence_codes') or [])
|
| 6317 |
+
if str(code or '').strip().upper() in {
|
| 6318 |
+
'KK_RUSSIAN_LEXICAL_INTRUSION','KK_RUSSIAN_SYNTACTIC_INTRUSION'
|
| 6319 |
+
}
|
| 6320 |
+
]
|
| 6321 |
+
if kk_intrusion:
|
| 6322 |
+
repair_evidence['contamination_evidence_codes'] = kk_intrusion[:2]
|
| 6323 |
+
self._record_translation_assurance(
|
| 6324 |
+
'KK_RUSSIAN_INTRUSION_REPAIR_EVIDENCE', target_lang=target_lang,
|
| 6325 |
+
block_id=f'SP-P{block_id}-{claim_id}', severity='repair', decision='REPAIR_LOCAL',
|
| 6326 |
+
detail={'classes': '|'.join(kk_intrusion)[:120]},
|
| 6327 |
+
)
|
| 6328 |
self._translation_claim_qa_evidence[evidence_key] = dict(repair_evidence)
|
| 6329 |
signature = self._translation_repair_evidence_signature(codes, repair_evidence)
|
| 6330 |
meta = self._translation_claim_repair_meta.setdefault(evidence_key, {
|
|
|
|
| 6525 |
raise DeepSeekRequestError('English manuscript contains no translatable content.')
|
| 6526 |
|
| 6527 |
glossary = self._build_translation_glossary(body, target_lang)
|
| 6528 |
+
self._active_translation_glossary = dict(glossary)
|
| 6529 |
body_units = [unit for unit in units if unit['kind'] == 'body']
|
| 6530 |
translations = {}
|
| 6531 |
token_budget = max(800, int(APP_CONFIG.get('translation_batch_tokens', 2500)))
|