Spaces:
Running
Running
R3.2.8.6 Hierarchical Multilingual Translation Assurance Compiler
Browse files- core/generation_runner.py +46 -11
- core/pipeline_agent.py +221 -80
core/generation_runner.py
CHANGED
|
@@ -21,7 +21,7 @@ from typing import Callable, Any
|
|
| 21 |
from config import APP_CONFIG, AMJ_PROMPTS, ASQ_PROMPTS
|
| 22 |
from core.checkpoint_store import TaskCheckpointStore
|
| 23 |
from core.meta_extractor import extract_golden_metadata
|
| 24 |
-
from core.pipeline_agent import SSciPipelineAgent
|
| 25 |
from core.production_guard import build_validated_package, replace_abstract
|
| 26 |
from core.qualitative_engine import get_qualitative_engine
|
| 27 |
from core.quantitative_engine import get_quantitative_engine
|
|
@@ -33,7 +33,7 @@ ProgressCallback = Callable[..., None]
|
|
| 33 |
MIN_FINAL_REFERENCES = 50
|
| 34 |
LITERATURE_EVIDENCE_TARGET = 75
|
| 35 |
CITATION_PLAN_TARGET = 70
|
| 36 |
-
TRANSLATION_CONTRACT_VERSION = '
|
| 37 |
|
| 38 |
|
| 39 |
def _clean_trailing_references(text, keywords=("reference", "references", "reference list", "参考文献")):
|
|
@@ -267,17 +267,18 @@ class GenerationRunner:
|
|
| 267 |
parts = []
|
| 268 |
if include_front:
|
| 269 |
record = mapping.get('__front_matter__') or {}
|
| 270 |
-
if record.get('qa_status')
|
| 271 |
parts.append(str(record['translated_text']).strip())
|
| 272 |
for key, _title in GenerationRunner._unique_top_level_sections(adaptive_structure):
|
| 273 |
record = mapping.get(key) or {}
|
| 274 |
-
if record.get('qa_status')
|
| 275 |
parts.append(str(record['translated_text']).strip())
|
| 276 |
return '\n\n'.join(part for part in parts if part).strip()
|
| 277 |
|
| 278 |
def _translate_frozen_unit(
|
| 279 |
self, agent, *, unit_key: str, unit_name: str, source_text: str, target_lang: str,
|
| 280 |
translation_sections: dict, translation_glossary: dict, adaptive_structure, include_front=False,
|
|
|
|
| 281 |
):
|
| 282 |
source = str(source_text or '').strip()
|
| 283 |
if not source:
|
|
@@ -287,7 +288,7 @@ class GenerationRunner:
|
|
| 287 |
fact_ledger_hash = self._translation_ledger_hash(agent, source)
|
| 288 |
existing = dict((translation_sections or {}).get(unit_key) or {})
|
| 289 |
if (
|
| 290 |
-
existing.get('qa_status')
|
| 291 |
and existing.get('source_hash') == source_hash
|
| 292 |
and existing.get('target_lang') == target_lang
|
| 293 |
and existing.get('translation_contract_version') == contract_version
|
|
@@ -297,11 +298,36 @@ class GenerationRunner:
|
|
| 297 |
return translation_sections, translation_glossary
|
| 298 |
partial = self._assemble_partial_translation(translation_sections, adaptive_structure, include_front=include_front)
|
| 299 |
self._publish_target_preview(partial, f'translating:{unit_name}', force=True, complete=False)
|
| 300 |
-
result = agent.translate_frozen_section(
|
| 301 |
-
source, target_lang, glossary=translation_glossary, section_name=unit_name, max_attempts=2
|
| 302 |
-
)
|
| 303 |
-
updated_glossary = dict(result.get('glossary') or translation_glossary or {})
|
| 304 |
updated_sections = dict(translation_sections or {})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 305 |
updated_sections[unit_key] = {
|
| 306 |
'source_hash': source_hash,
|
| 307 |
'target_lang': target_lang,
|
|
@@ -309,7 +335,9 @@ class GenerationRunner:
|
|
| 309 |
'translation_contract_version': str(result.get('translation_contract_version') or contract_version),
|
| 310 |
'fact_ledger_hash': str(result.get('fact_ledger_hash') or fact_ledger_hash),
|
| 311 |
'translated_text': str(result.get('text') or '').strip(),
|
| 312 |
-
'qa_status': 'passed',
|
|
|
|
|
|
|
| 313 |
'attempts': int(result.get('attempts') or 1),
|
| 314 |
'completed_at': datetime.now(timezone.utc).isoformat(),
|
| 315 |
}
|
|
@@ -1070,7 +1098,7 @@ Output only final academic prose. Do not report the word count."""
|
|
| 1070 |
agent, unit_key=unit_key, unit_name=unit_name, source_text=source_text,
|
| 1071 |
target_lang=target_lang, translation_sections=translation_sections,
|
| 1072 |
translation_glossary=translation_glossary, adaptive_structure=adaptive_structure,
|
| 1073 |
-
include_front=True,
|
| 1074 |
)
|
| 1075 |
partial = self._assemble_partial_translation(
|
| 1076 |
translation_sections, adaptive_structure, include_front=True
|
|
@@ -1085,6 +1113,13 @@ Output only final academic prose. Do not report the word count."""
|
|
| 1085 |
pct = min(96, 94 + int(index * 2 / max(1, len(final_units))))
|
| 1086 |
self._checkpoint(f'translation_{unit_key}', state, pct)
|
| 1087 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1088 |
full_draft_translated = self._assemble_partial_translation(
|
| 1089 |
translation_sections, adaptive_structure, include_front=True
|
| 1090 |
).rstrip()
|
|
|
|
| 21 |
from config import APP_CONFIG, AMJ_PROMPTS, ASQ_PROMPTS
|
| 22 |
from core.checkpoint_store import TaskCheckpointStore
|
| 23 |
from core.meta_extractor import extract_golden_metadata
|
| 24 |
+
from core.pipeline_agent import SSciPipelineAgent, PipelineCancelledError
|
| 25 |
from core.production_guard import build_validated_package, replace_abstract
|
| 26 |
from core.qualitative_engine import get_qualitative_engine
|
| 27 |
from core.quantitative_engine import get_quantitative_engine
|
|
|
|
| 33 |
MIN_FINAL_REFERENCES = 50
|
| 34 |
LITERATURE_EVIDENCE_TARGET = 75
|
| 35 |
CITATION_PLAN_TARGET = 70
|
| 36 |
+
TRANSLATION_CONTRACT_VERSION = 'hierarchical-multilingual-v1'
|
| 37 |
|
| 38 |
|
| 39 |
def _clean_trailing_references(text, keywords=("reference", "references", "reference list", "参考文献")):
|
|
|
|
| 267 |
parts = []
|
| 268 |
if include_front:
|
| 269 |
record = mapping.get('__front_matter__') or {}
|
| 270 |
+
if record.get('qa_status') in {'passed', 'passed_with_warning'} and record.get('translated_text'):
|
| 271 |
parts.append(str(record['translated_text']).strip())
|
| 272 |
for key, _title in GenerationRunner._unique_top_level_sections(adaptive_structure):
|
| 273 |
record = mapping.get(key) or {}
|
| 274 |
+
if record.get('qa_status') in {'passed', 'passed_with_warning'} and record.get('translated_text'):
|
| 275 |
parts.append(str(record['translated_text']).strip())
|
| 276 |
return '\n\n'.join(part for part in parts if part).strip()
|
| 277 |
|
| 278 |
def _translate_frozen_unit(
|
| 279 |
self, agent, *, unit_key: str, unit_name: str, source_text: str, target_lang: str,
|
| 280 |
translation_sections: dict, translation_glossary: dict, adaptive_structure, include_front=False,
|
| 281 |
+
defer_failures=True,
|
| 282 |
):
|
| 283 |
source = str(source_text or '').strip()
|
| 284 |
if not source:
|
|
|
|
| 288 |
fact_ledger_hash = self._translation_ledger_hash(agent, source)
|
| 289 |
existing = dict((translation_sections or {}).get(unit_key) or {})
|
| 290 |
if (
|
| 291 |
+
existing.get('qa_status') in {'passed', 'passed_with_warning'}
|
| 292 |
and existing.get('source_hash') == source_hash
|
| 293 |
and existing.get('target_lang') == target_lang
|
| 294 |
and existing.get('translation_contract_version') == contract_version
|
|
|
|
| 298 |
return translation_sections, translation_glossary
|
| 299 |
partial = self._assemble_partial_translation(translation_sections, adaptive_structure, include_front=include_front)
|
| 300 |
self._publish_target_preview(partial, f'translating:{unit_name}', force=True, complete=False)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 301 |
updated_sections = dict(translation_sections or {})
|
| 302 |
+
try:
|
| 303 |
+
result = agent.translate_frozen_section(
|
| 304 |
+
source, target_lang, glossary=translation_glossary, section_name=unit_name, max_attempts=2
|
| 305 |
+
)
|
| 306 |
+
except PipelineCancelledError:
|
| 307 |
+
raise
|
| 308 |
+
except Exception as error:
|
| 309 |
+
if not defer_failures:
|
| 310 |
+
raise
|
| 311 |
+
code_method = getattr(agent, 'translation_error_code', None)
|
| 312 |
+
error_code = str(code_method(error) if callable(code_method) else 'TRANSLATION_PENDING')
|
| 313 |
+
updated_sections[unit_key] = {
|
| 314 |
+
'source_hash': source_hash,
|
| 315 |
+
'target_lang': target_lang,
|
| 316 |
+
'glossary_version': self._glossary_version(translation_glossary),
|
| 317 |
+
'translation_contract_version': contract_version,
|
| 318 |
+
'fact_ledger_hash': fact_ledger_hash,
|
| 319 |
+
'translated_text': '',
|
| 320 |
+
'qa_status': 'repair_pending',
|
| 321 |
+
'error_code': error_code,
|
| 322 |
+
'attempts': int(existing.get('attempts') or 0) + 1,
|
| 323 |
+
'deferred_at': datetime.now(timezone.utc).isoformat(),
|
| 324 |
+
}
|
| 325 |
+
logging.warning(
|
| 326 |
+
'Translation deferred until final gate | unit=%s code=%s error=%s',
|
| 327 |
+
unit_name, error_code, type(error).__name__,
|
| 328 |
+
)
|
| 329 |
+
return updated_sections, translation_glossary
|
| 330 |
+
updated_glossary = dict(result.get('glossary') or translation_glossary or {})
|
| 331 |
updated_sections[unit_key] = {
|
| 332 |
'source_hash': source_hash,
|
| 333 |
'target_lang': target_lang,
|
|
|
|
| 335 |
'translation_contract_version': str(result.get('translation_contract_version') or contract_version),
|
| 336 |
'fact_ledger_hash': str(result.get('fact_ledger_hash') or fact_ledger_hash),
|
| 337 |
'translated_text': str(result.get('text') or '').strip(),
|
| 338 |
+
'qa_status': str(result.get('qa_status') or 'passed'),
|
| 339 |
+
'language_adapter': str(result.get('language_adapter') or ''),
|
| 340 |
+
'diagnostics': list(result.get('diagnostics') or [])[-20:],
|
| 341 |
'attempts': int(result.get('attempts') or 1),
|
| 342 |
'completed_at': datetime.now(timezone.utc).isoformat(),
|
| 343 |
}
|
|
|
|
| 1098 |
agent, unit_key=unit_key, unit_name=unit_name, source_text=source_text,
|
| 1099 |
target_lang=target_lang, translation_sections=translation_sections,
|
| 1100 |
translation_glossary=translation_glossary, adaptive_structure=adaptive_structure,
|
| 1101 |
+
include_front=True, defer_failures=False,
|
| 1102 |
)
|
| 1103 |
partial = self._assemble_partial_translation(
|
| 1104 |
translation_sections, adaptive_structure, include_front=True
|
|
|
|
| 1113 |
pct = min(96, 94 + int(index * 2 / max(1, len(final_units))))
|
| 1114 |
self._checkpoint(f'translation_{unit_key}', state, pct)
|
| 1115 |
|
| 1116 |
+
unresolved_units = [
|
| 1117 |
+
key for key, _name, _source in final_units
|
| 1118 |
+
if (translation_sections.get(key) or {}).get('qa_status') not in {'passed', 'passed_with_warning'}
|
| 1119 |
+
or not (translation_sections.get(key) or {}).get('translated_text')
|
| 1120 |
+
]
|
| 1121 |
+
if unresolved_units:
|
| 1122 |
+
raise RuntimeError('translation_final_gate_unresolved:' + ','.join(unresolved_units[:8]))
|
| 1123 |
full_draft_translated = self._assemble_partial_translation(
|
| 1124 |
translation_sections, adaptive_structure, include_front=True
|
| 1125 |
).rstrip()
|
core/pipeline_agent.py
CHANGED
|
@@ -56,7 +56,7 @@ class CitationAuditError(DeepSeekRequestError):
|
|
| 56 |
|
| 57 |
|
| 58 |
class SSciPipelineAgent:
|
| 59 |
-
TRANSLATION_CONTRACT_VERSION = '
|
| 60 |
|
| 61 |
def __init__(self, ds_key, s2_key=""):
|
| 62 |
self.ds_key = str(ds_key or "").strip()
|
|
@@ -120,6 +120,9 @@ class SSciPipelineAgent:
|
|
| 120 |
self.citation_map = {}
|
| 121 |
self.used_citations = set()
|
| 122 |
self.citation_audit_log = []
|
|
|
|
|
|
|
|
|
|
| 123 |
self.lock = threading.RLock()
|
| 124 |
|
| 125 |
self.lit_target_papers = APP_CONFIG["lit_target_papers"]
|
|
@@ -273,6 +276,7 @@ class SSciPipelineAgent:
|
|
| 273 |
"citation_map": self.citation_map,
|
| 274 |
"used_citations": list(self.used_citations),
|
| 275 |
"citation_audit_log": list(self.citation_audit_log),
|
|
|
|
| 276 |
"global_memory": self.global_memory,
|
| 277 |
"s2_access_mode": self._s2_access_mode,
|
| 278 |
"s2_logical_searches": int(self._s2_logical_searches),
|
|
@@ -294,6 +298,7 @@ class SSciPipelineAgent:
|
|
| 294 |
self.citation_map = state_dict.get("citation_map", {})
|
| 295 |
self.used_citations = set(state_dict.get("used_citations", []))
|
| 296 |
self.citation_audit_log = list(state_dict.get("citation_audit_log", []))
|
|
|
|
| 297 |
self.global_memory = state_dict.get("global_memory", self.global_memory)
|
| 298 |
loaded_mode = str(state_dict.get("s2_access_mode") or "").strip().lower()
|
| 299 |
expected_mode = "key" if self.s2_key else "anonymous"
|
|
@@ -3277,12 +3282,104 @@ Output only the repaired section."""
|
|
| 3277 |
)['abstract']
|
| 3278 |
|
| 3279 |
@staticmethod
|
| 3280 |
-
def
|
| 3281 |
-
|
| 3282 |
-
|
| 3283 |
-
|
| 3284 |
-
|
| 3285 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3286 |
|
| 3287 |
@staticmethod
|
| 3288 |
def _translation_prompt(target_lang):
|
|
@@ -3955,26 +4052,40 @@ Output only the repaired section."""
|
|
| 3955 |
mapping[acronym] = acronym
|
| 3956 |
return mapping
|
| 3957 |
|
| 3958 |
-
|
| 3959 |
-
|
| 3960 |
text = str(translated or '')
|
| 3961 |
compact = re.sub(r'\s+', '', text)
|
| 3962 |
if len(compact) < 80:
|
| 3963 |
return True
|
| 3964 |
-
|
| 3965 |
-
|
| 3966 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3967 |
return True
|
|
|
|
| 3968 |
cyrillic = re.findall(r'[А-Яа-яЁёӘәҒғҚқҢңӨөҰұҮүҺһІі]', text)
|
| 3969 |
if not cyrillic:
|
| 3970 |
-
raise DeepSeekRequestError('
|
| 3971 |
kazakh_specific = re.findall(r'[ӘәҒғҚқҢңӨөҰұҮүҺһІі]', text)
|
| 3972 |
-
if '
|
| 3973 |
minimum = max(2, int(len(cyrillic) * 0.002))
|
| 3974 |
if len(kazakh_specific) < minimum:
|
| 3975 |
-
|
| 3976 |
-
|
| 3977 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3978 |
return True
|
| 3979 |
|
| 3980 |
def validate_translation_semantic_equivalence(self, source, translated, target_lang, section_name=''):
|
|
@@ -4055,8 +4166,11 @@ SECTION: {section_name}
|
|
| 4055 |
'allowed_issue_codes': sorted(allowed_codes),
|
| 4056 |
'items': [{'id': row['id'], 'english': row['english'], 'target': row['target']} for row in rows],
|
| 4057 |
}
|
|
|
|
| 4058 |
system_prompt = f"""You are a bilingual academic translation QA classifier for {target_lang}.
|
| 4059 |
Review EACH claim independently. Do not rewrite and do not provide free-form explanations. Check semantic equivalence and language quality: omissions/additions, negation, uncertainty, association-versus-causation strength, terminology, grammar, language contamination, citation attachment, and acronym attachment.
|
|
|
|
|
|
|
| 4060 |
Return valid JSON only in this exact form: {{"items":[{{"id":"SP-P000001-C000001","ok":true,"codes":[]}}]}}.
|
| 4061 |
If ok=false, codes MUST contain only values from allowed_issue_codes. Never return numbered prose, examples, quotes, or corrective wording. Return every supplied id exactly once."""
|
| 4062 |
response = self.call_deepseek(
|
|
@@ -4184,37 +4298,61 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
|
|
| 4184 |
return parsed
|
| 4185 |
|
| 4186 |
def _validate_translation_block(self, source, translated, block_id, target_lang):
|
| 4187 |
-
|
| 4188 |
-
# private Fact Ledger. This routine now owns language/coverage and the
|
| 4189 |
-
# long-form/acronym adjacency check only; raw numeric Counter equality is
|
| 4190 |
-
# deliberately non-authoritative because it creates multilingual false
|
| 4191 |
-
# failures and can miss value swaps.
|
| 4192 |
-
for _long_form, acronym in self._translation_longform_acronym_pairs(source):
|
| 4193 |
-
if not re.search(rf'[((]\s*{re.escape(acronym)}\s*[))]', str(translated or '')):
|
| 4194 |
-
raise DeepSeekRequestError(
|
| 4195 |
-
f'Translation detached long-form/acronym pair in {block_id}: {acronym}.'
|
| 4196 |
-
)
|
| 4197 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4198 |
source_compact = re.sub(r'\s+', '', str(source or ''))
|
| 4199 |
target_compact = re.sub(r'\s+', '', str(translated or ''))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4200 |
if len(source_compact) >= 120:
|
| 4201 |
-
|
| 4202 |
-
|
| 4203 |
-
'translation_min_block_ratio_zh' if is_chinese else 'translation_min_block_ratio_cyrillic',
|
| 4204 |
-
0.28 if is_chinese else 0.65,
|
| 4205 |
-
))
|
| 4206 |
-
maximum = float(APP_CONFIG.get('translation_max_block_ratio', 2.20))
|
| 4207 |
ratio = len(target_compact) / max(1, len(source_compact))
|
| 4208 |
-
if ratio < minimum
|
| 4209 |
-
|
| 4210 |
-
|
| 4211 |
-
)
|
| 4212 |
-
|
| 4213 |
-
|
| 4214 |
-
if
|
| 4215 |
-
|
| 4216 |
-
|
| 4217 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4218 |
|
| 4219 |
def _validate_translation_block_protected(self, block_id, source, translated, target_lang, mapping=None):
|
| 4220 |
"""Validate one restored block after Fact Ledger integrity passed."""
|
|
@@ -4228,10 +4366,10 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
|
|
| 4228 |
protected_maps = {}
|
| 4229 |
claim_sources_by_block = {}
|
| 4230 |
for block_id, source in batch:
|
| 4231 |
-
|
| 4232 |
-
protected_batch.append((block_id, source, protected_source))
|
| 4233 |
-
protected_maps[block_id] =
|
| 4234 |
-
claim_sources_by_block[block_id] = claim_sources
|
| 4235 |
marked_source = '\n\n'.join(
|
| 4236 |
f'[[SP-P{block_id}]]\n{protected_source}'
|
| 4237 |
for block_id, _, protected_source in protected_batch
|
|
@@ -4399,12 +4537,21 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
|
|
| 4399 |
except Exception as error:
|
| 4400 |
last_error = error
|
| 4401 |
if attempt >= attempts:
|
| 4402 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4403 |
logging.warning(
|
| 4404 |
'Translation QA-only retry: section=%s attempt=%s/%s error=%s',
|
| 4405 |
section_name or 'section', attempt, attempts, self._safe_source_error(error),
|
| 4406 |
)
|
| 4407 |
-
|
| 4408 |
|
| 4409 |
def translate_frozen_section(self, section_text, target_lang, glossary=None, section_name='', max_attempts=2):
|
| 4410 |
"""Translate one frozen English unit and publish only after all QA passes."""
|
|
@@ -4413,6 +4560,8 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
|
|
| 4413 |
raise DeepSeekRequestError('Frozen English translation unit is empty.')
|
| 4414 |
if re.search(r'(?m)^#\s+References\s*$', value):
|
| 4415 |
raise DeepSeekRequestError('References must not be sent to the translation model.')
|
|
|
|
|
|
|
| 4416 |
frozen_glossary = self.extend_translation_glossary(value, target_lang, glossary)
|
| 4417 |
units = self._translation_body_units(value)
|
| 4418 |
body_units = [unit for unit in units if unit['kind'] == 'body']
|
|
@@ -4510,13 +4659,16 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
|
|
| 4510 |
output = '\n\n'.join(output_units).strip()
|
| 4511 |
self.validate_translation_alignment(value, output)
|
| 4512 |
self._validate_translation_language_quality(output, target_lang)
|
|
|
|
| 4513 |
return {
|
| 4514 |
'text': output,
|
| 4515 |
'glossary': frozen_glossary,
|
| 4516 |
'attempts': attempt,
|
| 4517 |
-
'qa_status': 'passed',
|
| 4518 |
'translation_contract_version': self.TRANSLATION_CONTRACT_VERSION,
|
| 4519 |
'fact_ledger_hash': self.translation_fact_ledger_hash(value),
|
|
|
|
|
|
|
| 4520 |
}
|
| 4521 |
except PipelineCancelledError:
|
| 4522 |
raise
|
|
@@ -4529,7 +4681,7 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
|
|
| 4529 |
or message.startswith('Translation alignment')
|
| 4530 |
or message.startswith('Translation changed ')
|
| 4531 |
or message.startswith('Translation protected ')
|
| 4532 |
-
or message.startswith('Translation
|
| 4533 |
or message.startswith('Translation block ')
|
| 4534 |
or message.startswith('Chinese translation quality validation failed')
|
| 4535 |
or message.startswith('Kazakh translation quality validation failed')
|
|
@@ -4600,54 +4752,43 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
|
|
| 4600 |
self.validate_translation_alignment(value, output)
|
| 4601 |
return output.strip()
|
| 4602 |
|
| 4603 |
-
@
|
| 4604 |
-
def validate_translation_alignment(english_text, translated_text):
|
| 4605 |
source = str(english_text or '').strip()
|
| 4606 |
target = str(translated_text or '').strip()
|
| 4607 |
if not target:
|
| 4608 |
raise DeepSeekRequestError('Target-language manuscript is empty.')
|
| 4609 |
-
en_tree = [level for level, _ in
|
| 4610 |
-
tr_tree = [level for level, _ in
|
| 4611 |
if en_tree != tr_tree:
|
| 4612 |
raise DeepSeekRequestError(
|
| 4613 |
f'Translation heading-tree mismatch: English={en_tree}, target={tr_tree}.'
|
| 4614 |
)
|
| 4615 |
-
#
|
| 4616 |
-
#
|
| 4617 |
-
#
|
| 4618 |
-
# values can preserve the same Counter.
|
| 4619 |
source_body = re.split(r'(?m)^#\s+References\s*$', source, maxsplit=1)[0]
|
| 4620 |
target_body = re.split(r'(?m)^#\s+(?:参考文献|Литература|Пайдаланылған әдебиеттер)\s*$', target, maxsplit=1)[0]
|
| 4621 |
source_chars = len(re.sub(r'\s+', '', source_body))
|
| 4622 |
target_chars = len(re.sub(r'\s+', '', target_body))
|
| 4623 |
-
if re.search(r'(?m)^#\s+参考文献\s*$', target):
|
| 4624 |
-
minimum_ratio = float(APP_CONFIG.get('translation_min_block_ratio_zh', 0.28))
|
| 4625 |
-
elif re.search(r'(?m)^#\s+(?:Литература|Пайдаланылған әдебиеттер)\s*$', target):
|
| 4626 |
-
minimum_ratio = float(APP_CONFIG.get('translation_min_block_ratio_cyrillic', 0.65))
|
| 4627 |
-
else:
|
| 4628 |
-
minimum_ratio = float(APP_CONFIG.get('translation_min_coverage_ratio', 0.25))
|
| 4629 |
ratio = target_chars / source_chars if source_chars else 0.0
|
| 4630 |
-
if source_chars and ratio <
|
| 4631 |
-
|
| 4632 |
-
f'Translation content coverage is too low: {ratio:.3f}.'
|
| 4633 |
-
)
|
| 4634 |
|
| 4635 |
-
# Russian and Kazakh both use Cyrillic, so a generic Cyrillic check is
|
| 4636 |
-
# insufficient. Long Kazakh manuscripts must contain a meaningful
|
| 4637 |
-
# number of Kazakh-specific letters; Russian manuscripts must not be
|
| 4638 |
-
# dominated by those letters. This is deterministic and adds no API cost.
|
| 4639 |
kazakh_specific = re.findall(r'[ӘәҒғҚқҢңӨөҰұҮүҺһІі]', target_body)
|
| 4640 |
cyrillic = re.findall(r'[А-Яа-яЁёӘәҒғҚқҢңӨөҰұҮүҺһІі]', target_body)
|
| 4641 |
-
if re.search(r'(?m)^#\s+Пайдаланылған әдебиеттер\s*$', target):
|
| 4642 |
minimum_kazakh = max(8, int(len(cyrillic) * 0.003))
|
| 4643 |
if len(kazakh_specific) < minimum_kazakh:
|
| 4644 |
-
|
| 4645 |
-
'Kazakh language
|
|
|
|
| 4646 |
)
|
| 4647 |
-
elif re.search(r'(?m)^#\s+Литература\s*$', target):
|
| 4648 |
-
if
|
| 4649 |
-
|
| 4650 |
-
'Russian language
|
|
|
|
| 4651 |
)
|
| 4652 |
return True
|
| 4653 |
|
|
|
|
| 56 |
|
| 57 |
|
| 58 |
class SSciPipelineAgent:
|
| 59 |
+
TRANSLATION_CONTRACT_VERSION = 'hierarchical-multilingual-v1'
|
| 60 |
|
| 61 |
def __init__(self, ds_key, s2_key=""):
|
| 62 |
self.ds_key = str(ds_key or "").strip()
|
|
|
|
| 120 |
self.citation_map = {}
|
| 121 |
self.used_citations = set()
|
| 122 |
self.citation_audit_log = []
|
| 123 |
+
# R3.2.8.6: translation validators emit hierarchical assurance evidence.
|
| 124 |
+
# Low-level diagnostics cannot independently terminate a manuscript.
|
| 125 |
+
self.translation_assurance_log = []
|
| 126 |
self.lock = threading.RLock()
|
| 127 |
|
| 128 |
self.lit_target_papers = APP_CONFIG["lit_target_papers"]
|
|
|
|
| 276 |
"citation_map": self.citation_map,
|
| 277 |
"used_citations": list(self.used_citations),
|
| 278 |
"citation_audit_log": list(self.citation_audit_log),
|
| 279 |
+
"translation_assurance_log": list(self.translation_assurance_log),
|
| 280 |
"global_memory": self.global_memory,
|
| 281 |
"s2_access_mode": self._s2_access_mode,
|
| 282 |
"s2_logical_searches": int(self._s2_logical_searches),
|
|
|
|
| 298 |
self.citation_map = state_dict.get("citation_map", {})
|
| 299 |
self.used_citations = set(state_dict.get("used_citations", []))
|
| 300 |
self.citation_audit_log = list(state_dict.get("citation_audit_log", []))
|
| 301 |
+
self.translation_assurance_log = list(state_dict.get("translation_assurance_log", []))
|
| 302 |
self.global_memory = state_dict.get("global_memory", self.global_memory)
|
| 303 |
loaded_mode = str(state_dict.get("s2_access_mode") or "").strip().lower()
|
| 304 |
expected_mode = "key" if self.s2_key else "anonymous"
|
|
|
|
| 3282 |
)['abstract']
|
| 3283 |
|
| 3284 |
@staticmethod
|
| 3285 |
+
def _translation_language_profile(target_lang):
|
| 3286 |
+
"""Language-specific rendering policy over one language-neutral claim/fact IR.
|
| 3287 |
+
|
| 3288 |
+
Surface length and script statistics are diagnostic evidence only.
|
| 3289 |
+
Fact/claim integrity remains language-neutral and authoritative.
|
| 3290 |
+
"""
|
| 3291 |
+
name = str(target_lang or '')
|
| 3292 |
+
if 'Russian' in name:
|
| 3293 |
+
return {
|
| 3294 |
+
'adapter_id': 'ru-academic-v1', 'language_code': 'ru',
|
| 3295 |
+
'morphology': 'inflectional', 'script': 'cyrillic',
|
| 3296 |
+
'surface_min_key': 'translation_min_block_ratio_cyrillic',
|
| 3297 |
+
'surface_min_default': 0.65, 'surface_max_default': 2.20,
|
| 3298 |
+
'term_policy': 'Allow grammatically required Russian case, gender, number, and inflection around translated terms while preserving concept identity and protected acronyms.',
|
| 3299 |
+
'contamination_policy': 'Treat sustained non-Russian language contamination as a repair signal; Kazakh-specific letters are diagnostic evidence, not a standalone failure rule.',
|
| 3300 |
+
}
|
| 3301 |
+
if 'Kazakh' in name:
|
| 3302 |
+
return {
|
| 3303 |
+
'adapter_id': 'kk-academic-v1', 'language_code': 'kk',
|
| 3304 |
+
'morphology': 'agglutinative', 'script': 'cyrillic',
|
| 3305 |
+
'surface_min_key': 'translation_min_block_ratio_cyrillic',
|
| 3306 |
+
'surface_min_default': 0.65, 'surface_max_default': 2.20,
|
| 3307 |
+
'term_policy': 'Translate directly from English into academic Kazakh. Allow case, possessive, person, plural and other productive suffixes to attach naturally to translated term stems while preserving concept identity and protected acronyms.',
|
| 3308 |
+
'contamination_policy': 'Treat sustained Russian contamination as a repair signal. Absence of Kazakh-specific letters in a short claim is never sufficient by itself to reject the claim.',
|
| 3309 |
+
}
|
| 3310 |
+
return {
|
| 3311 |
+
'adapter_id': 'zh-academic-v1', 'language_code': 'zh',
|
| 3312 |
+
'morphology': 'analytic', 'script': 'han',
|
| 3313 |
+
'surface_min_key': 'translation_min_block_ratio_zh',
|
| 3314 |
+
'surface_min_default': 0.28, 'surface_max_default': 2.20,
|
| 3315 |
+
'term_policy': 'Use stable academic Chinese terminology while preserving concept identity and protected acronyms. Natural Chinese reordering and sentence merging/splitting are allowed inside the same claim.',
|
| 3316 |
+
'contamination_policy': 'Treat sustained untranslated English prose as a repair signal; normal acronyms, model names and protected terms are allowed.',
|
| 3317 |
+
}
|
| 3318 |
+
|
| 3319 |
+
@classmethod
|
| 3320 |
+
def _translation_language_rule(cls, target_lang):
|
| 3321 |
+
profile = cls._translation_language_profile(target_lang)
|
| 3322 |
+
if profile['language_code'] == 'ru':
|
| 3323 |
+
core = 'Use idiomatic academic Russian with correct case government, gender-number agreement, aspect, punctuation, and terminology. Reorganize English syntax where necessary and avoid literal English calques.'
|
| 3324 |
+
elif profile['language_code'] == 'kk':
|
| 3325 |
+
core = 'Use idiomatic academic Kazakh with correct case and possessive suffixes, person/plural endings, vowel harmony, consonant assimilation, natural word order, and correct Kazakh-specific letters. Translate directly from English and avoid Russian syntactic calques.'
|
| 3326 |
+
else:
|
| 3327 |
+
core = 'Use formal academic Simplified Chinese with explicit concepts, natural Chinese syntax, clear hierarchy, and logically connected claims. Avoid carrying English nominalization and clause order into Chinese mechanically.'
|
| 3328 |
+
return core + ' ' + profile['term_policy'] + ' ' + profile['contamination_policy']
|
| 3329 |
+
|
| 3330 |
+
@classmethod
|
| 3331 |
+
def _compile_translation_ir(cls, source):
|
| 3332 |
+
"""Compile a private, language-neutral translation intermediate representation."""
|
| 3333 |
+
protected_source, fact_ledger, claim_sources = cls._protect_translation_claims(source)
|
| 3334 |
+
return {
|
| 3335 |
+
'ir_version': 'translation-ir-v1',
|
| 3336 |
+
'protected_source': protected_source,
|
| 3337 |
+
'fact_ledger': fact_ledger,
|
| 3338 |
+
'claim_sources': claim_sources,
|
| 3339 |
+
'claim_ids': list(claim_sources),
|
| 3340 |
+
}
|
| 3341 |
+
|
| 3342 |
+
def _record_translation_assurance(self, code, *, target_lang='', block_id='', severity='diagnostic', decision='PASS_WITH_WARNING', detail=None):
|
| 3343 |
+
row = {
|
| 3344 |
+
'code': str(code or 'TRANSLATION_DIAGNOSTIC'),
|
| 3345 |
+
'language': self._translation_language_profile(target_lang).get('language_code'),
|
| 3346 |
+
'block_id': str(block_id or ''),
|
| 3347 |
+
'severity': str(severity or 'diagnostic'),
|
| 3348 |
+
'decision': str(decision or 'PASS_WITH_WARNING'),
|
| 3349 |
+
}
|
| 3350 |
+
if isinstance(detail, dict):
|
| 3351 |
+
row['detail'] = {str(k): v for k, v in detail.items() if isinstance(v, (str, int, float, bool))}
|
| 3352 |
+
self.translation_assurance_log.append(row)
|
| 3353 |
+
logging.warning(
|
| 3354 |
+
'Translation assurance | code=%s language=%s block=%s severity=%s decision=%s detail=%s',
|
| 3355 |
+
row['code'], row['language'], row['block_id'], row['severity'], row['decision'], row.get('detail', {}),
|
| 3356 |
+
)
|
| 3357 |
+
return row
|
| 3358 |
+
|
| 3359 |
+
@staticmethod
|
| 3360 |
+
def _translation_assurance_decision(evidence):
|
| 3361 |
+
"""Central decision manager: validators provide evidence; this method owns consequence."""
|
| 3362 |
+
items = list(evidence or [])
|
| 3363 |
+
if any(str(item.get('severity')) == 'critical' for item in items):
|
| 3364 |
+
return 'HOLD_FINAL'
|
| 3365 |
+
if any(str(item.get('severity')) == 'repair' for item in items):
|
| 3366 |
+
return 'REPAIR_BLOCK'
|
| 3367 |
+
if items:
|
| 3368 |
+
return 'PASS_WITH_WARNING'
|
| 3369 |
+
return 'PASS'
|
| 3370 |
+
|
| 3371 |
+
@classmethod
|
| 3372 |
+
def translation_error_code(cls, error):
|
| 3373 |
+
message = str(error or '')
|
| 3374 |
+
if 'unowned factual content' in message or 'protected placeholder mismatch' in message or 'fact moved across claim boundary' in message or 'fact order changed' in message:
|
| 3375 |
+
return 'FACT_INTEGRITY'
|
| 3376 |
+
if 'claim marker mismatch' in message or 'marker mismatch' in message or 'heading-tree mismatch' in message:
|
| 3377 |
+
return 'STRUCTURE_INTEGRITY'
|
| 3378 |
+
if 'claim-local QA failed' in message or 'semantic' in message.lower() or 'caus' in message.lower():
|
| 3379 |
+
return 'SEMANTIC_INTEGRITY'
|
| 3380 |
+
if 'target-language signal' in message.lower() or 'returned unchanged' in message.lower():
|
| 3381 |
+
return 'LANGUAGE_QUALITY'
|
| 3382 |
+
return 'TRANSLATION_TECHNICAL'
|
| 3383 |
|
| 3384 |
@staticmethod
|
| 3385 |
def _translation_prompt(target_lang):
|
|
|
|
| 4052 |
mapping[acronym] = acronym
|
| 4053 |
return mapping
|
| 4054 |
|
| 4055 |
+
def _validate_translation_language_quality(self, translated, target_lang):
|
| 4056 |
+
"""Validate strong target-language identity; keep weak script heuristics diagnostic-only."""
|
| 4057 |
text = str(translated or '')
|
| 4058 |
compact = re.sub(r'\s+', '', text)
|
| 4059 |
if len(compact) < 80:
|
| 4060 |
return True
|
| 4061 |
+
profile = self._translation_language_profile(target_lang)
|
| 4062 |
+
if profile['language_code'] == 'zh':
|
| 4063 |
+
han = re.findall(r'[\u4e00-\u9fff]', text)
|
| 4064 |
+
if not han:
|
| 4065 |
+
raise DeepSeekRequestError('Translation target-language signal missing: Chinese text contains no Han characters.')
|
| 4066 |
+
if len(han) < max(12, int(len(compact) * 0.08)):
|
| 4067 |
+
self._record_translation_assurance(
|
| 4068 |
+
'ZH_LANGUAGE_SIGNAL_LOW', target_lang=target_lang, severity='diagnostic',
|
| 4069 |
+
decision='PASS_WITH_WARNING', detail={'han_chars': len(han), 'compact_chars': len(compact)},
|
| 4070 |
+
)
|
| 4071 |
return True
|
| 4072 |
+
|
| 4073 |
cyrillic = re.findall(r'[А-Яа-яЁёӘәҒғҚқҢңӨөҰұҮүҺһІі]', text)
|
| 4074 |
if not cyrillic:
|
| 4075 |
+
raise DeepSeekRequestError('Translation target-language signal missing: target contains no Cyrillic text.')
|
| 4076 |
kazakh_specific = re.findall(r'[ӘәҒғҚқҢңӨөҰұҮүҺһІі]', text)
|
| 4077 |
+
if profile['language_code'] == 'kk' and len(cyrillic) >= 200:
|
| 4078 |
minimum = max(2, int(len(cyrillic) * 0.002))
|
| 4079 |
if len(kazakh_specific) < minimum:
|
| 4080 |
+
self._record_translation_assurance(
|
| 4081 |
+
'KK_SPECIFIC_LETTER_SIGNAL_LOW', target_lang=target_lang, severity='diagnostic',
|
| 4082 |
+
decision='PASS_WITH_WARNING', detail={'kazakh_specific': len(kazakh_specific), 'cyrillic': len(cyrillic)},
|
| 4083 |
+
)
|
| 4084 |
+
if profile['language_code'] == 'ru' and cyrillic and len(kazakh_specific) / len(cyrillic) > 0.02:
|
| 4085 |
+
self._record_translation_assurance(
|
| 4086 |
+
'RU_KAZAKH_LETTER_SIGNAL_HIGH', target_lang=target_lang, severity='diagnostic',
|
| 4087 |
+
decision='PASS_WITH_WARNING', detail={'kazakh_specific': len(kazakh_specific), 'cyrillic': len(cyrillic)},
|
| 4088 |
+
)
|
| 4089 |
return True
|
| 4090 |
|
| 4091 |
def validate_translation_semantic_equivalence(self, source, translated, target_lang, section_name=''):
|
|
|
|
| 4166 |
'allowed_issue_codes': sorted(allowed_codes),
|
| 4167 |
'items': [{'id': row['id'], 'english': row['english'], 'target': row['target']} for row in rows],
|
| 4168 |
}
|
| 4169 |
+
profile = self._translation_language_profile(target_lang)
|
| 4170 |
system_prompt = f"""You are a bilingual academic translation QA classifier for {target_lang}.
|
| 4171 |
Review EACH claim independently. Do not rewrite and do not provide free-form explanations. Check semantic equivalence and language quality: omissions/additions, negation, uncertainty, association-versus-causation strength, terminology, grammar, language contamination, citation attachment, and acronym attachment.
|
| 4172 |
+
LANGUAGE ADAPTER: {profile['adapter_id']}; morphology={profile['morphology']}. {profile['term_policy']} {profile['contamination_policy']}
|
| 4173 |
+
For Russian, do not flag a terminology error merely because a translated term is inflected for grammatical case/number/gender. For Kazakh, do not flag a terminology error merely because productive suffixes attach to the correct term stem. For Kazakh language identity, absence of Kazakh-specific letters in one short claim is not sufficient evidence of Russian contamination.
|
| 4174 |
Return valid JSON only in this exact form: {{"items":[{{"id":"SP-P000001-C000001","ok":true,"codes":[]}}]}}.
|
| 4175 |
If ok=false, codes MUST contain only values from allowed_issue_codes. Never return numbered prose, examples, quotes, or corrective wording. Return every supplied id exactly once."""
|
| 4176 |
response = self.call_deepseek(
|
|
|
|
| 4298 |
return parsed
|
| 4299 |
|
| 4300 |
def _validate_translation_block(self, source, translated, block_id, target_lang):
|
| 4301 |
+
"""Hierarchical multilingual block assurance.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4302 |
|
| 4303 |
+
Fact/claim invariants are enforced earlier by the private ledger. Surface
|
| 4304 |
+
length and weak language statistics are diagnostic only; strong wrong-
|
| 4305 |
+
language/identity signals request bounded local repair.
|
| 4306 |
+
"""
|
| 4307 |
source_compact = re.sub(r'\s+', '', str(source or ''))
|
| 4308 |
target_compact = re.sub(r'\s+', '', str(translated or ''))
|
| 4309 |
+
profile = self._translation_language_profile(target_lang)
|
| 4310 |
+
evidence = []
|
| 4311 |
+
|
| 4312 |
+
if source_compact and source_compact.casefold() == target_compact.casefold():
|
| 4313 |
+
evidence.append({'code': 'UNCHANGED_SOURCE', 'severity': 'repair'})
|
| 4314 |
+
|
| 4315 |
if len(source_compact) >= 120:
|
| 4316 |
+
minimum = float(APP_CONFIG.get(profile['surface_min_key'], profile['surface_min_default']))
|
| 4317 |
+
maximum = float(APP_CONFIG.get('translation_max_block_ratio', profile['surface_max_default']))
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4318 |
ratio = len(target_compact) / max(1, len(source_compact))
|
| 4319 |
+
if ratio < minimum:
|
| 4320 |
+
evidence.append({'code': 'SURFACE_RATIO_LOW', 'severity': 'diagnostic', 'ratio': round(ratio, 4), 'threshold': minimum})
|
| 4321 |
+
elif ratio > maximum:
|
| 4322 |
+
evidence.append({'code': 'SURFACE_RATIO_HIGH', 'severity': 'diagnostic', 'ratio': round(ratio, 4), 'threshold': maximum})
|
| 4323 |
+
|
| 4324 |
+
if len(target_compact) >= 40:
|
| 4325 |
+
if profile['language_code'] == 'zh':
|
| 4326 |
+
han = len(re.findall(r'[\u4e00-\u9fff]', str(translated or '')))
|
| 4327 |
+
if han == 0:
|
| 4328 |
+
evidence.append({'code': 'TARGET_SCRIPT_MISSING', 'severity': 'repair'})
|
| 4329 |
+
else:
|
| 4330 |
+
cyrillic = re.findall(r'[А-Яа-яЁёӘәҒғҚқҢңӨөҰұҮүҺһІі]', str(translated or ''))
|
| 4331 |
+
if not cyrillic:
|
| 4332 |
+
evidence.append({'code': 'TARGET_SCRIPT_MISSING', 'severity': 'repair'})
|
| 4333 |
+
else:
|
| 4334 |
+
kazakh_specific = re.findall(r'[ӘәҒғҚқҢңӨөҰұҮүҺһІі]', str(translated or ''))
|
| 4335 |
+
if profile['language_code'] == 'kk' and len(cyrillic) >= 200:
|
| 4336 |
+
minimum_kk = max(2, int(len(cyrillic) * 0.002))
|
| 4337 |
+
if len(kazakh_specific) < minimum_kk:
|
| 4338 |
+
evidence.append({'code': 'KK_SPECIFIC_LETTER_SIGNAL_LOW', 'severity': 'diagnostic', 'kazakh_specific': len(kazakh_specific), 'cyrillic': len(cyrillic)})
|
| 4339 |
+
if profile['language_code'] == 'ru' and len(kazakh_specific) / max(1, len(cyrillic)) > 0.02:
|
| 4340 |
+
evidence.append({'code': 'RU_KAZAKH_LETTER_SIGNAL_HIGH', 'severity': 'diagnostic', 'kazakh_specific': len(kazakh_specific), 'cyrillic': len(cyrillic)})
|
| 4341 |
+
|
| 4342 |
+
decision = self._translation_assurance_decision(evidence)
|
| 4343 |
+
for item in evidence:
|
| 4344 |
+
self._record_translation_assurance(
|
| 4345 |
+
item.get('code'), target_lang=target_lang, block_id=block_id,
|
| 4346 |
+
severity=item.get('severity'),
|
| 4347 |
+
decision='REPAIR_BLOCK' if item.get('severity') == 'repair' else 'PASS_WITH_WARNING',
|
| 4348 |
+
detail={key: value for key, value in item.items() if key not in {'code', 'severity'}},
|
| 4349 |
+
)
|
| 4350 |
+
if decision == 'REPAIR_BLOCK':
|
| 4351 |
+
codes = [item.get('code') for item in evidence if item.get('severity') == 'repair']
|
| 4352 |
+
raise DeepSeekRequestError(
|
| 4353 |
+
f'Translation assurance requires local block repair in {block_id}: {codes[:5]}.'
|
| 4354 |
+
)
|
| 4355 |
+
return True
|
| 4356 |
|
| 4357 |
def _validate_translation_block_protected(self, block_id, source, translated, target_lang, mapping=None):
|
| 4358 |
"""Validate one restored block after Fact Ledger integrity passed."""
|
|
|
|
| 4366 |
protected_maps = {}
|
| 4367 |
claim_sources_by_block = {}
|
| 4368 |
for block_id, source in batch:
|
| 4369 |
+
ir = self._compile_translation_ir(source)
|
| 4370 |
+
protected_batch.append((block_id, source, ir['protected_source']))
|
| 4371 |
+
protected_maps[block_id] = ir['fact_ledger']
|
| 4372 |
+
claim_sources_by_block[block_id] = ir['claim_sources']
|
| 4373 |
marked_source = '\n\n'.join(
|
| 4374 |
f'[[SP-P{block_id}]]\n{protected_source}'
|
| 4375 |
for block_id, _, protected_source in protected_batch
|
|
|
|
| 4537 |
except Exception as error:
|
| 4538 |
last_error = error
|
| 4539 |
if attempt >= attempts:
|
| 4540 |
+
self._record_translation_assurance(
|
| 4541 |
+
'SEMANTIC_QA_UNCERTAIN', target_lang=target_lang, block_id=section_name or 'section',
|
| 4542 |
+
severity='diagnostic', decision='PASS_WITH_WARNING',
|
| 4543 |
+
detail={'attempts': attempts, 'error_type': type(error).__name__},
|
| 4544 |
+
)
|
| 4545 |
+
logging.warning(
|
| 4546 |
+
'Translation semantic QA technical fallback: section=%s attempts=%s error=%s',
|
| 4547 |
+
section_name or 'section', attempts, self._safe_source_error(error),
|
| 4548 |
+
)
|
| 4549 |
+
return {}
|
| 4550 |
logging.warning(
|
| 4551 |
'Translation QA-only retry: section=%s attempt=%s/%s error=%s',
|
| 4552 |
section_name or 'section', attempt, attempts, self._safe_source_error(error),
|
| 4553 |
)
|
| 4554 |
+
return {}
|
| 4555 |
|
| 4556 |
def translate_frozen_section(self, section_text, target_lang, glossary=None, section_name='', max_attempts=2):
|
| 4557 |
"""Translate one frozen English unit and publish only after all QA passes."""
|
|
|
|
| 4560 |
raise DeepSeekRequestError('Frozen English translation unit is empty.')
|
| 4561 |
if re.search(r'(?m)^#\s+References\s*$', value):
|
| 4562 |
raise DeepSeekRequestError('References must not be sent to the translation model.')
|
| 4563 |
+
assurance_start = len(self.translation_assurance_log)
|
| 4564 |
+
profile = self._translation_language_profile(target_lang)
|
| 4565 |
frozen_glossary = self.extend_translation_glossary(value, target_lang, glossary)
|
| 4566 |
units = self._translation_body_units(value)
|
| 4567 |
body_units = [unit for unit in units if unit['kind'] == 'body']
|
|
|
|
| 4659 |
output = '\n\n'.join(output_units).strip()
|
| 4660 |
self.validate_translation_alignment(value, output)
|
| 4661 |
self._validate_translation_language_quality(output, target_lang)
|
| 4662 |
+
diagnostics = list(self.translation_assurance_log[assurance_start:])
|
| 4663 |
return {
|
| 4664 |
'text': output,
|
| 4665 |
'glossary': frozen_glossary,
|
| 4666 |
'attempts': attempt,
|
| 4667 |
+
'qa_status': 'passed_with_warning' if diagnostics else 'passed',
|
| 4668 |
'translation_contract_version': self.TRANSLATION_CONTRACT_VERSION,
|
| 4669 |
'fact_ledger_hash': self.translation_fact_ledger_hash(value),
|
| 4670 |
+
'language_adapter': profile['adapter_id'],
|
| 4671 |
+
'diagnostics': diagnostics[-20:],
|
| 4672 |
}
|
| 4673 |
except PipelineCancelledError:
|
| 4674 |
raise
|
|
|
|
| 4681 |
or message.startswith('Translation alignment')
|
| 4682 |
or message.startswith('Translation changed ')
|
| 4683 |
or message.startswith('Translation protected ')
|
| 4684 |
+
or message.startswith('Translation assurance requires local block repair')
|
| 4685 |
or message.startswith('Translation block ')
|
| 4686 |
or message.startswith('Chinese translation quality validation failed')
|
| 4687 |
or message.startswith('Kazakh translation quality validation failed')
|
|
|
|
| 4752 |
self.validate_translation_alignment(value, output)
|
| 4753 |
return output.strip()
|
| 4754 |
|
| 4755 |
+
@classmethod
|
| 4756 |
+
def validate_translation_alignment(cls, english_text, translated_text):
|
| 4757 |
source = str(english_text or '').strip()
|
| 4758 |
target = str(translated_text or '').strip()
|
| 4759 |
if not target:
|
| 4760 |
raise DeepSeekRequestError('Target-language manuscript is empty.')
|
| 4761 |
+
en_tree = [level for level, _ in cls._markdown_heading_tree(source)]
|
| 4762 |
+
tr_tree = [level for level, _ in cls._markdown_heading_tree(target)]
|
| 4763 |
if en_tree != tr_tree:
|
| 4764 |
raise DeepSeekRequestError(
|
| 4765 |
f'Translation heading-tree mismatch: English={en_tree}, target={tr_tree}.'
|
| 4766 |
)
|
| 4767 |
+
# R3.2.8.6: whole-unit character ratios and Cyrillic subtype signals are
|
| 4768 |
+
# diagnostics only. Claim markers + private Fact Ledger + semantic claim
|
| 4769 |
+
# QA are authoritative for completeness and fidelity.
|
|
|
|
| 4770 |
source_body = re.split(r'(?m)^#\s+References\s*$', source, maxsplit=1)[0]
|
| 4771 |
target_body = re.split(r'(?m)^#\s+(?:参考文献|Литература|Пайдаланылған әдебиеттер)\s*$', target, maxsplit=1)[0]
|
| 4772 |
source_chars = len(re.sub(r'\s+', '', source_body))
|
| 4773 |
target_chars = len(re.sub(r'\s+', '', target_body))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4774 |
ratio = target_chars / source_chars if source_chars else 0.0
|
| 4775 |
+
if source_chars and (ratio < 0.20 or ratio > 2.80):
|
| 4776 |
+
logging.warning('Translation whole-unit surface diagnostic | ratio=%.3f | nonfatal=true', ratio)
|
|
|
|
|
|
|
| 4777 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4778 |
kazakh_specific = re.findall(r'[ӘәҒғҚқҢңӨөҰұҮүҺһІі]', target_body)
|
| 4779 |
cyrillic = re.findall(r'[А-Яа-яЁёӘәҒғҚқҢңӨөҰұҮүҺһІі]', target_body)
|
| 4780 |
+
if re.search(r'(?m)^#\s+Пайдаланылған әдебиеттер\s*$', target) and len(cyrillic) >= 200:
|
| 4781 |
minimum_kazakh = max(8, int(len(cyrillic) * 0.003))
|
| 4782 |
if len(kazakh_specific) < minimum_kazakh:
|
| 4783 |
+
logging.warning(
|
| 4784 |
+
'Kazakh whole-unit language signal diagnostic | specific=%s cyrillic=%s | nonfatal=true',
|
| 4785 |
+
len(kazakh_specific), len(cyrillic),
|
| 4786 |
)
|
| 4787 |
+
elif re.search(r'(?m)^#\s+Литература\s*$', target) and cyrillic:
|
| 4788 |
+
if len(kazakh_specific) / len(cyrillic) > 0.02:
|
| 4789 |
+
logging.warning(
|
| 4790 |
+
'Russian whole-unit language signal diagnostic | kazakh_specific=%s cyrillic=%s | nonfatal=true',
|
| 4791 |
+
len(kazakh_specific), len(cyrillic),
|
| 4792 |
)
|
| 4793 |
return True
|
| 4794 |
|