Spaces:
Running
Running
R14-TQ2-LP1-R1 Protected Entity Ledger + Typed Lexical Authority + Final-Gate Progress
#61
by GuXiaopeng - opened
- app.py +5 -0
- core/generation_runner.py +25 -5
- core/pipeline_agent.py +318 -68
- i18n.py +20 -0
- smartpaper_restore_bridge/frontend/index.html +16 -2
app.py
CHANGED
|
@@ -775,6 +775,11 @@ def _watch_task_status_live(task: dict) -> dict | None:
|
|
| 775 |
"waiting": tr(str(st.session_state.get("sp_lang_code") or "zh"), "translation_waiting"),
|
| 776 |
"translating": tr(str(st.session_state.get("sp_lang_code") or "zh"), "translation_section_running"),
|
| 777 |
"qa": tr(str(st.session_state.get("sp_lang_code") or "zh"), "translation_section_qa"),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 778 |
"ready": tr(str(st.session_state.get("sp_lang_code") or "zh"), "translation_section_ready"),
|
| 779 |
"complete": tr(str(st.session_state.get("sp_lang_code") or "zh"), "translation_complete_label"),
|
| 780 |
},
|
|
|
|
| 775 |
"waiting": tr(str(st.session_state.get("sp_lang_code") or "zh"), "translation_waiting"),
|
| 776 |
"translating": tr(str(st.session_state.get("sp_lang_code") or "zh"), "translation_section_running"),
|
| 777 |
"qa": tr(str(st.session_state.get("sp_lang_code") or "zh"), "translation_section_qa"),
|
| 778 |
+
"final_gate": tr(str(st.session_state.get("sp_lang_code") or "zh"), "translation_final_gate"),
|
| 779 |
+
"repairing": tr(str(st.session_state.get("sp_lang_code") or "zh"), "translation_section_repairing"),
|
| 780 |
+
"repair_pending": tr(str(st.session_state.get("sp_lang_code") or "zh"), "translation_section_repair_pending"),
|
| 781 |
+
"adjudicating": tr(str(st.session_state.get("sp_lang_code") or "zh"), "translation_section_adjudicating"),
|
| 782 |
+
"counts": tr(str(st.session_state.get("sp_lang_code") or "zh"), "translation_progress_counts"),
|
| 783 |
"ready": tr(str(st.session_state.get("sp_lang_code") or "zh"), "translation_section_ready"),
|
| 784 |
"complete": tr(str(st.session_state.get("sp_lang_code") or "zh"), "translation_complete_label"),
|
| 785 |
},
|
core/generation_runner.py
CHANGED
|
@@ -43,7 +43,7 @@ CITATION_ROLE_RULES = {
|
|
| 43 |
"methods": (2, 280),
|
| 44 |
"discussion": (2, 220),
|
| 45 |
}
|
| 46 |
-
TRANSLATION_CONTRACT_VERSION = 'hierarchical-multilingual-
|
| 47 |
|
| 48 |
|
| 49 |
def _clean_trailing_references(text, keywords=("reference", "references", "reference list", "参考文献")):
|
|
@@ -1159,7 +1159,7 @@ class GenerationRunner:
|
|
| 1159 |
def _translate_frozen_unit(
|
| 1160 |
self, agent, *, unit_key: str, unit_name: str, source_text: str, target_lang: str,
|
| 1161 |
translation_sections: dict, translation_glossary: dict, adaptive_structure, include_front=False,
|
| 1162 |
-
defer_failures=True, english_commit_text: str | None = None,
|
| 1163 |
):
|
| 1164 |
source = str(source_text or '').strip()
|
| 1165 |
if not source:
|
|
@@ -1170,6 +1170,14 @@ class GenerationRunner:
|
|
| 1170 |
canonical_translation_source = identity['compiled_source']
|
| 1171 |
contract_version = TRANSLATION_CONTRACT_VERSION
|
| 1172 |
fact_ledger_hash = identity['fact_ledger_hash']
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1173 |
existing = dict((translation_sections or {}).get(unit_key) or {})
|
| 1174 |
if not isinstance(getattr(self, '_translation_progress_queued', None), set):
|
| 1175 |
self._translation_progress_queued = set()
|
|
@@ -1199,8 +1207,9 @@ class GenerationRunner:
|
|
| 1199 |
logging.info('Translation reuse miss | unit=%s phase=final_or_direct reasons=%s', unit_name, ','.join(reasons) or 'unknown')
|
| 1200 |
self._translation_progress_inflight.add(str(unit_key))
|
| 1201 |
partial = self._assemble_partial_translation(translation_sections, adaptive_structure, include_front=include_front)
|
|
|
|
| 1202 |
self._publish_target_preview(
|
| 1203 |
-
partial,
|
| 1204 |
english_commit_text=english_commit_text,
|
| 1205 |
)
|
| 1206 |
updated_sections = dict(translation_sections or {})
|
|
@@ -1242,8 +1251,14 @@ class GenerationRunner:
|
|
| 1242 |
|
| 1243 |
def publish_translation_stage(stage):
|
| 1244 |
value = str(stage or '').strip().casefold()
|
| 1245 |
-
if value
|
| 1246 |
preview_stage = f'translation_qa:{unit_name}'
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1247 |
else:
|
| 1248 |
preview_stage = f'translating:{unit_name}'
|
| 1249 |
self._publish_target_preview(
|
|
@@ -1346,6 +1361,8 @@ class GenerationRunner:
|
|
| 1346 |
'glossary_version': self._glossary_version(translation_glossary),
|
| 1347 |
'translation_contract_version': contract_version,
|
| 1348 |
'fact_ledger_hash': fact_ledger_hash,
|
|
|
|
|
|
|
| 1349 |
'translated_text': '',
|
| 1350 |
'qa_status': 'repair_pending',
|
| 1351 |
'claim_states': dict(getattr(agent, '_last_translation_claim_states', {}) or {}),
|
|
@@ -1376,6 +1393,8 @@ class GenerationRunner:
|
|
| 1376 |
'glossary_version': self._glossary_version(updated_glossary),
|
| 1377 |
'translation_contract_version': str(result.get('translation_contract_version') or contract_version),
|
| 1378 |
'fact_ledger_hash': result_fact_ledger_hash,
|
|
|
|
|
|
|
| 1379 |
'translated_text': str(result.get('text') or '').strip(),
|
| 1380 |
'qa_status': str(result.get('qa_status') or 'passed'),
|
| 1381 |
'language_adapter': str(result.get('language_adapter') or ''),
|
|
@@ -2508,7 +2527,7 @@ Output only final academic prose. Do not report the word count."""
|
|
| 2508 |
target_lang=target_lang, translation_sections=translation_sections,
|
| 2509 |
translation_glossary=translation_glossary, adaptive_structure=adaptive_structure,
|
| 2510 |
include_front=True, defer_failures=False,
|
| 2511 |
-
english_commit_text=full_draft_en,
|
| 2512 |
)
|
| 2513 |
partial = self._assemble_partial_translation(
|
| 2514 |
translation_sections, adaptive_structure, include_front=True
|
|
@@ -2571,6 +2590,7 @@ Output only final academic prose. Do not report the word count."""
|
|
| 2571 |
literature_metrics=agent.get_source_metrics(),
|
| 2572 |
quality_diagnostics=quality_diagnostics,
|
| 2573 |
)
|
|
|
|
| 2574 |
output_filename = f"SmartPaper_{bj_time}.zip"
|
| 2575 |
pending = self.output_dir / (".pending_" + output_filename)
|
| 2576 |
pending.write_bytes(package.zip_bytes)
|
|
|
|
| 43 |
"methods": (2, 280),
|
| 44 |
"discussion": (2, 220),
|
| 45 |
}
|
| 46 |
+
TRANSLATION_CONTRACT_VERSION = 'hierarchical-multilingual-v8'
|
| 47 |
|
| 48 |
|
| 49 |
def _clean_trailing_references(text, keywords=("reference", "references", "reference list", "参考文献")):
|
|
|
|
| 1159 |
def _translate_frozen_unit(
|
| 1160 |
self, agent, *, unit_key: str, unit_name: str, source_text: str, target_lang: str,
|
| 1161 |
translation_sections: dict, translation_glossary: dict, adaptive_structure, include_front=False,
|
| 1162 |
+
defer_failures=True, english_commit_text: str | None = None, final_gate: bool = False,
|
| 1163 |
):
|
| 1164 |
source = str(source_text or '').strip()
|
| 1165 |
if not source:
|
|
|
|
| 1170 |
canonical_translation_source = identity['compiled_source']
|
| 1171 |
contract_version = TRANSLATION_CONTRACT_VERSION
|
| 1172 |
fact_ledger_hash = identity['fact_ledger_hash']
|
| 1173 |
+
entity_ledger_method = getattr(agent, 'translation_protected_entity_ledger_hash', None)
|
| 1174 |
+
protected_entity_ledger_hash = str(
|
| 1175 |
+
entity_ledger_method(canonical_translation_source, translation_glossary)
|
| 1176 |
+
if callable(entity_ledger_method) else ''
|
| 1177 |
+
)
|
| 1178 |
+
protected_entity_ledger_version = str(
|
| 1179 |
+
getattr(agent, 'PROTECTED_ENTITY_LEDGER_VERSION', 'protected-entity-ledger-v1') or 'protected-entity-ledger-v1'
|
| 1180 |
+
)
|
| 1181 |
existing = dict((translation_sections or {}).get(unit_key) or {})
|
| 1182 |
if not isinstance(getattr(self, '_translation_progress_queued', None), set):
|
| 1183 |
self._translation_progress_queued = set()
|
|
|
|
| 1207 |
logging.info('Translation reuse miss | unit=%s phase=final_or_direct reasons=%s', unit_name, ','.join(reasons) or 'unknown')
|
| 1208 |
self._translation_progress_inflight.add(str(unit_key))
|
| 1209 |
partial = self._assemble_partial_translation(translation_sections, adaptive_structure, include_front=include_front)
|
| 1210 |
+
initial_target_stage = f'translation_final_gate:{unit_name}' if final_gate else f'translating:{unit_name}'
|
| 1211 |
self._publish_target_preview(
|
| 1212 |
+
partial, initial_target_stage, force=True, complete=False,
|
| 1213 |
english_commit_text=english_commit_text,
|
| 1214 |
)
|
| 1215 |
updated_sections = dict(translation_sections or {})
|
|
|
|
| 1251 |
|
| 1252 |
def publish_translation_stage(stage):
|
| 1253 |
value = str(stage or '').strip().casefold()
|
| 1254 |
+
if value == 'qa':
|
| 1255 |
preview_stage = f'translation_qa:{unit_name}'
|
| 1256 |
+
elif value == 'repairing':
|
| 1257 |
+
preview_stage = f'translation_repairing:{unit_name}'
|
| 1258 |
+
elif value == 'adjudicating':
|
| 1259 |
+
preview_stage = f'translation_adjudicating:{unit_name}'
|
| 1260 |
+
elif final_gate:
|
| 1261 |
+
preview_stage = f'translation_final_gate:{unit_name}'
|
| 1262 |
else:
|
| 1263 |
preview_stage = f'translating:{unit_name}'
|
| 1264 |
self._publish_target_preview(
|
|
|
|
| 1361 |
'glossary_version': self._glossary_version(translation_glossary),
|
| 1362 |
'translation_contract_version': contract_version,
|
| 1363 |
'fact_ledger_hash': fact_ledger_hash,
|
| 1364 |
+
'protected_entity_ledger_version': protected_entity_ledger_version,
|
| 1365 |
+
'protected_entity_ledger_hash': protected_entity_ledger_hash,
|
| 1366 |
'translated_text': '',
|
| 1367 |
'qa_status': 'repair_pending',
|
| 1368 |
'claim_states': dict(getattr(agent, '_last_translation_claim_states', {}) or {}),
|
|
|
|
| 1393 |
'glossary_version': self._glossary_version(updated_glossary),
|
| 1394 |
'translation_contract_version': str(result.get('translation_contract_version') or contract_version),
|
| 1395 |
'fact_ledger_hash': result_fact_ledger_hash,
|
| 1396 |
+
'protected_entity_ledger_version': str(result.get('protected_entity_ledger_version') or protected_entity_ledger_version),
|
| 1397 |
+
'protected_entity_ledger_hash': str(result.get('protected_entity_ledger_hash') or protected_entity_ledger_hash),
|
| 1398 |
'translated_text': str(result.get('text') or '').strip(),
|
| 1399 |
'qa_status': str(result.get('qa_status') or 'passed'),
|
| 1400 |
'language_adapter': str(result.get('language_adapter') or ''),
|
|
|
|
| 2527 |
target_lang=target_lang, translation_sections=translation_sections,
|
| 2528 |
translation_glossary=translation_glossary, adaptive_structure=adaptive_structure,
|
| 2529 |
include_front=True, defer_failures=False,
|
| 2530 |
+
english_commit_text=full_draft_en, final_gate=True,
|
| 2531 |
)
|
| 2532 |
partial = self._assemble_partial_translation(
|
| 2533 |
translation_sections, adaptive_structure, include_front=True
|
|
|
|
| 2590 |
literature_metrics=agent.get_source_metrics(),
|
| 2591 |
quality_diagnostics=quality_diagnostics,
|
| 2592 |
)
|
| 2593 |
+
self._progress(98, "package_validation")
|
| 2594 |
output_filename = f"SmartPaper_{bj_time}.zip"
|
| 2595 |
pending = self.output_dir / (".pending_" + output_filename)
|
| 2596 |
pending.write_bytes(package.zip_bytes)
|
core/pipeline_agent.py
CHANGED
|
@@ -70,8 +70,9 @@ class CitationAuditError(DeepSeekRequestError):
|
|
| 70 |
|
| 71 |
|
| 72 |
class SSciPipelineAgent:
|
| 73 |
-
TRANSLATION_CONTRACT_VERSION = 'hierarchical-multilingual-
|
| 74 |
MAX_TRANSLATION_CLAIM_REPAIR_ATTEMPTS = 2
|
|
|
|
| 75 |
|
| 76 |
def __init__(self, ds_key, s2_key=""):
|
| 77 |
self.ds_key = str(ds_key or "").strip()
|
|
@@ -3884,6 +3885,209 @@ Output only the repaired section."""
|
|
| 3884 |
'Do not retain ordinary English words or phrases merely because they appear technical or familiar.'
|
| 3885 |
)
|
| 3886 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3887 |
@classmethod
|
| 3888 |
def _translation_language_rule(cls, target_lang):
|
| 3889 |
profile = cls._translation_language_profile(target_lang)
|
|
@@ -3918,7 +4122,7 @@ Output only the repaired section."""
|
|
| 3918 |
# statistics and numeric anchors.
|
| 3919 |
quote_pattern = re.compile(r'“[^”\n]{1,500}”|"[^"\n]{1,500}"')
|
| 3920 |
text = quote_pattern.sub(lambda match: ' ' if match.group(0) in source_value else match.group(0), text)
|
| 3921 |
-
text = cls.
|
| 3922 |
text = re.sub(r'\[\[SP-(?:P\d{6}|C\d{6}|T\d{6})\]\]', ' ', text)
|
| 3923 |
|
| 3924 |
# A glossary target may legitimately remain Latin (for example a
|
|
@@ -4018,88 +4222,85 @@ Output only the repaired section."""
|
|
| 4018 |
|
| 4019 |
@classmethod
|
| 4020 |
def _translation_lexical_purity_evidence(cls, source, target, target_lang, glossary=None):
|
| 4021 |
-
"""
|
| 4022 |
-
|
| 4023 |
-
|
| 4024 |
-
|
| 4025 |
-
|
| 4026 |
-
|
| 4027 |
-
|
| 4028 |
-
because the languages share Cyrillic script and overlapping vocabulary.
|
| 4029 |
"""
|
| 4030 |
profile = cls._translation_language_profile(target_lang)
|
| 4031 |
language_code = profile.get('language_code')
|
| 4032 |
if language_code not in {'zh', 'ru', 'kk'}:
|
| 4033 |
-
return {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4034 |
|
| 4035 |
source_value = str(source or '')
|
| 4036 |
text = cls._strip_translation_claim_markers(str(target or ''))
|
| 4037 |
-
|
| 4038 |
-
|
| 4039 |
-
|
| 4040 |
-
|
| 4041 |
-
|
| 4042 |
-
|
| 4043 |
-
english_function_words = {'a','an','and','as','at','by','for','from','in','of','on','or','the','to','with'}
|
| 4044 |
-
legal_latin_terms = set()
|
| 4045 |
-
for value in dict(glossary or {}).values():
|
| 4046 |
-
term = str(value or '').strip()
|
| 4047 |
-
words = re.findall(r'[A-Za-z][A-Za-z0-9+.#-]*', term)
|
| 4048 |
-
title_like = bool(words) and len(words) <= 5 and all(
|
| 4049 |
-
word.casefold() in english_function_words or word[:1].isupper() for word in words
|
| 4050 |
-
)
|
| 4051 |
-
technical_shape = bool(words) and any(
|
| 4052 |
-
any(ch.isdigit() for ch in word) or word.isupper()
|
| 4053 |
-
or (len(word) > 2 and any(ch.isupper() for ch in word[1:])) for word in words
|
| 4054 |
-
)
|
| 4055 |
-
if term and re.search(r'[A-Za-z]', term) and (title_like or technical_shape):
|
| 4056 |
-
legal_latin_terms.add(term)
|
| 4057 |
-
for term in sorted(legal_latin_terms, key=len, reverse=True):
|
| 4058 |
-
text = re.sub(re.escape(term), ' ', text, flags=re.I)
|
| 4059 |
-
|
| 4060 |
-
source_tokens = list(re.finditer(r'\b[A-Za-z][A-Za-z0-9+.#-]*\b', source_value))
|
| 4061 |
-
technical = set()
|
| 4062 |
-
proper = set()
|
| 4063 |
-
ordinary_sentence_starters = {
|
| 4064 |
-
'the','this','these','those','prior','previous','current','our','we','results','findings',
|
| 4065 |
-
'participants','evidence','analysis','study','research','however','therefore','because',
|
| 4066 |
-
}
|
| 4067 |
-
for match in source_tokens:
|
| 4068 |
-
token = match.group(0)
|
| 4069 |
-
if any(ch.isdigit() for ch in token) or token.isupper() or (len(token) > 2 and any(ch.isupper() for ch in token[1:])):
|
| 4070 |
-
technical.add(token)
|
| 4071 |
continue
|
| 4072 |
-
if
|
| 4073 |
-
|
| 4074 |
-
|
| 4075 |
-
text = re.sub(rf'(?<![A-Za-z0-9_]){re.escape(token)}(?![A-Za-z0-9_])', ' ', text)
|
| 4076 |
|
| 4077 |
-
#
|
| 4078 |
-
#
|
| 4079 |
-
|
| 4080 |
-
|
| 4081 |
-
candidate = re.sub(r'\s+', ' ', match.group(0)).strip()
|
| 4082 |
-
if not candidate:
|
| 4083 |
-
continue
|
| 4084 |
-
words = candidate.split()
|
| 4085 |
-
title_like = all(word.casefold() in english_function_words or word[:1].isupper() for word in words)
|
| 4086 |
-
if title_like and candidate.casefold() in re.sub(r'\s+', ' ', source_value).casefold():
|
| 4087 |
-
text = text[:match.start()] + (' ' * (match.end() - match.start())) + text[match.end():]
|
| 4088 |
|
|
|
|
| 4089 |
allow_words = {'et', 'al', 'vs'}
|
| 4090 |
residual = []
|
| 4091 |
-
for match in re.finditer(r"[A-Za-z][A-Za-
|
| 4092 |
token = match.group(0).strip("-'’–—")
|
| 4093 |
folded = token.casefold()
|
| 4094 |
if folded in allow_words or len(token) < 4:
|
| 4095 |
continue
|
| 4096 |
residual.append(token)
|
| 4097 |
unique = list(dict.fromkeys(residual))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4098 |
return {
|
| 4099 |
-
'confirmed':
|
| 4100 |
'tokens': unique[:8],
|
| 4101 |
'token_count': len(residual),
|
| 4102 |
'language_code': language_code,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4103 |
}
|
| 4104 |
|
| 4105 |
@classmethod
|
|
@@ -5325,15 +5526,56 @@ If ok=false, codes MUST contain only values from allowed_issue_codes. For SEMANT
|
|
| 5325 |
detail={'count': len(canonical_violations), 'sources': '|'.join(canonical_violations)[:160]},
|
| 5326 |
)
|
| 5327 |
contamination = None
|
| 5328 |
-
|
| 5329 |
-
|
| 5330 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5331 |
if 'LANGUAGE_CONTAMINATION' not in codes:
|
| 5332 |
codes.append('LANGUAGE_CONTAMINATION')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5333 |
self._record_translation_assurance(
|
| 5334 |
'LEXICAL_PURITY_CONFIRMED', target_lang=target_lang, block_id=row_id,
|
| 5335 |
severity='repair', decision='REPAIR_LOCAL',
|
| 5336 |
-
detail={
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5337 |
)
|
| 5338 |
if language_code == 'ru':
|
| 5339 |
contamination = self._russian_contamination_evidence(
|
|
@@ -6480,14 +6722,19 @@ Decision must be CONFIRMED_BLOCKING or NOT_CONFIRMED. evidence_code must be one
|
|
| 6480 |
source_claims[claim_id], current_target_claim, target_lang, frozen_glossary
|
| 6481 |
)
|
| 6482 |
if lexical.get('confirmed'):
|
|
|
|
| 6483 |
repair_evidence['detected_unprotected_english_tokens'] = list(lexical.get('tokens') or [])[:8]
|
| 6484 |
-
repair_evidence['contamination_evidence_codes'] = [
|
|
|
|
|
|
|
|
|
|
| 6485 |
self._record_translation_assurance(
|
| 6486 |
'LEXICAL_PURITY_REPAIR_EVIDENCE', target_lang=target_lang,
|
| 6487 |
block_id=f'SP-P{block_id}-{claim_id}', severity='repair', decision='REPAIR_LOCAL',
|
| 6488 |
detail={
|
| 6489 |
'token_count': int(lexical.get('token_count') or 0),
|
| 6490 |
'sample': ','.join(lexical.get('tokens') or [])[:120],
|
|
|
|
| 6491 |
},
|
| 6492 |
)
|
| 6493 |
elif self._translation_language_profile(target_lang).get('language_code') == 'kk':
|
|
@@ -6564,6 +6811,7 @@ Decision must be CONFIRMED_BLOCKING or NOT_CONFIRMED. evidence_code must be one
|
|
| 6564 |
detail={
|
| 6565 |
'token_count': int(post_repair_lexical.get('token_count') or 0),
|
| 6566 |
'sample': ','.join(post_repair_lexical.get('tokens') or [])[:120],
|
|
|
|
| 6567 |
},
|
| 6568 |
)
|
| 6569 |
segments[claim_id] = repaired_claim
|
|
@@ -6600,7 +6848,7 @@ Decision must be CONFIRMED_BLOCKING or NOT_CONFIRMED. evidence_code must be one
|
|
| 6600 |
if prior & current:
|
| 6601 |
meta['nonconvergent_state'] = 'exhausted_after_source_regenerate'
|
| 6602 |
if quality_issues:
|
| 6603 |
-
emit_stage('
|
| 6604 |
quality_issues = self._adjudicate_persistent_translation_issues(
|
| 6605 |
source_blocks, translations, quality_issues, target_lang, frozen_glossary,
|
| 6606 |
section_name=section_name or 'frozen section',
|
|
@@ -6643,6 +6891,8 @@ Decision must be CONFIRMED_BLOCKING or NOT_CONFIRMED. evidence_code must be one
|
|
| 6643 |
'qa_status': 'passed_with_warning' if diagnostics else 'passed',
|
| 6644 |
'translation_contract_version': self.TRANSLATION_CONTRACT_VERSION,
|
| 6645 |
'fact_ledger_hash': self.translation_fact_ledger_hash(value),
|
|
|
|
|
|
|
| 6646 |
'language_adapter': profile['adapter_id'],
|
| 6647 |
'diagnostics': diagnostics[-20:],
|
| 6648 |
'claim_states': dict(self._last_translation_claim_states),
|
|
|
|
| 70 |
|
| 71 |
|
| 72 |
class SSciPipelineAgent:
|
| 73 |
+
TRANSLATION_CONTRACT_VERSION = 'hierarchical-multilingual-v8'
|
| 74 |
MAX_TRANSLATION_CLAIM_REPAIR_ATTEMPTS = 2
|
| 75 |
+
PROTECTED_ENTITY_LEDGER_VERSION = 'protected-entity-ledger-v1'
|
| 76 |
|
| 77 |
def __init__(self, ds_key, s2_key=""):
|
| 78 |
self.ds_key = str(ds_key or "").strip()
|
|
|
|
| 3885 |
'Do not retain ordinary English words or phrases merely because they appear technical or familiar.'
|
| 3886 |
)
|
| 3887 |
|
| 3888 |
+
@staticmethod
|
| 3889 |
+
def _translation_ascii_boundary_pattern(surface):
|
| 3890 |
+
"""Return an ASCII-word-boundary pattern that remains safe next to CJK/Cyrillic.
|
| 3891 |
+
|
| 3892 |
+
Python ``\\b`` treats many non-Latin letters as word characters. A protected
|
| 3893 |
+
object such as ``ChatGPT`` can therefore fail to match in ``使用ChatGPT模型``.
|
| 3894 |
+
Translation lexical authority uses explicit ASCII boundaries instead.
|
| 3895 |
+
"""
|
| 3896 |
+
value = str(surface or '').strip()
|
| 3897 |
+
if not value:
|
| 3898 |
+
return None
|
| 3899 |
+
return re.compile(rf'(?<![A-Za-z0-9_]){re.escape(value)}(?![A-Za-z0-9_])', re.I)
|
| 3900 |
+
|
| 3901 |
+
@staticmethod
|
| 3902 |
+
def _translation_technical_entity_shape(value):
|
| 3903 |
+
token = str(value or '').strip()
|
| 3904 |
+
if not token or not re.search(r'[A-Za-z]', token):
|
| 3905 |
+
return False
|
| 3906 |
+
compact = re.sub(r'[^A-Za-z0-9]', '', token)
|
| 3907 |
+
return bool(
|
| 3908 |
+
any(ch.isdigit() for ch in compact)
|
| 3909 |
+
or (len(compact) >= 2 and compact.isupper())
|
| 3910 |
+
or (len(compact) > 2 and any(ch.isupper() for ch in compact[1:]))
|
| 3911 |
+
)
|
| 3912 |
+
|
| 3913 |
+
@classmethod
|
| 3914 |
+
def _translation_protected_entity_ledger(cls, source, glossary=None):
|
| 3915 |
+
"""Build a deterministic source-owned Latin entity ledger.
|
| 3916 |
+
|
| 3917 |
+
The ledger is derived from the current frozen source, never from a product
|
| 3918 |
+
tier or global user whitelist. Registry scope is the supplied frozen unit;
|
| 3919 |
+
each entry retains claim-local ownership so an entity occurring elsewhere in
|
| 3920 |
+
the manuscript cannot authorize insertion into another claim.
|
| 3921 |
+
"""
|
| 3922 |
+
source_value = str(source or '')
|
| 3923 |
+
entries = []
|
| 3924 |
+
seen = set()
|
| 3925 |
+
ordinary_sentence_starters = {
|
| 3926 |
+
'the','this','these','those','prior','previous','current','our','we','results','findings',
|
| 3927 |
+
'participants','evidence','analysis','study','research','however','therefore','because',
|
| 3928 |
+
'method','methods','discussion','introduction','conclusion','open',
|
| 3929 |
+
}
|
| 3930 |
+
compound_suffixes = {
|
| 3931 |
+
'based','style','derived','enabled','assisted','powered','driven','supported','mediated',
|
| 3932 |
+
'inspired','oriented','led','related','specific','level','type',
|
| 3933 |
+
}
|
| 3934 |
+
|
| 3935 |
+
def add(canonical, category, claim_id, surface=None, target_surface=None):
|
| 3936 |
+
canonical = str(canonical or '').strip()
|
| 3937 |
+
if not canonical or not re.search(r'[A-Za-z]', canonical):
|
| 3938 |
+
return
|
| 3939 |
+
surface = str(surface or canonical).strip()
|
| 3940 |
+
target_surface = str(target_surface or canonical).strip()
|
| 3941 |
+
key = (canonical.casefold(), str(category or ''), str(claim_id or ''), target_surface.casefold())
|
| 3942 |
+
if key in seen:
|
| 3943 |
+
return
|
| 3944 |
+
seen.add(key)
|
| 3945 |
+
entity_material = '|'.join(key).encode('utf-8')
|
| 3946 |
+
entries.append({
|
| 3947 |
+
'entity_id': 'E' + hashlib.sha256(entity_material).hexdigest()[:12],
|
| 3948 |
+
'canonical': canonical,
|
| 3949 |
+
'category': str(category or 'PROPER_NAME'),
|
| 3950 |
+
'source_surface': surface,
|
| 3951 |
+
'target_surface': target_surface,
|
| 3952 |
+
'claim_id': str(claim_id or 'C000001'),
|
| 3953 |
+
'retention_policy': 'exact_latin_identity',
|
| 3954 |
+
})
|
| 3955 |
+
|
| 3956 |
+
for claim in cls._translation_claim_units(source_value):
|
| 3957 |
+
claim_id = str(claim.get('claim_id') or 'C000001')
|
| 3958 |
+
claim_source = str(claim.get('text') or '')
|
| 3959 |
+
ir = cls._compile_translation_ir(claim_source)
|
| 3960 |
+
for manifest in (ir.get('fact_ledger') or {}).values():
|
| 3961 |
+
if not isinstance(manifest, dict):
|
| 3962 |
+
continue
|
| 3963 |
+
anchor_type = str(manifest.get('type') or '')
|
| 3964 |
+
surface = str(manifest.get('source') or '').strip()
|
| 3965 |
+
if anchor_type == 'NAME':
|
| 3966 |
+
add(surface, 'MODEL_TOOL', claim_id, surface)
|
| 3967 |
+
elif anchor_type == 'ACRONYM':
|
| 3968 |
+
add(surface, 'ACRONYM', claim_id, surface)
|
| 3969 |
+
elif anchor_type in {'HYPOTHESIS','QUOTE_ID'}:
|
| 3970 |
+
add(surface, 'VARIABLE', claim_id, surface)
|
| 3971 |
+
|
| 3972 |
+
token_pattern = re.compile(
|
| 3973 |
+
r'(?<![A-Za-z0-9_])([A-Za-z][A-Za-z0-9+.#]*(?:-[A-Za-z0-9+.#]+)*)(?![A-Za-z0-9_])'
|
| 3974 |
+
)
|
| 3975 |
+
for match in token_pattern.finditer(claim_source):
|
| 3976 |
+
token = match.group(1)
|
| 3977 |
+
if '-' in token:
|
| 3978 |
+
base, suffix = token.split('-', 1)
|
| 3979 |
+
suffix_folded = suffix.casefold()
|
| 3980 |
+
if suffix_folded in compound_suffixes and cls._translation_technical_entity_shape(base):
|
| 3981 |
+
add(base, 'MODEL_TOOL', claim_id, token, base)
|
| 3982 |
+
continue
|
| 3983 |
+
if suffix_folded in compound_suffixes and re.fullmatch(r'[A-Z][a-z]{2,}', base):
|
| 3984 |
+
add(base, 'PROPER_NAME', claim_id, token, base)
|
| 3985 |
+
continue
|
| 3986 |
+
if cls._translation_technical_entity_shape(token):
|
| 3987 |
+
add(token, 'MODEL_TOOL', claim_id, token)
|
| 3988 |
+
|
| 3989 |
+
# Conservative standalone proper-name support. Hyphen-adjacent words
|
| 3990 |
+
# are excluded here and handled only by the bounded compound rule above,
|
| 3991 |
+
# preventing ``Open-ended`` from authorizing a bare ``Open``.
|
| 3992 |
+
for match in re.finditer(r'(?<![A-Za-z0-9_-])([A-Z][a-z]{2,})(?![A-Za-z0-9_-])', claim_source):
|
| 3993 |
+
token = match.group(1)
|
| 3994 |
+
if token.casefold() in ordinary_sentence_starters:
|
| 3995 |
+
continue
|
| 3996 |
+
if match.start() == 0:
|
| 3997 |
+
continue
|
| 3998 |
+
add(token, 'PROPER_NAME', claim_id, token)
|
| 3999 |
+
|
| 4000 |
+
for source_term, target_term in dict(glossary or {}).items():
|
| 4001 |
+
source_term = str(source_term or '').strip()
|
| 4002 |
+
target_term = str(target_term or '').strip()
|
| 4003 |
+
if not source_term or source_term.casefold() not in claim_source.casefold():
|
| 4004 |
+
continue
|
| 4005 |
+
words = re.findall(r'[A-Za-z][A-Za-z0-9+.#-]*', target_term)
|
| 4006 |
+
title_like = bool(words) and len(words) <= 6 and all(
|
| 4007 |
+
word.casefold() in {'a','an','and','as','at','by','for','from','in','of','on','or','the','to','with'}
|
| 4008 |
+
or word[:1].isupper() for word in words
|
| 4009 |
+
)
|
| 4010 |
+
if target_term and re.search(r'[A-Za-z]', target_term) and (
|
| 4011 |
+
title_like or cls._translation_technical_entity_shape(target_term)
|
| 4012 |
+
):
|
| 4013 |
+
add(target_term, 'LEGAL_GLOSSARY', claim_id, source_term, target_term)
|
| 4014 |
+
|
| 4015 |
+
normalized_entries = sorted(
|
| 4016 |
+
entries,
|
| 4017 |
+
key=lambda row: (row['claim_id'], row['canonical'].casefold(), row['category'], row['target_surface'].casefold()),
|
| 4018 |
+
)
|
| 4019 |
+
material = json.dumps(normalized_entries, ensure_ascii=False, sort_keys=True, separators=(',', ':'))
|
| 4020 |
+
return {
|
| 4021 |
+
'version': cls.PROTECTED_ENTITY_LEDGER_VERSION,
|
| 4022 |
+
'entries': normalized_entries,
|
| 4023 |
+
'hash': hashlib.sha256(material.encode('utf-8')).hexdigest(),
|
| 4024 |
+
}
|
| 4025 |
+
|
| 4026 |
+
@classmethod
|
| 4027 |
+
def translation_protected_entity_ledger_hash(cls, source, glossary=None):
|
| 4028 |
+
return str(cls._translation_protected_entity_ledger(source, glossary).get('hash') or '')
|
| 4029 |
+
|
| 4030 |
+
@classmethod
|
| 4031 |
+
def _translation_lexical_nonentity_protected_pattern(cls):
|
| 4032 |
+
"""Protect non-lexical immutable objects without granting global entity authority.
|
| 4033 |
+
|
| 4034 |
+
Names/acronyms/model identifiers are intentionally excluded: they are legal
|
| 4035 |
+
only through the source-owned Protected Entity Ledger. This prevents a
|
| 4036 |
+
target-only ``ChatGPT``/``OpenAI``/tool name from being silently exempted.
|
| 4037 |
+
"""
|
| 4038 |
+
numeric = cls._translation_numeric_regex()
|
| 4039 |
+
date_pattern = cls._translation_date_regex()
|
| 4040 |
+
quantity_words = cls._translation_number_word_regex()
|
| 4041 |
+
compound_ci = r'(?i:(?:\d+(?:\.\d+)?%)?\s*CI\s*\[\s*[-+]?\.?\d+(?:\.\d+)?\s*,\s*[-+]?\.?\d+(?:\.\d+)?\s*\])'
|
| 4042 |
+
return re.compile(
|
| 4043 |
+
rf'(?:{date_pattern})'
|
| 4044 |
+
rf'|(?:{compound_ci})'
|
| 4045 |
+
rf'|(?:{quantity_words})'
|
| 4046 |
+
r'|(?i:https?://[^\s)>]+)'
|
| 4047 |
+
r'|(?i:\bdoi:\s*[^\s)>]+)'
|
| 4048 |
+
r'|(?i:\b10\.\d{4,9}/[-._;()/:A-Z0-9]+)'
|
| 4049 |
+
r'|(?i:\b(?:p|t|f|z|r|n)\s*(?:<=|>=|=|<|>|≤|≥)\s*[-+]?(?:\d+(?:\.\d+)?|\.\d+))'
|
| 4050 |
+
r'|(?:β|χ²|R²|ΔR²)\s*(?:<=|>=|=|<|>|≤|≥)\s*[-+]?(?:\d+(?:\.\d+)?|\.\d+)'
|
| 4051 |
+
r'|\([^()\n]*?(?:19|20)\d{2}[a-z]?[^()\n]*?\)'
|
| 4052 |
+
r'|`[^`\n]+`'
|
| 4053 |
+
r'|\$[^$\n]+\$'
|
| 4054 |
+
r'|(?:χ²|R²|ΔR²|η²|ω²|[αβγδΔχλμρστφηωΩ])'
|
| 4055 |
+
r'|\b(?:QUOTE|CASE|RESP|P)-\d+\b'
|
| 4056 |
+
rf'|(?:{numeric})'
|
| 4057 |
+
r'|\|'
|
| 4058 |
+
)
|
| 4059 |
+
|
| 4060 |
+
@staticmethod
|
| 4061 |
+
def _translation_sustained_latin_prose_evidence(text):
|
| 4062 |
+
value = str(text or '')
|
| 4063 |
+
matches = list(re.finditer(r"[A-Za-z][A-Za-z'’-]*", value))
|
| 4064 |
+
function_words = {
|
| 4065 |
+
'a','an','and','are','as','at','be','been','being','by','for','from','had','has','have',
|
| 4066 |
+
'in','into','is','it','of','on','or','that','the','these','this','those','to','was','were',
|
| 4067 |
+
'which','with','within','without','would','could','should','than','then','their','its',
|
| 4068 |
+
}
|
| 4069 |
+
allowed_gap = re.compile(r"^[\s,;:()\[\]{}.!?\"'“”‘’—–/\\-]*$")
|
| 4070 |
+
runs = []
|
| 4071 |
+
current = []
|
| 4072 |
+
previous_end = None
|
| 4073 |
+
for match in matches:
|
| 4074 |
+
if previous_end is None or allowed_gap.fullmatch(value[previous_end:match.start()] or ''):
|
| 4075 |
+
current.append(match)
|
| 4076 |
+
else:
|
| 4077 |
+
if current:
|
| 4078 |
+
runs.append(current)
|
| 4079 |
+
current = [match]
|
| 4080 |
+
previous_end = match.end()
|
| 4081 |
+
if current:
|
| 4082 |
+
runs.append(current)
|
| 4083 |
+
max_words = 0
|
| 4084 |
+
for run in runs:
|
| 4085 |
+
words = [item.group(0).casefold() for item in run]
|
| 4086 |
+
max_words = max(max_words, len(words))
|
| 4087 |
+
if (len(words) >= 4 and any(word in function_words for word in words)) or len(words) >= 7:
|
| 4088 |
+
return {'confirmed': True, 'max_run_words': len(words)}
|
| 4089 |
+
return {'confirmed': False, 'max_run_words': max_words}
|
| 4090 |
+
|
| 4091 |
@classmethod
|
| 4092 |
def _translation_language_rule(cls, target_lang):
|
| 4093 |
profile = cls._translation_language_profile(target_lang)
|
|
|
|
| 4122 |
# statistics and numeric anchors.
|
| 4123 |
quote_pattern = re.compile(r'“[^”\n]{1,500}”|"[^"\n]{1,500}"')
|
| 4124 |
text = quote_pattern.sub(lambda match: ' ' if match.group(0) in source_value else match.group(0), text)
|
| 4125 |
+
text = cls._translation_lexical_nonentity_protected_pattern().sub(' ', text)
|
| 4126 |
text = re.sub(r'\[\[SP-(?:P\d{6}|C\d{6}|T\d{6})\]\]', ' ', text)
|
| 4127 |
|
| 4128 |
# A glossary target may legitimately remain Latin (for example a
|
|
|
|
| 4222 |
|
| 4223 |
@classmethod
|
| 4224 |
def _translation_lexical_purity_evidence(cls, source, target, target_lang, glossary=None):
|
| 4225 |
+
"""Classify target Latin residue after claim-local protected-entity provenance.
|
| 4226 |
+
|
| 4227 |
+
Source-authorized entities are removed with ASCII-safe boundaries before
|
| 4228 |
+
lexical contamination authority is assigned. Genuine ordinary English
|
| 4229 |
+
residue remains fail-closed, while a target-invented technical entity is
|
| 4230 |
+
routed as an entity-provenance mismatch rather than mislabeled as an
|
| 4231 |
+
untranslated source lexeme.
|
|
|
|
| 4232 |
"""
|
| 4233 |
profile = cls._translation_language_profile(target_lang)
|
| 4234 |
language_code = profile.get('language_code')
|
| 4235 |
if language_code not in {'zh', 'ru', 'kk'}:
|
| 4236 |
+
return {
|
| 4237 |
+
'confirmed': False, 'tokens': [], 'token_count': 0,
|
| 4238 |
+
'language_code': language_code or '', 'subtype': '',
|
| 4239 |
+
'protected_entities': [], 'ledger_version': cls.PROTECTED_ENTITY_LEDGER_VERSION,
|
| 4240 |
+
'ledger_hash': '',
|
| 4241 |
+
}
|
| 4242 |
|
| 4243 |
source_value = str(source or '')
|
| 4244 |
text = cls._strip_translation_claim_markers(str(target or ''))
|
| 4245 |
+
ledger = cls._translation_protected_entity_ledger(source_value, glossary)
|
| 4246 |
+
retained = []
|
| 4247 |
+
for entry in ledger.get('entries') or []:
|
| 4248 |
+
surface = str(entry.get('target_surface') or entry.get('canonical') or '').strip()
|
| 4249 |
+
pattern = cls._translation_ascii_boundary_pattern(surface)
|
| 4250 |
+
if not pattern:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4251 |
continue
|
| 4252 |
+
if pattern.search(text):
|
| 4253 |
+
retained.append(str(entry.get('canonical') or surface))
|
| 4254 |
+
text = pattern.sub(' ', text)
|
|
|
|
| 4255 |
|
| 4256 |
+
# Non-lexical immutable objects remain protected. Names/acronyms/model
|
| 4257 |
+
# identifiers are excluded here and require claim-local ledger ownership.
|
| 4258 |
+
text = cls._translation_lexical_nonentity_protected_pattern().sub(' ', text)
|
| 4259 |
+
text = re.sub(r'\[\[SP-(?:P\d{6}|C\d{6}|T\d{6})\]\]', ' ', text)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4260 |
|
| 4261 |
+
sustained = cls._translation_sustained_latin_prose_evidence(text)
|
| 4262 |
allow_words = {'et', 'al', 'vs'}
|
| 4263 |
residual = []
|
| 4264 |
+
for match in re.finditer(r"[A-Za-z][A-Za-z0-9+.#]*(?:[-'’–—][A-Za-z0-9+.#]+)*", text):
|
| 4265 |
token = match.group(0).strip("-'’–—")
|
| 4266 |
folded = token.casefold()
|
| 4267 |
if folded in allow_words or len(token) < 4:
|
| 4268 |
continue
|
| 4269 |
residual.append(token)
|
| 4270 |
unique = list(dict.fromkeys(residual))
|
| 4271 |
+
if not unique:
|
| 4272 |
+
return {
|
| 4273 |
+
'confirmed': False,
|
| 4274 |
+
'tokens': [],
|
| 4275 |
+
'token_count': 0,
|
| 4276 |
+
'language_code': language_code,
|
| 4277 |
+
'subtype': 'PROTECTED_ENTITY_RETAINED' if retained else '',
|
| 4278 |
+
'protected_entities': list(dict.fromkeys(retained))[:8],
|
| 4279 |
+
'ledger_version': str(ledger.get('version') or cls.PROTECTED_ENTITY_LEDGER_VERSION),
|
| 4280 |
+
'ledger_hash': str(ledger.get('hash') or ''),
|
| 4281 |
+
}
|
| 4282 |
+
|
| 4283 |
+
source_folded = source_value.casefold()
|
| 4284 |
+
mismatch = [
|
| 4285 |
+
token for token in unique
|
| 4286 |
+
if cls._translation_technical_entity_shape(token) and token.casefold() not in source_folded
|
| 4287 |
+
]
|
| 4288 |
+
subtype = 'UNTRANSLATED_ORDINARY_LEXEME'
|
| 4289 |
+
if sustained.get('confirmed'):
|
| 4290 |
+
subtype = 'UNTRANSLATED_PROSE'
|
| 4291 |
+
elif mismatch:
|
| 4292 |
+
subtype = 'ENTITY_PROVENANCE_MISMATCH'
|
| 4293 |
return {
|
| 4294 |
+
'confirmed': True,
|
| 4295 |
'tokens': unique[:8],
|
| 4296 |
'token_count': len(residual),
|
| 4297 |
'language_code': language_code,
|
| 4298 |
+
'subtype': subtype,
|
| 4299 |
+
'protected_entities': list(dict.fromkeys(retained))[:8],
|
| 4300 |
+
'entity_provenance_mismatch_tokens': mismatch[:8],
|
| 4301 |
+
'ledger_version': str(ledger.get('version') or cls.PROTECTED_ENTITY_LEDGER_VERSION),
|
| 4302 |
+
'ledger_hash': str(ledger.get('hash') or ''),
|
| 4303 |
+
'max_latin_run_words': int(sustained.get('max_run_words') or 0),
|
| 4304 |
}
|
| 4305 |
|
| 4306 |
@classmethod
|
|
|
|
| 5526 |
detail={'count': len(canonical_violations), 'sources': '|'.join(canonical_violations)[:160]},
|
| 5527 |
)
|
| 5528 |
contamination = None
|
| 5529 |
+
lexical_subtype = str(lexical.get('subtype') or '')
|
| 5530 |
+
entity_mismatch = lexical_subtype == 'ENTITY_PROVENANCE_MISMATCH'
|
| 5531 |
+
deterministic_contamination = bool(lexical.get('confirmed')) and not entity_mismatch
|
| 5532 |
+
deterministic_issue = bool(canonical_violations) or entity_mismatch
|
| 5533 |
+
if lexical_subtype == 'PROTECTED_ENTITY_RETAINED' and model_contamination:
|
| 5534 |
+
self._record_translation_assurance(
|
| 5535 |
+
'PROTECTED_ENTITY_RETAINED', target_lang=target_lang, block_id=row_id,
|
| 5536 |
+
severity='diagnostic', decision='PASS_WITH_WARNING',
|
| 5537 |
+
detail={
|
| 5538 |
+
'count': len(lexical.get('protected_entities') or []),
|
| 5539 |
+
'sample': '|'.join(lexical.get('protected_entities') or [])[:120],
|
| 5540 |
+
},
|
| 5541 |
+
)
|
| 5542 |
+
if entity_mismatch:
|
| 5543 |
+
codes = [code for code in codes if code != 'LANGUAGE_CONTAMINATION']
|
| 5544 |
+
if 'ADDED_CLAIM' not in codes:
|
| 5545 |
+
codes.append('ADDED_CLAIM')
|
| 5546 |
+
evidence_row = self._translation_claim_qa_evidence.setdefault(
|
| 5547 |
+
(meta['block_id'], meta['claim_id']), {}
|
| 5548 |
+
)
|
| 5549 |
+
semantic = list(evidence_row.get('semantic_delta_codes') or [])
|
| 5550 |
+
if 'CLAIM_ADDED' not in semantic:
|
| 5551 |
+
semantic.append('CLAIM_ADDED')
|
| 5552 |
+
evidence_row['semantic_delta_codes'] = semantic[:6]
|
| 5553 |
+
evidence_row['entity_provenance_mismatch_tokens'] = list(
|
| 5554 |
+
lexical.get('entity_provenance_mismatch_tokens') or lexical.get('tokens') or []
|
| 5555 |
+
)[:8]
|
| 5556 |
+
self._record_translation_assurance(
|
| 5557 |
+
'ENTITY_PROVENANCE_MISMATCH', target_lang=target_lang, block_id=row_id,
|
| 5558 |
+
severity='repair', decision='REPAIR_LOCAL',
|
| 5559 |
+
detail={'sample': ','.join(lexical.get('entity_provenance_mismatch_tokens') or lexical.get('tokens') or [])[:120]},
|
| 5560 |
+
)
|
| 5561 |
+
elif deterministic_contamination:
|
| 5562 |
if 'LANGUAGE_CONTAMINATION' not in codes:
|
| 5563 |
codes.append('LANGUAGE_CONTAMINATION')
|
| 5564 |
+
evidence_row = self._translation_claim_qa_evidence.setdefault(
|
| 5565 |
+
(meta['block_id'], meta['claim_id']), {}
|
| 5566 |
+
)
|
| 5567 |
+
evidence_row['contamination_evidence_codes'] = [
|
| 5568 |
+
'UNTRANSLATED_PROSE' if lexical_subtype == 'UNTRANSLATED_PROSE' else 'UNTRANSLATED_LEXEME'
|
| 5569 |
+
]
|
| 5570 |
+
evidence_row['lexical_evidence_code'] = lexical_subtype or 'UNTRANSLATED_ORDINARY_LEXEME'
|
| 5571 |
self._record_translation_assurance(
|
| 5572 |
'LEXICAL_PURITY_CONFIRMED', target_lang=target_lang, block_id=row_id,
|
| 5573 |
severity='repair', decision='REPAIR_LOCAL',
|
| 5574 |
+
detail={
|
| 5575 |
+
'token_count': int(lexical.get('token_count') or 0),
|
| 5576 |
+
'sample': ','.join(lexical.get('tokens') or [])[:120],
|
| 5577 |
+
'subtype': lexical_subtype or 'UNTRANSLATED_ORDINARY_LEXEME',
|
| 5578 |
+
},
|
| 5579 |
)
|
| 5580 |
if language_code == 'ru':
|
| 5581 |
contamination = self._russian_contamination_evidence(
|
|
|
|
| 6722 |
source_claims[claim_id], current_target_claim, target_lang, frozen_glossary
|
| 6723 |
)
|
| 6724 |
if lexical.get('confirmed'):
|
| 6725 |
+
lexical_subtype = str(lexical.get('subtype') or 'UNTRANSLATED_ORDINARY_LEXEME')
|
| 6726 |
repair_evidence['detected_unprotected_english_tokens'] = list(lexical.get('tokens') or [])[:8]
|
| 6727 |
+
repair_evidence['contamination_evidence_codes'] = [
|
| 6728 |
+
'UNTRANSLATED_PROSE' if lexical_subtype == 'UNTRANSLATED_PROSE' else 'UNTRANSLATED_LEXEME'
|
| 6729 |
+
]
|
| 6730 |
+
repair_evidence['lexical_evidence_code'] = lexical_subtype
|
| 6731 |
self._record_translation_assurance(
|
| 6732 |
'LEXICAL_PURITY_REPAIR_EVIDENCE', target_lang=target_lang,
|
| 6733 |
block_id=f'SP-P{block_id}-{claim_id}', severity='repair', decision='REPAIR_LOCAL',
|
| 6734 |
detail={
|
| 6735 |
'token_count': int(lexical.get('token_count') or 0),
|
| 6736 |
'sample': ','.join(lexical.get('tokens') or [])[:120],
|
| 6737 |
+
'subtype': lexical_subtype,
|
| 6738 |
},
|
| 6739 |
)
|
| 6740 |
elif self._translation_language_profile(target_lang).get('language_code') == 'kk':
|
|
|
|
| 6811 |
detail={
|
| 6812 |
'token_count': int(post_repair_lexical.get('token_count') or 0),
|
| 6813 |
'sample': ','.join(post_repair_lexical.get('tokens') or [])[:120],
|
| 6814 |
+
'subtype': str(post_repair_lexical.get('subtype') or ''),
|
| 6815 |
},
|
| 6816 |
)
|
| 6817 |
segments[claim_id] = repaired_claim
|
|
|
|
| 6848 |
if prior & current:
|
| 6849 |
meta['nonconvergent_state'] = 'exhausted_after_source_regenerate'
|
| 6850 |
if quality_issues:
|
| 6851 |
+
emit_stage('adjudicating')
|
| 6852 |
quality_issues = self._adjudicate_persistent_translation_issues(
|
| 6853 |
source_blocks, translations, quality_issues, target_lang, frozen_glossary,
|
| 6854 |
section_name=section_name or 'frozen section',
|
|
|
|
| 6891 |
'qa_status': 'passed_with_warning' if diagnostics else 'passed',
|
| 6892 |
'translation_contract_version': self.TRANSLATION_CONTRACT_VERSION,
|
| 6893 |
'fact_ledger_hash': self.translation_fact_ledger_hash(value),
|
| 6894 |
+
'protected_entity_ledger_version': self.PROTECTED_ENTITY_LEDGER_VERSION,
|
| 6895 |
+
'protected_entity_ledger_hash': self.translation_protected_entity_ledger_hash(value, frozen_glossary),
|
| 6896 |
'language_adapter': profile['adapter_id'],
|
| 6897 |
'diagnostics': diagnostics[-20:],
|
| 6898 |
'claim_states': dict(self._last_translation_claim_states),
|
i18n.py
CHANGED
|
@@ -233,6 +233,11 @@ TEXTS: dict[str, dict[str, str]] = {
|
|
| 233 |
"draft_detail_current": "当前阶段:{stage}",
|
| 234 |
"translation_section_running": "正在翻译【{name}】…",
|
| 235 |
"translation_section_qa": "正在审核【{name}】的翻译完整性与语义一致性…",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 236 |
"translation_section_ready": "【{name}】翻译审核通过,正在显示译文…",
|
| 237 |
"translation_complete_label": "目标语言论文翻译完成。",
|
| 238 |
"busy_short": "⏳ 有用户正在生成论文,预计25–35分钟后可用。",
|
|
@@ -404,6 +409,11 @@ TEXTS: dict[str, dict[str, str]] = {
|
|
| 404 |
"draft_detail_current": "Current stage: {stage}",
|
| 405 |
"translation_section_running": "Translating 【{name}】…",
|
| 406 |
"translation_section_qa": "Checking translation integrity and semantic equivalence for 【{name}】…",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 407 |
"translation_section_ready": "Translation of 【{name}】 passed QA and is being displayed…",
|
| 408 |
"translation_complete_label": "The target-language paper translation is complete.",
|
| 409 |
"busy_short": "⏳ Another user is generating a paper. Please try again in about 25–35 minutes.",
|
|
@@ -575,6 +585,11 @@ TEXTS: dict[str, dict[str, str]] = {
|
|
| 575 |
"draft_detail_current": "Текущий этап: {stage}",
|
| 576 |
"translation_section_running": "Перевод раздела 【{name}】…",
|
| 577 |
"translation_section_qa": "Проверка полноты и смысловой эквивалентности перевода раздела 【{name}】…",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 578 |
"translation_section_ready": "Перевод раздела 【{name}】 прошёл проверку и отображается…",
|
| 579 |
"translation_complete_label": "Перевод статьи на целевой язык завершён.",
|
| 580 |
"busy_short": "⏳ Другой пользователь формирует статью. Повторите попытку примерно через 25–35 минут.",
|
|
@@ -746,6 +761,11 @@ TEXTS: dict[str, dict[str, str]] = {
|
|
| 746 |
"draft_detail_current": "Ағымдағы кезең: {stage}",
|
| 747 |
"translation_section_running": "【{name}】 бөлімі аударылуда…",
|
| 748 |
"translation_section_qa": "【{name}】 аудармасының толықтығы мен мағыналық сәйкестігі тексерілуде…",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 749 |
"translation_section_ready": "【{name}】 аудармасы тексеруден өтті және көрсетілуде…",
|
| 750 |
"translation_complete_label": "Мақаланың мақса��ты тілдегі аудармасы аяқталды.",
|
| 751 |
"busy_short": "⏳ Басқа пайдаланушы мақала дайындап жатыр. Шамамен 25–35 минуттан кейін қайталап көріңіз.",
|
|
|
|
| 233 |
"draft_detail_current": "当前阶段:{stage}",
|
| 234 |
"translation_section_running": "正在翻译【{name}】…",
|
| 235 |
"translation_section_qa": "正在审核【{name}】的翻译完整性与语义一致性…",
|
| 236 |
+
"translation_final_gate": "正在进行【{name}】最终翻译质检……",
|
| 237 |
+
"translation_section_repairing": "正在修复【{name}】翻译问题……",
|
| 238 |
+
"translation_section_repair_pending": "【{name}】仍有翻译问题,等待最终复核……",
|
| 239 |
+
"translation_section_adjudicating": "正在进行【{name}】最终翻译一致性确认……",
|
| 240 |
+
"translation_progress_counts": "{passed}/{total} 已通过 · {repair} 待修复 · {inflight} 正在处理",
|
| 241 |
"translation_section_ready": "【{name}】翻译审核通过,正在显示译文…",
|
| 242 |
"translation_complete_label": "目标语言论文翻译完成。",
|
| 243 |
"busy_short": "⏳ 有用户正在生成论文,预计25–35分钟后可用。",
|
|
|
|
| 409 |
"draft_detail_current": "Current stage: {stage}",
|
| 410 |
"translation_section_running": "Translating 【{name}】…",
|
| 411 |
"translation_section_qa": "Checking translation integrity and semantic equivalence for 【{name}】…",
|
| 412 |
+
"translation_final_gate": "Running final translation QA for 【{name}】…",
|
| 413 |
+
"translation_section_repairing": "Repairing the translation of 【{name}】…",
|
| 414 |
+
"translation_section_repair_pending": "Translation issues remain in 【{name}】 and are queued for final review…",
|
| 415 |
+
"translation_section_adjudicating": "Running final translation consistency confirmation for 【{name}】…",
|
| 416 |
+
"translation_progress_counts": "{passed}/{total} passed · {repair} pending repair · {inflight} in progress",
|
| 417 |
"translation_section_ready": "Translation of 【{name}】 passed QA and is being displayed…",
|
| 418 |
"translation_complete_label": "The target-language paper translation is complete.",
|
| 419 |
"busy_short": "⏳ Another user is generating a paper. Please try again in about 25–35 minutes.",
|
|
|
|
| 585 |
"draft_detail_current": "Текущий этап: {stage}",
|
| 586 |
"translation_section_running": "Перевод раздела 【{name}】…",
|
| 587 |
"translation_section_qa": "Проверка полноты и смысловой эквивалентности перевода раздела 【{name}】…",
|
| 588 |
+
"translation_final_gate": "Финальная проверка перевода раздела 【{name}】…",
|
| 589 |
+
"translation_section_repairing": "Исправление перевода раздела 【{name}】…",
|
| 590 |
+
"translation_section_repair_pending": "В разделе 【{name}】 остаются проблемы перевода; ожидается финальная проверка…",
|
| 591 |
+
"translation_section_adjudicating": "Финальное подтверждение согласованности перевода раздела 【{name}】…",
|
| 592 |
+
"translation_progress_counts": "пройдено {passed}/{total} · ожидают исправления {repair} · обрабатывается {inflight}",
|
| 593 |
"translation_section_ready": "Перевод раздела 【{name}】 прошёл проверку и отображается…",
|
| 594 |
"translation_complete_label": "Перевод статьи на целевой язык завершён.",
|
| 595 |
"busy_short": "⏳ Другой пользователь формирует статью. Повторите попытку примерно через 25–35 минут.",
|
|
|
|
| 761 |
"draft_detail_current": "Ағымдағы кезең: {stage}",
|
| 762 |
"translation_section_running": "【{name}】 бөлімі аударылуда…",
|
| 763 |
"translation_section_qa": "【{name}】 аудармасының толықтығы мен мағыналық сәйкестігі тексерілуде…",
|
| 764 |
+
"translation_final_gate": "【{name}】 аудармасының соңғы сапа тексеруі орындалуда…",
|
| 765 |
+
"translation_section_repairing": "【{name}】 аудармасындағы мәселе түзетілуде…",
|
| 766 |
+
"translation_section_repair_pending": "【{name}】 аудармасында мәселе қалды; соңғы тексеруді күтуде…",
|
| 767 |
+
"translation_section_adjudicating": "【{name}】 аудармасының соңғы сәйкестігі расталуда…",
|
| 768 |
+
"translation_progress_counts": "{passed}/{total} өтті · {repair} түзетуді күтуде · {inflight} өңделуде",
|
| 769 |
"translation_section_ready": "【{name}】 аудармасы тексеруден өтті және көрсетілуде…",
|
| 770 |
"translation_complete_label": "Мақаланың мақса��ты тілдегі аудармасы аяқталды.",
|
| 771 |
"busy_short": "⏳ Басқа пайдаланушы мақала дайындап жатыр. Шамамен 25–35 минуттан кейін қайталап көріңіз.",
|
smartpaper_restore_bridge/frontend/index.html
CHANGED
|
@@ -377,10 +377,16 @@
|
|
| 377 |
let value = String(map.waiting || "");
|
| 378 |
if (stage.startsWith("translating:")) {
|
| 379 |
value = formatTemplate(map.translating || value, {name:stage.slice("translating:".length)});
|
|
|
|
|
|
|
| 380 |
} else if (stage.startsWith("translation_qa:")) {
|
| 381 |
value = formatTemplate(map.qa || value, {name:stage.slice("translation_qa:".length)});
|
|
|
|
|
|
|
| 382 |
} else if (stage.startsWith("translation_repair_pending:")) {
|
| 383 |
-
value = formatTemplate(map.qa || value, {name:stage.slice("translation_repair_pending:".length)});
|
|
|
|
|
|
|
| 384 |
} else if (stage.startsWith("translation_ready:")) {
|
| 385 |
value = formatTemplate(map.ready || value, {name:stage.slice("translation_ready:".length)});
|
| 386 |
} else if (stage === "translation_complete") {
|
|
@@ -393,7 +399,15 @@
|
|
| 393 |
? payload.target_progress : {};
|
| 394 |
const total = Math.max(0, asCount(targetProgress.total));
|
| 395 |
const qaPassed = Math.max(0, Math.min(total || Number.MAX_SAFE_INTEGER, asCount(targetProgress.qa_passed)));
|
| 396 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 397 |
if (!value) return;
|
| 398 |
const hasTargetText = Boolean(String(payload && payload.target_text || "").trim()) || Boolean(channel.authoritative);
|
| 399 |
if (hasTargetText) {
|
|
|
|
| 377 |
let value = String(map.waiting || "");
|
| 378 |
if (stage.startsWith("translating:")) {
|
| 379 |
value = formatTemplate(map.translating || value, {name:stage.slice("translating:".length)});
|
| 380 |
+
} else if (stage.startsWith("translation_final_gate:")) {
|
| 381 |
+
value = formatTemplate(map.final_gate || map.qa || value, {name:stage.slice("translation_final_gate:".length)});
|
| 382 |
} else if (stage.startsWith("translation_qa:")) {
|
| 383 |
value = formatTemplate(map.qa || value, {name:stage.slice("translation_qa:".length)});
|
| 384 |
+
} else if (stage.startsWith("translation_repairing:")) {
|
| 385 |
+
value = formatTemplate(map.repairing || map.qa || value, {name:stage.slice("translation_repairing:".length)});
|
| 386 |
} else if (stage.startsWith("translation_repair_pending:")) {
|
| 387 |
+
value = formatTemplate(map.repair_pending || map.qa || value, {name:stage.slice("translation_repair_pending:".length)});
|
| 388 |
+
} else if (stage.startsWith("translation_adjudicating:")) {
|
| 389 |
+
value = formatTemplate(map.adjudicating || map.qa || value, {name:stage.slice("translation_adjudicating:".length)});
|
| 390 |
} else if (stage.startsWith("translation_ready:")) {
|
| 391 |
value = formatTemplate(map.ready || value, {name:stage.slice("translation_ready:".length)});
|
| 392 |
} else if (stage === "translation_complete") {
|
|
|
|
| 399 |
? payload.target_progress : {};
|
| 400 |
const total = Math.max(0, asCount(targetProgress.total));
|
| 401 |
const qaPassed = Math.max(0, Math.min(total || Number.MAX_SAFE_INTEGER, asCount(targetProgress.qa_passed)));
|
| 402 |
+
const repairPending = Math.max(0, Math.min(total || Number.MAX_SAFE_INTEGER, asCount(targetProgress.repair_pending)));
|
| 403 |
+
const inFlight = Math.max(0, Math.min(total || Number.MAX_SAFE_INTEGER, asCount(targetProgress.in_flight)));
|
| 404 |
+
if (total > 0) {
|
| 405 |
+
const countsTemplate = String(map.counts || "");
|
| 406 |
+
const counts = countsTemplate
|
| 407 |
+
? formatTemplate(countsTemplate, {passed:qaPassed, total, repair:repairPending, inflight:inFlight})
|
| 408 |
+
: `${qaPassed}/${total}`;
|
| 409 |
+
value = `${value} · ${counts}`;
|
| 410 |
+
}
|
| 411 |
if (!value) return;
|
| 412 |
const hasTargetText = Boolean(String(payload && payload.target_text || "").trim()) || Boolean(channel.authoritative);
|
| 413 |
if (hasTargetText) {
|