Spaces:
Running
Running
R14-KK-G1 Grammar Evidence Continuity + Adaptive Translation Batch Controller
#58
by GuXiaopeng - opened
- core/generation_runner.py +218 -30
- core/pipeline_agent.py +142 -37
core/generation_runner.py
CHANGED
|
@@ -34,10 +34,16 @@ from utils.file_parser import extract_text_from_files
|
|
| 34 |
|
| 35 |
ProgressCallback = Callable[..., None]
|
| 36 |
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 41 |
|
| 42 |
|
| 43 |
def _clean_trailing_references(text, keywords=("reference", "references", "reference list", "参考文献")):
|
|
@@ -1493,6 +1499,66 @@ class GenerationRunner:
|
|
| 1493 |
"ratio": round(ratio, 4), "status": status, "recommendation": recommendation, "nonfatal": True,
|
| 1494 |
}
|
| 1495 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1496 |
def _resolve_paths(self, items) -> list[str]:
|
| 1497 |
output = []
|
| 1498 |
for item in items or []:
|
|
@@ -1615,16 +1681,16 @@ class GenerationRunner:
|
|
| 1615 |
refs_text = str(state.get("refs_text") or "")
|
| 1616 |
if not refs_text:
|
| 1617 |
self._progress(8, "S2 + OpenAlex")
|
|
|
|
| 1618 |
refs_text = agent.fetch_references_parallel(
|
| 1619 |
topic,
|
| 1620 |
keywords,
|
| 1621 |
-
min_required=
|
| 1622 |
progress_callback=lambda: self._progress(9, "S2 + OpenAlex"),
|
| 1623 |
)
|
| 1624 |
-
if
|
| 1625 |
-
raise
|
| 1626 |
-
|
| 1627 |
-
)
|
| 1628 |
if user_lit_context:
|
| 1629 |
core_refs = agent.parse_user_core_literature(user_lit_context)
|
| 1630 |
if core_refs:
|
|
@@ -1699,29 +1765,86 @@ class GenerationRunner:
|
|
| 1699 |
if jit_texts:
|
| 1700 |
rag_engine._initialize_chunks([jit_texts])
|
| 1701 |
self._progress(15, "citation_planning")
|
| 1702 |
-
|
| 1703 |
-
adaptive_structure,
|
| 1704 |
)
|
| 1705 |
-
|
| 1706 |
-
|
| 1707 |
-
|
|
|
|
| 1708 |
)
|
| 1709 |
-
|
| 1710 |
-
|
|
|
|
|
|
|
|
|
|
| 1711 |
for items in citation_plan.values()
|
| 1712 |
for item in items
|
| 1713 |
if item.get("tag")
|
| 1714 |
}
|
| 1715 |
-
|
| 1716 |
-
|
| 1717 |
-
|
| 1718 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1719 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1720 |
state.update(
|
| 1721 |
{
|
| 1722 |
"adaptive_structure": adaptive_structure,
|
| 1723 |
"quote_invocation_plan": quote_invocation_plan,
|
| 1724 |
"citation_plan": citation_plan,
|
|
|
|
|
|
|
|
|
|
| 1725 |
"research_logic_contract": research_logic_contract,
|
| 1726 |
"working_abstract": working_abstract,
|
| 1727 |
"working_keywords": working_keywords,
|
|
@@ -1753,6 +1876,16 @@ class GenerationRunner:
|
|
| 1753 |
state.get("research_logic_contract") or agent.global_memory.get("research_logic_contract") or ""
|
| 1754 |
)
|
| 1755 |
citation_plan = state.get("citation_plan") if isinstance(state.get("citation_plan"), dict) else {}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1756 |
translation_sections = dict(state.get('translation_sections') or {})
|
| 1757 |
translation_glossary = dict(state.get('translation_glossary') or {})
|
| 1758 |
canonical_concept_ledger = state.get('canonical_concept_ledger')
|
|
@@ -1866,9 +1999,20 @@ class GenerationRunner:
|
|
| 1866 |
if not citation_plan:
|
| 1867 |
self._progress(15, "citation_planning")
|
| 1868 |
citation_plan = agent.build_claim_evidence_plan(
|
| 1869 |
-
adaptive_structure, topic, synthesis_result, mode=mode,
|
|
|
|
| 1870 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1871 |
state["citation_plan"] = citation_plan
|
|
|
|
|
|
|
|
|
|
| 1872 |
|
| 1873 |
quote_invocation_plan = (
|
| 1874 |
dict(state.get("quote_invocation_plan") or {}) if mode == "asq" else {}
|
|
@@ -2105,9 +2249,13 @@ Output only final academic prose. Do not report the word count."""
|
|
| 2105 |
used_after = set(agent.used_citations)
|
| 2106 |
survived = used_after - used_before
|
| 2107 |
if assigned_tag_list and not is_quarantined:
|
|
|
|
| 2108 |
required_survival = min(
|
| 2109 |
len(assigned_tag_list),
|
| 2110 |
-
max(
|
|
|
|
|
|
|
|
|
|
| 2111 |
)
|
| 2112 |
if len(survived) < required_survival:
|
| 2113 |
missing_items = [
|
|
@@ -2127,6 +2275,13 @@ Output only final academic prose. Do not report the word count."""
|
|
| 2127 |
excluded_tags=used_after | {str(item.get("tag") or "") for item in recovery_items},
|
| 2128 |
)
|
| 2129 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2130 |
supplemental_raw = agent.build_citation_recovery_paragraph(
|
| 2131 |
recovery_items[:needed], section_name=display_name, section_context=en_resolved
|
| 2132 |
)
|
|
@@ -2140,6 +2295,15 @@ Output only final academic prose. Do not report the word count."""
|
|
| 2140 |
en_resolved = en_resolved.rstrip() + "\n\n" + supplemental_resolved.strip()
|
| 2141 |
used_after = set(agent.used_citations)
|
| 2142 |
survived = used_after - used_before
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2143 |
if mode == "asq" and section_id == "findings":
|
| 2144 |
en_resolved = self._render_planned_qualitative_quotes(
|
| 2145 |
en_resolved,
|
|
@@ -2188,6 +2352,10 @@ Output only final academic prose. Do not report the word count."""
|
|
| 2188 |
"context_tail": context_tail,
|
| 2189 |
"accumulated_deviation": accumulated_deviation,
|
| 2190 |
"quality_diagnostics": quality_diagnostics,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2191 |
"agent_state": agent.export_state(),
|
| 2192 |
"data_metadata": data_metadata,
|
| 2193 |
}
|
|
@@ -2215,18 +2383,33 @@ Output only final academic prose. Do not report the word count."""
|
|
| 2215 |
if ref and ref not in seen_refs:
|
| 2216 |
references.append(f"- {ref}")
|
| 2217 |
seen_refs.add(ref)
|
| 2218 |
-
if
|
| 2219 |
-
raise
|
| 2220 |
-
|
| 2221 |
-
|
| 2222 |
-
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2223 |
full_draft_en = full_draft_en.rstrip() + "\n\n# References\n" + "\n\n".join(references)
|
| 2224 |
actual_words = self._english_count_for_target(full_draft_en)
|
| 2225 |
self._upsert_quality_diagnostic(
|
| 2226 |
quality_diagnostics, self._validate_target_length(full_draft_en, target_words)
|
| 2227 |
)
|
| 2228 |
body_only, _reference_tail = agent.validate_reference_tail(
|
| 2229 |
-
full_draft_en, minimum_references=
|
| 2230 |
)
|
| 2231 |
agent.validate_manuscript_structure(body_only, adaptive_structure, translated=False)
|
| 2232 |
agent.enforce_resolved_claim_specific_citations(body_only)
|
|
@@ -2245,7 +2428,7 @@ Output only final academic prose. Do not report the word count."""
|
|
| 2245 |
full_draft_en = replace_abstract(full_draft_en, final_abstract)
|
| 2246 |
full_draft_en = self._replace_keywords(full_draft_en, final_keywords)
|
| 2247 |
body_only, _reference_tail = agent.validate_reference_tail(
|
| 2248 |
-
full_draft_en, minimum_references=
|
| 2249 |
)
|
| 2250 |
agent.validate_manuscript_structure(body_only, adaptive_structure, translated=False)
|
| 2251 |
agent.enforce_resolved_claim_specific_citations(body_only)
|
|
@@ -2282,6 +2465,11 @@ Output only final academic prose. Do not report the word count."""
|
|
| 2282 |
"final_abstract": final_abstract,
|
| 2283 |
"final_keywords": final_keywords,
|
| 2284 |
"front_matter_phase": "final",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2285 |
"quality_diagnostics": quality_diagnostics,
|
| 2286 |
"english_finalized": True,
|
| 2287 |
"translation_complete": False,
|
|
@@ -2305,7 +2493,7 @@ Output only final academic prose. Do not report the word count."""
|
|
| 2305 |
)
|
| 2306 |
self._shutdown_translation_worker(wait=True)
|
| 2307 |
body_only, reference_tail = agent.validate_reference_tail(
|
| 2308 |
-
full_draft_en, minimum_references=
|
| 2309 |
)
|
| 2310 |
section_order = self._unique_top_level_sections(adaptive_structure)
|
| 2311 |
if not section_order:
|
|
|
|
| 34 |
|
| 35 |
ProgressCallback = Callable[..., None]
|
| 36 |
|
| 37 |
+
CITATION_SEMANTIC_SURVIVAL_RATIO = 0.78
|
| 38 |
+
CITATION_ROLE_RULES = {
|
| 39 |
+
"introduction": (3, 180),
|
| 40 |
+
"theory": (3, 180),
|
| 41 |
+
"theoretical_framing": (3, 180),
|
| 42 |
+
"methodology": (2, 280),
|
| 43 |
+
"methods": (2, 280),
|
| 44 |
+
"discussion": (2, 220),
|
| 45 |
+
}
|
| 46 |
+
TRANSLATION_CONTRACT_VERSION = 'hierarchical-multilingual-v6'
|
| 47 |
|
| 48 |
|
| 49 |
def _clean_trailing_references(text, keywords=("reference", "references", "reference list", "参考文献")):
|
|
|
|
| 1499 |
"ratio": round(ratio, 4), "status": status, "recommendation": recommendation, "nonfatal": True,
|
| 1500 |
}
|
| 1501 |
|
| 1502 |
+
@staticmethod
|
| 1503 |
+
def _citation_eligible_section_ids(mode: str) -> set[str]:
|
| 1504 |
+
return (
|
| 1505 |
+
{"introduction", "theoretical_framing", "methods", "discussion"}
|
| 1506 |
+
if str(mode).lower() == "asq"
|
| 1507 |
+
else {"introduction", "theory", "methodology", "discussion"}
|
| 1508 |
+
)
|
| 1509 |
+
|
| 1510 |
+
@classmethod
|
| 1511 |
+
def _adaptive_citation_requirements(cls, tasks, mode: str):
|
| 1512 |
+
"""Derive evidence demand and semantic-survival reserve from manuscript units."""
|
| 1513 |
+
allowed = cls._citation_eligible_section_ids(mode)
|
| 1514 |
+
minimums = {}
|
| 1515 |
+
reserve_targets = {}
|
| 1516 |
+
for task in tasks or []:
|
| 1517 |
+
section_id = str(task.get("section_id") or "")
|
| 1518 |
+
display_name = str(task.get("display_name") or "").strip()
|
| 1519 |
+
if section_id not in allowed or not display_name:
|
| 1520 |
+
continue
|
| 1521 |
+
base, words_per_source = CITATION_ROLE_RULES.get(section_id, (2, 240))
|
| 1522 |
+
target_words = max(1, int(task.get("target_words") or 1))
|
| 1523 |
+
minimum = max(base, int(math.ceil(target_words / max(1, words_per_source))))
|
| 1524 |
+
reserve = max(
|
| 1525 |
+
minimum,
|
| 1526 |
+
int(math.ceil(minimum / max(0.01, CITATION_SEMANTIC_SURVIVAL_RATIO))),
|
| 1527 |
+
)
|
| 1528 |
+
minimums[display_name] = minimum
|
| 1529 |
+
reserve_targets[display_name] = reserve
|
| 1530 |
+
return minimums, reserve_targets
|
| 1531 |
+
|
| 1532 |
+
@staticmethod
|
| 1533 |
+
def _adaptive_literature_candidate_target(target_words: int, agent) -> int:
|
| 1534 |
+
"""Adaptive retrieval aspiration; never a final reference-count quota."""
|
| 1535 |
+
dynamic = max(24, int(math.ceil(max(1, int(target_words or 1)) / 90.0)))
|
| 1536 |
+
configured_cap = max(1, int(getattr(agent, "lit_target_papers", dynamic) or dynamic))
|
| 1537 |
+
return min(configured_cap, dynamic)
|
| 1538 |
+
|
| 1539 |
+
@staticmethod
|
| 1540 |
+
def _citation_task_context(task, research_logic_contract="", synthesis_result="") -> str:
|
| 1541 |
+
parts = [
|
| 1542 |
+
str(task.get("display_name") or ""),
|
| 1543 |
+
str(task.get("required_content") or ""),
|
| 1544 |
+
" ".join(str(value) for value in (task.get("concepts") or []) if str(value).strip()),
|
| 1545 |
+
str(research_logic_contract or "")[:1600],
|
| 1546 |
+
str(synthesis_result or "")[:1600],
|
| 1547 |
+
]
|
| 1548 |
+
return re.sub(r"\s+", " ", " ".join(part for part in parts if part)).strip()
|
| 1549 |
+
|
| 1550 |
+
@staticmethod
|
| 1551 |
+
def _literature_insufficient_error(shortfalls) -> RuntimeError:
|
| 1552 |
+
details = "; ".join(
|
| 1553 |
+
f"{name}={actual}/{required}"
|
| 1554 |
+
for name, actual, required in (shortfalls or [])[:8]
|
| 1555 |
+
)
|
| 1556 |
+
return RuntimeError(
|
| 1557 |
+
"E_LITERATURE_INSUFFICIENT: high-relevance claim-specific literature is insufficient "
|
| 1558 |
+
f"for the planned manuscript units ({details}). Please revise the research topic or "
|
| 1559 |
+
"keywords and try again; unrelated references will not be used to satisfy a quota."
|
| 1560 |
+
)
|
| 1561 |
+
|
| 1562 |
def _resolve_paths(self, items) -> list[str]:
|
| 1563 |
output = []
|
| 1564 |
for item in items or []:
|
|
|
|
| 1681 |
refs_text = str(state.get("refs_text") or "")
|
| 1682 |
if not refs_text:
|
| 1683 |
self._progress(8, "S2 + OpenAlex")
|
| 1684 |
+
literature_candidate_target = self._adaptive_literature_candidate_target(target_words, agent)
|
| 1685 |
refs_text = agent.fetch_references_parallel(
|
| 1686 |
topic,
|
| 1687 |
keywords,
|
| 1688 |
+
min_required=literature_candidate_target,
|
| 1689 |
progress_callback=lambda: self._progress(9, "S2 + OpenAlex"),
|
| 1690 |
)
|
| 1691 |
+
if not agent.citation_map:
|
| 1692 |
+
raise self._literature_insufficient_error([("retrieved literature pool", 0, 1)])
|
| 1693 |
+
state["literature_candidate_target"] = literature_candidate_target
|
|
|
|
| 1694 |
if user_lit_context:
|
| 1695 |
core_refs = agent.parse_user_core_literature(user_lit_context)
|
| 1696 |
if core_refs:
|
|
|
|
| 1765 |
if jit_texts:
|
| 1766 |
rag_engine._initialize_chunks([jit_texts])
|
| 1767 |
self._progress(15, "citation_planning")
|
| 1768 |
+
citation_minimums, citation_reserve_targets = self._adaptive_citation_requirements(
|
| 1769 |
+
adaptive_structure, mode
|
| 1770 |
)
|
| 1771 |
+
citation_plan_target = sum(citation_reserve_targets.values())
|
| 1772 |
+
citation_plan = agent.build_claim_evidence_plan(
|
| 1773 |
+
adaptive_structure, topic, synthesis_result, mode=mode,
|
| 1774 |
+
target_unique=max(1, citation_plan_target),
|
| 1775 |
)
|
| 1776 |
+
|
| 1777 |
+
# Fill unit-specific reserve gaps before expensive drafting. Existing
|
| 1778 |
+
# retrieved evidence is preferred; bounded contextual JIT is last resort.
|
| 1779 |
+
planned_tags = {
|
| 1780 |
+
str(item.get("tag") or "")
|
| 1781 |
for items in citation_plan.values()
|
| 1782 |
for item in items
|
| 1783 |
if item.get("tag")
|
| 1784 |
}
|
| 1785 |
+
task_by_name = {
|
| 1786 |
+
str(task.get("display_name") or ""): task
|
| 1787 |
+
for task in adaptive_structure
|
| 1788 |
+
if str(task.get("display_name") or "")
|
| 1789 |
+
}
|
| 1790 |
+
for display_name, reserve_target in citation_reserve_targets.items():
|
| 1791 |
+
planned_items = citation_plan.setdefault(display_name, [])
|
| 1792 |
+
shortfall = max(0, reserve_target - len(planned_items))
|
| 1793 |
+
if shortfall <= 0:
|
| 1794 |
+
continue
|
| 1795 |
+
section_context = self._citation_task_context(
|
| 1796 |
+
task_by_name.get(display_name, {}), research_logic_contract, synthesis_result
|
| 1797 |
+
)
|
| 1798 |
+
recovery_items = agent.select_contextual_citation_recovery_items(
|
| 1799 |
+
[], section_name=display_name, section_context=section_context,
|
| 1800 |
+
needed=shortfall, excluded_tags=planned_tags,
|
| 1801 |
)
|
| 1802 |
+
for item in recovery_items:
|
| 1803 |
+
tag = str(item.get("tag") or "")
|
| 1804 |
+
if tag and tag not in planned_tags:
|
| 1805 |
+
planned_items.append(item)
|
| 1806 |
+
planned_tags.add(tag)
|
| 1807 |
+
# External JIT is not used merely to hit the reserve target.
|
| 1808 |
+
# It is triggered only when the unit remains below its actual
|
| 1809 |
+
# evidence-demand minimum after exhausting already retrieved
|
| 1810 |
+
# relevant literature.
|
| 1811 |
+
minimum_shortfall = max(
|
| 1812 |
+
0, int(citation_minimums.get(display_name, 0)) - len(planned_items)
|
| 1813 |
+
)
|
| 1814 |
+
if minimum_shortfall > 0:
|
| 1815 |
+
jit_items = agent.fetch_contextual_jit_recovery_items(
|
| 1816 |
+
topic=topic, section_name=display_name, section_context=section_context,
|
| 1817 |
+
needed=minimum_shortfall, excluded_tags=planned_tags,
|
| 1818 |
+
)
|
| 1819 |
+
for item in jit_items:
|
| 1820 |
+
tag = str(item.get("tag") or "")
|
| 1821 |
+
if tag and tag not in planned_tags:
|
| 1822 |
+
planned_items.append(item)
|
| 1823 |
+
planned_tags.add(tag)
|
| 1824 |
+
|
| 1825 |
+
shortfalls = [
|
| 1826 |
+
(name, len(citation_plan.get(name) or []), required)
|
| 1827 |
+
for name, required in citation_minimums.items()
|
| 1828 |
+
if len(citation_plan.get(name) or []) < required
|
| 1829 |
+
]
|
| 1830 |
+
if shortfalls:
|
| 1831 |
+
raise self._literature_insufficient_error(shortfalls)
|
| 1832 |
+
logging.info(
|
| 1833 |
+
"Adaptive citation feasibility passed: minimum_total=%s reserve_target=%s planned_unique=%s units=%s",
|
| 1834 |
+
sum(citation_minimums.values()), citation_plan_target, len(planned_tags), len(citation_minimums),
|
| 1835 |
+
)
|
| 1836 |
+
canonical_concept_ledger = self._build_canonical_concept_ledger(
|
| 1837 |
+
mode, data_metadata, keywords=keywords, adaptive_structure=adaptive_structure,
|
| 1838 |
+
research_logic_contract=research_logic_contract,
|
| 1839 |
+
)
|
| 1840 |
state.update(
|
| 1841 |
{
|
| 1842 |
"adaptive_structure": adaptive_structure,
|
| 1843 |
"quote_invocation_plan": quote_invocation_plan,
|
| 1844 |
"citation_plan": citation_plan,
|
| 1845 |
+
"citation_minimums": citation_minimums,
|
| 1846 |
+
"citation_reserve_targets": citation_reserve_targets,
|
| 1847 |
+
"citation_validated_tags_by_unit": {},
|
| 1848 |
"research_logic_contract": research_logic_contract,
|
| 1849 |
"working_abstract": working_abstract,
|
| 1850 |
"working_keywords": working_keywords,
|
|
|
|
| 1876 |
state.get("research_logic_contract") or agent.global_memory.get("research_logic_contract") or ""
|
| 1877 |
)
|
| 1878 |
citation_plan = state.get("citation_plan") if isinstance(state.get("citation_plan"), dict) else {}
|
| 1879 |
+
citation_minimums = dict(state.get("citation_minimums") or {})
|
| 1880 |
+
citation_reserve_targets = dict(state.get("citation_reserve_targets") or {})
|
| 1881 |
+
citation_validated_tags_by_unit = {
|
| 1882 |
+
str(name): list(tags or [])
|
| 1883 |
+
for name, tags in dict(state.get("citation_validated_tags_by_unit") or {}).items()
|
| 1884 |
+
}
|
| 1885 |
+
if not citation_minimums:
|
| 1886 |
+
citation_minimums, citation_reserve_targets = self._adaptive_citation_requirements(
|
| 1887 |
+
adaptive_structure, mode
|
| 1888 |
+
)
|
| 1889 |
translation_sections = dict(state.get('translation_sections') or {})
|
| 1890 |
translation_glossary = dict(state.get('translation_glossary') or {})
|
| 1891 |
canonical_concept_ledger = state.get('canonical_concept_ledger')
|
|
|
|
| 1999 |
if not citation_plan:
|
| 2000 |
self._progress(15, "citation_planning")
|
| 2001 |
citation_plan = agent.build_claim_evidence_plan(
|
| 2002 |
+
adaptive_structure, topic, synthesis_result, mode=mode,
|
| 2003 |
+
target_unique=max(1, sum(citation_reserve_targets.values())),
|
| 2004 |
)
|
| 2005 |
+
shortfalls = [
|
| 2006 |
+
(name, len(citation_plan.get(name) or []), required)
|
| 2007 |
+
for name, required in citation_minimums.items()
|
| 2008 |
+
if len(citation_plan.get(name) or []) < required
|
| 2009 |
+
]
|
| 2010 |
+
if shortfalls:
|
| 2011 |
+
raise self._literature_insufficient_error(shortfalls)
|
| 2012 |
state["citation_plan"] = citation_plan
|
| 2013 |
+
state["citation_minimums"] = citation_minimums
|
| 2014 |
+
state["citation_reserve_targets"] = citation_reserve_targets
|
| 2015 |
+
state["citation_validated_tags_by_unit"] = citation_validated_tags_by_unit
|
| 2016 |
|
| 2017 |
quote_invocation_plan = (
|
| 2018 |
dict(state.get("quote_invocation_plan") or {}) if mode == "asq" else {}
|
|
|
|
| 2249 |
used_after = set(agent.used_citations)
|
| 2250 |
survived = used_after - used_before
|
| 2251 |
if assigned_tag_list and not is_quarantined:
|
| 2252 |
+
adaptive_minimum = max(1, int(citation_minimums.get(display_name, 1)))
|
| 2253 |
required_survival = min(
|
| 2254 |
len(assigned_tag_list),
|
| 2255 |
+
max(
|
| 2256 |
+
adaptive_minimum,
|
| 2257 |
+
int(math.ceil(len(assigned_tag_list) * CITATION_SEMANTIC_SURVIVAL_RATIO)),
|
| 2258 |
+
),
|
| 2259 |
)
|
| 2260 |
if len(survived) < required_survival:
|
| 2261 |
missing_items = [
|
|
|
|
| 2275 |
excluded_tags=used_after | {str(item.get("tag") or "") for item in recovery_items},
|
| 2276 |
)
|
| 2277 |
)
|
| 2278 |
+
planned_tags_for_unit = {str(item.get("tag") or "") for item in planned_items}
|
| 2279 |
+
for recovery_item in recovery_items[:needed]:
|
| 2280 |
+
tag = str(recovery_item.get("tag") or "")
|
| 2281 |
+
if tag and tag not in planned_tags_for_unit:
|
| 2282 |
+
planned_items.append(recovery_item)
|
| 2283 |
+
planned_tags_for_unit.add(tag)
|
| 2284 |
+
citation_plan[display_name] = planned_items
|
| 2285 |
supplemental_raw = agent.build_citation_recovery_paragraph(
|
| 2286 |
recovery_items[:needed], section_name=display_name, section_context=en_resolved
|
| 2287 |
)
|
|
|
|
| 2295 |
en_resolved = en_resolved.rstrip() + "\n\n" + supplemental_resolved.strip()
|
| 2296 |
used_after = set(agent.used_citations)
|
| 2297 |
survived = used_after - used_before
|
| 2298 |
+
if assigned_tag_list and not is_quarantined:
|
| 2299 |
+
adaptive_minimum = max(1, int(citation_minimums.get(display_name, 1)))
|
| 2300 |
+
if len(survived) < adaptive_minimum:
|
| 2301 |
+
raise self._literature_insufficient_error(
|
| 2302 |
+
[(display_name, len(survived), adaptive_minimum)]
|
| 2303 |
+
)
|
| 2304 |
+
citation_validated_tags_by_unit[display_name] = sorted(
|
| 2305 |
+
survived, key=agent._citation_tag_number
|
| 2306 |
+
)
|
| 2307 |
if mode == "asq" and section_id == "findings":
|
| 2308 |
en_resolved = self._render_planned_qualitative_quotes(
|
| 2309 |
en_resolved,
|
|
|
|
| 2352 |
"context_tail": context_tail,
|
| 2353 |
"accumulated_deviation": accumulated_deviation,
|
| 2354 |
"quality_diagnostics": quality_diagnostics,
|
| 2355 |
+
"citation_plan": citation_plan,
|
| 2356 |
+
"citation_minimums": citation_minimums,
|
| 2357 |
+
"citation_reserve_targets": citation_reserve_targets,
|
| 2358 |
+
"citation_validated_tags_by_unit": citation_validated_tags_by_unit,
|
| 2359 |
"agent_state": agent.export_state(),
|
| 2360 |
"data_metadata": data_metadata,
|
| 2361 |
}
|
|
|
|
| 2383 |
if ref and ref not in seen_refs:
|
| 2384 |
references.append(f"- {ref}")
|
| 2385 |
seen_refs.add(ref)
|
| 2386 |
+
if not references:
|
| 2387 |
+
raise self._literature_insufficient_error([("final validated references", 0, 1)])
|
| 2388 |
+
final_shortfalls = []
|
| 2389 |
+
for name, minimum in citation_minimums.items():
|
| 2390 |
+
validated_tags = set(citation_validated_tags_by_unit.get(name) or [])
|
| 2391 |
+
if not validated_tags:
|
| 2392 |
+
validated_tags = {
|
| 2393 |
+
str(item.get("tag") or "")
|
| 2394 |
+
for item in (citation_plan.get(name) or [])
|
| 2395 |
+
if str(item.get("tag") or "") in agent.used_citations
|
| 2396 |
+
}
|
| 2397 |
+
if len(validated_tags) < int(minimum):
|
| 2398 |
+
final_shortfalls.append((name, len(validated_tags), int(minimum)))
|
| 2399 |
+
if final_shortfalls:
|
| 2400 |
+
raise self._literature_insufficient_error(final_shortfalls)
|
| 2401 |
+
final_reference_count = len(references)
|
| 2402 |
+
logging.info(
|
| 2403 |
+
"Adaptive citation final Gate passed: validated_references=%s units=%s",
|
| 2404 |
+
final_reference_count, len(citation_minimums),
|
| 2405 |
+
)
|
| 2406 |
full_draft_en = full_draft_en.rstrip() + "\n\n# References\n" + "\n\n".join(references)
|
| 2407 |
actual_words = self._english_count_for_target(full_draft_en)
|
| 2408 |
self._upsert_quality_diagnostic(
|
| 2409 |
quality_diagnostics, self._validate_target_length(full_draft_en, target_words)
|
| 2410 |
)
|
| 2411 |
body_only, _reference_tail = agent.validate_reference_tail(
|
| 2412 |
+
full_draft_en, minimum_references=max(1, final_reference_count)
|
| 2413 |
)
|
| 2414 |
agent.validate_manuscript_structure(body_only, adaptive_structure, translated=False)
|
| 2415 |
agent.enforce_resolved_claim_specific_citations(body_only)
|
|
|
|
| 2428 |
full_draft_en = replace_abstract(full_draft_en, final_abstract)
|
| 2429 |
full_draft_en = self._replace_keywords(full_draft_en, final_keywords)
|
| 2430 |
body_only, _reference_tail = agent.validate_reference_tail(
|
| 2431 |
+
full_draft_en, minimum_references=max(1, final_reference_count)
|
| 2432 |
)
|
| 2433 |
agent.validate_manuscript_structure(body_only, adaptive_structure, translated=False)
|
| 2434 |
agent.enforce_resolved_claim_specific_citations(body_only)
|
|
|
|
| 2465 |
"final_abstract": final_abstract,
|
| 2466 |
"final_keywords": final_keywords,
|
| 2467 |
"front_matter_phase": "final",
|
| 2468 |
+
"final_reference_count": final_reference_count,
|
| 2469 |
+
"citation_plan": citation_plan,
|
| 2470 |
+
"citation_minimums": citation_minimums,
|
| 2471 |
+
"citation_reserve_targets": citation_reserve_targets,
|
| 2472 |
+
"citation_validated_tags_by_unit": citation_validated_tags_by_unit,
|
| 2473 |
"quality_diagnostics": quality_diagnostics,
|
| 2474 |
"english_finalized": True,
|
| 2475 |
"translation_complete": False,
|
|
|
|
| 2493 |
)
|
| 2494 |
self._shutdown_translation_worker(wait=True)
|
| 2495 |
body_only, reference_tail = agent.validate_reference_tail(
|
| 2496 |
+
full_draft_en, minimum_references=max(1, int(state.get("final_reference_count") or 1))
|
| 2497 |
)
|
| 2498 |
section_order = self._unique_top_level_sections(adaptive_structure)
|
| 2499 |
if not section_order:
|
core/pipeline_agent.py
CHANGED
|
@@ -62,7 +62,7 @@ class CitationAuditError(DeepSeekRequestError):
|
|
| 62 |
|
| 63 |
|
| 64 |
class SSciPipelineAgent:
|
| 65 |
-
TRANSLATION_CONTRACT_VERSION = 'hierarchical-multilingual-
|
| 66 |
|
| 67 |
def __init__(self, ds_key, s2_key=""):
|
| 68 |
self.ds_key = str(ds_key or "").strip()
|
|
@@ -2698,9 +2698,9 @@ INPUT:
|
|
| 2698 |
|
| 2699 |
At most one query is issued to each literature backend. New references
|
| 2700 |
are admitted only when title/abstract evidence has distinctive lexical
|
| 2701 |
-
overlap with the current manuscript unit; otherwise the
|
| 2702 |
-
is allowed to fail rather than padding the
|
| 2703 |
-
literature.
|
| 2704 |
"""
|
| 2705 |
needed = max(0, int(needed or 0))
|
| 2706 |
if needed <= 0:
|
|
@@ -3751,6 +3751,88 @@ Output only the repaired section."""
|
|
| 3751 |
'contamination_policy': 'Treat sustained untranslated English prose as a repair signal; normal acronyms, model names and protected terms are allowed.',
|
| 3752 |
}
|
| 3753 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3754 |
@staticmethod
|
| 3755 |
def _translation_lexical_retention_contract(target_lang):
|
| 3756 |
"""Single prompt-side authority for legal Latin retention in ZH/RU/KK.
|
|
@@ -5519,6 +5601,7 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
|
|
| 5519 |
if not str(error).startswith('Translation marker mismatch:'):
|
| 5520 |
raise
|
| 5521 |
if len(batch) > 1:
|
|
|
|
| 5522 |
found_ids = set(re.findall(r'(?m)^\[\[SP-P(\d{6})\]\]\s*$', str(result or '')))
|
| 5523 |
if not found_ids:
|
| 5524 |
logging.warning(
|
|
@@ -5601,6 +5684,7 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
|
|
| 5601 |
)
|
| 5602 |
restored[block_id] = translated_with_claims
|
| 5603 |
continue
|
|
|
|
| 5604 |
logging.warning(
|
| 5605 |
'Translation protected block recovery: block=SP-P%s error=%s; strict isolated retry.',
|
| 5606 |
block_id, self._safe_source_error(error),
|
|
@@ -5961,10 +6045,21 @@ Each code must be one of: {sorted(allowed)}. Return one or more specific dimensi
|
|
| 5961 |
'contamination_evidence_codes': kk_typed[:2],
|
| 5962 |
})
|
| 5963 |
continue
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5964 |
rows.append({
|
| 5965 |
'id': f'SP-P{block_id}-{claim_id}', 'block_id': block_id,
|
| 5966 |
'claim_id': claim_id, 'code': code, 'english': source_claims[claim_id],
|
| 5967 |
'target': target_claims[claim_id], 'deterministic_confirmed': False,
|
|
|
|
| 5968 |
})
|
| 5969 |
|
| 5970 |
confirmed = {}
|
|
@@ -5987,7 +6082,10 @@ Each code must be one of: {sorted(allowed)}. Return one or more specific dimensi
|
|
| 5987 |
allowed_evidence = {
|
| 5988 |
'CLAIM_MISSING','CLAIM_ADDED','POLARITY_CHANGED','CAUSALITY_CHANGED',
|
| 5989 |
'UNCERTAINTY_CHANGED','MEANING_CHANGED','CITATION_MOVED','ACRONYM_MISATTACHED',
|
| 5990 |
-
'GRAMMATICAL_VIOLATION','
|
|
|
|
|
|
|
|
|
|
| 5991 |
'KK_RUSSIAN_LEXICAL_INTRUSION','KK_RUSSIAN_SYNTACTIC_INTRUSION',
|
| 5992 |
'ACCEPTABLE_VARIANT','MORPHOLOGICAL_VARIANT','STYLISTIC_PREFERENCE',
|
| 5993 |
'QA_DISAGREEMENT','NO_BLOCKING_EVIDENCE',
|
|
@@ -6007,6 +6105,7 @@ Each code must be one of: {sorted(allowed)}. Return one or more specific dimensi
|
|
| 6007 |
'tokens': list((row.get('lexical_evidence') or {}).get('tokens') or [])[:8],
|
| 6008 |
},
|
| 6009 |
'contamination_evidence_codes': list(row.get('contamination_evidence_codes') or [])[:2],
|
|
|
|
| 6010 |
} for row in model_rows],
|
| 6011 |
}
|
| 6012 |
lexical_contract = self._translation_lexical_retention_contract(target_lang)
|
|
@@ -6015,7 +6114,8 @@ This is verification-only: do not rewrite, suggest wording, or emit free-form ex
|
|
| 6015 |
{lexical_contract}
|
| 6016 |
For Russian, allow required case/number/gender inflection and retain sustained non-Russian prose as an additional contamination signal. For Kazakh, allow productive case, possessive, person/plural and vowel-harmony suffixes on the correct term stem, but treat clearly Russian lexical/syntactic substitution as blocking only when the pair itself supports that conclusion. For Chinese, allow natural reordering and sentence reshaping inside the same claim while preserving meaning.
|
| 6017 |
For LANGUAGE_CONTAMINATION, deterministic_lexical is authoritative for ordinary English/Latin residue: when confirmed=false and token_count=0, you MUST NOT confirm UNTRANSLATED_LEXEME or UNTRANSLATED_PROSE. For Kazakh, a non-Latin contamination blocker may be confirmed only when contamination_evidence_codes contains KK_RUSSIAN_LEXICAL_INTRUSION or KK_RUSSIAN_SYNTACTIC_INTRUSION, and the evidence_code must match one of those supplied typed codes. The per-item glossary lists the frozen target terms that are legal for that source claim.
|
| 6018 |
-
|
|
|
|
| 6019 |
Decision must be CONFIRMED_BLOCKING or NOT_CONFIRMED. evidence_code must be one of: {sorted(allowed_evidence)}. Return every supplied id+issue_code pair exactly once."""
|
| 6020 |
try:
|
| 6021 |
response = self.call_deepseek(
|
|
@@ -6051,6 +6151,23 @@ Decision must be CONFIRMED_BLOCKING or NOT_CONFIRMED. evidence_code must be one
|
|
| 6051 |
if typed and evidence_code not in typed and decision == 'CONFIRMED_BLOCKING':
|
| 6052 |
decision = 'NOT_CONFIRMED'
|
| 6053 |
evidence_code = 'QA_DISAGREEMENT'
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6054 |
if decision == 'CONFIRMED_BLOCKING':
|
| 6055 |
confirmed.setdefault(row['block_id'], {}).setdefault(row['claim_id'], []).append(row['code'])
|
| 6056 |
self._record_translation_assurance(
|
|
@@ -6132,25 +6249,15 @@ Decision must be CONFIRMED_BLOCKING or NOT_CONFIRMED. evidence_code must be one
|
|
| 6132 |
if not body_units:
|
| 6133 |
raise DeepSeekRequestError('Frozen English translation unit contains no translatable body.')
|
| 6134 |
token_budget = max(800, min(2000, int(APP_CONFIG.get('translation_batch_tokens', 2500))))
|
| 6135 |
-
|
| 6136 |
-
|
| 6137 |
-
|
| 6138 |
-
tokens = max(1, self._count_tokens(unit['text']))
|
| 6139 |
-
if current and current_tokens + tokens > token_budget:
|
| 6140 |
-
batches.append(current); current = []; current_tokens = 0
|
| 6141 |
-
current.append((block_id, unit['text'])); current_tokens += tokens
|
| 6142 |
-
if current:
|
| 6143 |
-
batches.append(current)
|
| 6144 |
system_prompt = (
|
| 6145 |
self._translation_prompt(target_lang)
|
| 6146 |
+ '\nUse this frozen terminology glossary consistently. Never remap an existing term:\n'
|
| 6147 |
+ json.dumps(frozen_glossary, ensure_ascii=False)
|
| 6148 |
)
|
| 6149 |
-
source_blocks = {
|
| 6150 |
-
block_id: source
|
| 6151 |
-
for batch in batches
|
| 6152 |
-
for block_id, source in batch
|
| 6153 |
-
}
|
| 6154 |
attempts = max(1, int(max_attempts or 1))
|
| 6155 |
last_error = None
|
| 6156 |
self._last_translation_claim_states = {}
|
|
@@ -6205,8 +6312,15 @@ Decision must be CONFIRMED_BLOCKING or NOT_CONFIRMED. evidence_code must be one
|
|
| 6205 |
sum(len(v or {}) for v in quality_issues.values()),
|
| 6206 |
)
|
| 6207 |
else:
|
| 6208 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6209 |
translations.update(self._translate_marked_batch_with_recovery(batch, system_prompt, target_lang))
|
|
|
|
|
|
|
| 6210 |
emit_stage('qa')
|
| 6211 |
quality_issues = self._audit_translation_blocks_with_retry(
|
| 6212 |
source_blocks, translations, target_lang, frozen_glossary,
|
|
@@ -6528,29 +6642,20 @@ Decision must be CONFIRMED_BLOCKING or NOT_CONFIRMED. evidence_code must be one
|
|
| 6528 |
self._active_translation_glossary = dict(glossary)
|
| 6529 |
body_units = [unit for unit in units if unit['kind'] == 'body']
|
| 6530 |
translations = {}
|
| 6531 |
-
token_budget = max(800, int(APP_CONFIG.get('translation_batch_tokens', 2500)))
|
| 6532 |
-
|
| 6533 |
-
current = []
|
| 6534 |
-
current_tokens = 0
|
| 6535 |
-
for index, unit in enumerate(body_units, start=1):
|
| 6536 |
-
block_id = f'{index:06d}'
|
| 6537 |
-
tokens = max(1, self._count_tokens(unit['text']))
|
| 6538 |
-
if current and current_tokens + tokens > token_budget:
|
| 6539 |
-
batches.append(current)
|
| 6540 |
-
current = []
|
| 6541 |
-
current_tokens = 0
|
| 6542 |
-
current.append((block_id, unit['text']))
|
| 6543 |
-
current_tokens += tokens
|
| 6544 |
-
if current:
|
| 6545 |
-
batches.append(current)
|
| 6546 |
|
| 6547 |
system_prompt = self._translation_prompt(target_lang) + '\nUse this terminology glossary consistently:\n' + json.dumps(glossary, ensure_ascii=False)
|
| 6548 |
-
|
| 6549 |
self._check_cancelled()
|
|
|
|
|
|
|
| 6550 |
parsed = self._translate_marked_batch_with_recovery(
|
| 6551 |
batch, system_prompt, target_lang
|
| 6552 |
)
|
| 6553 |
translations.update(parsed)
|
|
|
|
|
|
|
| 6554 |
|
| 6555 |
output_units = []
|
| 6556 |
body_index = 0
|
|
|
|
| 62 |
|
| 63 |
|
| 64 |
class SSciPipelineAgent:
|
| 65 |
+
TRANSLATION_CONTRACT_VERSION = 'hierarchical-multilingual-v6'
|
| 66 |
|
| 67 |
def __init__(self, ds_key, s2_key=""):
|
| 68 |
self.ds_key = str(ds_key or "").strip()
|
|
|
|
| 2698 |
|
| 2699 |
At most one query is issued to each literature backend. New references
|
| 2700 |
are admitted only when title/abstract evidence has distinctive lexical
|
| 2701 |
+
overlap with the current manuscript unit; otherwise the adaptive
|
| 2702 |
+
evidence-sufficiency Gate is allowed to fail rather than padding the
|
| 2703 |
+
manuscript with irrelevant literature.
|
| 2704 |
"""
|
| 2705 |
needed = max(0, int(needed or 0))
|
| 2706 |
if needed <= 0:
|
|
|
|
| 3751 |
'contamination_policy': 'Treat sustained untranslated English prose as a repair signal; normal acronyms, model names and protected terms are allowed.',
|
| 3752 |
}
|
| 3753 |
|
| 3754 |
+
def _translation_batch_damage_score(self, target_lang):
|
| 3755 |
+
"""Runtime-only structural damage score used to shrink later translation batches."""
|
| 3756 |
+
language_code = self._translation_language_profile(target_lang).get('language_code') or 'zh'
|
| 3757 |
+
scores = getattr(self, '_translation_batch_damage_scores', None)
|
| 3758 |
+
if not isinstance(scores, dict):
|
| 3759 |
+
scores = {}
|
| 3760 |
+
self._translation_batch_damage_scores = scores
|
| 3761 |
+
return max(0, int(scores.get(language_code) or 0))
|
| 3762 |
+
|
| 3763 |
+
def _record_translation_batch_damage(self, target_lang, reason='structure'):
|
| 3764 |
+
language_code = self._translation_language_profile(target_lang).get('language_code') or 'zh'
|
| 3765 |
+
scores = getattr(self, '_translation_batch_damage_scores', None)
|
| 3766 |
+
if not isinstance(scores, dict):
|
| 3767 |
+
scores = {}
|
| 3768 |
+
self._translation_batch_damage_scores = scores
|
| 3769 |
+
scores[language_code] = min(6, max(0, int(scores.get(language_code) or 0)) + 1)
|
| 3770 |
+
logging.warning(
|
| 3771 |
+
'Translation adaptive batch damage | language=%s score=%s reason=%s',
|
| 3772 |
+
language_code, scores[language_code], str(reason or 'structure')[:80],
|
| 3773 |
+
)
|
| 3774 |
+
return scores[language_code]
|
| 3775 |
+
|
| 3776 |
+
def _relax_translation_batch_damage(self, target_lang):
|
| 3777 |
+
language_code = self._translation_language_profile(target_lang).get('language_code') or 'zh'
|
| 3778 |
+
scores = getattr(self, '_translation_batch_damage_scores', None)
|
| 3779 |
+
if not isinstance(scores, dict):
|
| 3780 |
+
return 0
|
| 3781 |
+
scores[language_code] = max(0, int(scores.get(language_code) or 0) - 1)
|
| 3782 |
+
return scores[language_code]
|
| 3783 |
+
|
| 3784 |
+
def _translation_batch_block_cost(self, source, target_lang):
|
| 3785 |
+
"""Estimate output/structure pressure without another model call."""
|
| 3786 |
+
text = str(source or '')
|
| 3787 |
+
source_tokens = max(1, self._count_tokens(text))
|
| 3788 |
+
ir = self._compile_translation_ir(text)
|
| 3789 |
+
claim_count = max(1, len(ir.get('claim_ids') or []))
|
| 3790 |
+
anchor_count = len(ir.get('fact_ledger') or {})
|
| 3791 |
+
language_code = self._translation_language_profile(target_lang).get('language_code') or 'zh'
|
| 3792 |
+
# Output pressure differs by target morphology/script, while source tokens
|
| 3793 |
+
# remain the common baseline. These are planning weights, not quality Gates.
|
| 3794 |
+
output_factor = {'zh': 0.90, 'ru': 1.12, 'kk': 1.32}.get(language_code, 1.0)
|
| 3795 |
+
structural_overhead = claim_count * 24 + anchor_count * 18
|
| 3796 |
+
damage_score = self._translation_batch_damage_score(target_lang)
|
| 3797 |
+
damage_multiplier = 1.0 + min(0.90, 0.18 * damage_score)
|
| 3798 |
+
risk_cost = int(max(1, round((source_tokens * output_factor + structural_overhead) * damage_multiplier)))
|
| 3799 |
+
return risk_cost, {
|
| 3800 |
+
'source_tokens': source_tokens,
|
| 3801 |
+
'claim_count': claim_count,
|
| 3802 |
+
'anchor_count': anchor_count,
|
| 3803 |
+
'risk_cost': risk_cost,
|
| 3804 |
+
'damage_score': damage_score,
|
| 3805 |
+
}
|
| 3806 |
+
|
| 3807 |
+
def _take_adaptive_translation_batch(self, pending, target_lang, configured_budget):
|
| 3808 |
+
"""Take the next ordered batch using output pressure + structural density."""
|
| 3809 |
+
remaining = list(pending or [])
|
| 3810 |
+
if not remaining:
|
| 3811 |
+
return [], []
|
| 3812 |
+
budget = max(800, min(2200, int(configured_budget or 2000)))
|
| 3813 |
+
batch, total_cost = [], 0
|
| 3814 |
+
aggregate = {'source_tokens': 0, 'claim_count': 0, 'anchor_count': 0}
|
| 3815 |
+
for item in remaining:
|
| 3816 |
+
block_id, source = item
|
| 3817 |
+
cost, metrics = self._translation_batch_block_cost(source, target_lang)
|
| 3818 |
+
if batch and total_cost + cost > budget:
|
| 3819 |
+
break
|
| 3820 |
+
batch.append((block_id, source))
|
| 3821 |
+
total_cost += cost
|
| 3822 |
+
for key in aggregate:
|
| 3823 |
+
aggregate[key] += int(metrics.get(key) or 0)
|
| 3824 |
+
if total_cost >= budget:
|
| 3825 |
+
break
|
| 3826 |
+
if not batch:
|
| 3827 |
+
batch = [remaining[0]]
|
| 3828 |
+
logging.info(
|
| 3829 |
+
'Translation adaptive batch | language=%s blocks=%s source_tokens=%s claims=%s anchors=%s risk=%s budget=%s damage=%s',
|
| 3830 |
+
self._translation_language_profile(target_lang).get('language_code'), len(batch),
|
| 3831 |
+
aggregate['source_tokens'], aggregate['claim_count'], aggregate['anchor_count'],
|
| 3832 |
+
total_cost, budget, self._translation_batch_damage_score(target_lang),
|
| 3833 |
+
)
|
| 3834 |
+
return batch, remaining[len(batch):]
|
| 3835 |
+
|
| 3836 |
@staticmethod
|
| 3837 |
def _translation_lexical_retention_contract(target_lang):
|
| 3838 |
"""Single prompt-side authority for legal Latin retention in ZH/RU/KK.
|
|
|
|
| 5601 |
if not str(error).startswith('Translation marker mismatch:'):
|
| 5602 |
raise
|
| 5603 |
if len(batch) > 1:
|
| 5604 |
+
self._record_translation_batch_damage(target_lang, 'paragraph_marker_mismatch')
|
| 5605 |
found_ids = set(re.findall(r'(?m)^\[\[SP-P(\d{6})\]\]\s*$', str(result or '')))
|
| 5606 |
if not found_ids:
|
| 5607 |
logging.warning(
|
|
|
|
| 5684 |
)
|
| 5685 |
restored[block_id] = translated_with_claims
|
| 5686 |
continue
|
| 5687 |
+
self._record_translation_batch_damage(target_lang, 'protected_or_claim_integrity')
|
| 5688 |
logging.warning(
|
| 5689 |
'Translation protected block recovery: block=SP-P%s error=%s; strict isolated retry.',
|
| 5690 |
block_id, self._safe_source_error(error),
|
|
|
|
| 6045 |
'contamination_evidence_codes': kk_typed[:2],
|
| 6046 |
})
|
| 6047 |
continue
|
| 6048 |
+
repair_evidence = dict(
|
| 6049 |
+
getattr(self, '_translation_claim_qa_evidence', {}).get((block_id, claim_id), {}) or {}
|
| 6050 |
+
)
|
| 6051 |
+
grammar_evidence_codes = []
|
| 6052 |
+
if code == 'GRAMMAR':
|
| 6053 |
+
grammar_evidence_codes = [
|
| 6054 |
+
str(item or '').strip().upper()
|
| 6055 |
+
for item in (repair_evidence.get('grammar_evidence_codes') or [])
|
| 6056 |
+
if str(item or '').strip()
|
| 6057 |
+
] or ['GRAMMAR_UNSPECIFIED']
|
| 6058 |
rows.append({
|
| 6059 |
'id': f'SP-P{block_id}-{claim_id}', 'block_id': block_id,
|
| 6060 |
'claim_id': claim_id, 'code': code, 'english': source_claims[claim_id],
|
| 6061 |
'target': target_claims[claim_id], 'deterministic_confirmed': False,
|
| 6062 |
+
'grammar_evidence_codes': grammar_evidence_codes[:4],
|
| 6063 |
})
|
| 6064 |
|
| 6065 |
confirmed = {}
|
|
|
|
| 6082 |
allowed_evidence = {
|
| 6083 |
'CLAIM_MISSING','CLAIM_ADDED','POLARITY_CHANGED','CAUSALITY_CHANGED',
|
| 6084 |
'UNCERTAINTY_CHANGED','MEANING_CHANGED','CITATION_MOVED','ACRONYM_MISATTACHED',
|
| 6085 |
+
'GRAMMATICAL_VIOLATION','CASE_GOVERNMENT','AGREEMENT','MORPHOLOGY','VERB_FORM',
|
| 6086 |
+
'SYNTAX_STRUCTURE','PUNCTUATION','SYNTACTIC_COMPLETENESS','GRAMMAR_UNSPECIFIED',
|
| 6087 |
+
'WORD_ORDER_UNGRAMMATICAL','WORD_ORDER_SEMANTIC_SCOPE_CHANGED',
|
| 6088 |
+
'GLOSSARY_CONCEPT_MISMATCH','UNTRANSLATED_PROSE','UNTRANSLATED_LEXEME',
|
| 6089 |
'KK_RUSSIAN_LEXICAL_INTRUSION','KK_RUSSIAN_SYNTACTIC_INTRUSION',
|
| 6090 |
'ACCEPTABLE_VARIANT','MORPHOLOGICAL_VARIANT','STYLISTIC_PREFERENCE',
|
| 6091 |
'QA_DISAGREEMENT','NO_BLOCKING_EVIDENCE',
|
|
|
|
| 6105 |
'tokens': list((row.get('lexical_evidence') or {}).get('tokens') or [])[:8],
|
| 6106 |
},
|
| 6107 |
'contamination_evidence_codes': list(row.get('contamination_evidence_codes') or [])[:2],
|
| 6108 |
+
'grammar_evidence_codes': list(row.get('grammar_evidence_codes') or [])[:4],
|
| 6109 |
} for row in model_rows],
|
| 6110 |
}
|
| 6111 |
lexical_contract = self._translation_lexical_retention_contract(target_lang)
|
|
|
|
| 6114 |
{lexical_contract}
|
| 6115 |
For Russian, allow required case/number/gender inflection and retain sustained non-Russian prose as an additional contamination signal. For Kazakh, allow productive case, possessive, person/plural and vowel-harmony suffixes on the correct term stem, but treat clearly Russian lexical/syntactic substitution as blocking only when the pair itself supports that conclusion. For Chinese, allow natural reordering and sentence reshaping inside the same claim while preserving meaning.
|
| 6116 |
For LANGUAGE_CONTAMINATION, deterministic_lexical is authoritative for ordinary English/Latin residue: when confirmed=false and token_count=0, you MUST NOT confirm UNTRANSLATED_LEXEME or UNTRANSLATED_PROSE. For Kazakh, a non-Latin contamination blocker may be confirmed only when contamination_evidence_codes contains KK_RUSSIAN_LEXICAL_INTRUSION or KK_RUSSIAN_SYNTACTIC_INTRUSION, and the evidence_code must match one of those supplied typed codes. The per-item glossary lists the frozen target terms that are legal for that source claim.
|
| 6117 |
+
For GRAMMAR, grammar_evidence_codes is the typed defect history and must remain continuous into this final decision. Never use generic GRAMMATICAL_VIOLATION as final blocking evidence when a typed grammar history exists. If WORD_ORDER is supplied, non-canonical or stylistically different ordering is NOT blocking. Confirm only when the target itself is ungrammatical because of ordering (WORD_ORDER_UNGRAMMATICAL) or when ordering changes modifier/negation/modality/condition/relation scope relative to the English claim (WORD_ORDER_SEMANTIC_SCOPE_CHANGED). For CASE_GOVERNMENT, AGREEMENT, MORPHOLOGY, VERB_FORM, SYNTAX_STRUCTURE, PUNCTUATION, or SYNTACTIC_COMPLETENESS, a blocking evidence_code must match the supplied typed class. If only GRAMMAR_UNSPECIFIED was supplied, you may narrow it to one of those specific classes or one of the two WORD_ORDER consequence codes; do not confirm a generic grammar label.
|
| 6118 |
+
Return valid JSON only: {{"items":[{{"id":"SP-P000001-C000001","issue_code":"GRAMMAR","decision":"CONFIRMED_BLOCKING","evidence_code":"AGREEMENT"}}]}}.
|
| 6119 |
Decision must be CONFIRMED_BLOCKING or NOT_CONFIRMED. evidence_code must be one of: {sorted(allowed_evidence)}. Return every supplied id+issue_code pair exactly once."""
|
| 6120 |
try:
|
| 6121 |
response = self.call_deepseek(
|
|
|
|
| 6151 |
if typed and evidence_code not in typed and decision == 'CONFIRMED_BLOCKING':
|
| 6152 |
decision = 'NOT_CONFIRMED'
|
| 6153 |
evidence_code = 'QA_DISAGREEMENT'
|
| 6154 |
+
if row['code'] == 'GRAMMAR' and decision == 'CONFIRMED_BLOCKING':
|
| 6155 |
+
typed = {str(code or '').strip().upper() for code in (row.get('grammar_evidence_codes') or []) if str(code or '').strip()}
|
| 6156 |
+
specific_grammar = {
|
| 6157 |
+
'CASE_GOVERNMENT','AGREEMENT','MORPHOLOGY','VERB_FORM',
|
| 6158 |
+
'SYNTAX_STRUCTURE','PUNCTUATION','SYNTACTIC_COMPLETENESS',
|
| 6159 |
+
}
|
| 6160 |
+
if typed == {'GRAMMAR_UNSPECIFIED'} or not typed:
|
| 6161 |
+
matching = set(specific_grammar) | {
|
| 6162 |
+
'WORD_ORDER_UNGRAMMATICAL','WORD_ORDER_SEMANTIC_SCOPE_CHANGED'
|
| 6163 |
+
}
|
| 6164 |
+
else:
|
| 6165 |
+
matching = set(typed) & specific_grammar
|
| 6166 |
+
if 'WORD_ORDER' in typed:
|
| 6167 |
+
matching.update({'WORD_ORDER_UNGRAMMATICAL','WORD_ORDER_SEMANTIC_SCOPE_CHANGED'})
|
| 6168 |
+
if evidence_code not in matching:
|
| 6169 |
+
decision = 'NOT_CONFIRMED'
|
| 6170 |
+
evidence_code = 'QA_DISAGREEMENT'
|
| 6171 |
if decision == 'CONFIRMED_BLOCKING':
|
| 6172 |
confirmed.setdefault(row['block_id'], {}).setdefault(row['claim_id'], []).append(row['code'])
|
| 6173 |
self._record_translation_assurance(
|
|
|
|
| 6249 |
if not body_units:
|
| 6250 |
raise DeepSeekRequestError('Frozen English translation unit contains no translatable body.')
|
| 6251 |
token_budget = max(800, min(2000, int(APP_CONFIG.get('translation_batch_tokens', 2500))))
|
| 6252 |
+
pending_blocks = [
|
| 6253 |
+
(f'{index:06d}', unit['text']) for index, unit in enumerate(body_units, start=1)
|
| 6254 |
+
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6255 |
system_prompt = (
|
| 6256 |
self._translation_prompt(target_lang)
|
| 6257 |
+ '\nUse this frozen terminology glossary consistently. Never remap an existing term:\n'
|
| 6258 |
+ json.dumps(frozen_glossary, ensure_ascii=False)
|
| 6259 |
)
|
| 6260 |
+
source_blocks = {block_id: source for block_id, source in pending_blocks}
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6261 |
attempts = max(1, int(max_attempts or 1))
|
| 6262 |
last_error = None
|
| 6263 |
self._last_translation_claim_states = {}
|
|
|
|
| 6312 |
sum(len(v or {}) for v in quality_issues.values()),
|
| 6313 |
)
|
| 6314 |
else:
|
| 6315 |
+
pending = list(pending_blocks)
|
| 6316 |
+
while pending:
|
| 6317 |
+
batch, pending = self._take_adaptive_translation_batch(
|
| 6318 |
+
pending, target_lang, token_budget
|
| 6319 |
+
)
|
| 6320 |
+
damage_before = self._translation_batch_damage_score(target_lang)
|
| 6321 |
translations.update(self._translate_marked_batch_with_recovery(batch, system_prompt, target_lang))
|
| 6322 |
+
if self._translation_batch_damage_score(target_lang) == damage_before and damage_before > 0:
|
| 6323 |
+
self._relax_translation_batch_damage(target_lang)
|
| 6324 |
emit_stage('qa')
|
| 6325 |
quality_issues = self._audit_translation_blocks_with_retry(
|
| 6326 |
source_blocks, translations, target_lang, frozen_glossary,
|
|
|
|
| 6642 |
self._active_translation_glossary = dict(glossary)
|
| 6643 |
body_units = [unit for unit in units if unit['kind'] == 'body']
|
| 6644 |
translations = {}
|
| 6645 |
+
token_budget = max(800, min(2200, int(APP_CONFIG.get('translation_batch_tokens', 2500))))
|
| 6646 |
+
pending = [(f'{index:06d}', unit['text']) for index, unit in enumerate(body_units, start=1)]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6647 |
|
| 6648 |
system_prompt = self._translation_prompt(target_lang) + '\nUse this terminology glossary consistently:\n' + json.dumps(glossary, ensure_ascii=False)
|
| 6649 |
+
while pending:
|
| 6650 |
self._check_cancelled()
|
| 6651 |
+
batch, pending = self._take_adaptive_translation_batch(pending, target_lang, token_budget)
|
| 6652 |
+
damage_before = self._translation_batch_damage_score(target_lang)
|
| 6653 |
parsed = self._translate_marked_batch_with_recovery(
|
| 6654 |
batch, system_prompt, target_lang
|
| 6655 |
)
|
| 6656 |
translations.update(parsed)
|
| 6657 |
+
if self._translation_batch_damage_score(target_lang) == damage_before and damage_before > 0:
|
| 6658 |
+
self._relax_translation_batch_damage(target_lang)
|
| 6659 |
|
| 6660 |
output_units = []
|
| 6661 |
body_index = 0
|