# -*- coding: utf-8 -*-
"""R3-Free headless SmartPaper generation runner.
The runner contains no Streamlit calls. It is safe to execute in a subprocess
whose lifetime is independent from Streamlit ScriptRunner reruns and browser
connections. Checkpoints are task-scoped, immutable-by-sequence JSON snapshots.
"""
from __future__ import annotations
import copy
import hashlib
import inspect
import json
import logging
import math
import os
import queue
import re
import threading
import time
from concurrent.futures import Future
from datetime import datetime, timezone, timedelta
from pathlib import Path
from typing import Callable, Any
from config import APP_CONFIG, AMJ_PROMPTS, ASQ_PROMPTS
from core.checkpoint_store import TaskCheckpointStore
from core.meta_extractor import extract_golden_metadata
from core.model_gateway import resolve_plan_policy_snapshot
from core.pipeline_agent import SSciPipelineAgent, PipelineCancelledError
from core.production_guard import build_validated_package, build_recovery_package, replace_abstract
from core.qualitative_engine import get_qualitative_engine
from core.quantitative_engine import get_quantitative_engine
from core.rag_engine import initialize_rag
from core.study_design_contract import (
build_user_research_intent,
keyword_policy_from_contract,
apply_keyword_policy,
parse_keyword_terms,
)
from utils.file_parser import extract_text_from_files
ProgressCallback = Callable[..., None]
CITATION_SEMANTIC_SURVIVAL_RATIO = 0.78
EVIDENCE_COMPLETION_MAX_PER_TOP_LEVEL = 4
CITATION_ROLE_RULES = {
"introduction": (3, 180),
"theory": (3, 180),
"theoretical_framing": (3, 180),
"methodology": (2, 280),
"methods": (2, 280),
"discussion": (2, 220),
}
TRANSLATION_CONTRACT_VERSION = 'hierarchical-multilingual-v9'
FINAL_CITATION_LEDGER_VERSION = 'resolved-final-ledger-v1'
ENGLISH_SECTION_SEAL_VERSION = 'english-section-integrity-seal-v1'
# S0-R1-C5.5: one shared private-vocabulary registry is consumed by both
# Writer-facing sanitization and final reader-facing validation. Keeping a
# single registry prevents the historical drift where a phrase was allowed
# into the Writer prompt/output but rejected later by the purity Gate.
AUTHOR_FACING_PRIVATE_VOCABULARY = (
(r"\bfrozen\s+research\s+evidence\s+packet\b", "study evidence"),
(r"\bfrozen\s+evidence\s+packet\b", "study evidence"),
(r"\bevidence\s+packet\b", "study evidence"),
(r"\b(?:study\s+)?evidence\s+ledger\b", "study evidence record"),
(r"\bfrozen\s+(?:research\s+)?evidence\b", "study evidence"),
(r"\bfrozen\s+study\s+evidence\b", "study evidence"),
(r"\bfrozen\s+(?:research\s+|study\s+)?(facts?|numbers?|design|setting|sample|measures?|results?|themes?|quotations?|procedures?)\b", r"documented \1"),
(r"\b(?:study|result)-owned\s+facts?\b", "study-specific facts"),
(r"\bfrozen\s+task-level\s+concepts?\b", "canonical research concepts"),
(r"\bfrozen\s+theme\s+summary\b", "theme summary"),
(r"\bfrozen\s+thematic\s+structure\b", "thematic structure"),
(r"\bfrozen\s+statistical\s+evidence\b", "study statistical evidence"),
(r"\bfrozen\s+measurement\s+design\b", "documented measurement design"),
(r"\bfrozen\s+association\s+model\b", "specified association model"),
(r"\bfrozen\s+bootstrap\s+and\s+collinearity\s+checks\b", "documented bootstrap and collinearity checks"),
)
def _clean_trailing_references(text, keywords=("reference", "references", "reference list", "参考文献")):
lines = str(text or "").split("\n")
for i in range(len(lines) - 1, max(-1, len(lines) - 100), -1):
if any(re.match(rf"^##?\s*{kw}", lines[i], re.IGNORECASE) for kw in keywords):
return "\n".join(lines[:i]).strip()
return str(text or "").strip()
def _clean_markdown_headings(text):
return re.sub(r"(?m)^#+\s+(.*)$", r"**\1**", str(text or ""))
def _regex_sub_literal(pattern, replacement, text, *, flags=0, count=0):
r"""Run regex matching while treating replacement bytes as literal text.
Dynamic evidence/RAG payloads may legitimately contain backslashes such as
``\h`` or LaTeX commands. Passing those strings directly as ``re.sub``'s
replacement template can raise ``re.error: bad escape``. A callable
replacement preserves the payload byte-for-byte and disables replacement
template interpretation without changing regex matching semantics.
"""
literal = str(replacement if replacement is not None else "")
source = str(text if text is not None else "")
return re.sub(pattern, lambda _match, value=literal: value, source, count=count, flags=flags)
def _canonical_request_hash(request_data: dict[str, Any]) -> str:
material = dict(request_data or {})
material.pop("request_hash", None)
encoded = json.dumps(material, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
return hashlib.sha256(encoded).hexdigest()
def _normalize_target_manuscript_typography(text: str, target_lang: str) -> str:
"""Apply only deterministic language-local typography after semantic QA.
R14-TQ2-PQ1-R1 fixes English-style spaces around Chinese full-width
punctuation without touching Russian/Kazakh spacing, references, semantic
content, or code-fenced material. This runs only after the target manuscript
has passed translation QA; it is presentation normalization, not translation.
"""
value = str(text or "")
language = str(target_lang or "")
if "chinese" not in language.casefold() and "中文" not in language:
return value
value = value.replace("\r\n", "\n").replace("\r", "\n")
# The English References tail is immutable data, not target-language prose.
# Chinese punctuation normalization therefore stops before References.
reference_match = re.search(r"(?m)^#\s+参考文献\s*$", value)
frozen_reference_tail = ""
if reference_match:
frozen_reference_tail = value[reference_match.start():]
value = value[:reference_match.start()]
lines = value.split("\n")
normalized = []
in_fence = False
for line in lines:
stripped = line.lstrip()
if stripped.startswith("```"):
in_fence = not in_fence
normalized.append(line)
continue
if in_fence:
normalized.append(line)
continue
current = re.sub(r"[ \t]+([，。！？；：])", r"\1", line)
current = re.sub(r"([，。！？；：])[ \t]+(?=\S)", r"\1", current)
normalized.append(current)
normalized_body = "\n".join(normalized)
return normalized_body + frozen_reference_tail
class _OrderedTranslationWorker:
"""One daemon worker owned by the headless Executor process.
It intentionally has no checkpoint authority. Results are returned through
Future objects and are checkpointed only by GenerationRunner's main thread.
"""
def __init__(self, handler):
self._handler = handler
self._queue = queue.Queue()
self._stop = threading.Event()
# Lifecycle closure and task cancellation are deliberately distinct.
# A graceful drain/close before the strict final translation gate must
# not poison the translation agent with E_TASK_CANCELLED.
self._cancel = threading.Event()
self._thread = threading.Thread(
target=self._run,
name="smartpaper-translation-worker",
daemon=True,
)
self._thread.start()
@property
def stop_event(self):
return self._stop
@property
def cancel_event(self):
return self._cancel
def submit(self, job: dict[str, Any]) -> Future:
future = Future()
self._queue.put((future, dict(job)))
return future
def _run(self):
while True:
item = self._queue.get()
try:
if item is None:
return
future, job = item
if not future.set_running_or_notify_cancel():
continue
try:
result = self._handler(job)
except BaseException as error:
future.set_exception(error)
else:
future.set_result(result)
finally:
self._queue.task_done()
def close(self, *, wait: bool = False) -> None:
if self._stop.is_set():
if wait and self._thread.is_alive():
self._thread.join(timeout=2.0)
return
# wait=True is the normal final-gate path after all queued work has
# been collected. Only an abortive/non-waiting shutdown signals the
# translation agent to cancel an in-flight model call.
if not wait:
self._cancel.set()
self._stop.set()
while True:
try:
item = self._queue.get_nowait()
except queue.Empty:
break
try:
if item is not None:
future, _job = item
future.cancel()
finally:
self._queue.task_done()
self._queue.put(None)
if wait and self._thread.is_alive():
self._thread.join(timeout=2.0)
class GenerationRunner:
def __init__(
self,
*,
task_dir: str | os.PathLike,
request_data: dict[str, Any],
execution_epoch: int,
executor_instance_id: str = "",
confirmed_checkpoint_seq: int | None = None,
confirmed_checkpoint_stage: str = "",
trusted_task_policy_snapshot: dict[str, Any] | None = None,
progress_callback: ProgressCallback | None = None,
cancel_checker: Callable[[], bool] | None = None,
):
self.task_dir = Path(task_dir)
self.request = dict(request_data)
self.execution_epoch = int(execution_epoch)
self.executor_instance_id = str(executor_instance_id or "")
self.confirmed_checkpoint_seq = (
None if confirmed_checkpoint_seq is None else max(0, int(confirmed_checkpoint_seq))
)
self.confirmed_checkpoint_stage = str(confirmed_checkpoint_stage or "")[:120]
self.trusted_task_policy_snapshot = copy.deepcopy(
dict(trusted_task_policy_snapshot or {})
)
self.progress_callback = progress_callback or (lambda **_: None)
self.cancel_checker = cancel_checker or (lambda: False)
expected_hash = str(self.request.get("request_hash") or "")
actual_hash = _canonical_request_hash(self.request)
if not expected_hash or expected_hash != actual_hash:
raise RuntimeError("request_hash_mismatch")
self.request_hash = expected_hash
self.checkpoints = TaskCheckpointStore(
self.task_dir,
self.request_hash,
self.execution_epoch,
task_id=str(self.request.get("task_id") or "") if self.executor_instance_id else "",
executor_instance_id=self.executor_instance_id,
confirmed_checkpoint_seq=self.confirmed_checkpoint_seq,
confirmed_checkpoint_stage=self.confirmed_checkpoint_stage,
)
self.output_dir = self.task_dir / "outputs"
self.output_dir.mkdir(parents=True, exist_ok=True)
self.agent = None
self.translation_agent = None
self._preview_lock = threading.RLock()
preview_token = str(self.request.get("preview_token") or "").strip()
self.preview_token = preview_token if re.fullmatch(r"[A-Za-z0-9_-]{32,128}", preview_token) else ""
self.preview_path = None
self._last_preview_write = 0.0
self._preview_english_text = ""
self._preview_english_stage = ""
self._preview_english_complete = False
self._preview_target_text = ""
self._preview_target_stage = "translation_waiting"
self._preview_target_complete = False
self._preview_target_english_commit_text = ""
self._translation_worker = None
self._translation_worker_sections = {}
self._translation_worker_glossary = {}
self._translation_worker_jobs = []
self._translation_worker_collected = 0
self._translation_worker_pending = set()
self._translation_worker_context = {}
# Display-only observability. These counters never decide task success,
# checkpoint authority, quota, or translation reuse.
self._translation_progress_sections = {}
self._translation_progress_queued = set()
self._translation_progress_inflight = set()
self._translation_progress_total = 0
self._qualitative_quote_sources = {}
# R14-TQ2-PQ1-R3-R1: evidence coverage is a quality signal, not a
# manuscript-completeness authority. It is persisted in checkpoints and
# exposed to the browser only as a stable status code.
self._preview_evidence_status = "normal"
self._evidence_coverage_events = []
if self.preview_token:
preview_dir = Path(__file__).resolve().parents[1] / "static" / "previews"
preview_dir.mkdir(parents=True, exist_ok=True)
self.preview_path = preview_dir / f"{self.preview_token}.json"
# Best-effort cleanup of stale display-only previews; task/checkpoint
# authority is unaffected by this cache.
cutoff = time.time() - 24 * 3600
for old_preview in preview_dir.glob("*.json"):
try:
if old_preview.stat().st_mtime < cutoff:
old_preview.unlink()
except OSError:
pass
def _compute_request_counts(self) -> tuple[int, int, int]:
"""Return UI metrics: DeepSeek requests and eligible literature results."""
agent = self.agent
if agent is None:
return 0, 0, 0
metrics = getattr(agent, "source_metrics", {}) or {}
s2_metric = metrics.get("s2", {}) if isinstance(metrics, dict) else {}
oa_metric = metrics.get("oa", {}) if isinstance(metrics, dict) else {}
translator_calls = int(getattr(getattr(self, "translation_agent", None), "ds_calls", 0) or 0)
return (
int(getattr(agent, "ds_calls", 0) or 0) + translator_calls,
int(s2_metric.get("eligible_result_count", 0) or 0),
int(oa_metric.get("eligible_result_count", 0) or 0),
)
def _progress(self, pct: int, node: str, *, checkpoint_seq=None, checkpoint_stage=None) -> None:
ds_calls, s2_calls, oa_calls = self._compute_request_counts()
self.progress_callback(
progress_pct=max(0, min(100, int(pct))),
current_node=str(node or "")[:240],
checkpoint_seq=checkpoint_seq,
checkpoint_stage=checkpoint_stage,
ds_calls=ds_calls,
s2_calls=s2_calls,
oa_calls=oa_calls,
)
def _checkpoint(self, stage: str, state: dict[str, Any], pct: int, *, display_node: str | None = None) -> None:
seq, normalized_stage = self.checkpoints.save(stage, state)
self._progress(
pct, display_node or normalized_stage,
checkpoint_seq=seq, checkpoint_stage=normalized_stage,
)
@staticmethod
def _preview_visible_text(value: str) -> str:
text = str(value or "")
# Never expose hidden reasoning. Raw CITE placeholders are also hidden
# from the user because they have not passed semantic citation QA yet;
# the authoritative manuscript buffer remains untouched.
text = re.sub(r"(?is).*?(?:|$)", "", text)
text = re.sub(r"\s*\[?CITE-\d+\]?\s*", " ", text, flags=re.I)
text = re.sub(r"[ \t]{2,}", " ", text)
return text.strip()
@staticmethod
def _build_qualitative_quote_ownership(tasks, data_metadata):
"""Freeze one and only one Findings owner for every qualitative quote."""
evidence_by_tag = {
str(item.get("tag") or "").strip(): dict(item)
for item in (data_metadata or {}).get("evidence", [])
if isinstance(item, dict) and str(item.get("tag") or "").strip()
}
owners = {}
for task in tasks or []:
if str(task.get("section_id") or "") != "findings":
continue
owner = str(task.get("display_name") or "").strip()
for item in task.get("theme_evidence") or []:
if not isinstance(item, dict):
continue
tag = str(item.get("tag") or "").strip()
if not tag:
continue
if tag not in evidence_by_tag:
raise RuntimeError(f"Qualitative quotation ownership references unknown tag: {tag}")
if tag in owners:
raise RuntimeError(
f"Qualitative quotation has multiple Findings owners: {tag} -> {owners[tag]!r}, {owner!r}"
)
owners[tag] = owner
missing = sorted(set(evidence_by_tag) - set(owners))
if missing:
raise RuntimeError(f"Qualitative quotations lack a Findings owner: {missing[:8]}")
return owners
@staticmethod
def _qualitative_task_key(task):
task = task or {}
explicit = str(task.get("task_key") or "").strip()
if explicit:
return explicit
section_id = str(task.get("section_id") or task.get("section") or "section").strip()
stable_unit = str(task.get("theme_id") or task.get("subsection_id") or section_id).strip()
part_index = int(task.get("part_index") or 0) + 1
return f"{section_id}:{stable_unit}:part:{part_index:02d}"
@classmethod
def _build_qualitative_quote_invocation_plan(cls, tasks, data_metadata):
"""Freeze one deterministic body invocation for every retained quote.
Logical ownership is a stable Theme object. Runtime Part chunks only
select where the already-owned quote is rendered; they never become the
quote owner and can never change cardinality.
"""
metadata = data_metadata or {}
evidence_by_tag = {
str(item.get("tag") or "").strip(): dict(item)
for item in metadata.get("evidence", [])
if isinstance(item, dict) and str(item.get("tag") or "").strip()
}
theme_id_by_title = {}
for index, theme in enumerate(metadata.get("themes") or [], start=1):
if not isinstance(theme, dict):
continue
title = re.sub(r"\s+", " ", str(theme.get("title") or "")).strip().casefold()
if title:
theme_id_by_title[title] = str(theme.get("theme_id") or f"THEME-{index:02d}").strip()
plan = {}
task_slots = {}
for task in tasks or []:
if str(task.get("section_id") or "") != "findings":
continue
task_key = cls._qualitative_task_key(task)
task_theme_id = str(task.get("theme_id") or "").strip()
task_theme_title = str(task.get("theme_title") or task.get("subsection") or "").strip()
for item in task.get("theme_evidence") or []:
if not isinstance(item, dict):
continue
tag = str(item.get("tag") or "").strip()
if not tag:
continue
evidence = evidence_by_tag.get(tag)
if evidence is None:
raise RuntimeError(f"Qualitative quote plan references unknown evidence: {tag}")
if tag in plan:
raise RuntimeError(f"Qualitative quote plan has duplicate render assignment: {tag}")
owner_theme_id = str(
task_theme_id
or evidence.get("theme_id")
or theme_id_by_title.get(
re.sub(r"\s+", " ", str(evidence.get("theme") or task_theme_title)).strip().casefold()
)
or ""
).strip()
if not owner_theme_id:
raise RuntimeError(f"Qualitative quote lacks a stable theme owner: {tag}")
quote = str(evidence.get("quote") or "").strip()
if not quote:
raise RuntimeError(f"Qualitative quote is empty: {tag}")
quote_hash = str(evidence.get("quote_hash") or "").strip() or hashlib.sha256(
re.sub(r"\s+", " ", quote).strip().encode("utf-8")
).hexdigest()
slot_ordinal = task_slots.get(task_key, 0) + 1
task_slots[task_key] = slot_ordinal
plan[tag] = {
"quote_id": tag,
"quote_hash": quote_hash,
"owner_theme_id": owner_theme_id,
"owner_theme_title": str(evidence.get("theme") or task_theme_title).strip(),
"render_task_key": task_key,
"render_part_index": int(task.get("part_index") or 0),
"render_part_count": int(task.get("part_count") or 1),
"slot_ordinal": slot_ordinal,
"cardinality": 1,
}
missing = sorted(set(evidence_by_tag) - set(plan))
extra = sorted(set(plan) - set(evidence_by_tag))
if missing or extra:
raise RuntimeError(
f"Qualitative quote invocation plan mismatch: missing={missing[:8]} extra={extra[:8]}"
)
cls._validate_qualitative_quote_invocation_plan(plan, tasks, metadata)
return plan
@classmethod
def _validate_qualitative_quote_invocation_plan(cls, plan, tasks, data_metadata):
metadata = data_metadata or {}
evidence_by_tag = {
str(item.get("tag") or "").strip(): dict(item)
for item in metadata.get("evidence", [])
if isinstance(item, dict) and str(item.get("tag") or "").strip()
}
task_by_key = {cls._qualitative_task_key(task): task for task in tasks or []}
if set(plan or {}) != set(evidence_by_tag):
raise RuntimeError("Qualitative quote invocation plan does not match the frozen evidence ledger.")
for tag, record in (plan or {}).items():
if int(record.get("cardinality") or 0) != 1:
raise RuntimeError(f"Qualitative quote cardinality must equal one: {tag}")
owner_theme_id = str(record.get("owner_theme_id") or "").strip()
if not owner_theme_id or "part " in owner_theme_id.casefold():
raise RuntimeError(f"Qualitative quote owner is not a stable Theme object: {tag}")
task_key = str(record.get("render_task_key") or "").strip()
task = task_by_key.get(task_key)
if not task or str(task.get("section_id") or "") != "findings":
raise RuntimeError(f"Qualitative quote render task is invalid: {tag} -> {task_key!r}")
task_theme_id = str(task.get("theme_id") or "").strip()
if task_theme_id and task_theme_id != owner_theme_id:
raise RuntimeError(f"Qualitative quote render task crossed theme ownership: {tag}")
evidence = evidence_by_tag[tag]
quote = str(evidence.get("quote") or "").strip()
expected_hash = str(evidence.get("quote_hash") or "").strip() or hashlib.sha256(
re.sub(r"\s+", " ", quote).strip().encode("utf-8")
).hexdigest()
if str(record.get("quote_hash") or "") != expected_hash:
raise RuntimeError(f"Qualitative quote hash drifted after planning: {tag}")
return True
@staticmethod
def _strip_model_controlled_qualitative_quotes(text, data_metadata):
"""Remove model-emitted quote controls/literals before Executor rendering.
This is a containment boundary, not the quote control plane: quotation
presence and cardinality come exclusively from the frozen invocation
plan. The model's attempted tags/literals therefore have zero authority.
"""
value = str(text or "")
value = re.sub(r"(?)", block)
]
if not anchor_indices:
anchor_indices = [len(blocks) - 1] if blocks else []
insertions = {}
for index, record in enumerate(scheduled):
tag = str(record.get("quote_id") or "").strip()
quote = str((evidence_by_tag.get(tag) or {}).get("quote") or "").strip()
if not quote:
raise RuntimeError(f"Planned qualitative quotation is unavailable at render time: {tag}")
if anchor_indices:
anchor_pos = min(
len(anchor_indices) - 1,
max(0, int(((index + 1) * len(anchor_indices)) / (len(scheduled) + 1))),
)
block_index = anchor_indices[anchor_pos]
insertions.setdefault(block_index, []).append(f'"{quote}"')
else:
blocks.append(f'"{quote}"')
rendered_blocks = []
for index, block in enumerate(blocks):
rendered_blocks.append(block)
rendered_blocks.extend(insertions.get(index, []))
rendered = "\n\n".join(rendered_blocks).strip()
for record in scheduled:
tag = str(record.get("quote_id") or "").strip()
quote = str((evidence_by_tag.get(tag) or {}).get("quote") or "").strip()
if rendered.count(quote) != 1:
raise RuntimeError(f"Qualitative deterministic quote rendering failed: {tag}")
return rendered
@staticmethod
def _validate_qualitative_quote_rendering(text, quote_invocation_plan, data_metadata):
body_text = re.split(
r"(?m)^#\s+Appendix(?:[:\s].*)?$", str(text or ""), maxsplit=1
)[0]
unresolved = sorted(set(re.findall(r"\bQUOTE-\d+\b", body_text, flags=re.I)))
if unresolved:
raise RuntimeError(f"Unresolved qualitative quote controls in body: {unresolved[:8]}")
normalized_body = re.sub(r"\s+", " ", body_text).casefold()
evidence_by_tag = {
str(item.get("tag") or "").strip(): dict(item)
for item in (data_metadata or {}).get("evidence", [])
if isinstance(item, dict) and str(item.get("tag") or "").strip()
}
for tag, record in (quote_invocation_plan or {}).items():
quote = re.sub(r"\s+", " ", str((evidence_by_tag.get(tag) or {}).get("quote") or "")).strip().casefold()
expected = int(record.get("cardinality") or 0)
actual = normalized_body.count(quote) if quote else 0
if actual != expected:
raise RuntimeError(
f"Qualitative quote invocation invariant failed: {tag} actual={actual} planned={expected}"
)
return True
@staticmethod
def _redact_qualitative_quotes(text, data_metadata):
"""Hide prior verbatim quote text from cross-unit continuity context."""
value = str(text or "")
quotes = sorted(
{
str(item.get("quote") or "").strip()
for item in (data_metadata or {}).get("evidence", [])
if isinstance(item, dict) and str(item.get("quote") or "").strip()
},
key=len,
reverse=True,
)
for quote in quotes:
value = value.replace(f'"{quote}"', "[prior qualitative quotation omitted]")
value = value.replace(quote, "[prior qualitative quotation omitted]")
return value
@classmethod
def _bind_owned_qualitative_quotes(
cls,
text,
*,
display_name,
data_metadata,
quote_ownership,
prior_manuscript="",
):
"""Bind only the current Findings unit's owned quotation tags.
Other manuscript units are not allowed to contain frozen quote text or
quote tags. A direct verbatim reproduction of an owned quote is
normalized back to its tag before deterministic binding, preserving the
frozen wording without letting model prose create a second copy.
"""
value = str(text or "")
evidence_by_tag = {
str(item.get("tag") or "").strip(): dict(item)
for item in (data_metadata or {}).get("evidence", [])
if isinstance(item, dict) and str(item.get("tag") or "").strip()
}
allowed = {
tag for tag, owner in (quote_ownership or {}).items()
if str(owner) == str(display_name)
}
raw_tags = set(re.findall(r"\bQUOTE-\d+\b", value))
unauthorized_tags = sorted(raw_tags - allowed)
if unauthorized_tags:
raise RuntimeError(
f"Qualitative quotation tag used outside owning Findings unit {display_name!r}: {unauthorized_tags[:8]}"
)
# Quote text itself is also fenced. Owned verbatim text is converted
# back to a tag and then bound once; unowned verbatim text fails early.
for tag, item in evidence_by_tag.items():
quote = str(item.get("quote") or "").strip()
if not quote or quote not in value:
continue
if tag not in allowed:
raise RuntimeError(
f"Qualitative quotation text used outside owning Findings unit: {tag} in {display_name!r}"
)
value = value.replace(f'"{quote}"', tag)
value = value.replace(quote, tag)
# A quote already present in frozen prior prose cannot be rebound later,
# even after resume. This makes ownership idempotent without new state.
normalized_prior = re.sub(r"\s+", " ", str(prior_manuscript or "")).casefold()
for tag in sorted(allowed):
quote = str((evidence_by_tag.get(tag) or {}).get("quote") or "").strip()
if quote and re.sub(r"\s+", " ", quote).casefold() in normalized_prior:
raise RuntimeError(f"Qualitative quotation already used before owning unit: {tag}")
for tag in sorted(allowed):
tag_pattern = rf"(? 1:
raise RuntimeError(
f"Qualitative quotation repeated within owning Findings unit: {tag} count={count}"
)
if count == 1:
quote = str((evidence_by_tag.get(tag) or {}).get("quote") or "").strip()
value = re.sub(
tag_pattern,
lambda _match, frozen_quote=quote: f'"{frozen_quote}"',
value,
count=1,
)
# No unresolved quote tag may escape into citation QA or final output.
unresolved = sorted(set(re.findall(r"\bQUOTE-\d+\b", value)))
if unresolved:
raise RuntimeError(f"Unresolved qualitative quotation tags: {unresolved[:8]}")
return value
@staticmethod
def _qualitative_quote_source_map(data_metadata):
return {
str(item.get("tag") or "").strip(): str(item.get("quote") or "").strip()
for item in (data_metadata or {}).get("evidence", [])
if isinstance(item, dict)
and str(item.get("tag") or "").strip()
and str(item.get("quote") or "").strip()
}
@staticmethod
def _qualitative_quote_ledger_identity_ok(
record, *, target_lang: str, request_hash: str, source_quote_hashes: dict,
task_id: str = "", translation_contract_version: str = TRANSLATION_CONTRACT_VERSION,
) -> bool:
"""Type-safe identity gate for task-level target-language Quote reuse.
Missing scalar identity fields remain backward-compatible with pre-R4
checkpoints. Present fields must be strings and match exactly. Quote
hashes must be a mapping and match the current frozen English evidence.
execution_epoch is deliberately not part of logical identity so a newer
epoch can reuse the same QA-passed target rendering for the same task.
"""
if not isinstance(record, dict) or not isinstance(source_quote_hashes, dict):
return False
def scalar_matches(stored, current: str) -> bool:
if stored is None or stored == "":
return True
return isinstance(stored, str) and stored == str(current or "")
stored_hashes = record.get("source_quote_hashes")
if stored_hashes is None or stored_hashes == "":
hashes_match = True
else:
hashes_match = isinstance(stored_hashes, dict) and stored_hashes == source_quote_hashes
return (
scalar_matches(record.get("task_id"), task_id)
and scalar_matches(record.get("target_lang"), target_lang)
and scalar_matches(record.get("request_hash"), request_hash)
and scalar_matches(record.get("translation_contract_version"), translation_contract_version)
and hashes_match
)
@classmethod
def _prepare_qualitative_translation_source(cls, source, quote_sources, quote_ledger):
"""Replace every frozen qualitative quote with a typed semantic slot.
QUOTE identity is never inferred from translated narrative surface text.
Missing target objects are classified as pending and translated through
the dedicated typed-object path before the surrounding narrative runs.
"""
value = str(source or "")
pending = []
reused = []
for tag, quote in sorted((quote_sources or {}).items()):
wrapped = f'"{quote}"'
tagged = f'{wrapped} [{tag}]'
if tagged in value:
value = value.replace(tagged, f"[{tag}]")
(reused if tag in (quote_ledger or {}) else pending).append(tag)
continue
if wrapped not in value:
continue
value = value.replace(wrapped, f"[{tag}]")
(reused if tag in (quote_ledger or {}) else pending).append(tag)
return value, pending, reused
@classmethod
def _finalize_qualitative_translation_quotes(cls, translated, pending, reused, quote_ledger):
value = str(translated or "")
ledger = dict(quote_ledger or {})
ordered_tags = list(dict.fromkeys(list(pending or []) + list(reused or [])))
for tag in ordered_tags:
rendered_quote = str(ledger.get(tag) or "").strip()
if not rendered_quote:
raise RuntimeError(f"Qualitative quote translation ledger missing target text: {tag}")
pattern = rf"(? dict[str, int]:
"""Return non-authoritative logical target-unit progress for the UI."""
mapping = dict(getattr(self, '_translation_progress_sections', {}) or {})
context = getattr(self, '_translation_worker_context', {}) or {}
tasks = context.get('adaptive_structure') or [] if isinstance(context, dict) else []
logical_keys = []
if tasks:
logical_keys = ['__front_matter__'] + [
key for key, _title in self._unique_top_level_sections(tasks)
]
logical_keys = list(dict.fromkeys(key for key in logical_keys if key))
total = int(getattr(self, '_translation_progress_total', 0) or len(logical_keys))
if total <= 0 and logical_keys:
total = len(logical_keys)
qa_passed = 0
repair_pending = 0
inflight = set(getattr(self, '_translation_progress_inflight', set()) or set())
for key in logical_keys:
record = mapping.get(key) or {}
status = str(record.get('qa_status') or '')
if key not in inflight and status in {'passed', 'passed_with_warning'} and record.get('translated_text'):
qa_passed += 1
elif status == 'repair_pending':
repair_pending += 1
queued = len(set(getattr(self, '_translation_progress_queued', set()) or set()).intersection(logical_keys))
in_flight = len(inflight.intersection(logical_keys))
return {
'qa_passed': min(qa_passed, total) if total else qa_passed,
'repair_pending': min(repair_pending, total) if total else repair_pending,
'queued': min(queued, total) if total else queued,
'in_flight': min(in_flight, total) if total else in_flight,
'total': max(0, total),
}
def _write_preview_payload(self, *, force: bool = False) -> None:
with self._preview_lock:
if self.preview_path is None:
return
now = time.monotonic()
if not force and now - self._last_preview_write < 0.45:
return
payload = {
"task_id": str(self.request.get("task_id") or ""),
"request_hash": self.request_hash,
"execution_epoch": self.execution_epoch,
"stage": str(self._preview_english_stage or "")[:240],
"text": self._preview_visible_text(self._preview_english_text),
"complete": bool(self._preview_english_complete),
"target_stage": str(self._preview_target_stage or "")[:240],
"target_text": self._preview_visible_text(self._preview_target_text),
"target_complete": bool(self._preview_target_complete),
"target_english_commit_text": self._preview_visible_text(
self._preview_target_english_commit_text
),
"target_progress": self._translation_progress_snapshot(),
"evidence_status": str(self._preview_evidence_status or "normal")[:40],
"updated_at": datetime.now(timezone.utc).isoformat(),
}
temporary = self.preview_path.with_suffix(".json.tmp")
temporary.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
os.replace(temporary, self.preview_path)
self._last_preview_write = now
def _publish_preview(self, text: str, stage: str, *, force: bool = False, complete: bool = False) -> None:
with self._preview_lock:
self._preview_english_text = str(text or "")
self._preview_english_stage = str(stage or "")
self._preview_english_complete = bool(complete)
self._write_preview_payload(force=force)
def _publish_target_preview(
self, text: str, stage: str, *, force: bool = True, complete: bool = False,
english_commit_text: str | None = None,
) -> None:
with self._preview_lock:
self._preview_target_text = str(text or "")
self._preview_target_stage = str(stage or "")
self._preview_target_complete = bool(complete)
if english_commit_text is not None:
self._preview_target_english_commit_text = str(english_commit_text or "")
self._write_preview_payload(force=force)
def _stream_text(self, generator, *, preview_prefix: str | None = None, preview_stage: str = "") -> str:
chunks = []
for chunk in generator:
value = str(chunk or "")
if not value:
continue
chunks.append(value)
if preview_prefix is not None:
self._publish_preview(preview_prefix + "".join(chunks), preview_stage, force=False)
text = "".join(chunks)
if preview_prefix is not None:
self._publish_preview(preview_prefix + text, preview_stage, force=True)
return text
@staticmethod
def _text_hash(value: str) -> str:
normalized = re.sub(r'\s+', ' ', str(value or '')).strip()
return hashlib.sha256(normalized.encode('utf-8')).hexdigest()
@staticmethod
def _unique_top_level_sections(tasks):
output = []
seen = set()
for task in tasks or []:
key = str(task.get('section_id') or task.get('section') or '').strip()
title = str(task.get('section') or '').strip()
if key and title and key not in seen:
seen.add(key); output.append((key, title))
return output
@staticmethod
def _extract_top_level_section(manuscript: str, title: str) -> str:
value = str(manuscript or '')
match = re.search(
rf'(?ms)^#\s+{re.escape(str(title))}\s*$\n(.*?)(?=^#\s+[^#].*?$|\Z)', value
)
if not match:
return ''
return f"# {title}\n" + match.group(1).strip()
@staticmethod
def _normalize_orchestrator_owned_headings(text: str, task: dict) -> str:
"""Restore immutable H1/H2 structure after any model-owned repair path.
Writer and repair models own prose only. The orchestrator owns the
expected top-level and subsection headings described by the generation
task. This deterministic normalizer removes only exact markdown
heading lines for those owned titles, then restores the canonical
heading(s) required for this unit. All non-heading text is preserved.
The operation is intentionally idempotent: N(N(x)) == N(x).
"""
value = str(text or "")
task = dict(task or {})
section_title = str(task.get("section") or "").strip()
subsection_title = str(task.get("subsection") or "").strip()
section_first = bool(task.get("section_first")) and bool(section_title)
subsection_first = bool(task.get("subsection_first")) and bool(subsection_title)
owned_titles = [title for title in (section_title, subsection_title) if title]
if not owned_titles:
return value
patterns = [
re.compile(rf"^[ \\t]*#{{1,6}}[ \\t]+{re.escape(title)}[ \\t]*$")
for title in owned_titles
]
kept_lines = []
for line in value.splitlines(keepends=True):
visible = line.rstrip("\r\n")
if any(pattern.fullmatch(visible) for pattern in patterns):
continue
kept_lines.append(line)
body = "".join(kept_lines).lstrip("\r\n")
canonical = []
if section_first:
canonical.append(f"# {section_title}")
if subsection_first:
canonical.append(f"## {subsection_title}")
if not canonical:
return body
prefix = "\n\n".join(canonical)
return prefix if not body else prefix + "\n\n" + body
@staticmethod
def _english_prefix_through_top_level(manuscript: str, title: str) -> str:
value = str(manuscript or '')
match = re.search(
rf'(?ms)^#\s+{re.escape(str(title))}\s*$\n.*?(?=^#\s+[^#].*?$|\Z)', value
)
return value[:match.end()].strip() if match else value.strip()
@staticmethod
def _extract_front_matter(manuscript: str, first_section_title: str) -> str:
value = str(manuscript or '')
match = re.search(rf'(?m)^#\s+{re.escape(str(first_section_title))}\s*$', value)
return value[:match.start()].strip() if match else ''
@staticmethod
def _extract_body_from_first_section(manuscript: str, first_section_title: str) -> str:
value = str(manuscript or '')
match = re.search(rf'(?m)^#\s+{re.escape(str(first_section_title))}\s*$', value)
return value[match.start():].strip() if match else value.strip()
@staticmethod
def _repair_validate_study_fact_contract(
mode: str, agent, quant_engine, text: str, data_metadata: dict, *, section_name: str
):
"""S0-R1 compatibility gateway: inspect only and never mutate prose."""
value = str(text or '').strip()
diagnostics = []
if str(mode or '') == 'asq':
diagnostics = agent.inspect_qualitative_study_fact_contract(
value, data_metadata, section_name=section_name
)
elif str(mode or '') == 'amj':
diagnostics = quant_engine.inspect_quantitative_study_fact_contract(
value, data_metadata, section_name=section_name
)
diagnostics = list(diagnostics or [])
scope_inspector = getattr(agent, "inspect_study_design_scope_contract", None)
if callable(scope_inspector):
diagnostics.extend(scope_inspector(value, data_metadata, section_name=section_name) or [])
return value, diagnostics
@staticmethod
def _study_fact_violations(mode: str, agent, quant_engine, text: str, data_metadata: dict, *, section_name: str):
rows = []
if str(mode or '') == 'asq':
rows.extend(agent.inspect_qualitative_study_fact_contract(text, data_metadata, section_name=section_name) or [])
elif str(mode or '') == 'amj':
rows.extend(quant_engine.inspect_quantitative_study_fact_contract(text, data_metadata, section_name=section_name) or [])
scope_inspector = getattr(agent, "inspect_study_design_scope_contract", None)
if callable(scope_inspector):
rows.extend(scope_inspector(text, data_metadata, section_name=section_name) or [])
return list(rows)
@staticmethod
def _final_abstract_normalized_text(value: str) -> str:
"""Normalize reader-facing English prose for deterministic finding coverage checks."""
return re.sub(r"[^a-z0-9]+", " ", str(value or "").casefold()).strip()
@classmethod
def _final_abstract_identity_present(cls, sentence: str, label: str) -> bool:
needle = cls._final_abstract_normalized_text(label)
haystack = cls._final_abstract_normalized_text(sentence)
return bool(needle and haystack and needle in haystack)
@staticmethod
def _final_abstract_result_sentences(text: str) -> list[str]:
"""Return sentences that actually assert a frozen empirical finding.
Deliberately exclude vague contribution language such as "our findings
contribute". A Final Abstract must report what the study found, not
merely say that findings exist.
"""
sentences = [
part.strip()
for part in re.split(r"(?<=[.!?])\s+|\n+", str(text or ""))
if str(part or "").strip()
]
finding_pattern = re.compile(
r"(?i)(?:"
r"\b(?:we|the\s+study|the\s+analysis|the\s+results?|our\s+analysis)\s+"
r"(?:find|finds|found|show|shows|showed|indicate|indicates|indicated|reveal|reveals|revealed|identify|identifies|identified|support|supports|supported)\b|"
r"\bfindings?\s+(?:show|shows|showed|indicate|indicates|indicated|reveal|reveals|revealed|identify|identifies|identified|suggest|suggests|suggested)\b|"
r"\b(?:was|were|is|are)\s+(?:positively|negatively|significantly|non[- ]?significantly|not\s+significantly)\s+(?:associated|related)\b|"
r"\b(?:association|relationship|interaction|indirect\s+(?:association|effect)|theme|themes|pattern|patterns|mechanism|mechanisms)\b[^.!?]{0,140}\b(?:was|were|is|are|emerged|identified|supported|unsupported|significant|non[- ]?significant|nonzero)\b|"
r"\bH\d+\s+(?:was|were|is|are)\s+(?:supported|unsupported|not\s+supported|rejected)\b|"
r"\bdid\s+not\s+(?:significantly\s+)?(?:moderate|mediate|change|alter)\b"
r")"
)
result_sentences = []
prospective = re.compile(
r"(?i)\b(?:we\s+(?:propose|hypothesize|hypothesise|predict|expect|test)|"
r"hypothes(?:is|es|ized|ised)|this\s+study\s+(?:examines|investigates|tests)|"
r"we\s+further\s+hypothesize)\b"
)
retrospective = re.compile(
r"(?i)\b(?:results?|findings?\s+(?:show|indicate|reveal|identify|suggest)|"
r"analysis\s+(?:show|indicate|reveal|identify)|found|revealed|indicated|"
r"supported|unsupported|non[- ]?significant|not\s+significant|nonzero|received\s+no\s+support|"
r"did\s+not\s+(?:significantly\s+)?(?:moderate|mediate|change|alter))\b"
)
for sentence in sentences:
if not finding_pattern.search(sentence):
# Explicit frozen-status language can be result-bearing even
# without a finite 'was/were' construction.
if not re.search(
r"(?i)\b(?:nonzero\s+indirect\s+(?:association|effect)|"
r"unsupported\s+(?:interaction|moderation|association)|"
r"supported\s+(?:interaction|moderation|association))\b",
sentence,
):
continue
if prospective.search(sentence) and not retrospective.search(sentence):
continue
result_sentences.append(sentence)
return result_sentences
@staticmethod
def _final_abstract_ci_status(row: dict) -> str:
ci = list((row or {}).get("ci") or [])
if len(ci) != 2:
return ""
try:
low, high = float(ci[0]), float(ci[1])
except (TypeError, ValueError):
return ""
return "supported" if (low > 0.0 or high < 0.0) else "unsupported"
@staticmethod
def _final_abstract_support_language(sentence: str, status: str) -> bool:
value = str(sentence or "")
if status == "unsupported":
return bool(re.search(
r"(?i)\b(?:not\s+supported|unsupported|no\s+support|did\s+not\s+support|"
r"non[- ]?significant|not\s+significant|not\s+significantly\s+(?:associated|related)|"
r"no\s+(?:meaningful\s+)?(?:association|relationship|interaction|moderation|effect|evidence)|"
r"did\s+not\s+(?:significantly\s+)?(?:moderate|mediate|differ|vary|change|alter)|"
r"confidence\s+interval[^.!?]{0,80}(?:included|crossed)\s+zero)\b",
value,
))
if status == "supported":
return bool(re.search(
r"(?i)\b(?:supported|significant|nonzero|positive(?:ly)?\s+(?:association|associated|relationship|related)|"
r"negative(?:ly)?\s+(?:association|associated|relationship|related)|"
r"was\s+associated|were\s+associated|was\s+related|were\s+related)\b",
value,
))
return True
@staticmethod
def _final_abstract_metric_light(text: str) -> tuple[str, list[dict]]:
"""Remove nonessential inferential statistics from a Final Abstract.
This is a completion-resilience transform, not a correctness waiver.
It is used only after a frozen Final Abstract has already been generated
and a strict study-fact validator reports a metric-specific defect. The
substantive finding sentence is preserved while beta/CI/p-value detail
that is optional in an abstract is removed. Sample-size/design facts are
intentionally untouched.
"""
value = str(text or "")
original = value
# Parenthetical inferential bundles are the common case:
# (β = .21, 95% CI [...], p = .03)
metric_bundle = re.compile(
r"\s*\((?=[^()]{0,220}(?:β|standardized\s+estimate|(?:\d{2}%\s*)?CI\b|confidence\s+interval|\bp\s*[=<>]))[^()]*\)",
flags=re.I,
)
value = metric_bundle.sub("", value)
# Bare inline statistics are also optional in the abstract. Keep the
# surrounding finding language and remove only the metric surface.
value = re.sub(
r"(?i)(?:,\s*)?(?:β|standardized\s+estimate)\s*(?:=|:)?\s*[-+]?\d+(?:\.\d+)?",
"",
value,
)
value = re.sub(
r"(?i)(?:,\s*)?(?:\d{2}%\s*)?(?:CI|confidence\s+interval)\s*[:=]?\s*[\[(][^\])]{1,80}[\])]?",
"",
value,
)
value = re.sub(r"(?i)(?:,\s*)?\bp\s*(?:=|<|>)\s*\.?\d+(?:\.\d+)?", "", value)
# Typography cleanup only; never alter construct identities or finding
# polarity/support language.
value = re.sub(r"\s+,", ",", value)
value = re.sub(r",\s*,+", ",", value)
value = re.sub(r"\(\s*\)", "", value)
value = re.sub(r"\s+([.;:])", r"\1", value)
value = re.sub(r" {2,}", " ", value)
value = re.sub(r"\n{3,}", "\n\n", value).strip()
diagnostics = []
if value != original.strip():
diagnostics.append({
"type": "final_abstract_metric_light_recovery",
"reason": (
"Optional inferential statistics were removed from the Final Abstract after strict metric validation could not safely bind them."
),
})
return value, diagnostics
@classmethod
def _canonical_final_abstract_findings(
cls, mode: str, data_metadata: dict, *, hypothesis_ledger: dict | None = None,
violation_types: set[str] | None = None,
) -> list[str]:
"""Return fact-safe, metric-light finding sentences from frozen ledgers.
These sentences are a last-resort completion fallback after DeepSeek has
already received bounded local-repair opportunities. They use only
frozen relation/theme identities and support status; no new statistic,
method, participant, or causal claim is invented.
"""
meta = dict(data_metadata or {})
wanted = set(violation_types or set())
out: list[str] = []
if str(mode or "").strip().lower() == "amj":
ledger = dict(hypothesis_ledger or meta.get("hypothesis_ledger") or {})
evidence = dict(meta.get("association_results") or {})
if not evidence:
evidence = dict((meta.get("quantitative_evidence_ledger") or {}).get("association_results") or {})
relation_results = dict(evidence.get("relation_results") or {})
indirect_results = dict(evidence.get("indirect_results") or {})
items = [dict(x) for x in (ledger.get("items") or []) if isinstance(x, dict)]
def relation_sentence(item: dict) -> str:
predictor = str(item.get("predictor") or "").strip()
outcome = str(item.get("outcome") or "").strip()
moderator = str(item.get("moderator") or "").strip()
relation = str(item.get("relation") or "").strip().lower()
row = dict(relation_results.get(str(item.get("relation_id") or "")) or {})
if not predictor or not outcome or not row:
return ""
status = cls._final_abstract_ci_status(row)
estimate = row.get("estimate")
try:
estimate_value = float(estimate)
except (TypeError, ValueError):
estimate_value = 0.0
if relation == "moderation" and moderator:
if status == "unsupported":
return f"{moderator} did not significantly moderate the {predictor}–{outcome} association."
if status == "supported":
return f"{moderator} significantly moderated the {predictor}–{outcome} association."
return f"The analysis assessed whether {moderator} moderated the {predictor}–{outcome} association."
if status == "unsupported":
return f"{predictor} was not significantly associated with {outcome}."
direction = "positively" if estimate_value >= 0 else "negatively"
return f"{predictor} was {direction} associated with {outcome}."
if wanted.intersection({
"final_abstract_missing_unsupported_finding",
"final_abstract_missing_moderation_finding",
"final_abstract_missing_empirical_finding",
}):
unsupported = []
moderation = []
for item in items:
row = dict(relation_results.get(str(item.get("relation_id") or "")) or {})
status = cls._final_abstract_ci_status(row)
if status == "unsupported":
unsupported.append(item)
if str(item.get("relation") or "").strip().lower() == "moderation":
moderation.append(item)
if "final_abstract_missing_unsupported_finding" in wanted:
out.extend(filter(None, (relation_sentence(item) for item in unsupported)))
if "final_abstract_missing_moderation_finding" in wanted and moderation:
sentence = relation_sentence(moderation[0])
if sentence:
out.append(sentence)
if "final_abstract_missing_empirical_finding" in wanted:
# Preserve representativeness even when the abstract already
# contains null/unsupported findings. Prefer one supported
# non-moderation relation so the fallback communicates a
# central positive/negative empirical result rather than
# becoming a list of only non-significant findings.
supported = [
item for item in items
if str(item.get("relation") or "").strip().lower() != "moderation"
and cls._final_abstract_ci_status(
dict(relation_results.get(str(item.get("relation_id") or "")) or {})
) == "supported"
]
candidate = next(
(item for item in supported if str(item.get("outcome") or "").strip()),
supported[0] if supported else None,
)
if candidate is None and not out:
candidate = next(
(item for item in items if relation_results.get(str(item.get("relation_id") or ""))),
None,
)
sentence = relation_sentence(candidate or {})
if sentence:
if sentence not in out:
out.append(sentence)
if "final_abstract_missing_indirect_finding" in wanted:
indirect_rows = [dict(x) for x in (ledger.get("indirect_analyses") or []) if isinstance(x, dict)]
selected = next((x for x in indirect_rows if x.get("numbered_hypothesis")), indirect_rows[0] if indirect_rows else None)
if selected:
path = [str(x or "").strip() for x in (selected.get("path") or []) if str(x or "").strip()]
effect = dict(indirect_results.get(str(selected.get("effect_id") or "")) or {})
if len(path) >= 3 and effect:
status = cls._final_abstract_ci_status(effect)
try:
estimate_value = float(effect.get("estimate"))
except (TypeError, ValueError):
estimate_value = 0.0
if status == "unsupported":
out.append(
f"The indirect association from {path[0]} to {path[-1]} through {path[1]} was not significant."
)
else:
direction = "positive" if estimate_value >= 0 else "negative"
out.append(
f"The indirect association from {path[0]} to {path[-1]} through {path[1]} was {direction} and significant."
)
elif str(mode or "").strip().lower() == "asq":
if wanted.intersection({
"final_abstract_missing_qualitative_finding",
"final_abstract_missing_qualitative_theme_findings",
}):
themes = [dict(x) for x in (meta.get("themes") or []) if isinstance(x, dict)]
labels = []
for theme in themes:
label = str(theme.get("title") or "").strip()
if not label:
concepts = [str(x or "").strip() for x in (theme.get("concepts") or []) if str(x or "").strip()]
label = concepts[0] if concepts else ""
if label and label not in labels:
labels.append(label)
if len(labels) >= 2:
break
if len(labels) >= 2:
out.append(f"The analysis identified {labels[0]} and {labels[1]} as central themes.")
elif len(labels) == 1:
out.append(f"The analysis identified {labels[0]} as a central theme.")
deduped = []
seen = set()
for sentence in out:
key = cls._final_abstract_normalized_text(sentence)
if key and key not in seen:
deduped.append(sentence)
seen.add(key)
return deduped
@classmethod
def _final_abstract_resilience_repair(
cls, mode: str, text: str, violations, data_metadata: dict, *, hypothesis_ledger: dict | None = None,
) -> tuple[str, list[dict]]:
"""Provider-neutral last-mile repair for recoverable Final-Abstract defects."""
current = str(text or "").strip()
rows = [dict(x) for x in (violations or []) if isinstance(x, dict)]
kinds = {str(x.get("type") or "") for x in rows}
diagnostics: list[dict] = []
metric_kinds = {"quantitative_metric_owner_unresolved", "quantitative_metric_mismatch", "unfrozen_p_value"}
if str(mode or "").strip().lower() == "amj" and kinds.intersection(metric_kinds):
current, metric_diagnostics = cls._final_abstract_metric_light(current)
diagnostics.extend(metric_diagnostics)
finding_kinds = {
"final_abstract_missing_empirical_finding",
"final_abstract_missing_unsupported_finding",
"final_abstract_missing_moderation_finding",
"final_abstract_missing_indirect_finding",
"final_abstract_missing_qualitative_finding",
"final_abstract_missing_qualitative_theme_findings",
}
if kinds.intersection(finding_kinds):
sentences = cls._canonical_final_abstract_findings(
mode, data_metadata, hypothesis_ledger=hypothesis_ledger,
violation_types=kinds.intersection(finding_kinds),
)
haystack = cls._final_abstract_normalized_text(current)
added = []
for sentence in sentences:
key = cls._final_abstract_normalized_text(sentence)
if key and key not in haystack:
added.append(sentence)
if added:
current = (current.rstrip() + " " + " ".join(added)).strip()
diagnostics.append({
"type": "final_abstract_frozen_finding_fallback",
"count": len(added),
"reason": "Frozen finding identities were added after bounded model repair left only abstract-coverage defects.",
})
return current, diagnostics
@classmethod
def _deterministic_final_abstract(
cls, mode: str, topic: str, data_metadata: dict, *, hypothesis_ledger: dict | None = None,
) -> str:
"""Build a conservative provider-free Final Abstract from frozen authority.
Delivery-first rule: this fallback is used only after bounded model/front-
matter repair has failed. It never invents coefficients, sample sizes,
methods, or causal claims; it uses the user topic plus frozen finding/theme
ledgers already owned by the completed manuscript.
"""
clean_topic = re.sub(r"\s+", " ", str(topic or "")).strip().rstrip(".")
if str(mode or "").strip().lower() == "amj":
lead = f"This survey-based quantitative study examines {clean_topic}." if clean_topic else (
"This survey-based quantitative study examines the associations specified in the frozen research model."
)
method = (
"The analysis evaluates the prespecified construct associations using the completed study evidence and reports the results conservatively."
)
wanted = {
"final_abstract_missing_empirical_finding",
"final_abstract_missing_unsupported_finding",
"final_abstract_missing_moderation_finding",
"final_abstract_missing_indirect_finding",
}
else:
lead = f"This qualitative study examines {clean_topic}." if clean_topic else (
"This qualitative study examines the focal management phenomenon defined by the completed manuscript."
)
method = (
"The analysis develops an evidence-bounded account from the completed qualitative study and preserves the study's documented design and evidence identities."
)
wanted = {
"final_abstract_missing_qualitative_finding",
"final_abstract_missing_qualitative_theme_findings",
}
findings = cls._canonical_final_abstract_findings(
mode, data_metadata, hypothesis_ledger=hypothesis_ledger, violation_types=wanted,
)
if not findings:
findings = [
"The principal findings and their boundary conditions are reported directly from the completed analysis without adding unsupported numerical or causal claims."
]
contribution = (
"The study clarifies the focal relationships and boundary conditions in a form that remains aligned with the completed evidence and research design."
if str(mode or "").strip().lower() == "amj"
else "The study offers a structured interpretation of the focal process while keeping the claims aligned with the completed qualitative evidence."
)
return " ".join([lead, method, *findings, contribution]).strip()
@classmethod
def _deterministic_research_logic_contract(
cls, mode: str, topic: str, keywords: str, data_metadata: dict, *, hypothesis_ledger: dict | None = None,
) -> str:
"""Provider-free planning contract used when research-logic generation fails.
This is an internal planning anchor, not reader-facing manuscript prose. It
intentionally stays conservative: no result direction, coefficient, sample
value, causal claim, or bibliographic assertion is introduced. Quantitative
H-identities are copied only from the frozen ledger.
"""
clean_topic = re.sub(r"\s+", " ", str(topic or "")).strip()
clean_keywords = ", ".join(parse_keyword_terms(keywords, explicit_user=True))
if str(mode or "").strip().casefold() == "amj":
parts = [
f"Research focus: {clean_topic}." if clean_topic else "Research focus: the management problem defined by the frozen user request.",
"Design: survey-based quantitative research.",
"Planning rule: develop the management problem and theoretical rationale without predicting coefficients, significance, support, or causal effects.",
"Evidence rule: use only the documented study design and completed analysis as authority for empirical facts and results.",
]
items = [item for item in (dict(hypothesis_ledger or {}).get("items") or []) if isinstance(item, dict)]
for item in items:
hid = re.sub(r"\s+", " ", str(item.get("hypothesis_id") or "")).strip().upper()
predictor = re.sub(r"\s+", " ", str(item.get("predictor") or "")).strip()
outcome = re.sub(r"\s+", " ", str(item.get("outcome") or "")).strip()
moderator = re.sub(r"\s+", " ", str(item.get("moderator") or "")).strip()
relation = str(item.get("relation") or "association").strip().casefold()
if not hid or not predictor or not outcome:
continue
if relation == "moderation" and moderator:
parts.append(
f"{hid} defines {moderator} as the moderator of the association between {predictor} and {outcome}."
)
else:
parts.append(f"{hid} defines the association between {predictor} and {outcome}.")
else:
parts = [
f"Research focus: {clean_topic}." if clean_topic else "Research focus: the management phenomenon defined by the frozen user request.",
"Design: qualitative research.",
"Planning rule: examine the organizational process, meanings, mechanisms, and boundary conditions without predicting an emergent resolution.",
"Evidence rule: use only the documented qualitative setting, data sources, and completed evidence; do not invent participants, quotations, counts, or findings.",
]
if clean_keywords:
parts.insert(1, f"User keywords: {clean_keywords}.")
return " ".join(part for part in parts if str(part or "").strip()).strip()
@classmethod
def _deterministic_working_front_matter(
cls, mode: str, topic: str, keywords: str, data_metadata: dict, *, hypothesis_ledger: dict | None = None,
) -> dict:
"""Build provisional front matter without a provider call.
Working front matter is only a continuity anchor before body drafting. The
fallback therefore avoids empirical findings entirely and prefers frozen
user keyword authority. Final front matter remains independently rebuilt
from the completed manuscript later.
"""
clean_topic = re.sub(r"\s+", " ", str(topic or "")).strip().rstrip(".")
candidate_keywords = parse_keyword_terms(keywords, explicit_user=True)
resolved_keywords = cls._canonical_quantitative_front_matter_keywords(
mode, candidate_keywords, data_metadata
)
if not resolved_keywords:
resolved_keywords = cls._front_matter_keyword_fallback_terms(data_metadata)
if not resolved_keywords:
resolved_keywords = ["Management Research"]
if str(mode or "").strip().casefold() == "amj":
lead = (
f"This survey-based quantitative study examines {clean_topic}."
if clean_topic else
"This survey-based quantitative study examines the management relationships defined by the user request."
)
abstract = " ".join([
lead,
"The manuscript develops the theoretical rationale for the prespecified research model and evaluates the documented associations using the completed study evidence.",
"At this planning stage, no coefficient, significance test, hypothesis support status, or causal conclusion is anticipated.",
"The final abstract will be reconstructed from the completed analysis and frozen evidence before delivery.",
])
else:
lead = (
f"This qualitative study examines {clean_topic}."
if clean_topic else
"This qualitative study examines the management phenomenon defined by the user request."
)
abstract = " ".join([
lead,
"The manuscript uses the documented qualitative study design to examine the focal process, meanings, mechanisms, and boundary conditions.",
"At this planning stage, no theme, quotation, participant detail, or substantive finding is anticipated beyond the frozen study design.",
"The final abstract will be reconstructed from the completed qualitative evidence before delivery.",
])
return {"abstract": abstract.strip(), "keywords": list(resolved_keywords)}
@classmethod
def _deterministic_hypothesis_reconcile(cls, text: str, ledger: dict) -> tuple[str, list[dict]]:
"""Reconcile H-identities without model rewriting or result-driven remapping.
Unsafe H-labelled sentences are removed rather than semantically rewritten;
missing canonical identities are then appended from the frozen ledger. This
keeps P1 identity integrity while avoiding a task kill for mechanically
repairable H-label defects.
"""
value = cls._normalize_hypothesis_ordinal_aliases(str(text or ""), ledger)
rows = cls._hypothesis_contract_violations(value, ledger, section_name="Full Manuscript", require_all=True)
diagnostics: list[dict] = []
if not rows:
return value, diagnostics
# Remove whole sentences whose H label is unknown or attached to the wrong
# construct pairing. Violation previews are intentionally capped, so
# match them as normalized prefixes against the original sentence spans
# rather than deleting only the truncated preview and leaving a prose tail.
bad_sentence_previews = []
unknown_ids = set()
for row in rows:
kind = str(row.get("type") or "")
if kind == "hypothesis_contract_pairing_mismatch" and str(row.get("sentence") or "").strip():
bad_sentence_previews.append(re.sub(r"\s+", " ", str(row.get("sentence") or "")).strip())
if kind == "hypothesis_contract_unknown_id":
unknown_ids.update(str(x or "").upper() for x in (row.get("observed") or []) if str(x or "").strip())
spans_to_remove = []
for start, end, sentence in SSciPipelineAgent._sentence_spans(value):
normalized_sentence = re.sub(r"\s+", " ", str(sentence or "")).strip()
pairing_match = any(
normalized_sentence == preview
or normalized_sentence.startswith(preview)
or preview.startswith(normalized_sentence)
for preview in bad_sentence_previews if preview
)
unknown_match = bool(unknown_ids) and any(
re.search(rf"\b{re.escape(hid)}\b", sentence, flags=re.I) for hid in unknown_ids
)
if pairing_match or unknown_match:
spans_to_remove.append((start, end, pairing_match, unknown_match))
removed_pairing = 0
removed_unknown = 0
for start, end, pairing_match, unknown_match in reversed(spans_to_remove):
value = value[:start] + value[end:]
removed_pairing += int(pairing_match)
removed_unknown += int(unknown_match)
if removed_pairing:
diagnostics.append({
"type": "hypothesis_pairing_sentence_removed",
"count": removed_pairing,
"reason": "Sentence(s) attaching a frozen H-identifier to the wrong construct pairing were removed before deterministic H-identity reconstruction.",
})
if removed_unknown:
diagnostics.append({
"type": "unknown_hypothesis_sentence_removed",
"count": removed_unknown,
"reason": "Sentence(s) containing H-identifiers outside the frozen hypothesis ledger were removed before deterministic H-identity reconstruction.",
})
# A residual unknown token can still exist in a heading/range that the
# sentence scanner did not classify as prose. Remove the identifier only
# as a final mechanical cleanup; substantive H-content is never rewritten.
if unknown_ids:
pattern = r"\b(?:" + "|".join(re.escape(x) for x in sorted(unknown_ids, key=len, reverse=True)) + r")\b"
value, count = re.subn(pattern, "", value, flags=re.I)
if count:
diagnostics.append({
"type": "unknown_hypothesis_identifier_removed",
"count": count,
"reason": "Residual unknown H-identifiers were removed; only the frozen hypothesis ledger may own H-label identity.",
})
value = re.sub(r"[ \t]{2,}", " ", value)
value = re.sub(r"\n{3,}", "\n\n", value).strip()
remaining = cls._hypothesis_contract_violations(value, ledger, section_name="Full Manuscript", require_all=True)
missing = []
for row in remaining:
if str(row.get("type") or "") == "hypothesis_contract_missing_id":
missing.extend(str(x or "").upper() for x in (row.get("expected") or []) if str(x or "").strip())
observed = {str(x or "").upper() for x in (row.get("observed") or []) if str(x or "").strip()}
missing = [x for x in missing if x not in observed]
if missing:
items = {
str(item.get("hypothesis_id") or "").strip().upper(): item
for item in (dict(ledger or {}).get("items") or []) if isinstance(item, dict)
}
lines = []
for hid in dict.fromkeys(missing):
item = dict(items.get(hid) or {})
predictor = re.sub(r"\s+", " ", str(item.get("predictor") or "")).strip()
outcome = re.sub(r"\s+", " ", str(item.get("outcome") or "")).strip()
moderator = re.sub(r"\s+", " ", str(item.get("moderator") or "")).strip()
relation = str(item.get("relation") or "association").strip().lower()
if not predictor or not outcome:
continue
if relation == "moderation" and moderator:
lines.append(f"{hid} states that {moderator} moderates the association between {predictor} and {outcome}.")
else:
lines.append(f"{hid} states that {predictor} is associated with {outcome}.")
if lines:
paragraph = "Hypothesis identity summary: " + " ".join(lines)
# Put the deterministic summary at the end of the theory section
# when that structural boundary is available; otherwise append to
# the frozen body. No new empirical result is introduced.
match = re.search(r"(?m)^#\s+(?:Methodology|Methods|Research Setting and Methods)\s*$", value)
if match:
value = value[:match.start()].rstrip() + "\n\n" + paragraph + "\n\n" + value[match.start():].lstrip()
else:
value = value.rstrip() + "\n\n" + paragraph
diagnostics.append({
"type": "missing_hypothesis_identity_reconstructed",
"count": len(lines),
"reason": "Missing H-identifiers were reconstructed only from the frozen hypothesis ledger.",
})
return value, diagnostics
@classmethod
def _final_abstract_finding_violations(
cls, mode: str, text: str, data_metadata: dict, *, hypothesis_ledger: dict | None = None
) -> list[dict]:
"""S1-R2 deterministic Final-Abstract frozen-finding coverage gate.
The gate is language-neutral at the manuscript-authority layer: English
Final Front Matter is frozen once, then ZH/RU/KK are semantic
translations of that exact source. Product tier is intentionally not an
input because Standard and Flagship share this correctness contract.
"""
value = str(text or "").strip()
meta = dict(data_metadata or {})
result_sentences = cls._final_abstract_result_sentences(value)
rows: list[dict] = []
if str(mode or "").strip().lower() == "amj":
ledger = dict(hypothesis_ledger or meta.get("hypothesis_ledger") or {})
evidence = dict(meta.get("association_results") or {})
if not evidence:
evidence = dict((meta.get("quantitative_evidence_ledger") or {}).get("association_results") or {})
relation_results = dict(evidence.get("relation_results") or {})
indirect_results = dict(evidence.get("indirect_results") or {})
def relation_sentence(item: dict, *, require_status: bool = True) -> bool:
relation_id = str(item.get("relation_id") or "")
frozen = dict(relation_results.get(relation_id) or {})
status = cls._final_abstract_ci_status(frozen)
identities = [item.get("predictor"), item.get("outcome")]
if str(item.get("relation") or "") == "moderation":
identities.append(item.get("moderator"))
for sentence in result_sentences:
if not all(cls._final_abstract_identity_present(sentence, label) for label in identities if label):
continue
if str(item.get("relation") or "") == "moderation" and not re.search(r"(?i)\b(?:moderat|interaction)\w*\b", sentence):
continue
if require_status and status and not cls._final_abstract_support_language(sentence, status):
continue
return True
return False
numbered = [dict(item) for item in (ledger.get("items") or []) if isinstance(item, dict)]
unsupported = []
moderation = []
for item in numbered:
frozen = dict(relation_results.get(str(item.get("relation_id") or "")) or {})
status = cls._final_abstract_ci_status(frozen)
if status == "unsupported":
unsupported.append(item)
if str(item.get("relation") or "") == "moderation":
moderation.append(item)
# Unsupported numbered hypotheses are interpretation-critical and
# must never disappear from the Final Abstract.
for item in unsupported:
if relation_sentence(item, require_status=True):
continue
rows.append({
"type": "final_abstract_missing_unsupported_finding",
"hypothesis_id": str(item.get("hypothesis_id") or ""),
"reason": (
"The Final Abstract omits the frozen unsupported/non-significant status of a numbered hypothesis."
),
"repair_instruction": (
"State the unsupported finding concisely in ordinary results language, preserving the exact construct identities. "
"Do not add coefficients unless indispensable."
),
})
# A focal moderation result is a main boundary-condition finding even
# when supported; require one moderation result sentence when present.
if moderation and not any(relation_sentence(item, require_status=True) for item in moderation):
item = moderation[0]
rows.append({
"type": "final_abstract_missing_moderation_finding",
"hypothesis_id": str(item.get("hypothesis_id") or ""),
"reason": "The Final Abstract does not report the frozen moderation/interaction finding.",
"repair_instruction": (
"Report whether the frozen moderator changed the focal association, using the exact predictor, moderator, and outcome identities. "
"Keep the statement concise and evidence-bounded."
),
})
# Preserve the main mechanism finding when an indirect analysis was
# actually compiled. If several unnumbered paths exist, one main
# frozen path is sufficient; numbered indirect hypotheses remain
# independently governed by the hypothesis contract.
indirect_rows = [dict(item) for item in (ledger.get("indirect_analyses") or []) if isinstance(item, dict)]
if indirect_rows:
selected = next((item for item in indirect_rows if item.get("numbered_hypothesis")), indirect_rows[0])
effect = dict(indirect_results.get(str(selected.get("effect_id") or "")) or {})
status = cls._final_abstract_ci_status(effect)
path = list(selected.get("path") or [])
indirect_found = False
for sentence in result_sentences:
if not re.search(r"(?i)\b(?:indirect|mediat)\w*\b", sentence):
continue
required_labels = []
if path:
required_labels.extend([path[0], path[-1]])
if len(path) > 2:
required_labels.append(path[1])
if required_labels and not all(cls._final_abstract_identity_present(sentence, x) for x in required_labels if x):
continue
if status and not cls._final_abstract_support_language(sentence, status):
continue
indirect_found = True
break
if not indirect_found:
rows.append({
"type": "final_abstract_missing_indirect_finding",
"reason": "The Final Abstract omits the main frozen indirect/mediation finding.",
"repair_instruction": (
"Summarize the frozen indirect association through the identified mediator in one concise findings sentence. "
"Do not claim causal mediation for observational data and do not add unnecessary statistics."
),
})
# Every empirical Final Abstract needs at least one actual result
# sentence. Design/hypothesis prose plus 'our findings contribute'
# is not sufficient.
if relation_results and not result_sentences:
rows.append({
"type": "final_abstract_missing_empirical_finding",
"reason": "The Final Abstract describes the design/hypotheses but states no frozen empirical result.",
"repair_instruction": (
"Add one or two concise sentences stating the main frozen empirical findings before the contribution. "
"Prefer support/non-support and substantive direction over a list of coefficients."
),
})
return rows
if str(mode or "").strip().lower() == "asq":
themes = [dict(item) for item in (meta.get("themes") or []) if isinstance(item, dict)]
if not themes:
if not result_sentences:
rows.append({
"type": "final_abstract_missing_qualitative_finding",
"reason": "The Final Abstract states no empirical qualitative finding.",
"repair_instruction": "Add a concise findings sentence grounded in the frozen qualitative evidence; do not invent a theme or process claim.",
})
return rows
covered = set()
for index, theme in enumerate(themes):
anchors = [str(theme.get("title") or "").strip()]
anchors.extend(str(x or "").strip() for x in (theme.get("concepts") or []))
anchors = [x for x in anchors if x]
for sentence in result_sentences:
if any(cls._final_abstract_identity_present(sentence, anchor) for anchor in anchors):
covered.add(index)
break
required = min(2, len(themes))
if len(covered) < required:
rows.append({
"type": "final_abstract_missing_qualitative_theme_findings",
"observed": len(covered),
"expected": required,
"reason": (
"The Final Abstract does not report enough frozen qualitative theme/concept findings to function as a results-bearing final abstract."
),
"repair_instruction": (
f"State the main qualitative findings in one or two concise sentences grounded in at least {required} frozen theme/concept identities. "
"Do not merely say that findings contribute, and do not invent themes, quotations, participants, or procedures."
),
})
return rows
return rows
def _delivery_first_final_abstract_language_guard(
self, *, mode: str, agent, quant_engine, final_abstract: str, reader_topic_en: str,
final_title: str, data_metadata: dict, hypothesis_ledger: dict | None, quality_notes: list,
) -> tuple[str, bool]:
"""Repair residual non-English script without turning a reader-surface defect into P0 failure.
Returns ``(abstract, rebuilt)``. Explicit P1 integrity validators remain
authoritative; only the language-surface defect is downgraded when a
provider-neutral clean reconstruction cannot be proven safe.
"""
current = str(final_abstract or "").strip()
if not agent._english_reader_surface_has_foreign_script(current):
return current, False
language_fallback = self._deterministic_final_abstract(
mode, reader_topic_en or final_title, data_metadata, hypothesis_ledger=hypothesis_ledger
)
language_rows = self._study_fact_violations(
mode, agent, quant_engine, language_fallback, data_metadata, section_name="Final Abstract"
)
if hypothesis_ledger:
language_rows.extend(self._hypothesis_contract_violations(
language_fallback, hypothesis_ledger, section_name="Final Abstract"
))
language_rows.extend(self._final_abstract_finding_violations(
mode, language_fallback, data_metadata, hypothesis_ledger=hypothesis_ledger
))
fatal_language_rows, advisory_language_rows = self._partition_section_violations(
language_rows, section_name="Final Abstract", mode=mode, exhausted=True
)
if (
language_fallback
and not agent._english_reader_surface_has_foreign_script(language_fallback)
and not fatal_language_rows
):
self._record_adaptive_completion_residual(
quality_notes,
advisory_language_rows + [{
"type": "final_english_front_matter_language_fallback",
"reason": "Residual non-English script in the Final Abstract was replaced by a deterministic English reconstruction from frozen research authority."
}],
section_name="Final Abstract",
stage="delivery_first_front_matter_language_fallback",
)
return language_fallback, True
self._record_adaptive_completion_residual(
quality_notes, [{
"type": "final_english_front_matter_language_warning",
"reason": "Final Abstract still contains non-English script after deterministic language fallback; delivery was preserved under the P0 Delivery-First contract."
}],
section_name="Final Abstract",
stage="delivery_first_front_matter_language_warning",
)
return current, False
@staticmethod
def _section_retry_feedback(violations, *, limit=8) -> str:
rows = []
for item in list(violations or [])[:max(1, int(limit))]:
if not isinstance(item, dict):
rows.append(str(item)[:500])
continue
kind = str(item.get('type') or 'validation_issue')
detail = str(item.get('repair_instruction') or item.get('reason') or item.get('preview') or '')
if kind == 'causal_language_violation' and not detail:
detail = (
"Rewrite only the causal-strength wording. Preserve the study fact and analytical meaning, "
"but state association/covariation rather than causal identification. Explicit limitation "
"phrases such as 'restricted to associations, not causal effects' and 'cannot establish "
"causality' are valid and should not be weakened further."
)
if kind == 'citation_evidence_ownership_mismatch':
detail = (
"Split the sentence by evidence ownership. Keep the study-owned fact as a separate statement grounded only in the study evidence and without an external citation. "
"State any methodological principle separately and attach only one directly supporting METHOD_AUTHORITY citation. Keep author inference separate, cautious, and uncited unless a source directly supports that exact inference."
)
actor = str(item.get('actor') or '')
expected = item.get('expected')
observed = item.get('observed')
parts = [kind]
if actor:
parts.append(f'actor={actor}')
if observed not in (None, ''):
parts.append(f'observed={observed}')
if expected not in (None, ''):
parts.append(f'expected={expected}')
if detail:
parts.append(detail[:360])
rows.append(' | '.join(parts))
return '\n'.join(f'- {row}' for row in rows)
@staticmethod
def _section_forbids_external_citations(section_title: str = "", section_id: str = "") -> bool:
"""One authority for manuscript units that must remain citation-free.
P3-A4 closes a historic split where Results/Findings were declared
citation-free in the Writer prompt but did not enter the same quarantine
validation path as Conclusion/Appendix-style units.
"""
title = str(section_title or '').casefold()
sid = str(section_id or '').strip().casefold()
return (
any(word in title for word in ("abstract", "keyword", "conclusion", "references", "appendix"))
or sid in {"results", "findings"}
)
@staticmethod
def _section_violation_severity(row, *, section_name: str = "", mode: str = "", exhausted: bool = False) -> str:
"""Map one validation issue to REPAIRABLE/FATAL/ADVISORY.
Before the one local retry, all substantive correctness issues remain
repairable. After retry exhaustion only issues that can change study
facts, core inference, hypothesis identity, core evidence support, or
citation integrity may terminate the manuscript. Minor reader-facing
style/background defects become advisory notes instead of destroying an
otherwise complete paper.
"""
item = dict(row or {}) if isinstance(row, dict) else {"type": "unknown_validation_issue"}
kind = str(item.get("type") or "unknown_validation_issue")
section = str(section_name or "")
if not exhausted:
if kind == "top_journal_critic":
return "ADVISORY"
return "REPAIRABLE"
# Delivery-first resilience: uncertain noncanonical citation-like forms
# get a DeepSeek repair opportunity, but they are not allowed to terminate
# an otherwise complete manuscript after bounded repair. Definite
# authority/scope violations remain fail-closed below.
if kind in {
"citation_noncanonical_unverified_marker",
"citation_suspected_in_quarantined_unit",
}:
return "ADVISORY"
always_fatal = {
"causal_language_violation",
"sample_size_mismatch", "sample_count_mismatch", "scale_points_mismatch",
"construct_role_mismatch", "quantitative_metric_mismatch", "unfrozen_p_value",
"unsupported_quantitative_method", "actor_data_source_relation_mismatch",
"unsupported_study_procedure", "enumeration_contract_invalid",
"hypothesis_contract_missing_id", "hypothesis_contract_ordinal_alias",
"hypothesis_contract_pairing_mismatch", "hypothesis_contract_unknown_id",
"citation_outside_frozen_section_whitelist",
"citation_resolved_marker_outside_frozen_section_whitelist",
"citation_unknown_tag", "citation_resolved_marker_without_authority",
"citation_resolved_marker_ambiguous", "citation_noncanonical_resolved_marker",
"citation_forbidden_in_quarantined_unit",
"authorial_ai_meta_trace",
"study_design_scope_mismatch", "unsupported_focal_construct", "unsupported_relation_claim",
"unsupported_moderation_claim", "unsupported_mediation_claim", "unsupported_temporal_scope",
"unsupported_design_tradition", "unsupported_unit_of_analysis", "unsupported_data_source_claim",
"unsupported_process_claim",
"final_abstract_missing_empirical_finding",
"final_abstract_missing_unsupported_finding",
"final_abstract_missing_moderation_finding",
"final_abstract_missing_indirect_finding",
"final_abstract_missing_qualitative_finding",
"final_abstract_missing_qualitative_theme_findings",
}
if kind in always_fatal:
return "FATAL"
if kind in {"author_facing_internal_workflow_language", "top_journal_critic"}:
return "ADVISORY"
if kind in {"citation_claim_split_required", "citation_unsupported_claim", "citation_support_unverified", "citation_evidence_ownership_mismatch"}:
core_citation_section = bool(re.search(
r"(?i)(?:Theoretical Mechanisms|Hypothesis Development|Methodology|Results?|Findings?)",
section,
))
return "FATAL" if core_citation_section else "ADVISORY"
# Delivery-First hardening: unknown/new validators are not allowed to
# create an accidental terminal path merely because a developer forgot
# to register their severity. Only explicit P1/hard-failure identities
# above may remain fatal after bounded repair.
return "ADVISORY"
@staticmethod
def _section_violation_lineage(row) -> str:
"""Return a stable repair lineage for one section validation defect.
R4 assigns the bounded rewrite entitlement to the defect lineage rather
than to the whole Section. Closely related validators share a lineage
so spelling variants of the same integrity problem cannot manufacture
fresh retries.
"""
item = dict(row or {}) if isinstance(row, dict) else {"type": "unknown_validation_issue"}
kind = str(item.get("type") or "unknown_validation_issue")
groups = {
"citation_scope": {
"citation_outside_frozen_section_whitelist",
"citation_resolved_marker_outside_frozen_section_whitelist",
},
"citation_authority": {
"citation_unknown_tag",
"citation_resolved_marker_without_authority",
"citation_resolved_marker_ambiguous",
"citation_noncanonical_resolved_marker",
"citation_forbidden_in_quarantined_unit",
},
"citation_style": {
"citation_noncanonical_unverified_marker",
"citation_suspected_in_quarantined_unit",
},
"citation_ownership": {"citation_evidence_ownership_mismatch"},
"citation_support": {"citation_unsupported_claim", "citation_support_unverified"},
"citation_compound": {"citation_claim_split_required"},
"causal_scope": {"causal_language_violation"},
"quant_method": {"unsupported_quantitative_method", "quantitative_metric_mismatch", "unfrozen_p_value"},
"study_fact": {"sample_size_mismatch", "sample_count_mismatch", "scale_points_mismatch",
"construct_role_mismatch", "actor_data_source_relation_mismatch", "unsupported_study_procedure"},
"enumeration": {"enumeration_contract_invalid", "enumeration_contract_mismatch"},
"hypothesis": {"hypothesis_contract_missing_id", "hypothesis_contract_ordinal_alias",
"hypothesis_contract_pairing_mismatch", "hypothesis_contract_unknown_id"},
"workflow_trace": {"authorial_ai_meta_trace", "author_facing_internal_workflow_language"},
"study_design_scope": {
"study_design_scope_mismatch", "unsupported_focal_construct", "unsupported_relation_claim",
"unsupported_moderation_claim", "unsupported_mediation_claim", "unsupported_temporal_scope",
"unsupported_design_tradition", "unsupported_unit_of_analysis", "unsupported_data_source_claim",
"unsupported_process_claim",
},
}
for lineage, kinds in groups.items():
if kind in kinds:
return lineage
return kind
@staticmethod
def _strip_unscoped_citation_identities(value):
"""Remove raw citation capability tokens from non-authoritative context."""
return re.sub(r"\[CITE-\d+\]", "", str(value or ""), flags=re.I)
@staticmethod
def _section_citation_capability_context(agent, planned_items, max_chars=12000):
"""Expose only citation identities already frozen for this manuscript unit.
Retrieval visibility is not citation authority. R6-R10 therefore builds
writer-facing targeted context from the unit's frozen claim-evidence plan
instead of the global RAG pool, which may contain METHOD_AUTHORITY tags
owned by another unit.
"""
rows = []
seen = set()
for item in list(planned_items or []):
if not isinstance(item, dict):
continue
tag = str(item.get("tag") or "").strip()
if not tag or tag in seen:
continue
metadata = dict(getattr(agent, "citation_map", {}).get(tag) or {})
evidence = str(item.get("evidence") or metadata.get("evidence") or "").strip()
claim = str(item.get("claim") or "").strip()
if not evidence:
continue
seen.add(tag)
row = f"{tag} AUTHORIZED CLAIM: {claim}\nEVIDENCE: {evidence}" if claim else f"{tag} EVIDENCE: {evidence}"
rows.append(row)
value = "\n\n--- Authorized Section Evidence ---\n".join(rows)
return value[:max(0, int(max_chars))]
@staticmethod
def _section_regressed_lineages(current_lineages, cleared_lineages, regression_attempts):
"""Return previously-cleared lineages reintroduced by a later sibling repair."""
current = {str(value) for value in (current_lineages or set()) if str(value)}
cleared = {str(value) for value in (cleared_lineages or set()) if str(value)}
attempts = dict(regression_attempts or {})
return {name for name in current & cleared if int(attempts.get(name, 0)) < 1}
@classmethod
def _section_retry_decision(cls, rows, lineage_attempts, *, hard_cap_reached: bool = False):
"""Return (may_retry, exhausted_lineages, new_lineages).
Each lineage receives one rewrite entitlement. A lineage that survives
its own rewrite remains fail-closed. A genuinely new lineage discovered
only after a previous repair receives one bounded chance, subject to the
Section hard cap.
"""
lineages = {cls._section_violation_lineage(row) for row in (rows or [])}
exhausted = {name for name in lineages if int(lineage_attempts.get(name, 0)) >= 1}
fresh = lineages - exhausted
# R6-R3 fairness closure: an exhausted lineage may not steal the bounded
# repair entitlement of a genuinely fresh lineage discovered in the same
# validation round. The Section hard cap remains the global stop.
return bool(fresh) and not hard_cap_reached, exhausted, fresh
@classmethod
def _partition_section_violations(cls, rows, *, section_name: str = "", mode: str = "", exhausted: bool = False):
blocking, advisory = [], []
for row in list(rows or []):
severity = cls._section_violation_severity(
row, section_name=section_name, mode=mode, exhausted=exhausted
)
if severity == "ADVISORY":
advisory.append(row)
else:
blocking.append(row)
return blocking, advisory
@classmethod
def _raise_if_fatal_completion_residual(
cls, rows, *, section_name: str = "", mode: str = "", error_code: str = "final_validation_unresolved"
) -> None:
"""Fail closed only for residuals already classified as FATAL.
Adaptive completion may retain advisory diagnostics, but it must never
downgrade hypothesis identity or other fatal correctness contracts into
a shippable manuscript.
"""
fatal = [
row for row in list(rows or [])
if cls._section_violation_severity(
row, section_name=section_name, mode=mode, exhausted=True
) == "FATAL"
]
if fatal:
raise RuntimeError(f"{error_code}:{fatal[:4]}")
@classmethod
def _partition_section_violations_by_lineage(
cls, rows, lineage_attempts, *, section_name: str = "", mode: str = "",
hard_cap_reached: bool = False,
):
"""Classify each defect using only its own repair-exhaustion state.
R6-R3 prevents a previously exhausted sibling lineage from globally
changing the severity of a newly discovered defect. When the Section
hard cap is reached every lineage is evaluated as exhausted.
"""
blocking, advisory = [], []
for row in list(rows or []):
lineage = cls._section_violation_lineage(row)
lineage_exhausted = bool(hard_cap_reached) or int(lineage_attempts.get(lineage, 0)) >= 1
severity = cls._section_violation_severity(
row, section_name=section_name, mode=mode, exhausted=lineage_exhausted
)
if severity == "ADVISORY":
advisory.append(row)
else:
blocking.append(row)
return blocking, advisory
@classmethod
def _persistent_citation_authority_rows(cls, rows) -> list[dict]:
"""Return citation identity/scope defects that may never cross a Section seal.
P3-A4 intentionally keeps this narrower than general validation failure.
Citation support/style advisories retain their existing adaptive-completion
behavior; only source identity and unit-scope authority stay fail-closed.
"""
return [
row for row in list(rows or [])
if isinstance(row, dict)
and cls._section_violation_lineage(row) in {"citation_scope", "citation_authority"}
]
@staticmethod
def _record_section_advisories(quality_notes: list, rows, *, section_name: str = "") -> None:
"""Append reader-facing, nonfatal notes without exposing control vocabulary."""
for row in list(rows or []):
if not isinstance(row, dict):
continue
kind = str(row.get("type") or "quality_advisory")
if kind == "citation_claim_split_required":
detail = "A non-core literature sentence may combine more than one externally verifiable proposition."
action = "Before formal submission, consider splitting that sentence and checking the citation attached to each proposition."
elif kind in {"citation_unsupported_claim", "citation_support_unverified"}:
detail = "A non-core literature statement could not be fully verified against the available retrieved evidence."
action = "Review the statement and its supporting literature before formal submission; the manuscript was completed because the issue does not alter the study's core results or design."
elif kind == "author_facing_internal_workflow_language":
detail = "A minor reader-facing wording issue remained after automatic revision."
action = "Review the indicated passage for ordinary academic phrasing before formal submission."
elif kind == "authorial_ai_meta_trace":
detail = "A minor author-facing wording pattern may benefit from manual stylistic review."
action = "Review the indicated passage and remove any wording that does not read as ordinary scholarly prose."
elif kind == "top_journal_critic":
detail = str(row.get("reason") or "An optional journal-style refinement was identified.")[:500]
action = "This is a non-blocking style suggestion; revise only if it improves conciseness, accuracy, or rigor."
else:
detail = "A nonfatal quality issue was recorded for author review."
action = "Review the indicated section before formal submission if additional polishing is desired."
note = {
"type": "section_quality_advisory",
"severity": "ADVISORY",
"section": str(section_name or "Manuscript"),
"issue_type": kind,
"detail": detail,
"user_action": action,
}
signature = (note["section"], note["issue_type"], note["detail"])
if any(
(str(existing.get("section") or ""), str(existing.get("issue_type") or ""), str(existing.get("detail") or "")) == signature
for existing in quality_notes if isinstance(existing, dict)
):
continue
quality_notes.append(note)
@staticmethod
def _record_adaptive_completion_residual(quality_notes: list, rows, *, section_name: str, stage: str) -> None:
kinds = sorted({str((row or {}).get("type") or "validation_issue") for row in list(rows or []) if isinstance(row, dict)})
note = {
"type": "adaptive_completion_residual",
"severity": "ADVISORY",
"section": str(section_name or "Manuscript"),
"issue_type": ",".join(kinds[:8]) or "validation_issue",
"detail": (
"SmartPaper completed the manuscript after bounded adaptive repair. "
"A conservative validator still reported a residual issue, so it is recorded for optional author review instead of terminating generation."
),
"user_action": "Review this passage before formal submission if additional refinement is desired.",
"stage": str(stage or "adaptive_completion"),
}
sig = (note["section"], note["issue_type"], note["stage"])
if not any(
(str(x.get("section") or ""), str(x.get("issue_type") or ""), str(x.get("stage") or "")) == sig
for x in quality_notes if isinstance(x, dict)
):
quality_notes.append(note)
def _adaptive_repair_section_after_exhaustion(
self, *, agent, quant_engine, text: str, violations, mode: str, section_id: str,
section_name: str, data_metadata: dict, study_evidence: str, evidence_contract: str,
hypothesis_ledger: dict, task: dict, quote_invocation_plan: dict, quality_notes: list,
allowed_citation_tags=None, quarantine=False,
):
"""A2 last-mile semantic repair after lineage retry exhaustion.
Content defects are recoverable by default. The model gets the locked
study/evidence context, then deterministic validators re-check the exact
returned unit. If conservative diagnostics still disagree after bounded
repair, the best complete unit is retained and the residual is recorded
in Appendix quality notes rather than killing the whole paper.
"""
current = str(text or '').strip()
remaining = list(violations or [])
passes = max(1, int((APP_CONFIG.get("adaptive_completion") or {}).get("section_repair_passes", 2)))
scopes = ("paragraph", "section", "section")
for idx in range(passes):
if self.cancel_checker():
raise RuntimeError("task_cancel_requested")
repaired = agent.adaptive_repair_academic_unit(
current, violations=remaining, section_id=section_id, section_name=section_name,
mode=mode, study_evidence=study_evidence, evidence_contract=evidence_contract,
repair_scope=scopes[min(idx, len(scopes) - 1)],
)
if not str(repaired or '').strip():
repaired = current
if mode == "amj" and hypothesis_ledger:
repaired = self._normalize_hypothesis_ordinal_aliases(repaired, hypothesis_ledger)
if mode == "asq" and str(section_id or '').lower() == "findings":
repaired = self._strip_model_controlled_qualitative_quotes(repaired, data_metadata)
repaired = self._render_planned_qualitative_quotes(
repaired, task=task, data_metadata=data_metadata, quote_invocation_plan=quote_invocation_plan,
)
if mode == "amj" and data_metadata.get("quantitative_table_manifest"):
if str(section_id or '').lower() == "results":
repaired = self._apply_quantitative_table_contract(repaired, data_metadata, section_name)
else:
repaired = self._strip_model_markdown_tables(repaired)
repaired, _enum_rows = agent.repair_and_validate_enumeration_contract(
repaired, section_name=section_name
)
# P3-A4: every model-owned last-mile rewrite must re-enter the same
# citation authority + claim-specific semantic Gate before it can be
# considered sealable. Otherwise an unrelated adaptive repair could
# reintroduce a pre-resolved/unknown/wrong-scope citation after the
# earlier Section citation audit had already passed.
repaired = agent.audit_and_resolve_citations(
repaired, section_name=section_name, section_id=section_id,
quarantine=quarantine, allowed_tags=allowed_citation_tags,
)
adaptive_citation_rows = copy.deepcopy(
list(getattr(agent, "_last_citation_semantic_violations", []) or [])
)
rows = self._section_validation_bundle(
mode, agent, quant_engine, repaired, data_metadata, section_id=section_id,
section_name=section_name, evidence_contract=evidence_contract, include_critic=False,
citation_violations=adaptive_citation_rows, hypothesis_ledger=hypothesis_ledger,
)
blocking, advisory = self._partition_section_violations(
rows, section_name=section_name, mode=mode, exhausted=True
)
if advisory:
self._record_section_advisories(quality_notes, advisory, section_name=section_name)
current = repaired
remaining = blocking
if not blocking:
return current, []
if remaining:
self._record_adaptive_completion_residual(
quality_notes, remaining, section_name=section_name, stage="section_repair_exhausted"
)
return current, remaining
def _adaptive_repair_front_matter(
self, *, agent, quant_engine, abstract_text: str, violations, mode: str,
data_metadata: dict, study_evidence: str, hypothesis_ledger: dict, quality_notes: list,
label: str,
):
current = str(abstract_text or '').strip()
remaining = list(violations or [])
passes = max(1, int((APP_CONFIG.get("adaptive_completion") or {}).get("front_matter_repair_passes", 2)))
for idx in range(passes):
repaired = agent.adaptive_repair_academic_unit(
current, violations=remaining, section_id="abstract", section_name=label, mode=mode,
study_evidence=study_evidence, evidence_contract="",
repair_scope="paragraph" if idx == 0 else "section",
)
repaired = str(repaired or current).strip()
if hypothesis_ledger:
repaired = self._normalize_hypothesis_ordinal_aliases(repaired, hypothesis_ledger)
rows = self._study_fact_violations(
mode, agent, quant_engine, repaired, data_metadata, section_name=label
)
if hypothesis_ledger:
rows.extend(self._hypothesis_contract_violations(
repaired, hypothesis_ledger, section_name=label
))
if str(label or "").strip().casefold() == "final abstract":
rows.extend(self._final_abstract_finding_violations(
mode, repaired, data_metadata, hypothesis_ledger=hypothesis_ledger
))
if rows:
resilient, resilience_notes = self._final_abstract_resilience_repair(
mode, repaired, rows, data_metadata, hypothesis_ledger=hypothesis_ledger
)
if resilient != repaired:
repaired = resilient
rows = self._study_fact_violations(
mode, agent, quant_engine, repaired, data_metadata, section_name=label
)
if hypothesis_ledger:
rows.extend(self._hypothesis_contract_violations(
repaired, hypothesis_ledger, section_name=label
))
rows.extend(self._final_abstract_finding_violations(
mode, repaired, data_metadata, hypothesis_ledger=hypothesis_ledger
))
if resilience_notes:
self._record_adaptive_completion_residual(
quality_notes, resilience_notes, section_name=label,
stage="front_matter_resilience_recovery"
)
current, remaining = repaired, rows
if not remaining:
return current, []
if remaining:
self._record_adaptive_completion_residual(
quality_notes, remaining, section_name=label, stage="front_matter_repair_exhausted"
)
return current, remaining
@staticmethod
def _citation_violation_tags(rows) -> set:
"""Extract only citation tags owned by the rejected claim sentence.
S0-R1-C1 uses this set as a bounded retry exclusion. Reasons may mention
comparison sources, so only explicit tag fields and rejected sentences are
inspected; this avoids excluding unrelated frozen evidence.
"""
tags = set()
for row in rows or []:
if not isinstance(row, dict):
continue
owned = ' '.join(str(row.get(key) or '') for key in ('tag', 'citation', 'sentence'))
for raw in re.findall(r'\[?CITE-\d+\]?', owned, flags=re.I):
token = raw.upper().strip('[]')
tags.add(f'[{token}]')
return tags
@staticmethod
def _author_facing_internal_workflow_violations(text: str) -> list[dict]:
"""Detect reader-facing leakage using the same registry as sanitization."""
value = str(text or "")
rows = []
for pattern, _replacement in AUTHOR_FACING_PRIVATE_VOCABULARY:
for match in re.finditer(pattern, value, flags=re.I):
start = max(0, match.start() - 80)
end = min(len(value), match.end() + 120)
preview = re.sub(r"\s+", " ", value[start:end]).strip()
rows.append({
"type": "author_facing_internal_workflow_language",
"preview": preview[:320],
"reason": "Private generation-control vocabulary surfaced in reader-facing manuscript prose.",
"repair_instruction": (
"Rewrite the sentence as ordinary academic prose describing the study, evidence, method, or limitation directly."
),
})
if len(rows) >= 6:
return rows
return rows
@staticmethod
def _writer_visible_evidence_context(text: str) -> str:
"""Return a reader-safe Writer view using the shared private registry."""
value = str(text or "")
# Formatting-only labels are sanitized before the phrase registry.
value = re.sub(r"(?im)^(#{1,6}\s+)Frozen\s+", r"\1", value)
value = re.sub(r"(?im)^\|\s*Field\s*\|\s*Frozen value\s*\|$", "| Field | Study value |", value)
for pattern, replacement in AUTHOR_FACING_PRIVATE_VOCABULARY:
value = re.sub(pattern, replacement, value, flags=re.I)
value = re.sub(r"(?im);\s*the Executor\b[^\n]*", ".", value)
value = re.sub(r"(?im)^\s*the Executor\b[^\n]*$", "", value)
return value
@staticmethod
def _hypothesis_contract_directive(ledger: dict) -> str:
record = dict(ledger or {})
items = [item for item in (record.get("items") or []) if isinstance(item, dict)]
if not items:
return ""
lines = []
for item in items:
hid = str(item.get("hypothesis_id") or "").strip()
predictor = str(item.get("predictor") or "").strip()
outcome = str(item.get("outcome") or "").strip()
moderator = str(item.get("moderator") or "").strip()
relation = str(item.get("relation") or "association").strip().casefold()
if hid and predictor and outcome:
if relation == "moderation" and moderator:
lines.append(f"{hid}: {moderator} moderates the association between {predictor} and {outcome}.")
else:
lines.append(f"{hid}: {predictor} is associated with {outcome}.")
if not lines:
return ""
indirect = dict(record.get("indirect_analysis") or {})
indirect_line = ""
if indirect:
p = str(indirect.get("predictor") or "").strip()
m = str(indirect.get("mediator") or "").strip()
o = str(indirect.get("outcome") or "").strip()
if p and m and o:
indirect_line = (
f"The estimated indirect association {p} -> {m} -> {o} is an analysis target, not a separately numbered hypothesis."
)
rendered = " ".join(lines + ([indirect_line] if indirect_line else []))
return (
"IMMUTABLE HYPOTHESIS CONTRACT: " + rendered
+ " Use H1/H2/H3-style identifiers exactly as listed whenever hypotheses are defined or referenced. "
"Do not renumber, reorder, create an additional hypothesis, replace an H-label with 'first/second/third hypothesis', "
"or change the construct pairing to fit observed results. The contract is result-independent."
)
@staticmethod
def _hypothesis_memory_text(ledger: dict) -> str:
"""Render the sole AMJ H1-Hn memory from the frozen ledger."""
lines = []
for item in (dict(ledger or {}).get("items") or []):
if not isinstance(item, dict):
continue
hid = str(item.get("hypothesis_id") or "").strip().upper()
predictor = re.sub(r"\s+", " ", str(item.get("predictor") or "")).strip()
outcome = re.sub(r"\s+", " ", str(item.get("outcome") or "")).strip()
moderator = str(item.get("moderator") or "").strip()
relation = str(item.get("relation") or "association").strip().casefold()
if hid and predictor and outcome:
if relation == "moderation" and moderator:
lines.append(f"{hid}: {moderator} moderates the association between {predictor} and {outcome}.")
else:
lines.append(f"{hid}: {predictor} is associated with {outcome}.")
return "\n".join(lines)
@staticmethod
def _normalize_hypothesis_ordinal_aliases(text: str, ledger: dict) -> str:
"""Map safe ordinal aliases to frozen H-identifiers without semantic rewrite."""
value = str(text or "")
items = [item for item in (dict(ledger or {}).get("items") or []) if isinstance(item, dict)]
if not items:
return value
words = ["first", "second", "third", "fourth", "fifth", "sixth"]
for index, item in enumerate(items[: len(words)]):
hid = str(item.get("hypothesis_id") or "").strip().upper()
if not hid:
continue
word = words[index]
value = re.sub(rf"\b(?:the\s+)?{word}\s+hypothesis\b", hid, value, flags=re.I)
return value
@classmethod
def _hypothesis_contract_violations(
cls, text: str, ledger: dict, *, section_name: str = "", require_all: bool = False
) -> list[dict]:
record = dict(ledger or {})
items = [item for item in (record.get("items") or []) if isinstance(item, dict)]
if not items:
return []
value = str(text or "")
expected = {
str(item.get("hypothesis_id") or "").upper(): item
for item in items if str(item.get("hypothesis_id") or "").strip()
}
if not expected:
return []
rows = []
observed = {f"H{n}" for n in re.findall(r"\bH(\d{1,2})\b", value, flags=re.I)}
unknown = sorted(observed - set(expected))
if unknown:
rows.append({
"type": "hypothesis_contract_unknown_id",
"observed": unknown,
"expected": sorted(expected),
"reason": "The manuscript introduced a hypothesis identifier outside the frozen result-independent hypothesis ledger.",
"repair_instruction": "Use only the frozen H-identifiers and keep the indirect association unnumbered unless the ledger explicitly numbers it.",
})
ordinal = re.search(r"\b(?:first|second|third|fourth|fifth|sixth)\s+hypothesis\b", value, flags=re.I)
if ordinal:
rows.append({
"type": "hypothesis_contract_ordinal_alias",
"preview": re.sub(r"\s+", " ", value[max(0, ordinal.start()-80):ordinal.end()+120]).strip()[:320],
"reason": "Ordinal hypothesis aliases can silently reorder the canonical H1-Hn identity.",
"repair_instruction": "Refer to hypotheses only by their frozen H1-Hn identifiers.",
})
all_constructs = {
re.sub(r"\s+", " ", str(item.get(key) or "")).strip()
for item in items for key in ("predictor", "outcome", "moderator")
if str(item.get(key) or "").strip()
}
for start, end, sentence in SSciPipelineAgent._sentence_spans(value):
ids = list(dict.fromkeys(
f"H{n}" for n in re.findall(r"\bH(\d{1,2})\b", sentence, flags=re.I)
))
# Collective references such as "H1-H3 are summarized below"
# are not definitions and must not be mistaken for remapping.
if len(ids) != 1:
continue
hid = ids[0]
item = expected.get(hid)
if not item:
continue
predictor = re.sub(r"\s+", " ", str(item.get("predictor") or "")).strip()
outcome = re.sub(r"\s+", " ", str(item.get("outcome") or "")).strip()
moderator = re.sub(r"\s+", " ", str(item.get("moderator") or "")).strip()
lower_sentence = re.sub(r"\s+", " ", sentence).casefold()
mentioned_constructs = {
term for term in all_constructs if term.casefold() in lower_sentence
}
# Pairing identity is global, not limited to the theory-definition
# sections. Require at least two named constructs so a bare reference
# such as "H1 was supported" cannot be misclassified as remapping.
expected_terms = [term for term in (predictor, outcome, moderator) if term]
if len(mentioned_constructs) >= 2 and predictor and outcome and any(
term.casefold() not in lower_sentence for term in expected_terms
):
rows.append({
"type": "hypothesis_contract_pairing_mismatch",
"id": hid,
"sentence": re.sub(r"\s+", " ", sentence).strip()[:420],
"expected": f"{predictor} -> {outcome}",
"reason": "A canonical hypothesis label was attached to a different construct pairing.",
"repair_instruction": f"Define {hid} only for {predictor} and {outcome}, using association language.",
})
if require_all:
missing = sorted(set(expected) - observed)
if missing:
rows.append({
"type": "hypothesis_contract_missing_id",
"observed": sorted(observed),
"expected": sorted(expected),
"reason": "The completed hypothesis architecture omitted one or more frozen hypothesis identifiers.",
"repair_instruction": "Preserve every canonical H1-Hn identity from theory through the integrated model and downstream reporting.",
})
return rows[:8]
@staticmethod
def _strip_model_markdown_tables(text: str) -> str:
"""Remove writer-created Markdown tables before deterministic AMJ table binding."""
lines = str(text or "").splitlines()
kept = []
index = 0
while index < len(lines):
line = lines[index]
if re.match(r"^\s*(?:#{1,6}\s*)?Table\s+\d+\s*[:.\-]", line, flags=re.I):
index += 1
while index < len(lines) and not lines[index].strip():
index += 1
if index < len(lines) and lines[index].lstrip().startswith("|"):
while index < len(lines) and lines[index].lstrip().startswith("|"):
index += 1
if index < len(lines) and re.match(r"^\s*\*?Note\s*:", lines[index], flags=re.I):
index += 1
continue
if line.lstrip().startswith("|"):
start = index
while index < len(lines) and lines[index].lstrip().startswith("|"):
index += 1
block = lines[start:index]
if len(block) >= 2 and re.match(r"^\s*\|?\s*:?-{3,}", block[1]):
continue
kept.extend(block)
continue
kept.append(line)
index += 1
return re.sub(r"\n{3,}", "\n\n", "\n".join(kept)).strip()
@classmethod
def _apply_quantitative_table_contract(cls, text: str, data_metadata: dict, display_name: str) -> str:
manifest = dict((data_metadata or {}).get("quantitative_table_manifest") or {})
tables = [item for item in (manifest.get("tables") or []) if isinstance(item, dict)]
if not tables:
return str(text or "")
value = cls._strip_model_markdown_tables(text)
assigned = []
for item in tables:
target = str(item.get("target_subsection") or "").strip()
if target and target.casefold() in str(display_name or "").casefold():
assigned.append(item)
if not assigned:
return value
blocks = []
for item in sorted(assigned, key=lambda row: int(row.get("number") or 0)):
number = int(item.get("number") or 0)
title = str(item.get("title") or "").strip()
markdown = str(item.get("markdown") or "").strip()
note = str(item.get("note") or "").strip()
if not number or not title or not markdown:
raise RuntimeError(f"quantitative_table_manifest_invalid:{display_name}")
block = f"Table {number}: {title}\n{markdown}"
if note:
block += f"\n\n*Note: {note}*"
blocks.append(block)
return (value.rstrip() + "\n\n" + "\n\n".join(blocks)).strip()
@staticmethod
def _snapshot_section_citation_state(agent, citation_plan: dict, unit_name: str) -> dict:
"""Snapshot only Section-owned citation state so a rejected attempt cannot leak forward."""
return {
"used_citations": set(getattr(agent, "used_citations", set()) or set()),
"citation_audit_log": copy.deepcopy(list(getattr(agent, "citation_audit_log", []) or [])),
"citation_map": copy.deepcopy(dict(getattr(agent, "citation_map", {}) or {})),
"planned_items": copy.deepcopy(list(dict(citation_plan or {}).get(unit_name, []) or [])),
}
@staticmethod
def _restore_section_citation_state(agent, citation_plan: dict, unit_name: str, snapshot: dict) -> None:
"""Restore citation ownership after an invalid Section attempt without rewinding API counters."""
snap = dict(snapshot or {})
lock = getattr(agent, "lock", None)
if lock is None:
agent.used_citations = set(snap.get("used_citations") or set())
agent.citation_audit_log = copy.deepcopy(list(snap.get("citation_audit_log") or []))
agent.citation_map = copy.deepcopy(dict(snap.get("citation_map") or {}))
else:
with lock:
agent.used_citations = set(snap.get("used_citations") or set())
agent.citation_audit_log = copy.deepcopy(list(snap.get("citation_audit_log") or []))
agent.citation_map = copy.deepcopy(dict(snap.get("citation_map") or {}))
citation_plan[unit_name] = copy.deepcopy(list(snap.get("planned_items") or []))
setattr(agent, "_last_citation_semantic_violations", [])
@classmethod
def _section_validation_bundle(
cls, mode: str, agent, quant_engine, text: str, data_metadata: dict, *,
section_id: str, section_name: str, evidence_contract: str = "", include_critic: bool = True,
citation_violations=None, hypothesis_ledger=None,
) -> list[dict]:
"""Collect typed S0-R1 validation feedback without mutating manuscript prose."""
value = str(text or "")
violations = []
if include_critic:
verdict = agent.critique_section(
value, section_id=section_id, section_name=section_name, mode=mode,
evidence_contract=evidence_contract,
)
if isinstance(verdict, dict) and verdict.get("pass") is False:
violations.append({
"type": "top_journal_critic",
"reason": str(verdict.get("critical_issue") or "")[:800],
"repair_instruction": str(verdict.get("repair_instruction") or "")[:1200],
})
violations.extend(list(agent.inspect_authorial_ai_meta(value) or []))
violations.extend(cls._study_fact_violations(
mode, agent, quant_engine, value, data_metadata, section_name=section_name
))
violations.extend(cls._author_facing_internal_workflow_violations(value))
if str(mode or "").lower() == "amj":
violations.extend(cls._hypothesis_contract_violations(
value, hypothesis_ledger or {}, section_name=section_name,
require_all=bool(re.search(r"Integrated Research Model", str(section_name or ""), flags=re.I)),
))
violations.extend(copy.deepcopy(list(citation_violations or [])))
return violations
def _checkpoint_section_retry_exhausted(
self, state: dict, *, section_id: str, section_name: str, violations, pct: int, attempts: int = 2
) -> None:
typed = copy.deepcopy(list(violations or []))
state["section_retry_exhausted"] = {
"section_id": str(section_id or ""),
"section_name": str(section_name or ""),
"attempts": max(1, int(attempts or 1)),
"violations": typed[:12],
"semantic_integrity_corruption": False,
}
self._checkpoint(
"section_retry_exhausted", state, pct,
display_node=f"section_retry_exhausted:{section_name}",
)
@staticmethod
def _section_integrity_hash(value: str) -> str:
exact = str(value or "").replace("\r\n", "\n").replace("\r", "\n").strip()
return hashlib.sha256(exact.encode("utf-8")).hexdigest()
@classmethod
def _build_section_integrity_seal(cls, section_key: str, section_title: str, section_text: str) -> dict:
return {
"version": ENGLISH_SECTION_SEAL_VERSION,
"section_key": str(section_key or ""),
"section_title": str(section_title or ""),
"sha256": cls._section_integrity_hash(section_text),
}
def _refresh_section_integrity_seals_after_delivery_repair(
self, manuscript: str, section_order, section_integrity_seals: dict,
) -> list[tuple[str, str, str]]:
"""Reseal only body sections changed by a deterministic P0 repair.
G1 permits a narrow deterministic hypothesis-identity repair after the
ordinary section seals have been created. Any changed section must be
resealed immediately so resume/checkpoint authority never points at stale
bytes. The returned section text also lets the caller invalidate/requeue
target translation from the exact repaired English source.
"""
changed: list[tuple[str, str, str]] = []
seals = section_integrity_seals
for section_key, section_title in list(section_order or []):
section_text = self._extract_top_level_section(manuscript, section_title)
if not section_text:
raise RuntimeError(f"delivery_first_reseal_source_missing:{section_title}")
current = dict(seals.get(section_key) or {})
updated = self._build_section_integrity_seal(section_key, section_title, section_text)
if str(current.get("sha256") or "") != str(updated.get("sha256") or ""):
seals[section_key] = updated
changed.append((str(section_key), str(section_title), section_text))
return changed
def _requeue_resealed_translation_sections(
self, *, changed_sections, mode: str, quant_engine, data_metadata: dict,
translation_agent, translation_sections: dict, english_commit_text: str,
) -> int:
"""Requeue translations whose frozen English source changed after repair."""
queued = 0
for section_key, section_title, section_text in list(changed_sections or []):
source_text = section_text
if (
str(mode or "").strip().casefold() == "amj"
and str(section_key).strip().casefold() == "results"
and dict(data_metadata or {}).get("quantitative_table_manifest")
):
source_text = quant_engine._strip_target_quantitative_markdown_tables(section_text)
if self._enqueue_translation_unit(
translation_agent,
unit_key=section_key,
unit_name=section_title,
source_text=source_text,
translation_sections=translation_sections,
english_commit_text=english_commit_text,
):
queued += 1
return queued
@staticmethod
def _replace_top_level_section(manuscript: str, title: str, replacement: str) -> str:
value = str(manuscript or "")
pattern = re.compile(
rf'(?ms)^#\s+{re.escape(str(title))}\s*$\n.*?(?=^#\s+[^#].*?$|\Z)'
)
matches = list(pattern.finditer(value))
if len(matches) != 1:
raise RuntimeError(f"top_level_section_replace_count_invalid:{title}:{len(matches)}")
match = matches[0]
rendered = str(replacement or "").strip()
return (value[:match.start()] + rendered + "\n\n" + value[match.end():].lstrip("\n")).strip() + "\n\n"
@staticmethod
def _remove_top_level_section(manuscript: str, title: str) -> str:
value = str(manuscript or "")
pattern = re.compile(rf'(?ms)^#\s+{re.escape(str(title))}\s*$\n.*?(?=^#\s+[^#].*?$|\Z)')
matches = list(pattern.finditer(value))
if len(matches) != 1:
raise RuntimeError(f"top_level_section_remove_count_invalid:{title}:{len(matches)}")
match = matches[0]
return (value[:match.start()] + value[match.end():].lstrip("\n")).strip() + "\n\n"
@classmethod
def _validate_section_integrity_seal(cls, seal: dict, section_text: str, section_title: str) -> None:
record = dict(seal or {})
if record.get("version") != ENGLISH_SECTION_SEAL_VERSION:
raise RuntimeError(f"english_section_integrity_seal_version_invalid:{section_title}")
expected = str(record.get("sha256") or "")
actual = cls._section_integrity_hash(section_text)
if not expected or expected != actual:
raise RuntimeError(f"english_section_integrity_seal_mismatch:{section_title}")
@staticmethod
def _apply_quantitative_front_matter_contract(mode: str, topic: str, keywords: str, data_metadata: dict):
"""Preserve original research seeds through retrieval and reasoning.
Reader-facing AMJ title/keywords are reconciled only after the empirical
scope and completed manuscript are frozen.
"""
return str(topic or "").strip(), str(keywords or "").strip()
@staticmethod
def _front_matter_language_projection_hash(topic: str, keywords) -> str:
terms = parse_keyword_terms(keywords, explicit_user=True)
payload = json.dumps(
{"title": str(topic or "").strip(), "keywords": terms},
ensure_ascii=False, sort_keys=True, separators=(",", ":"),
).encode("utf-8")
return hashlib.sha256(payload).hexdigest()
def _resolve_english_front_matter_projection(
self, agent, state: dict, data_metadata: dict, source_topic: str, source_keywords
) -> dict:
"""Resolve/cache a language-only English projection without mutating user intent."""
meta = dict(data_metadata or {})
contract = dict(meta.get("study_design_contract") or meta.get("front_matter_contract") or {})
intent = dict(contract.get("user_research_intent") or meta.get("user_research_intent") or {})
semantic_title = str(intent.get("source_topic") or source_topic or "").strip()
semantic_keywords = list(intent.get("source_keywords") or parse_keyword_terms(source_keywords, explicit_user=True))
source_hash = self._front_matter_language_projection_hash(semantic_title, semantic_keywords)
expected_count = len(semantic_keywords)
cached = dict(
state.get("front_matter_language_projection")
or meta.get("front_matter_language_projection")
or {}
)
if (
cached.get("version") == "front-matter-language-authority-v1"
and cached.get("source_hash") == source_hash
):
try:
validated = agent.validate_reader_facing_english_projection(
cached, expected_keyword_count=expected_count
)
return {
"version": "front-matter-language-authority-v1",
"source_hash": source_hash,
"english_title": validated["title"],
"english_keywords": list(validated["keywords"]),
"translation_applied": bool(cached.get("translation_applied")),
}
except Exception as exc:
logging.warning(
"Cached English front-matter language projection invalid; rebuilding | error=%s",
type(exc).__name__,
)
resolved = agent.normalize_reader_facing_front_matter_to_english(
semantic_title, semantic_keywords
)
validated = agent.validate_reader_facing_english_projection(
resolved, expected_keyword_count=expected_count
)
projection = {
"version": "front-matter-language-authority-v1",
"source_hash": source_hash,
"english_title": validated["title"],
"english_keywords": list(validated["keywords"]),
"translation_applied": bool(resolved.get("translation_applied")),
}
# Presentation projection is additive metadata only. Original source_topic
# and source_keywords inside user_research_intent remain byte-for-byte authority.
data_metadata["front_matter_language_projection"] = dict(projection)
state["front_matter_language_projection"] = dict(projection)
state["data_metadata"] = data_metadata
return projection
@staticmethod
def _resolved_front_matter_keyword_policy(data_metadata: dict):
"""Resolve one policy object for Working/Final/retry/resume/fallback paths."""
meta = dict(data_metadata or {})
contract = dict(meta.get("study_design_contract") or meta.get("front_matter_contract") or {})
if not contract and isinstance(meta.get("user_research_intent"), dict):
contract = {"user_research_intent": dict(meta.get("user_research_intent") or {})}
return keyword_policy_from_contract(contract)
@staticmethod
def _front_matter_keyword_fallback_terms(data_metadata: dict):
"""Derive recoverable English reader keywords from frozen study-design authority."""
meta = dict(data_metadata or {})
projection = dict(meta.get("front_matter_language_projection") or {})
projected = [str(item).strip() for item in (projection.get("english_keywords") or []) if str(item).strip()]
if projected:
return projected
contract = dict(meta.get("study_design_contract") or {})
values = []
quant = dict(contract.get("quantitative") or {})
for row in quant.get("constructs") or []:
if isinstance(row, dict):
label = re.sub(r"\s+", " ", str(row.get("label") or "")).strip()
if label:
values.append(label)
qual = dict(contract.get("qualitative") or {})
if qual:
for candidate in (
qual.get("phenomenon"),
qual.get("design_tradition"),
):
value = re.sub(r"\s+", " ", str(candidate or "")).strip()
if value:
values.append(value)
for theme in meta.get("themes") or []:
if isinstance(theme, dict):
value = re.sub(r"\s+", " ", str(theme.get("title") or "")).strip()
if value:
values.append(value)
return parse_keyword_terms(values, explicit_user=False)
@staticmethod
def _user_authoritative_front_matter_keywords(data_metadata: dict):
"""Return explicit user keywords only when the resolved policy says USER_EXPLICIT."""
meta = dict(data_metadata or {})
contract = dict(meta.get("study_design_contract") or meta.get("front_matter_contract") or {})
intent = dict(contract.get("user_research_intent") or meta.get("user_research_intent") or {})
policy = keyword_policy_from_contract(contract)
if str(policy.get("authority_source") or "").upper() != "USER_EXPLICIT":
return []
projection = dict(meta.get("front_matter_language_projection") or {})
authoritative = list(projection.get("english_keywords") or intent.get("source_keywords") or [])
return apply_keyword_policy(
authoritative,
policy,
authoritative=authoritative,
)
@staticmethod
def _canonical_quantitative_front_matter_keywords(mode: str, candidate_keywords, data_metadata: dict):
"""Apply one keyword policy across quantitative/qualitative and all product tiers."""
meta = dict(data_metadata or {})
contract = dict(meta.get("study_design_contract") or meta.get("front_matter_contract") or {})
intent = dict(contract.get("user_research_intent") or meta.get("user_research_intent") or {})
policy = keyword_policy_from_contract(contract)
projection = dict(meta.get("front_matter_language_projection") or {})
authoritative = list(projection.get("english_keywords") or intent.get("source_keywords") or [])
values = apply_keyword_policy(
candidate_keywords,
policy,
authoritative=authoritative if str(policy.get("authority_source") or "").upper() == "USER_EXPLICIT" else None,
)
return values
@staticmethod
def _replace_title(manuscript: str, title: str) -> str:
value = str(manuscript or "")
rendered = re.sub(r"\s+", " ", str(title or "")).strip()
if not rendered:
return value
title_replacement = f"# {rendered}"
updated, count = re.subn(
r"(?m)^#\s+.+$",
lambda _match, replacement=title_replacement: replacement,
value,
count=1,
)
if count == 1:
return updated
logging.warning("Manuscript title heading missing during final reconciliation; reconstructing canonical H1 instead of failing generation.")
return f"# {rendered}\n\n" + value.lstrip("\n")
@staticmethod
def _replace_keywords(manuscript: str, keywords) -> str:
value = str(manuscript or '')
if isinstance(keywords, (list, tuple)):
rendered = ', '.join(str(item).strip() for item in keywords if str(item).strip())
else:
rendered = str(keywords or '').strip()
pattern = re.compile(r'(?ms)(^##\s+Keywords\s*$\n)(.*?)(?=^#\s+[^#].*?$|\Z)')
matches = list(pattern.finditer(value))
if len(matches) == 1:
match = matches[0]
return value[:match.start()] + match.group(1) + rendered + '\n\n' + value[match.end():].lstrip('\n')
logging.warning("Keywords heading missing or ambiguous during final reconciliation; rebuilding front-matter keyword block without aborting generation | count=%s", len(matches))
abstract_pattern = re.compile(r'(?ms)(^##\s+Abstract\s*$\n.*?)(?=^#\s+[^#].*?$|\Z)')
abstract_match = abstract_pattern.search(value)
block = f"## Keywords\n{rendered}\n\n"
if abstract_match:
return value[:abstract_match.end()].rstrip() + "\n\n" + block + value[abstract_match.end():].lstrip("\n")
title_match = re.search(r'(?m)^#\s+.+$', value)
if title_match:
return value[:title_match.end()] + "\n\n" + block + value[title_match.end():].lstrip("\n")
return block + value.lstrip("\n")
@staticmethod
def _target_front_matter_language_code(target_lang: str) -> str:
value = str(target_lang or '').strip().casefold().replace('_', '-')
if value.startswith('zh') or 'chinese' in value or '中文' in value or '简体中文' in value:
return 'zh'
if value.startswith('ru') or 'russian' in value or 'рус' in value:
return 'ru'
if value.startswith('kk') or value.startswith('kz') or 'kazakh' in value or 'қазақ' in value:
return 'kk'
if value.startswith('en') or 'english' in value:
return 'en'
return value
@classmethod
def _target_keyword_heading_aliases(cls, target_lang: str) -> list[str]:
code = cls._target_front_matter_language_code(target_lang)
if code == 'zh':
return ['关键词', '关键字']
if code == 'ru':
return ['Ключевые слова']
if code == 'kk':
return ['Түйін сөздер', 'Кілт сөздер']
return ['Keywords']
@classmethod
def _rebind_target_user_intent_front_matter(cls, manuscript: str, target_lang: str, data_metadata: dict, glossary: dict):
"""Preserve user keyword identity and explicit relation type in target front matter."""
value = str(manuscript or '')
meta = dict(data_metadata or {})
contract = dict(meta.get('study_design_contract') or {})
intent = dict(contract.get('user_research_intent') or meta.get('user_research_intent') or {})
projection = dict(meta.get('front_matter_language_projection') or {})
source_keywords = list(
projection.get('english_keywords')
or intent.get('source_keywords')
or contract.get('source_keywords')
or []
)
mapping = dict(glossary or {})
diagnostics = []
if source_keywords:
target_keywords = [re.sub(r'\s+', ' ', str(mapping.get(src) or '')).strip() for src in source_keywords]
if all(target_keywords):
aliases = cls._target_keyword_heading_aliases(target_lang)
heading_re = re.compile(r'(?mi)^##\s+(?:' + '|'.join(re.escape(x) for x in aliases) + r')\s*$')
match = heading_re.search(value)
if match:
next_heading = re.search(r'(?m)^#{1,3}\s+', value[match.end():])
end = match.end() + (next_heading.start() if next_heading else len(value[match.end():]))
rendered = ', '.join(target_keywords)
value = value[:match.end()] + '\n' + rendered + '\n\n' + value[end:].lstrip('\n')
diagnostics.append({'type':'target_user_keywords_rebound','count':len(target_keywords)})
explicit = set(intent.get('explicit_relation_types') or [])
if explicit:
title_match = re.search(r'(?m)^#\s+(.+?)\s*$', value)
if title_match:
title = title_match.group(1).strip()
target_intent = build_user_research_intent(title, '', mode=str(contract.get('mode') or ''))
missing = explicit - set(target_intent.get('explicit_relation_types') or [])
if missing:
quant = dict(contract.get('quantitative') or {})
construct_rows = [row for row in (quant.get('constructs') or []) if isinstance(row, dict)]
role_label = {}
for role in ('mediator', 'moderator'):
values = [str(row.get('label') or '') for row in construct_rows if role in (row.get('roles') or []) and row.get('label')]
if len(values) == 1:
role_label[role] = re.sub(r'\s+', ' ', str(mapping.get(values[0]) or values[0])).strip()
code = cls._target_front_matter_language_code(target_lang)
suffixes = []
if 'moderation' in missing:
label = role_label.get('moderator') or ''
if code.startswith('zh'):
suffixes.append(f'{label}的调节作用' if label else '调节作用')
elif code.startswith('ru'):
suffixes.append(f'модерирующая роль переменной «{label}»' if label else 'модерирующая роль')
elif code.startswith('kk') or code.startswith('kz'):
suffixes.append(f'«{label}» айнымалысының модераторлық рөлі' if label else 'модераторлық рөл')
else:
suffixes.append(f'the moderating role of {label}' if label else 'moderation')
if missing.intersection({'mediation', 'serial_mediation', 'parallel_mediation'}):
label = role_label.get('mediator') or ''
if code.startswith('zh'):
suffixes.append(f'{label}的中介作用' if label else '中介作用')
elif code.startswith('ru'):
suffixes.append(f'медиирующая роль переменной «{label}»' if label else 'медиирующая роль')
elif code.startswith('kk') or code.startswith('kz'):
suffixes.append(f'«{label}» айнымалысының медиаторлық рөлі' if label else 'медиаторлық рөл')
else:
suffixes.append(f'the mediating role of {label}' if label else 'mediation')
if suffixes:
separator = '：' if code.startswith('zh') else ': '
joiner = '；' if code.startswith('zh') else '; '
repaired_title = title.rstrip(' .;:：') + separator + joiner.join(suffixes)
value = value[:title_match.start(1)] + repaired_title + value[title_match.end(1):]
diagnostics.append({'type':'target_user_relation_title_rebound','missing':sorted(missing)})
return value, diagnostics
@staticmethod
def _section_is_complete(tasks, completed_display_names, section_key: str) -> bool:
relevant = [
str(task.get('display_name') or '') for task in (tasks or [])
if str(task.get('section_id') or task.get('section') or '') == str(section_key)
]
done = set(str(item) for item in (completed_display_names or []))
return bool(relevant) and all(name in done for name in relevant)
@staticmethod
def _glossary_version(glossary) -> str:
encoded = json.dumps(dict(glossary or {}), ensure_ascii=False, sort_keys=True).encode('utf-8')
return hashlib.sha256(encoded).hexdigest()[:16]
@staticmethod
def _build_canonical_concept_ledger(
mode: str, data_metadata: dict, *, keywords='', adaptive_structure=None, research_logic_contract=''
) -> dict:
"""Freeze task-level research concept identity before body translation.
This ledger is intentionally conservative: quantitative construct names
come directly from frozen metadata; qualitative concept/theme labels come
from the Evidence Ledger/outline. Lower-case hyphenated concepts used in
the frozen research logic are added because the legacy Title-Case glossary
discovery cannot see them. No semantic synonym merging is performed.
"""
items = []
index_by_key = {}
def add(value, kind, policy='FLEXIBLE_TERM'):
term = re.sub(r'\s+', ' ', str(value or '')).strip(' \t\r\n,;:.')
if not term or len(term) < 3 or len(term) > 100:
return
words = re.findall(r"[A-Za-zÀ-ÖØ-öø-ÿА-Яа-яЁёӘәҒғҚқҢңӨөҰұҮүҺһІі][\w’'\-]*", term, flags=re.UNICODE)
if not words or len(words) > 10:
return
key = re.sub(r'[–—-]+', '-', term).casefold()
normalized_policy = 'STRICT_CONSTRUCT' if str(policy or '').upper() == 'STRICT_CONSTRUCT' else 'FLEXIBLE_TERM'
if key in index_by_key:
row = items[index_by_key[key]]
if normalized_policy == 'STRICT_CONSTRUCT' and row.get('policy') != 'STRICT_CONSTRUCT':
row['policy'] = 'STRICT_CONSTRUCT'
row['kind'] = str(kind or row.get('kind') or 'concept')
return
index_by_key[key] = len(items)
items.append({'source': term, 'kind': str(kind or 'concept'), 'policy': normalized_policy})
meta = dict(data_metadata or {})
contract = dict(meta.get('study_design_contract') or {})
user_intent = dict(contract.get('user_research_intent') or meta.get('user_research_intent') or {})
if str(mode or '') == 'amj':
construct_items = [row for row in (meta.get('construct_items') or []) if isinstance(row, dict)]
if construct_items:
for row in construct_items:
roles = list(row.get('roles') or [])
role_label = '_'.join(str(role) for role in roles if str(role)) or 'construct'
add(row.get('label'), f'quantitative_{role_label}', 'STRICT_CONSTRUCT')
else:
constructs = meta.get('constructs') if isinstance(meta.get('constructs'), dict) else {}
for role in ('predictor', 'mediator', 'moderator', 'outcome', 'control', 'covariate'):
add(constructs.get(role), f'quantitative_{role}', 'STRICT_CONSTRUCT')
else:
for theme in (meta.get('themes') or []):
if not isinstance(theme, dict):
continue
add(theme.get('title'), 'qualitative_theme', 'STRICT_CONSTRUCT')
for concept in (theme.get('concepts') or []):
add(concept, 'qualitative_concept', 'STRICT_CONSTRUCT')
for row in (meta.get('evidence') or []):
if isinstance(row, dict):
add(row.get('concept'), 'qualitative_concept', 'STRICT_CONSTRUCT')
for task in (adaptive_structure or []):
if not isinstance(task, dict):
continue
for concept in (task.get('concepts') or []):
add(concept, 'outline_concept', 'FLEXIBLE_TERM')
# R7-R2C-R5: user keywords are research-intent identities, not expendable
# planning seeds. Freeze them for multilingual inheritance while still
# distinguishing them from empirical construct roles.
projection = dict(meta.get('front_matter_language_projection') or {})
intent_keywords = list(projection.get('english_keywords') or user_intent.get('source_keywords') or [])
if not intent_keywords:
intent_keywords = parse_keyword_terms(keywords, explicit_user=True)
for seed in intent_keywords:
if 1 <= len(str(seed).split()) <= 10:
add(seed, 'user_intent_keyword', 'STRICT_CONSTRUCT')
# Capture explicit lower-case hyphenated academic expressions only when
# they recur or carry a research-concept cue. This closes the
# context-awareness/context-aware blind spot without freezing generic
# methodological adjectives such as cross-sectional by default.
hyphenated = re.findall(r'(?= 2 or parts.intersection(concept_cues):
add(term, 'logic_hyphenated_concept', 'FLEXIBLE_TERM')
return {
'version': 'concept-ledger-v2',
'items': items,
'explicit_relation_types': list(user_intent.get('explicit_relation_types') or []),
'source_topic': str(user_intent.get('source_topic') or contract.get('source_topic') or ''),
'source_keywords': list(user_intent.get('source_keywords') or contract.get('source_keywords') or []),
}
@staticmethod
def _canonical_concept_writer_directive(ledger: dict) -> str:
terms = [str(item.get('source') or '').strip() for item in (dict(ledger or {}).get('items') or []) if isinstance(item, dict)]
terms = [term for term in terms if term]
if not terms:
return ''
rendered = '; '.join(terms[:40])
relation_types = [str(item).strip() for item in (dict(ledger or {}).get('explicit_relation_types') or []) if str(item).strip()]
relation_directive = ''
if relation_types:
relation_directive = (
' USER RESEARCH INTENT: Preserve these explicitly requested study relations throughout drafting and target-language translation: '
+ ', '.join(relation_types)
+ '. Do not downgrade a mediator/moderator or other explicit relation into a generic contextual term merely because direct prior-literature support is sparse. Prior-literature claims still require real citation evidence; novel study logic may be stated cautiously as author inference without inventing a source.'
)
return (
'CANONICAL CONCEPT IDENTITY: The following research labels are frozen task-level concepts: '
+ rendered
+ '. When referring to the same construct or concept, preserve its canonical English label; '
'do not silently replace it with a near-synonym that changes concept identity. Ordinary grammatical '
'articles, possessives, and surrounding syntax remain free. Distinct listed labels are not synonyms unless the evidence explicitly says so.'
+ relation_directive
)
@staticmethod
def _translation_ledger_hash(agent, source: str) -> str:
method = getattr(agent, 'translation_fact_ledger_hash', None)
if callable(method):
return str(method(source) or '')
# Test doubles / legacy adapters still get deterministic invalidation.
return hashlib.sha256(str(source or '').encode('utf-8')).hexdigest()
def _translation_identity_bundle(self, agent, source: str) -> dict[str, str]:
"""Return one canonical identity domain for translation reuse.
The frozen English source remains the manuscript authority, while the
Translation Compiler may consume a canonicalized representation (for
example qualitative quote literals become QUOTE semantic slots). Cache
identity must therefore persist both hashes and derive the private Fact
Ledger only from the exact compiled source sent to the translator.
"""
raw_source = str(source or '').strip()
compiled_source, _pending, _reused = self._prepare_qualitative_translation_source(
raw_source, getattr(self, '_qualitative_quote_sources', {}), {}
)
compiled_source = str(compiled_source or '').strip()
return {
'source_hash': self._text_hash(raw_source),
'compiled_source': compiled_source,
'compiled_source_hash': self._text_hash(compiled_source),
'fact_ledger_hash': self._translation_ledger_hash(agent, compiled_source),
}
@staticmethod
def _assemble_partial_translation(translation_sections, adaptive_structure, *, include_front=False) -> str:
mapping = translation_sections if isinstance(translation_sections, dict) else {}
parts = []
if include_front:
record = mapping.get('__front_matter__') or {}
if record.get('qa_status') in {'passed', 'passed_with_warning'} and record.get('translated_text'):
parts.append(str(record['translated_text']).strip())
for key, _title in GenerationRunner._unique_top_level_sections(adaptive_structure):
record = mapping.get(key) or {}
if record.get('qa_status') in {'passed', 'passed_with_warning'} and record.get('translated_text'):
parts.append(str(record['translated_text']).strip())
return '\n\n'.join(part for part in parts if part).strip()
def _start_translation_worker(
self, agent, *, translation_sections, translation_glossary,
adaptive_structure, target_lang,
) -> None:
if self._translation_worker is not None:
return
self.translation_agent = agent
self._translation_worker_sections = dict(translation_sections or {})
self._translation_worker_glossary = dict(translation_glossary or {})
self._translation_worker_context = {
'adaptive_structure': list(adaptive_structure or []),
'target_lang': str(target_lang or ''),
}
logical_keys = ['__front_matter__'] + [
key for key, _title in self._unique_top_level_sections(adaptive_structure)
]
logical_keys = list(dict.fromkeys(key for key in logical_keys if key))
self._translation_progress_total = len(logical_keys)
self._translation_progress_sections = dict(translation_sections or {})
self._translation_progress_queued = {
key for key in logical_keys if key in self._translation_progress_sections
}
self._translation_progress_inflight = set()
self._translation_worker = _OrderedTranslationWorker(self._run_translation_job)
self._write_preview_payload(force=True)
def _run_translation_job(self, job: dict[str, Any]) -> dict[str, Any]:
if self.translation_agent is None:
raise RuntimeError('translation_worker_agent_missing')
context = dict(self._translation_worker_context or {})
sections, glossary = self._translate_frozen_unit(
self.translation_agent,
unit_key=str(job.get('unit_key') or ''),
unit_name=str(job.get('unit_name') or ''),
source_text=str(job.get('source_text') or ''),
target_lang=str(context.get('target_lang') or ''),
translation_sections=self._translation_worker_sections,
translation_glossary=self._translation_worker_glossary,
adaptive_structure=context.get('adaptive_structure') or [],
include_front=True,
defer_failures=True,
english_commit_text=str(job.get('english_commit_text') or ''),
)
self._translation_worker_sections = dict(sections or {})
self._translation_worker_glossary = dict(glossary or {})
return {
'unit_key': str(job.get('unit_key') or ''),
'source_hash': str(job.get('source_hash') or ''),
'translation_sections': dict(self._translation_worker_sections),
'translation_glossary': dict(self._translation_worker_glossary),
}
def _enqueue_translation_unit(
self, agent, *, unit_key: str, unit_name: str, source_text: str,
translation_sections: dict, english_commit_text: str,
) -> bool:
source = str(source_text or '').strip()
if not source or self._translation_worker is None:
return False
identity = self._translation_identity_bundle(agent, source)
source_hash = identity['source_hash']
compiled_source_hash = identity['compiled_source_hash']
fact_ledger_hash = identity['fact_ledger_hash']
entity_ledger_method = getattr(agent, 'translation_protected_entity_ledger_hash', None)
protected_entity_ledger_hash = str(
entity_ledger_method(identity['compiled_source'], self._translation_worker_glossary)
if callable(entity_ledger_method) else ''
)
protected_entity_ledger_version = str(
getattr(agent, 'PROTECTED_ENTITY_LEDGER_VERSION', 'protected-entity-ledger-v1')
or 'protected-entity-ledger-v1'
)
existing = dict((translation_sections or {}).get(unit_key) or {})
if not isinstance(getattr(self, '_translation_progress_queued', None), set):
self._translation_progress_queued = set()
if not isinstance(getattr(self, '_translation_progress_inflight', None), set):
self._translation_progress_inflight = set()
# Do not replace the worker-owned display snapshot with a possibly stale
# main-thread copy here; collection or the worker completion path updates
# _translation_progress_sections monotonically.
self._translation_progress_queued.add(str(unit_key))
if (
existing.get('qa_status') in {'passed', 'passed_with_warning'}
and existing.get('source_hash') == source_hash
and existing.get('compiled_source_hash') == compiled_source_hash
and existing.get('target_lang') == str(self._translation_worker_context.get('target_lang') or '')
and existing.get('translation_contract_version') == TRANSLATION_CONTRACT_VERSION
and existing.get('fact_ledger_hash') == fact_ledger_hash
and existing.get('protected_entity_ledger_version') == protected_entity_ledger_version
and existing.get('protected_entity_ledger_hash') == protected_entity_ledger_hash
and existing.get('translated_text')
):
return False
if existing and existing.get('translated_text'):
reasons = []
if existing.get('source_hash') != source_hash: reasons.append('english_source_hash')
if existing.get('compiled_source_hash') != compiled_source_hash: reasons.append('compiled_source_hash')
if existing.get('fact_ledger_hash') != fact_ledger_hash: reasons.append('fact_ledger_hash')
if existing.get('protected_entity_ledger_version') != protected_entity_ledger_version: reasons.append('protected_entity_ledger_version')
if existing.get('protected_entity_ledger_hash') != protected_entity_ledger_hash: reasons.append('protected_entity_ledger_hash')
if existing.get('translation_contract_version') != TRANSLATION_CONTRACT_VERSION: reasons.append('contract_version')
if existing.get('target_lang') != str(self._translation_worker_context.get('target_lang') or ''): reasons.append('target_lang')
if existing.get('qa_status') not in {'passed', 'passed_with_warning'}: reasons.append('qa_status')
logging.info('Translation reuse miss | unit=%s phase=enqueue reasons=%s', unit_name, ','.join(reasons) or 'unknown')
pending_key = (str(unit_key), source_hash, compiled_source_hash, fact_ledger_hash, protected_entity_ledger_version, protected_entity_ledger_hash)
if pending_key in self._translation_worker_pending:
return False
self._translation_progress_inflight.add(str(unit_key))
future = self._translation_worker.submit({
'unit_key': str(unit_key),
'unit_name': str(unit_name),
'source_text': source,
'source_hash': source_hash,
'compiled_source_hash': compiled_source_hash,
'fact_ledger_hash': fact_ledger_hash,
'english_commit_text': str(english_commit_text or ''),
})
self._translation_worker_pending.add(pending_key)
self._translation_worker_jobs.append({
'pending_key': pending_key,
'future': future,
})
self._write_preview_payload(force=True)
return True
def _collect_translation_results(self, translation_sections, translation_glossary, *, block=False):
sections = dict(translation_sections or {})
glossary = dict(translation_glossary or {})
while self._translation_worker_collected < len(self._translation_worker_jobs):
item = self._translation_worker_jobs[self._translation_worker_collected]
future = item['future']
if not block and not future.done():
break
result = future.result()
sections = dict(result.get('translation_sections') or sections)
glossary = dict(result.get('translation_glossary') or glossary)
self._translation_progress_sections = dict(sections)
self._translation_worker_pending.discard(item['pending_key'])
self._translation_worker_collected += 1
return sections, glossary
def _shutdown_translation_worker(self, *, wait=False) -> None:
worker = self._translation_worker
if worker is None:
return
worker.close(wait=bool(wait))
self._translation_worker = None
def _translate_frozen_unit(
self, agent, *, unit_key: str, unit_name: str, source_text: str, target_lang: str,
translation_sections: dict, translation_glossary: dict, adaptive_structure, include_front=False,
defer_failures=True, english_commit_text: str | None = None, final_gate: bool = False,
):
source = str(source_text or '').strip()
if not source:
return translation_sections, translation_glossary
identity = self._translation_identity_bundle(agent, source)
source_hash = identity['source_hash']
compiled_source_hash = identity['compiled_source_hash']
canonical_translation_source = identity['compiled_source']
contract_version = TRANSLATION_CONTRACT_VERSION
fact_ledger_hash = identity['fact_ledger_hash']
entity_ledger_method = getattr(agent, 'translation_protected_entity_ledger_hash', None)
protected_entity_ledger_hash = str(
entity_ledger_method(canonical_translation_source, translation_glossary)
if callable(entity_ledger_method) else ''
)
protected_entity_ledger_version = str(
getattr(agent, 'PROTECTED_ENTITY_LEDGER_VERSION', 'protected-entity-ledger-v1') or 'protected-entity-ledger-v1'
)
existing = dict((translation_sections or {}).get(unit_key) or {})
if not isinstance(getattr(self, '_translation_progress_queued', None), set):
self._translation_progress_queued = set()
if not isinstance(getattr(self, '_translation_progress_inflight', None), set):
self._translation_progress_inflight = set()
self._translation_progress_sections = dict(translation_sections or {})
self._translation_progress_queued.add(str(unit_key))
if (
existing.get('qa_status') in {'passed', 'passed_with_warning'}
and existing.get('source_hash') == source_hash
and existing.get('compiled_source_hash') == compiled_source_hash
and existing.get('target_lang') == target_lang
and existing.get('translation_contract_version') == contract_version
and existing.get('fact_ledger_hash') == fact_ledger_hash
and existing.get('protected_entity_ledger_version') == protected_entity_ledger_version
and existing.get('protected_entity_ledger_hash') == protected_entity_ledger_hash
and existing.get('translated_text')
):
self._translation_progress_inflight.discard(str(unit_key))
return translation_sections, translation_glossary
if existing and existing.get('translated_text'):
reasons = []
if existing.get('source_hash') != source_hash: reasons.append('english_source_hash')
if existing.get('compiled_source_hash') != compiled_source_hash: reasons.append('compiled_source_hash')
if existing.get('fact_ledger_hash') != fact_ledger_hash: reasons.append('fact_ledger_hash')
if existing.get('protected_entity_ledger_version') != protected_entity_ledger_version: reasons.append('protected_entity_ledger_version')
if existing.get('protected_entity_ledger_hash') != protected_entity_ledger_hash: reasons.append('protected_entity_ledger_hash')
if existing.get('translation_contract_version') != contract_version: reasons.append('contract_version')
if existing.get('target_lang') != target_lang: reasons.append('target_lang')
if existing.get('qa_status') not in {'passed', 'passed_with_warning'}: reasons.append('qa_status')
logging.info('Translation reuse miss | unit=%s phase=final_or_direct reasons=%s', unit_name, ','.join(reasons) or 'unknown')
self._translation_progress_inflight.add(str(unit_key))
partial = self._assemble_partial_translation(translation_sections, adaptive_structure, include_front=include_front)
initial_target_stage = f'translation_final_gate:{unit_name}' if final_gate else f'translating:{unit_name}'
self._publish_target_preview(
partial, initial_target_stage, force=True, complete=False,
english_commit_text=english_commit_text,
)
updated_sections = dict(translation_sections or {})
raw_quote_ledger_record = updated_sections.get("__quote_ledger__")
if raw_quote_ledger_record is not None and not isinstance(raw_quote_ledger_record, dict):
logging.warning(
"Qualitative quote translation ledger ignored malformed record type=%s",
type(raw_quote_ledger_record).__name__,
)
quote_ledger_record = (
dict(raw_quote_ledger_record) if isinstance(raw_quote_ledger_record, dict) else {}
)
current_quote_hashes = {
tag: hashlib.sha256(re.sub(r"\s+", " ", quote).strip().encode("utf-8")).hexdigest()
for tag, quote in getattr(self, "_qualitative_quote_sources", {}).items()
}
current_request_hash = str(getattr(self, "request_hash", "") or "")
current_task_id = str(getattr(getattr(self, "task_dir", None), "name", "") or "")
ledger_identity_ok = self._qualitative_quote_ledger_identity_ok(
quote_ledger_record,
task_id=current_task_id,
target_lang=target_lang,
request_hash=current_request_hash,
translation_contract_version=TRANSLATION_CONTRACT_VERSION,
source_quote_hashes=current_quote_hashes,
)
stored_translations = quote_ledger_record.get("translations")
quote_ledger = (
dict(stored_translations)
if ledger_identity_ok and isinstance(stored_translations, dict)
else {}
)
translation_source, pending_quote_tags, reused_quote_tags = self._prepare_qualitative_translation_source(
source, getattr(self, "_qualitative_quote_sources", {}), quote_ledger
)
translation_source = str(translation_source or '').strip()
if translation_source != canonical_translation_source:
raise RuntimeError('Canonical translation source changed with Quote Ledger state.')
def publish_translation_stage(stage):
value = str(stage or '').strip().casefold()
if value == 'qa':
preview_stage = f'translation_qa:{unit_name}'
elif value == 'repairing':
preview_stage = f'translation_repairing:{unit_name}'
elif value == 'adjudicating':
preview_stage = f'translation_adjudicating:{unit_name}'
elif final_gate:
preview_stage = f'translation_final_gate:{unit_name}'
else:
preview_stage = f'translating:{unit_name}'
self._publish_target_preview(
partial, preview_stage, force=True, complete=False,
english_commit_text=english_commit_text,
)
try:
if pending_quote_tags:
quote_translate_method = getattr(agent, 'translate_qualitative_quote_objects', None)
if not callable(quote_translate_method):
raise RuntimeError('Typed qualitative quote translation entry point is unavailable.')
quote_objects = {
tag: getattr(self, '_qualitative_quote_sources', {}).get(tag, '')
for tag in pending_quote_tags
}
quote_result = quote_translate_method(
quote_objects,
target_lang,
glossary=translation_glossary,
max_attempts=2,
stage_callback=publish_translation_stage,
)
returned_quotes = (quote_result or {}).get('translations')
if not isinstance(returned_quotes, dict) or set(returned_quotes) != set(pending_quote_tags):
raise RuntimeError(
'Typed qualitative quote translation identity mismatch: '
f'expected={sorted(pending_quote_tags)} got={sorted(returned_quotes) if isinstance(returned_quotes, dict) else []}'
)
for tag in pending_quote_tags:
rendered_quote = str(returned_quotes.get(tag) or '').strip()
if not rendered_quote:
raise RuntimeError(f'Typed qualitative quote translation is empty: {tag}')
quote_ledger[tag] = rendered_quote
returned_glossary = (quote_result or {}).get('glossary')
if isinstance(returned_glossary, dict):
translation_glossary = dict(returned_glossary)
updated_sections['__quote_ledger__'] = {
'task_id': getattr(getattr(self, "task_dir", None), "name", ""),
'request_hash': current_request_hash,
'writer_execution_epoch': int(getattr(self, "execution_epoch", 0) or 0),
'target_lang': target_lang,
'translation_contract_version': TRANSLATION_CONTRACT_VERSION,
'source_quote_hashes': current_quote_hashes,
'translations': dict(quote_ledger),
'updated_at': datetime.now(timezone.utc).isoformat(),
}
translate_method = agent.translate_frozen_section
call_kwargs = {
'glossary': translation_glossary, 'section_name': unit_name, 'max_attempts': 2,
}
try:
parameters = list(inspect.signature(translate_method).parameters.values())
supports_stage_callback = any(
item.name == 'stage_callback' or item.kind == inspect.Parameter.VAR_KEYWORD
for item in parameters
)
supports_claim_states = any(
item.name == 'claim_states' or item.kind == inspect.Parameter.VAR_KEYWORD
for item in parameters
)
except (TypeError, ValueError):
supports_stage_callback = False
supports_claim_states = False
if supports_stage_callback:
call_kwargs['stage_callback'] = publish_translation_stage
if supports_claim_states and isinstance(existing.get('claim_states'), dict):
call_kwargs['claim_states'] = dict(existing.get('claim_states') or {})
result = translate_method(translation_source, target_lang, **call_kwargs)
translated_text = str(result.get('text') or '').strip()
if pending_quote_tags or reused_quote_tags:
translated_text, quote_ledger = self._finalize_qualitative_translation_quotes(
translated_text, pending_quote_tags, reused_quote_tags, quote_ledger
)
agent.validate_translation_alignment(source, translated_text)
result = dict(result)
result['text'] = translated_text
updated_sections['__quote_ledger__'] = {
'task_id': getattr(getattr(self, "task_dir", None), "name", ""),
'request_hash': current_request_hash,
'writer_execution_epoch': int(getattr(self, "execution_epoch", 0) or 0),
'target_lang': target_lang,
'translation_contract_version': TRANSLATION_CONTRACT_VERSION,
'source_quote_hashes': current_quote_hashes,
'translations': dict(quote_ledger),
'updated_at': datetime.now(timezone.utc).isoformat(),
}
except PipelineCancelledError:
raise
except Exception as error:
# Persist repair state for both intermediate and strict final gates.
# The caller owns checkpoint authority and decides whether to stop.
code_method = getattr(agent, 'translation_error_code', None)
error_code = str(code_method(error) if callable(code_method) else 'TRANSLATION_PENDING')
updated_sections[unit_key] = {
'source_hash': source_hash,
'compiled_source_hash': compiled_source_hash,
'target_lang': target_lang,
'glossary_version': self._glossary_version(translation_glossary),
'translation_contract_version': contract_version,
'fact_ledger_hash': fact_ledger_hash,
'protected_entity_ledger_version': protected_entity_ledger_version,
'protected_entity_ledger_hash': protected_entity_ledger_hash,
'translated_text': '',
'qa_status': 'repair_pending',
'claim_states': dict(getattr(agent, '_last_translation_claim_states', {}) or {}),
'claim_state_version': 'translation-claim-state-v2',
'error_code': error_code,
'attempts': int(existing.get('attempts') or 0) + 1,
'deferred_at': datetime.now(timezone.utc).isoformat(),
}
logging.warning(
'Translation repair pending | unit=%s final_gate=%s code=%s error=%s',
unit_name, bool(final_gate), error_code, type(error).__name__,
)
self._translation_progress_sections = dict(updated_sections)
self._translation_progress_inflight.discard(str(unit_key))
self._publish_target_preview(
partial, f'translation_repair_pending:{unit_name}', force=True, complete=False,
english_commit_text=english_commit_text,
)
return updated_sections, translation_glossary
updated_glossary = dict(result.get('glossary') or translation_glossary or {})
result_fact_ledger_hash = str(result.get('fact_ledger_hash') or fact_ledger_hash)
if result_fact_ledger_hash != fact_ledger_hash:
raise RuntimeError('Translation result Fact Ledger identity differs from canonical compiled source.')
updated_sections[unit_key] = {
'source_hash': source_hash,
'compiled_source_hash': compiled_source_hash,
'target_lang': target_lang,
'glossary_version': self._glossary_version(updated_glossary),
'translation_contract_version': str(result.get('translation_contract_version') or contract_version),
'fact_ledger_hash': result_fact_ledger_hash,
'protected_entity_ledger_version': str(result.get('protected_entity_ledger_version') or protected_entity_ledger_version),
'protected_entity_ledger_hash': str(result.get('protected_entity_ledger_hash') or protected_entity_ledger_hash),
'translated_text': str(result.get('text') or '').strip(),
'qa_status': str(result.get('qa_status') or 'passed'),
'language_adapter': str(result.get('language_adapter') or ''),
'diagnostics': list(result.get('diagnostics') or [])[-20:],
'claim_states': dict(result.get('claim_states') or {}),
'claim_state_version': str(result.get('claim_state_version') or 'translation-claim-state-v2'),
'attempts': int(result.get('attempts') or 1),
'completed_at': datetime.now(timezone.utc).isoformat(),
}
partial = self._assemble_partial_translation(updated_sections, adaptive_structure, include_front=include_front)
self._translation_progress_sections = dict(updated_sections)
self._translation_progress_inflight.discard(str(unit_key))
self._publish_target_preview(
partial, f'translation_ready:{unit_name}', force=True, complete=False,
english_commit_text=english_commit_text,
)
return updated_sections, updated_glossary
@staticmethod
def _adaptive_recovery_language_code(target_lang: str) -> str:
value = str(target_lang or '').strip().casefold().replace('_', '-')
if value.startswith('zh') or 'chinese' in value or '中文' in value or '简体' in value:
return 'zh'
if value.startswith('ru') or 'russian' in value or 'рус' in value:
return 'ru'
if value.startswith('kk') or value.startswith('kz') or 'kazakh' in value or 'қазақ' in value:
return 'kk'
return 'other'
def _validate_adaptive_recovery_text(
self, agent, *, source_text: str, translated_text: str, target_lang: str,
section_name: str, glossary=None,
) -> None:
"""Fail closed on adaptive recovery text that bypassed strict compiler QA.
Adaptive fallbacks exist to preserve delivery after bounded strict-QA
exhaustion, not to waive language or semantic integrity. Short quotes and
Appendix cells are explicitly re-audited because the ordinary validators
intentionally treat very short strings as diagnostic-only.
"""
source = str(source_text or '').strip()
translated = str(translated_text or '').strip()
if not source or not translated:
raise RuntimeError('adaptive_recovery_text_missing')
# Deterministic script signal for genuinely translatable English lexical
# material. Protected acronyms/numeric-only cells are allowed to remain
# unchanged and therefore do not require a target-script character.
words = re.findall(r'[A-Za-z]{2,}', source)
translatable_words = [word for word in words if not (word.isupper() and len(word) <= 6)]
protected_surface = bool(
source == translated
and re.fullmatch(r'[A-Za-z][A-Za-z0-9._+\-]{1,40}', source)
and (source.isupper() or any(char.isupper() for char in source[1:]))
)
if translatable_words and not protected_surface:
language_code = self._adaptive_recovery_language_code(target_lang)
if language_code == 'zh' and not re.search(r'[\u4e00-\u9fff]', translated):
raise RuntimeError('adaptive_recovery_target_language_signal_missing:zh')
if language_code in {'ru', 'kk'} and not re.search(
r'[А-Яа-яЁёӘәҒғҚқҢңӨөҰұҮүҺһІі]', translated
):
raise RuntimeError(
f'adaptive_recovery_target_language_signal_missing:{language_code}'
)
language_validator = getattr(agent, '_validate_translation_language_quality', None)
if callable(language_validator):
language_validator(translated, target_lang)
# The ordinary semantic-equivalence validator intentionally short-circuits
# short strings. For those exact objects, require a fresh provider-neutral
# binary audit so a non-empty but unrelated translation cannot be accepted.
short_object = len(source) < 120 or len(translated) < 80
if short_object:
call_model = getattr(agent, 'call_model', None)
extractor = getattr(agent, '_extract_json_from_text', None)
if not callable(call_model):
raise RuntimeError('adaptive_recovery_short_semantic_audit_unavailable')
prompt = (
'Audit semantic equivalence for one short academic translation object. '
'Return JSON only: {"ok":true|false,"issues":["short issue"]}. '
'Reject if the TARGET is not actually written in the requested target language except '
'for source-owned acronyms, proper names, identifiers, or numbers; also reject if it '
'omits/adds substantive meaning, changes actor/object/relation, negation, uncertainty, '
'causality strength, temporal scope, or quotation meaning. Do not rewrite.\n'
f'SECTION: {section_name}\nTARGET_LANGUAGE: {target_lang}\n'
f'{source}\n{translated}'
)
response = call_model(
'You are a conservative bilingual academic translation equivalence auditor.',
prompt, max_tokens=700, temp=0.0, model_role='reasoning', thinking=False,
allow_fallback=False, logical_node='ordinary_translation_qa',
)
if callable(extractor):
parsed = extractor(response)
else:
try:
parsed = json.loads(str(response or ''))
except Exception:
parsed = None
if not isinstance(parsed, dict) or parsed.get('ok') is not True:
issues = parsed.get('issues') if isinstance(parsed, dict) else []
detail = '; '.join(str(item) for item in (issues or [])[:4])
raise RuntimeError(
'adaptive_recovery_short_semantic_equivalence_failed:' + (detail[:300] or 'unconfirmed')
)
return
semantic_validator = getattr(agent, 'validate_translation_semantic_equivalence', None)
if not callable(semantic_validator):
raise RuntimeError('adaptive_recovery_semantic_validator_unavailable')
semantic_validator(
source, translated, target_lang, section_name=section_name,
)
@staticmethod
def _rebind_target_qualitative_appendix_once(
manuscript: str, canonical_appendix: str, reference_heading: str,
) -> str:
"""Deterministically replace/append the final typed Appendix exactly once."""
value = str(manuscript or '').strip()
appendix = str(canonical_appendix or '').strip()
if not appendix:
raise RuntimeError('target_qualitative_appendix_rebind_source_missing')
heading_match = re.match(r'(?m)^#\s+([^\n]+)\s*$', appendix)
if not heading_match:
raise RuntimeError('target_qualitative_appendix_rebind_heading_missing')
appendix_heading = heading_match.group(1).strip()
ref_heading = str(reference_heading or '').strip()
reference_tail = ''
body = value
if ref_heading:
ref_match = re.search(rf'(?m)^#\s+{re.escape(ref_heading)}\s*$', value)
if ref_match:
body = value[:ref_match.start()].rstrip()
reference_tail = value[ref_match.start():].strip()
appendix_match = re.search(rf'(?m)^#\s+{re.escape(appendix_heading)}\s*$', body)
if appendix_match:
appendix_tail = body[appendix_match.end():]
if re.search(r'(?m)^#\s+[^#].*$', appendix_tail):
raise RuntimeError('target_qualitative_appendix_rebind_nonterminal_section')
body = body[:appendix_match.start()].rstrip()
pieces = [part for part in (body, appendix, reference_tail) if str(part or '').strip()]
return '\n\n'.join(str(part).strip() for part in pieces).strip()
def _recover_qualitative_quote_ledger(
self, agent, *, sections: dict, target_lang: str, translation_glossary: dict,
required_tags, stage_callback=None,
):
"""Recover only missing/stale typed qualitative Quote objects.
Quote identity remains server-owned. Recovery is language/tier neutral:
required IDs come from the frozen Appendix manifest, source text comes
from the frozen evidence ledger, and only missing target objects are
translated. A failed batch is isolated to individual Quote objects so
one brittle item cannot discard already recoverable evidence.
"""
updated_sections = sections if isinstance(sections, dict) else {}
glossary = dict(translation_glossary or {})
quote_sources = dict(getattr(self, '_qualitative_quote_sources', {}) or {})
required = {
str(tag or '').strip() for tag in (required_tags or []) if str(tag or '').strip()
}
missing_sources = sorted(tag for tag in required if not str(quote_sources.get(tag) or '').strip())
if missing_sources:
raise RuntimeError(f'qualitative_quote_source_missing:{missing_sources[:8]}')
current_quote_hashes = {
tag: hashlib.sha256(re.sub(r"\s+", " ", quote).strip().encode("utf-8")).hexdigest()
for tag, quote in quote_sources.items()
}
current_request_hash = str(getattr(self, 'request_hash', '') or '')
current_task_id = str(getattr(getattr(self, 'task_dir', None), 'name', '') or '')
raw_record = updated_sections.get('__quote_ledger__')
record = dict(raw_record) if isinstance(raw_record, dict) else {}
identity_ok = self._qualitative_quote_ledger_identity_ok(
record,
task_id=current_task_id,
target_lang=target_lang,
request_hash=current_request_hash,
translation_contract_version=TRANSLATION_CONTRACT_VERSION,
source_quote_hashes=current_quote_hashes,
)
stored = record.get('translations')
quote_map = dict(stored) if identity_ok and isinstance(stored, dict) else {}
recovery_events = (
list(record.get('recovery_events') or [])
if identity_ok and isinstance(record.get('recovery_events'), list) else []
)
def persist():
updated_sections['__quote_ledger__'] = {
'task_id': current_task_id,
'request_hash': current_request_hash,
'writer_execution_epoch': int(getattr(self, 'execution_epoch', 0) or 0),
'target_lang': target_lang,
'translation_contract_version': TRANSLATION_CONTRACT_VERSION,
'source_quote_hashes': current_quote_hashes,
'translations': dict(quote_map),
'recovery_events': list(recovery_events)[-40:],
'updated_at': datetime.now(timezone.utc).isoformat(),
}
missing = sorted(tag for tag in required if not str(quote_map.get(tag) or '').strip())
if not missing:
if required and (not identity_ok or raw_record is not updated_sections.get('__quote_ledger__')):
persist()
return updated_sections, glossary, quote_map, recovery_events
translate_quotes = getattr(agent, 'translate_qualitative_quote_objects', None)
if not callable(translate_quotes):
raise RuntimeError('Typed qualitative quote translation entry point is unavailable.')
def merge_result(result, expected_tags, route):
nonlocal glossary
returned = (result or {}).get('translations')
expected = set(expected_tags)
if not isinstance(returned, dict) or set(returned) != expected:
raise RuntimeError(
'Typed qualitative quote translation identity mismatch: '
f'expected={sorted(expected)} got={sorted(returned) if isinstance(returned, dict) else []}'
)
for tag in expected_tags:
rendered = str(returned.get(tag) or '').strip()
if not rendered:
raise RuntimeError(f'Typed qualitative quote translation is empty: {tag}')
quote_map[tag] = rendered
returned_glossary = (result or {}).get('glossary')
if isinstance(returned_glossary, dict):
glossary = dict(returned_glossary)
recovery_events.append({
'route': str(route), 'quote_ids': sorted(expected),
'at': datetime.now(timezone.utc).isoformat(),
})
persist()
# Fast path: one typed batch. If any object destabilizes the strict
# compiler, isolate only the missing objects instead of losing the batch.
try:
batch_result = translate_quotes(
{tag: quote_sources[tag] for tag in missing},
target_lang, glossary=glossary, max_attempts=2,
stage_callback=stage_callback,
)
merge_result(batch_result, missing, 'typed_batch')
except PipelineCancelledError:
raise
except Exception as batch_error:
logging.warning(
'Qualitative Quote Ledger batch recovery isolated | missing=%s error=%s',
len(missing), type(batch_error).__name__,
)
for tag in missing:
if str(quote_map.get(tag) or '').strip():
continue
source_quote = str(quote_sources.get(tag) or '').strip()
try:
one_result = translate_quotes(
{tag: source_quote}, target_lang,
glossary=glossary, max_attempts=2,
stage_callback=stage_callback,
)
merge_result(one_result, [tag], 'typed_single')
continue
except PipelineCancelledError:
raise
except Exception as strict_error:
fallback_method = getattr(agent, 'adaptive_translate_fallback', None)
if not callable(fallback_method):
raise strict_error
fallback = fallback_method(
source_quote, target_lang,
section_name=f'Qualitative Quote {tag}', glossary=glossary,
)
adaptive_text = str((fallback or {}).get('text') or '').strip()
if not adaptive_text:
raise RuntimeError(f'Adaptive qualitative quote translation is empty: {tag}') from strict_error
self._validate_adaptive_recovery_text(
agent, source_text=source_quote, translated_text=adaptive_text,
target_lang=target_lang, section_name=f'Qualitative Quote {tag}',
glossary=glossary,
)
render_method = getattr(agent, '_render_qualitative_quote_translation', None)
rendered = adaptive_text
if callable(render_method):
rendered = str(render_method(adaptive_text, target_lang) or '').strip()
if not rendered:
raise RuntimeError(f'Adaptive qualitative quote rendering is empty: {tag}') from strict_error
quote_map[tag] = rendered
recovery_events.append({
'route': 'adaptive_single', 'quote_ids': [tag],
'prior_error': type(strict_error).__name__,
'at': datetime.now(timezone.utc).isoformat(),
})
persist()
unresolved = sorted(tag for tag in required if not str(quote_map.get(tag) or '').strip())
if unresolved:
raise RuntimeError(f'qualitative_appendix_quote_ledger_incomplete:{unresolved[:8]}')
return updated_sections, glossary, quote_map, recovery_events
def _adaptive_translate_qualitative_unit_with_quotes(
self, agent, *, source_text: str, target_lang: str, section_name: str,
translation_sections: dict, translation_glossary: dict,
):
"""Delivery-first fallback that preserves one shared Quote identity.
Generic whole-unit fallback is allowed for qualitative narrative only
after verbatim evidence is replaced by stable Quote slots. Missing Quote
objects are recovered from frozen evidence, then injected back into the
fallback result. This keeps Findings and Appendix on the same translated
quotation instead of letting two independent model calls paraphrase the
same verbatim evidence differently.
"""
source = str(source_text or '').strip()
canonical_source, pending_tags, reused_tags = self._prepare_qualitative_translation_source(
source, getattr(self, '_qualitative_quote_sources', {}), {}
)
canonical_source = str(canonical_source or '').strip()
required_tags = list(dict.fromkeys(list(pending_tags or []) + list(reused_tags or [])))
sections = translation_sections if isinstance(translation_sections, dict) else {}
glossary = dict(translation_glossary or {})
quote_map = {}
recovery_events = []
if required_tags:
sections, glossary, quote_map, recovery_events = self._recover_qualitative_quote_ledger(
agent, sections=sections, target_lang=target_lang,
translation_glossary=glossary, required_tags=required_tags,
)
fallback_method = getattr(agent, 'adaptive_translate_fallback', None)
if not callable(fallback_method):
raise RuntimeError('Adaptive translation fallback entry point is unavailable.')
fallback = fallback_method(
canonical_source, target_lang, section_name=section_name, glossary=glossary
)
translated = str((fallback or {}).get('text') or '').strip()
if not translated:
raise RuntimeError('adaptive_qualitative_translation_output_missing')
if required_tags:
translated, quote_map = self._finalize_qualitative_translation_quotes(
translated, required_tags, [], quote_map
)
alignment_validator = getattr(agent, 'validate_translation_alignment', None)
if callable(alignment_validator):
alignment_validator(source, translated)
self._validate_adaptive_recovery_text(
agent, source_text=source, translated_text=translated,
target_lang=target_lang, section_name=section_name, glossary=glossary,
)
result = dict(fallback or {})
result['text'] = translated
if recovery_events:
result['qualitative_quote_recovery_events'] = list(recovery_events)[-40:]
return result, sections, glossary
def _translate_qualitative_appendix_unit(
self, agent, qual_engine, *, unit_key: str, source_text: str, target_lang: str,
translation_sections: dict, translation_glossary: dict, data_metadata: dict,
):
"""Translate Appendix cells, then bind the six-column table deterministically."""
source = str(source_text or '').strip()
manifest = qual_engine.build_appendix_manifest(data_metadata)
if not qual_engine._validate_appendix_manifest(manifest):
raise RuntimeError('qualitative_appendix_manifest_invalid')
manifest_hash = str(manifest.get('manifest_hash') or '')
identity = self._translation_identity_bundle(agent, source)
entity_hash_method = getattr(agent, 'translation_protected_entity_ledger_hash', None)
protected_entity_ledger_hash = str(
entity_hash_method(identity['compiled_source'], translation_glossary)
if callable(entity_hash_method) else ''
)
protected_entity_ledger_version = str(
getattr(agent, 'PROTECTED_ENTITY_LEDGER_VERSION', 'protected-entity-ledger-v1')
or 'protected-entity-ledger-v1'
)
sections = dict(translation_sections or {})
existing = dict(sections.get(unit_key) or {})
if (
existing.get('qa_status') in {'passed', 'passed_with_warning'}
and existing.get('source_hash') == identity['source_hash']
and existing.get('target_lang') == target_lang
and existing.get('qualitative_appendix_binding_version') == qual_engine.TARGET_APPENDIX_BINDING_VERSION
and existing.get('qualitative_appendix_manifest_hash') == manifest_hash
and existing.get('translated_text')
):
qual_engine.validate_bound_target_appendix(existing['translated_text'], manifest, target_lang)
return sections, translation_glossary
required_quotes = {
str(row.get('quote_id') or '').strip()
for row in manifest.get('rows') or []
if str(row.get('quote_id') or '').strip()
}
sections, translation_glossary, quote_map, quote_recovery_events = (
self._recover_qualitative_quote_ledger(
agent, sections=sections, target_lang=target_lang,
translation_glossary=translation_glossary, required_tags=required_quotes,
)
)
translate_cells = getattr(agent, 'translate_qualitative_appendix_cells', None)
if not callable(translate_cells):
raise RuntimeError('Typed qualitative Appendix cell translation entry point is unavailable.')
try:
prebound_cells, model_cells = qual_engine.appendix_translation_plan(
manifest, translation_glossary
)
appendix_cell_recovery_events = []
if model_cells:
try:
result = translate_cells(
model_cells, target_lang,
glossary=translation_glossary, max_attempts=2,
claim_states=(existing.get('appendix_cell_claim_states') if isinstance(existing.get('appendix_cell_claim_states'), dict) else None),
)
model_translations = dict((result or {}).get('translations') or {})
if set(model_translations) != set(model_cells):
raise RuntimeError('qualitative_appendix_target_cell_translation_identity_mismatch')
updated_glossary = dict((result or {}).get('glossary') or translation_glossary or {})
except PipelineCancelledError:
raise
except Exception as batch_error:
logging.warning(
'Qualitative Appendix cell batch recovery isolated | cells=%s error=%s',
len(model_cells), type(batch_error).__name__,
)
model_translations = {}
updated_glossary = dict(translation_glossary or {})
for cell_id, source_cell in model_cells.items():
try:
one_result = translate_cells(
{cell_id: source_cell}, target_lang,
glossary=updated_glossary, max_attempts=2,
)
one_map = dict((one_result or {}).get('translations') or {})
if set(one_map) != {cell_id} or not str(one_map.get(cell_id) or '').strip():
raise RuntimeError(
f'qualitative_appendix_target_cell_translation_identity_mismatch:{cell_id}'
)
model_translations[cell_id] = str(one_map[cell_id]).strip()
one_glossary = (one_result or {}).get('glossary')
if isinstance(one_glossary, dict):
updated_glossary = dict(one_glossary)
appendix_cell_recovery_events.append({
'route': 'typed_single', 'cell_id': str(cell_id),
'at': datetime.now(timezone.utc).isoformat(),
})
except PipelineCancelledError:
raise
except Exception as strict_error:
fallback_method = getattr(agent, 'adaptive_translate_fallback', None)
if not callable(fallback_method):
raise strict_error
fallback = fallback_method(
str(source_cell or ''), target_lang,
section_name=f'Qualitative Appendix Cell {cell_id}',
glossary=updated_glossary,
)
rendered_cell = re.sub(
r'\s+', ' ', str((fallback or {}).get('text') or '')
).strip()
if not rendered_cell:
raise RuntimeError(
f'qualitative_appendix_target_cell_translation_empty:{cell_id}'
) from strict_error
self._validate_adaptive_recovery_text(
agent, source_text=str(source_cell or ''),
translated_text=rendered_cell, target_lang=target_lang,
section_name=f'Qualitative Appendix Cell {cell_id}',
glossary=updated_glossary,
)
model_translations[cell_id] = rendered_cell
appendix_cell_recovery_events.append({
'route': 'adaptive_single', 'cell_id': str(cell_id),
'prior_error': type(strict_error).__name__,
'at': datetime.now(timezone.utc).isoformat(),
})
result = {
'qa_status': 'passed_with_warning',
'attempts': int(existing.get('attempts') or 0) + 1,
'claim_states': {},
'appendix_cell_contract_version': 'typed-qualitative-appendix-cells-v2',
}
else:
result = {
'qa_status': 'passed', 'attempts': 1, 'claim_states': {},
'appendix_cell_contract_version': 'typed-qualitative-appendix-cells-v2',
}
model_translations = {}
updated_glossary = dict(translation_glossary or {})
cell_translations = {**dict(prebound_cells), **dict(model_translations)}
expected_cell_ids = set(qual_engine.appendix_translation_cells(manifest))
if set(cell_translations) != expected_cell_ids:
raise RuntimeError('qualitative_appendix_target_cell_binding_incomplete')
translated_text = qual_engine.bind_target_language_appendix(
manifest, target_lang, cell_translations, quote_map
)
qual_engine.validate_bound_target_appendix(translated_text, manifest, target_lang)
except PipelineCancelledError:
raise
except Exception as error:
code_method = getattr(agent, 'translation_error_code', None)
error_code = str(code_method(error) if callable(code_method) else 'TRANSLATION_PENDING')
sections[unit_key] = {
'source_hash': identity['source_hash'],
'compiled_source_hash': identity['compiled_source_hash'],
'target_lang': target_lang,
'translation_contract_version': TRANSLATION_CONTRACT_VERSION,
'fact_ledger_hash': identity['fact_ledger_hash'],
'protected_entity_ledger_version': protected_entity_ledger_version,
'protected_entity_ledger_hash': protected_entity_ledger_hash,
'translated_text': '',
'qa_status': 'repair_pending',
'attempts': int(existing.get('attempts') or 0) + 1,
'error_code': error_code,
'qualitative_appendix_binding_version': qual_engine.TARGET_APPENDIX_BINDING_VERSION,
'qualitative_appendix_manifest_hash': manifest_hash,
'appendix_cell_claim_states': dict(getattr(agent, '_last_translation_claim_states', {}) or {}),
'updated_at': datetime.now(timezone.utc).isoformat(),
}
logging.warning(
'Qualitative Appendix typed translation repair pending | unit=%s code=%s error=%s',
unit_key, error_code, type(error).__name__,
)
return sections, translation_glossary
sections[unit_key] = {
'source_hash': identity['source_hash'],
'compiled_source_hash': identity['compiled_source_hash'],
'target_lang': target_lang,
'glossary_version': self._glossary_version(updated_glossary),
'translation_contract_version': TRANSLATION_CONTRACT_VERSION,
'fact_ledger_hash': identity['fact_ledger_hash'],
'protected_entity_ledger_version': protected_entity_ledger_version,
'protected_entity_ledger_hash': protected_entity_ledger_hash,
'translated_text': translated_text,
'qa_status': str((result or {}).get('qa_status') or 'passed'),
'attempts': int((result or {}).get('attempts') or 1),
'qualitative_appendix_binding_version': qual_engine.TARGET_APPENDIX_BINDING_VERSION,
'qualitative_appendix_manifest_hash': manifest_hash,
'appendix_cell_contract_version': str((result or {}).get('appendix_cell_contract_version') or ''),
'appendix_cell_claim_states': dict((result or {}).get('claim_states') or {}),
'appendix_prebound_label_count': len(prebound_cells),
'quote_ledger_recovery_events': list(quote_recovery_events)[-40:],
'appendix_cell_recovery_events': list(appendix_cell_recovery_events)[-80:],
'updated_at': datetime.now(timezone.utc).isoformat(),
}
return sections, updated_glossary
@staticmethod
def _bounded_target(base_target: int, accumulated_deviation: float, remaining_including_current: int, safe_chunk: int) -> int:
base = max(80, int(base_target))
correction = -float(accumulated_deviation) / max(1, int(remaining_including_current))
lower = max(80, int(round(base * 0.75)))
upper = min(int(safe_chunk * 1.5), max(lower, int(round(base * 1.25))))
return max(lower, min(upper, int(round(base + correction))))
@staticmethod
def _english_count_for_target(manuscript: str) -> int:
body = re.split(r"(?m)^#\s+References\s*$", str(manuscript or ""), maxsplit=1)[0]
body = re.split(r"(?m)^#\s+Appendix: Data Structure and Evidence\s*$", body, maxsplit=1)[0]
# Markdown punctuation is ignored; contractions/hyphenated terms stay one token.
return len(re.findall(r"\b[A-Za-z0-9]+(?:[-'][A-Za-z0-9]+)*\b", body))
def _validate_target_length(self, manuscript: str, target_words: int) -> dict[str, Any]:
"""Return a nonfatal total-length quality diagnostic.
The user-selected target remains a planning input, not a completion gate.
Completeness is validated separately by structure, evidence, citation,
translation, DOCX, and ZIP checks.
"""
quality = APP_CONFIG.get("manuscript_validation", {})
minimum = int(target_words * float(quality.get("minimum_total_ratio", 0.97)))
maximum = int(target_words * float(quality.get("maximum_total_ratio", 1.03)))
short_notice = float(quality.get("total_short_notice_ratio", 0.70))
topic_review = float(quality.get("total_topic_review_ratio", 0.50))
actual = self._english_count_for_target(manuscript)
ratio = (actual / max(1, int(target_words)))
status = "normal"
recommendation = ""
if ratio < topic_review:
status = "very_short"
recommendation = (
"First verify that generation completed normally and no section/evidence is missing. "
"If the manuscript is complete, consider broadening the research scope or selecting a new research topic."
)
elif ratio < short_notice:
status = "short"
recommendation = (
"If the manuscript is structurally complete, consider broadening the theoretical scope, context, "
"research question, or research topic."
)
elif actual < minimum:
status = "slightly_short"
elif actual > maximum:
status = "long"
if status != "normal":
logging.warning(
"Manuscript length diagnostic | actual=%s | target=%s | ratio=%.3f | status=%s | nonfatal=true",
actual, target_words, ratio, status,
)
return {
"scope": "manuscript", "name": "English manuscript", "actual_words": actual,
"target_words": int(target_words), "ratio": round(ratio, 4), "status": status,
"recommendation": recommendation, "nonfatal": True,
}
@staticmethod
def _upsert_quality_diagnostic(items: list[dict[str, Any]], diagnostic: dict[str, Any]) -> None:
key = (str(diagnostic.get("scope") or ""), str(diagnostic.get("name") or ""))
items[:] = [
item for item in items
if (str(item.get("scope") or ""), str(item.get("name") or "")) != key
]
items.append(dict(diagnostic))
def _section_word_diagnostic(self, name: str, actual_words: int, target_words: int) -> dict[str, Any]:
validation = APP_CONFIG.get("manuscript_validation", {})
budget = APP_CONFIG.get("word_budget", {})
target = max(1, int(target_words))
actual = max(0, int(actual_words))
ratio = actual / target
short_notice = float(validation.get("section_short_notice_ratio", 0.60))
topic_review = float(validation.get("section_topic_review_ratio", 0.40))
preferred_min = float(budget.get("section_min_ratio", 0.92))
preferred_max = float(budget.get("section_max_ratio", 1.08))
status = "normal"
recommendation = ""
if ratio < topic_review:
status = "very_short"
recommendation = (
"First verify that this section completed normally and that evidence was available. "
"If it is complete, consider broadening the research scope or selecting a new research topic."
)
elif ratio < short_notice:
status = "short"
recommendation = (
"If the section is complete, consider broadening the theoretical scope, context, research question, "
"or research topic."
)
elif ratio < preferred_min:
status = "slightly_short"
elif ratio > preferred_max:
status = "long"
if status != "normal":
logging.warning(
"Section length diagnostic | section=%s | actual=%s | target=%s | ratio=%.3f | status=%s | nonfatal=true",
name, actual, target, ratio, status,
)
return {
"scope": "section", "name": str(name), "actual_words": actual, "target_words": target,
"ratio": round(ratio, 4), "status": status, "recommendation": recommendation, "nonfatal": True,
}
@staticmethod
def _citation_eligible_section_ids(mode: str) -> set[str]:
return (
{"introduction", "theoretical_framing", "methods", "discussion"}
if str(mode).lower() == "asq"
else {"introduction", "theory", "methodology", "discussion"}
)
@staticmethod
def _quota_driven_citation_recovery_allowed(section_id: str) -> bool:
"""Discussion evidence is claim-driven, never padded to a word-count target.
R6-R5 keeps claim-specific planner assignments, but disables reserve/JIT
top-ups whose only trigger is a numeric citation target. After drafting,
the existing bounded top-level completion may still attach a directly
validated source to an actual external claim.
"""
return str(section_id or '').strip().lower() != 'discussion'
# Backward-readable name used by R6-R5 replay documentation.
_pre_draft_citation_recovery_allowed = _quota_driven_citation_recovery_allowed
@classmethod
def _adaptive_citation_requirements(cls, tasks, mode: str):
"""Derive evidence demand and semantic-survival reserve from manuscript units."""
allowed = cls._citation_eligible_section_ids(mode)
minimums = {}
reserve_targets = {}
for task in tasks or []:
section_id = str(task.get("section_id") or "")
display_name = str(task.get("display_name") or "").strip()
if section_id not in allowed or not display_name:
continue
base, words_per_source = CITATION_ROLE_RULES.get(section_id, (2, 240))
target_words = max(1, int(task.get("target_words") or 1))
minimum = max(base, int(math.ceil(target_words / max(1, words_per_source))))
reserve = max(
minimum,
int(math.ceil(minimum / max(0.01, CITATION_SEMANTIC_SURVIVAL_RATIO))),
)
minimums[display_name] = minimum
reserve_targets[display_name] = reserve
return minimums, reserve_targets
@classmethod
def _top_level_citation_requirements(cls, tasks, mode: str):
"""Aggregate citation authority by academic section, never by technical chunk.
``Introduction (Part 1/2)`` and ``Introduction (Part 2/2)`` are generation
chunks, not separate scholarly evidence authorities. Planning may remain
unit-local, but fatal literature sufficiency is evaluated on the canonical
top-level academic section.
"""
allowed = cls._citation_eligible_section_ids(mode)
words_by_section = {}
for task in tasks or []:
section_id = str(task.get('section_id') or '')
if section_id not in allowed:
continue
words_by_section[section_id] = words_by_section.get(section_id, 0) + max(
1, int(task.get('target_words') or 1)
)
minimums = {}
reserve_targets = {}
for section_id, target_words in words_by_section.items():
base, words_per_source = CITATION_ROLE_RULES.get(section_id, (2, 240))
minimum = max(base, int(math.ceil(target_words / max(1, words_per_source))))
reserve = max(
minimum,
int(math.ceil(minimum / max(0.01, CITATION_SEMANTIC_SURVIVAL_RATIO))),
)
minimums[section_id] = minimum
reserve_targets[section_id] = reserve
return minimums, reserve_targets
@staticmethod
def _citation_unit_section_map(tasks):
return {
str(task.get('display_name') or ''): str(task.get('section_id') or '')
for task in (tasks or []) if str(task.get('display_name') or '').strip()
}
@staticmethod
def _top_level_unique_tag_counts(citation_plan, unit_section_map):
tags_by_section = {}
for unit_name, items in dict(citation_plan or {}).items():
section_id = str(unit_section_map.get(str(unit_name)) or '')
if not section_id:
continue
bucket = tags_by_section.setdefault(section_id, set())
for item in items or []:
tag = str((item or {}).get('tag') or '')
if tag:
bucket.add(tag)
return {section_id: len(tags) for section_id, tags in tags_by_section.items()}
@staticmethod
def _adaptive_literature_candidate_target(target_words: int, agent) -> int:
"""Adaptive retrieval aspiration; never a final reference-count quota."""
dynamic = max(24, int(math.ceil(max(1, int(target_words or 1)) / 90.0)))
configured_cap = max(1, int(getattr(agent, "lit_target_papers", dynamic) or dynamic))
return min(configured_cap, dynamic)
@staticmethod
def _citation_task_context(task, research_logic_contract="", synthesis_result="") -> str:
parts = [
str(task.get("display_name") or ""),
str(task.get("required_content") or ""),
" ".join(str(value) for value in (task.get("concepts") or []) if str(value).strip()),
str(research_logic_contract or "")[:1600],
str(synthesis_result or "")[:1600],
]
return re.sub(r"\s+", " ", " ".join(part for part in parts if part)).strip()
@staticmethod
def _literature_insufficient_error(shortfalls) -> RuntimeError:
details = "; ".join(
f"{name}={actual}/{required}"
for name, actual, required in (shortfalls or [])[:8]
)
return RuntimeError(
"E_LITERATURE_INSUFFICIENT: high-relevance claim-specific literature is insufficient "
f"for the planned manuscript units ({details}). Please revise the research topic or "
"keywords and try again; unrelated references will not be used to satisfy a quota."
)
@staticmethod
def _evidence_warning_detail() -> str:
return (
"Some externally oriented claims have limited direct high-relevance literature coverage. "
"This may reflect a newer or specialized topic, dispersed literature, search/keyword limits, "
"or limited prior evidence; this status does not determine the novelty, importance, or overall "
"research value of the topic. Unrelated sources were not used to satisfy a citation count."
)
def _record_evidence_coverage_event(self, state, shortfalls, *, phase: str, preview_status: str) -> None:
"""Persist a neutral evidence-coverage signal without stopping generation.
The event records only deterministic coverage counts. It never infers why
literature is sparse and never evaluates topic value. Duplicate events
are collapsed across resume/replay.
"""
rows = []
for name, actual, required in shortfalls or []:
rows.append({
"section": str(name or ""),
"actual": max(0, int(actual or 0)),
"required": max(0, int(required or 0)),
})
if not rows:
return
signature = (str(phase or ""), tuple((r["section"], r["actual"], r["required"]) for r in rows))
existing = list(state.get("evidence_coverage_events") or self._evidence_coverage_events or [])
seen = {
(str(item.get("phase") or ""), tuple(
(str(row.get("section") or ""), int(row.get("actual") or 0), int(row.get("required") or 0))
for row in (item.get("shortfalls") or []) if isinstance(row, dict)
))
for item in existing if isinstance(item, dict)
}
if signature not in seen:
existing.append({
"phase": str(phase or ""),
"shortfalls": rows,
"observed_at": datetime.now(timezone.utc).isoformat(),
})
self._evidence_coverage_events = existing
state["evidence_coverage_events"] = existing
state["evidence_risk_detected"] = True
self._preview_evidence_status = str(preview_status or "limited_risk")
self._write_preview_payload(force=True)
logging.warning(
"Evidence coverage warning | phase=%s | shortfalls=%s | nonfatal=true",
phase, rows,
)
@staticmethod
def _evidence_coverage_summary(*, status, early_risk, final_reference_count, minimums, validated_by_top_level, claims_by_top_level, events, method_authority_count=0, method_external_claims=0):
sections = []
for section_id, planned_minimum in dict(minimums or {}).items():
validated = len(set((validated_by_top_level or {}).get(section_id) or []))
external_claims = max(0, int((claims_by_top_level or {}).get(section_id, 0) or 0))
required = min(int(planned_minimum), external_claims) if external_claims else 0
sections.append({
"section": str(section_id),
"validated_direct_citations": validated,
"evidence_requirement": required,
"external_claims": external_claims,
"planning_minimum": int(planned_minimum),
})
normalized_status = str(status or "adequate")
return {
"status": normalized_status,
"submission_readiness": "review_required" if normalized_status == "limited" else "quality_gate_passed",
"early_risk_detected": bool(early_risk),
"final_reference_count": max(0, int(final_reference_count or 0)),
"sections": sections,
"methodology_authority": {
"validated_method_authority_citations": max(0, int(method_authority_count or 0)),
"methodology_external_claims": max(0, int(method_external_claims or 0)),
},
"events": list(events or []),
}
def _bounded_top_level_evidence_completion(
self, agent, *, topic: str, section_id: str, section_name: str, section_text: str,
required: int, existing_tags, excluded_tags=None,
) -> tuple[str, list[str]]:
"""Give one completed top-level section a single bounded evidence top-up.
This is not a prose-generation path. It issues at most one contextual JIT
retrieval and may attach only semantically validated citations to claims
that already exist. If no directly supporting source is found, the prose
remains unchanged and the manuscript later reports evidence-limited quality.
"""
value = str(section_text or "")
current = {str(tag or "").strip() for tag in (existing_tags or []) if str(tag or "").strip()}
needed = max(0, int(required or 0) - len(current))
needed = min(EVIDENCE_COMPLETION_MAX_PER_TOP_LEVEL, needed)
if needed <= 0 or not value.strip():
return value, []
before = set(getattr(agent, "used_citations", set()) or set())
items = agent.fetch_contextual_jit_recovery_items(
topic=topic,
section_name=section_name,
section_id=section_id,
section_context=value,
needed=needed,
excluded_tags=(before | set(excluded_tags or set())),
)
if not items:
logging.info(
"Bounded evidence completion found no eligible source | section=%s needed=%s",
section_name, needed,
)
return value, []
recovered = agent.recover_citations_on_existing_claims(
value, items, section_name=section_name, section_id=section_id, needed=needed
)
after = set(getattr(agent, "used_citations", set()) or set())
added = sorted(after - before, key=agent._citation_tag_number)
if recovered == value or not added:
logging.info(
"Bounded evidence completion made no validated attachment | section=%s candidates=%s",
section_name, len(items),
)
return value, []
logging.info(
"Bounded evidence completion accepted | section=%s added=%s required=%s",
section_name, len(added), required,
)
return recovered, added
@staticmethod
def _paragraph_token_set(value):
text = re.sub(r'\[CITE-\d+\]', ' ', str(value or ''), flags=re.I)
return set(re.findall(r'[A-Za-z][A-Za-z0-9_-]{2,}', text.lower()))
@classmethod
def _remove_severe_cross_unit_duplicates(cls, candidate, prior_text):
"""Drop only very high-confidence cross-unit duplicate paragraphs."""
previous = [p.strip() for p in re.split(r'\n\s*\n', str(prior_text or '')) if len(p.split()) >= 55 and not p.lstrip().startswith('#')]
kept, diagnostics = [], []
for paragraph in re.split(r'(\n\s*\n)', str(candidate or '')):
if re.fullmatch(r'\n\s*\n', paragraph or ''):
kept.append(paragraph); continue
value = str(paragraph or '').strip()
if len(value.split()) < 55 or value.startswith('#'):
kept.append(paragraph); continue
tokens = cls._paragraph_token_set(value)
citations = set(re.findall(r'\[CITE-\d+\]', value, flags=re.I))
numerics = set(re.findall(r'(? list[str]:
output = []
for item in items or []:
path = (self.task_dir / str(item)).resolve()
if self.task_dir.resolve() not in path.parents:
raise RuntimeError("input_path_escape")
if path.is_file():
output.append(str(path))
return output
def _validate_trusted_task_policy_for_execution(self, mode: str, target_words: int) -> dict[str, Any] | None:
snapshot = copy.deepcopy(self.trusted_task_policy_snapshot)
if not snapshot:
return None
resolved = snapshot.get("resolved_policy")
if not isinstance(resolved, dict):
raise RuntimeError("trusted_task_policy_snapshot_invalid")
task_id = str(snapshot.get("task_id") or "").strip()
request_task_id = str(self.request.get("task_id") or "").strip()
if not task_id or task_id != request_task_id:
raise RuntimeError("trusted_task_policy_task_mismatch")
tier = str(snapshot.get("product_tier") or "").strip().upper()
entitlement_class = str(snapshot.get("entitlement_class") or "").strip().upper()
if tier not in {"STANDARD", "FLAGSHIP"}:
raise RuntimeError("trusted_task_policy_product_tier_invalid")
# Keep runner validation aligned with the executor's trusted-policy
# contract. Entitlement authority comes from the SHA-bound resolved
# ProductPolicy, not from a duplicated runtime enum. This preserves
# fail-closed behavior while allowing FLAGSHIP_TRIAL when (and only
# when) the trusted trial policy explicitly authorizes it.
trial = resolved.get("trial") if isinstance(resolved.get("trial"), dict) else None
billing = resolved.get("billing") if isinstance(resolved.get("billing"), dict) else None
allowed_entitlements: list[str] = []
if trial and str(trial.get("entitlement_class") or "").strip():
allowed_entitlements.append(str(trial.get("entitlement_class") or "").strip().upper())
if billing and str(billing.get("entitlement_class") or "").strip():
paid_class = str(billing.get("entitlement_class") or "").strip().upper()
if paid_class not in allowed_entitlements:
allowed_entitlements.append(paid_class)
# Backward-compatible trusted snapshots created before the nested
# trial/billing objects were embedded still carry SHA-bound policy
# version links. Derive their same-tier classes from those links
# rather than reopening a duplicated global entitlement whitelist.
if not allowed_entitlements:
if str(resolved.get("trial_policy_version") or "").strip():
allowed_entitlements.append(f"{tier}_TRIAL")
if str(resolved.get("billing_policy_version") or "").strip():
allowed_entitlements.append(f"{tier}_PAID")
if not entitlement_class or entitlement_class not in allowed_entitlements:
raise RuntimeError("trusted_task_policy_entitlement_invalid")
if any(not value.startswith(f"{tier}_") for value in allowed_entitlements):
raise RuntimeError("trusted_task_policy_entitlement_mismatch")
if str(resolved.get("product_tier") or "").strip().upper() != tier:
raise RuntimeError("trusted_task_policy_product_tier_mismatch")
if str(resolved.get("snapshot_schema_version") or "") != str(snapshot.get("snapshot_schema_version") or ""):
raise RuntimeError("trusted_task_policy_schema_mismatch")
for field in ("product_policy_version", "word_count_policy_version", "model_policy_version", "cost_policy_version"):
if str(resolved.get(field) or "") != str(snapshot.get(field) or ""):
raise RuntimeError("trusted_task_policy_version_mismatch")
for field in ("billing_policy_version", "trial_policy_version"):
resolved_value = str(resolved.get(field) or "").strip() or None
snapshot_value = str(snapshot.get(field) or "").strip() or None
if resolved_value != snapshot_value:
raise RuntimeError("trusted_task_policy_version_mismatch")
if str(resolved.get("manuscript_mode") or "").strip().lower() != mode:
raise RuntimeError("trusted_task_policy_mode_mismatch")
minimum = int(snapshot.get("min_body_words") or 0)
maximum = int(snapshot.get("max_body_words") or 0)
if minimum <= 0 or maximum < minimum or not minimum <= target_words <= maximum:
raise RuntimeError("trusted_task_policy_word_range_mismatch")
word_count = resolved.get("word_count") if isinstance(resolved.get("word_count"), dict) else {}
if (
int(word_count.get("min_body_words") or 0) != minimum
or int(word_count.get("max_body_words") or 0) != maximum
or str(word_count.get("counting_contract_version") or "") != str(snapshot.get("counting_contract_version") or "")
):
raise RuntimeError("trusted_task_policy_word_policy_mismatch")
envelope = {
"task_id": task_id,
"snapshot_schema_version": str(snapshot.get("snapshot_schema_version") or ""),
"product_tier": str(snapshot.get("product_tier") or ""),
"product_policy_version": str(snapshot.get("product_policy_version") or ""),
"word_count_policy_version": str(snapshot.get("word_count_policy_version") or ""),
"model_policy_version": str(snapshot.get("model_policy_version") or ""),
"cost_policy_version": str(snapshot.get("cost_policy_version") or ""),
"billing_policy_version": str(snapshot.get("billing_policy_version") or "").strip() or None,
"trial_policy_version": str(snapshot.get("trial_policy_version") or "").strip() or None,
"entitlement_class": str(snapshot.get("entitlement_class") or "").strip() or None,
"min_body_words": minimum,
"max_body_words": maximum,
"counting_contract_version": str(snapshot.get("counting_contract_version") or ""),
"resolved_policy": resolved,
}
observed_sha = hashlib.sha256(
json.dumps(envelope, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
).hexdigest()
if observed_sha != str(snapshot.get("snapshot_sha256") or "").strip().lower():
raise RuntimeError("trusted_task_policy_snapshot_sha_mismatch")
model = resolved.get("model") if isinstance(resolved.get("model"), dict) else {}
routes = model.get("routes") if isinstance(model.get("routes"), dict) else {}
if not routes:
raise RuntimeError("trusted_task_policy_model_routes_missing")
if str(snapshot.get("model_policy_version") or "") not in {
"model-standard-v2", "model-flagship-v2"
}:
# Preserve the dormant R3A v1 fail-closed contract; only v2 may
# cross the new trusted logical-node execution seam.
raise RuntimeError("trusted_task_policy_model_route_contract_not_ready")
if str(model.get("provider_catalog_version") or "") != "deepseek-v4-concrete-v1":
raise RuntimeError("trusted_task_policy_provider_catalog_invalid")
return snapshot
@staticmethod
def _section_logical_node(mode: str, section_id: str, subsection_id: str | None = None) -> str | None:
"""Map academic topology to a tier-neutral writer logical node."""
mode = str(mode or "").strip().lower()
section_id = str(section_id or "").strip().lower()
subsection_id = str(subsection_id or "").strip().lower()
if mode == "amj":
if section_id in {"introduction", "theory"}:
return "core_section_writing"
if section_id == "methodology":
return "methods_writing"
if section_id == "results":
return "quantitative_results_writing"
if section_id == "discussion":
if subsection_id in {"practical_implications", "limitations"}:
return "ordinary_section_writing"
return "core_section_writing"
if section_id == "conclusion":
return "ordinary_section_writing"
elif mode == "asq":
if section_id in {"introduction", "theoretical_framing"}:
return "core_section_writing"
if section_id == "methods":
return "methods_writing"
if section_id == "findings":
if subsection_id == "emergent_model":
return "core_section_writing"
return "evidence_rich_findings_narration"
if section_id == "discussion":
if subsection_id in {"boundaries", "practical_implications"}:
return "ordinary_section_writing"
return "core_section_writing"
if section_id == "conclusion":
return "ordinary_section_writing"
if section_id == "appendix":
return None
raise RuntimeError(f"logical_node_mapping_missing:{mode}:{section_id}:{subsection_id}")
def _run_impl(self) -> dict[str, Any]:
req = self.request
mode = str(req.get("mode") or "").lower()
if mode not in {"amj", "asq"}:
raise RuntimeError("invalid_generation_mode")
target_words = int(req.get("target_words") or 0)
if not APP_CONFIG["target_words_min"] <= target_words <= APP_CONFIG["target_words_max"]:
raise RuntimeError("target_words_out_of_range")
trusted_task_policy_snapshot = self._validate_trusted_task_policy_for_execution(
mode, target_words
)
# A2: provider credentials are adapter-owned. DeepSeek remains the
# current default adapter, but the runner no longer hard-codes it as the
# only legal model provider.
ds_key = str(os.environ.get("DEEPSEEK_API_KEY") or "").strip()
s2_key = str(os.environ.get("S2_API_KEY") or "").strip()
plan_policy_snapshot = resolve_plan_policy_snapshot(APP_CONFIG, req)
if mode == "amj":
APP_CONFIG.update(AMJ_PROMPTS)
else:
APP_CONFIG.update(ASQ_PROMPTS)
agent = SSciPipelineAgent(ds_key, s2_key)
self.agent = agent
agent.set_model_policy_snapshot(plan_policy_snapshot)
if trusted_task_policy_snapshot is not None:
agent.set_trusted_product_policy_snapshot(trusted_task_policy_snapshot)
agent.set_cancel_checker(self.cancel_checker)
# DeepSeek/search activity marks the executor as genuinely alive; the
# heartbeat thread decides separately when to renew D1's lease.
agent.set_execution_progress_hook(lambda: self._progress_activity())
safe_chunk_words = int(agent.chunk_words_default)
quant_engine = get_quantitative_engine()
qual_engine = get_qualitative_engine(agent)
if self.confirmed_checkpoint_seq is None:
# Compatibility path for isolated unit helpers that do not represent
# a claimed production executor. Production always supplies Worker
# checkpoint metadata through task_executor.
state = self.checkpoints.load_latest() or {}
else:
state = self.checkpoints.load_confirmed() or {}
quality_diagnostics = list(state.get("quality_diagnostics") or [])
quality_notes = list(state.get("quality_notes") or [])
state.setdefault("plan_policy_snapshot", copy.deepcopy(plan_policy_snapshot))
if trusted_task_policy_snapshot is not None:
state.setdefault(
"trusted_task_policy_snapshot",
copy.deepcopy(trusted_task_policy_snapshot),
)
self._evidence_coverage_events = list(state.get("evidence_coverage_events") or [])
if str(state.get("evidence_status") or "") == "limited":
self._preview_evidence_status = "limited"
elif bool(state.get("evidence_risk_detected")):
self._preview_evidence_status = "limited_risk"
if state:
self._preview_english_text = str(state.get('full_draft_en') or '')
self._preview_target_text = str(state.get('full_draft_translated') or '')
logging.info(
"[SP_EXECUTOR] resume checkpoint seq=%s stage=%s",
state.get("checkpoint_seq"), state.get("checkpoint_stage"),
)
topic = str(req.get("topic") or "").strip()
keywords = str(req.get("keywords") or "").strip()
target_lang = str(req.get("target_lang") or "").strip()
output_suffix = str(req.get("output_suffix") or "Target").strip()
interface_language = str(req.get("interface_language") or "zh").strip()
reference_paths = self._resolve_paths(req.get("reference_files") or [])
data_paths = self._resolve_paths(req.get("data_files") or [])
interview_paths = self._resolve_paths(req.get("interview_files") or [])
base_evidence_contract = str(APP_CONFIG.get("evidence_integrity_contract") or "")
# Restore agent state before continuing any dependent step.
if isinstance(state.get("agent_state"), dict):
agent.load_state(state["agent_state"])
if state.get("english_finalized") and state.get("final_citation_ledger_version") != FINAL_CITATION_LEDGER_VERSION:
# R4-R3: an R4-R2 checkpoint may contain a correct resolved body but
# an empty References tail caused by rescanning already-resolved CITE
# tags. Re-enter only finalization/translation/package gates; completed
# English units and the same task/reservation remain authoritative.
logging.warning("Final citation ledger contract advanced; re-running finalization from frozen completed units.")
state["english_finalized"] = False
state["translation_complete"] = False
state["package_validated"] = False
if state.get("english_finalized") and state.get("english_section_seal_version") != ENGLISH_SECTION_SEAL_VERSION:
# R2 compatibility: upgrade old completed English sections locally.
# No writing-model call, new task, or new quota reservation is allowed.
logging.warning("English Section Integrity Seal contract advanced; re-validating frozen completed sections locally.")
state["english_finalized"] = False
state["translation_complete"] = False
state["package_validated"] = False
user_lit_context = str(state.get("user_lit_context") or "")
user_data_md = str(state.get("user_data_md") or "")
data_metadata = dict(state.get("data_metadata") or {})
if not state.get("evidence_ready"):
self._progress(3, "inputs_and_evidence")
if reference_paths:
user_lit_context = extract_text_from_files(reference_paths[:20])
all_paper_texts = [extract_text_from_files([path]) for path in reference_paths[:20]]
agent.global_memory["core_golden_metadata"] = extract_golden_metadata(agent, all_paper_texts)
if mode == "amj":
data_file = data_paths[0] if data_paths else None
# L2-R1: establish the validated English identity projection
# BEFORE a system-generated quantitative construct/statistical
# contract is frozen. The immutable source Title/Keywords remain
# semantic authority and are passed separately below.
upstream_projection = self._resolve_english_front_matter_projection(
agent, state, data_metadata, topic, keywords
)
internal_topic_en = str(
upstream_projection.get("english_title") or topic or ""
).strip()
internal_keywords_en = list(upstream_projection.get("english_keywords") or [])
_, user_data_md, _, data_metadata = quant_engine.resolve_empirical_data(
data_file, interface_language,
topic=topic, keywords=keywords,
internal_topic=internal_topic_en,
internal_keywords=internal_keywords_en,
agent=agent,
)
data_metadata["front_matter_language_projection"] = dict(upstream_projection)
else:
raw_interviews = extract_text_from_files(interview_paths[:1]) if interview_paths else ""
prewrite_started = time.monotonic()
prewrite_calls = agent.ds_calls
user_data_md, _, data_metadata = qual_engine.resolve_qualitative_data(topic, keywords, raw_interviews)
agent.record_prewrite_stage("qualitative_packet", prewrite_started, prewrite_calls)
state.update(
{
"topic": topic,
"keywords": keywords,
"target_lang": target_lang,
"target_words": target_words,
"mode": mode,
"interface_language": interface_language,
"user_lit_context": user_lit_context,
"user_data_md": user_data_md,
"data_metadata": data_metadata,
"agent_state": agent.export_state(),
"evidence_ready": True,
}
)
self._checkpoint("evidence_ready", state, 6)
if mode == "amj":
current_quant_ledger = dict(data_metadata.get("quantitative_evidence_ledger") or {})
expected_quant_version = str(getattr(quant_engine, "QUANTITATIVE_EVIDENCE_LEDGER_VERSION", ""))
if current_quant_ledger.get("version") != expected_quant_version:
data_metadata["quantitative_evidence_ledger"] = quant_engine.build_quantitative_evidence_ledger(
data_metadata, user_data_md
)
state["data_metadata"] = data_metadata
logging.warning(
"Quantitative Evidence Ledger contract advanced; rebuilt locally version=%s",
expected_quant_version,
)
self._checkpoint("quantitative_evidence_ledger_upgraded", state, 6)
if str(data_metadata.get("source_type") or "") == "system_generated":
table_manifest = dict(data_metadata.get("quantitative_table_manifest") or {})
expected_table_version = str(getattr(quant_engine, "QUANTITATIVE_TABLE_MANIFEST_VERSION", ""))
if table_manifest.get("version") != expected_table_version:
rebuilt_tables = quant_engine.build_table_manifest_from_evidence_markdown(user_data_md)
if not rebuilt_tables:
raise RuntimeError("quantitative_table_manifest_rebuild_failed")
data_metadata["quantitative_table_manifest"] = rebuilt_tables
logging.warning(
"Quantitative table manifest contract advanced; rebuilt locally version=%s",
expected_table_version,
)
hypothesis_ledger = dict(data_metadata.get("hypothesis_ledger") or {})
expected_hypothesis_version = str(getattr(quant_engine, "HYPOTHESIS_LEDGER_VERSION", ""))
if hypothesis_ledger.get("version") != expected_hypothesis_version:
study_contract = dict(data_metadata.get("study_design_contract") or {})
if study_contract:
rebuilt_hypotheses = quant_engine.build_hypothesis_ledger(study_contract)
else:
constructs = dict(data_metadata.get("constructs") or {})
rebuilt_hypotheses = quant_engine.build_hypothesis_ledger([
constructs.get("predictor"), constructs.get("mediator"), constructs.get("outcome")
])
if not rebuilt_hypotheses:
raise RuntimeError("quantitative_hypothesis_ledger_rebuild_failed")
data_metadata["hypothesis_ledger"] = rebuilt_hypotheses
logging.warning(
"Quantitative hypothesis ledger contract advanced; rebuilt locally version=%s",
expected_hypothesis_version,
)
front_contract = dict(data_metadata.get("front_matter_contract") or {})
expected_front_version = str(getattr(quant_engine, "FRONT_MATTER_CONTRACT_VERSION", ""))
if front_contract.get("version") != expected_front_version:
study_contract = dict(data_metadata.get("study_design_contract") or {})
if study_contract:
rebuilt_front = quant_engine.build_front_matter_contract(topic, keywords, study_contract)
else:
constructs = dict(data_metadata.get("constructs") or {})
rebuilt_front = quant_engine.build_front_matter_contract(
topic, keywords, [
constructs.get("predictor"), constructs.get("mediator"), constructs.get("outcome")
]
)
if rebuilt_front:
data_metadata["front_matter_contract"] = rebuilt_front
else:
logging.warning("Quantitative front-matter semantic contract could not be rebuilt; continuing with existing research seeds and degraded final reconciliation.")
prior_translation_sections = dict(state.get("translation_sections") or {})
prior_translation_sections.pop("__front_matter__", None)
state["translation_sections"] = prior_translation_sections
state["translation_complete"] = False
state["package_validated"] = False
logging.warning(
"Quantitative front-matter contract advanced; rebuilt locally version=%s",
expected_front_version,
)
state["data_metadata"] = data_metadata
source_topic, source_keywords = topic, keywords
topic, keywords = self._apply_quantitative_front_matter_contract(
mode, topic, keywords, data_metadata
)
state.pop("effective_topic", None)
state.pop("effective_keywords", None)
front_matter_projection = self._resolve_english_front_matter_projection(
agent, state, data_metadata, source_topic, source_keywords
)
reader_topic_en = str(front_matter_projection.get("english_title") or source_topic or topic or "").strip()
reader_keywords_en = list(front_matter_projection.get("english_keywords") or [])
reader_keywords_prompt = "; ".join(reader_keywords_en)
data_metadata["front_matter_language_projection"] = dict(front_matter_projection)
state["data_metadata"] = data_metadata
hypothesis_ledger = dict(data_metadata.get("hypothesis_ledger") or {}) if mode == "amj" else {}
hypothesis_contract = self._hypothesis_contract_directive(hypothesis_ledger)
if hypothesis_ledger:
# C5.1 single authority: legacy prose extraction may not redefine AMJ
# H1-Hn once the result-independent ledger exists. Downstream memory is
# populated deterministically from that ledger and survives checkpoints.
agent.global_memory["hypotheses"] = self._hypothesis_memory_text(hypothesis_ledger)
agent.global_memory["hypothesis_contract_locked"] = True
self._qualitative_quote_sources = (
self._qualitative_quote_source_map(data_metadata) if mode == "asq" else {}
)
# ASQ planning never needs access to verbatim quotations. Keep the
# research-logic/front-matter stage on a quote-free internal-evidence
# summary; verbatim text is disclosed later only to its owning Findings
# unit (or the appendix).
planning_data_md = (
qual_engine.build_scoped_evidence_md(data_metadata, include_themes=True)
if mode == "asq"
else user_data_md
)
refs_text = str(state.get("refs_text") or "")
if not refs_text:
self._progress(8, "S2 + OpenAlex")
literature_candidate_target = self._adaptive_literature_candidate_target(target_words, agent)
refs_text = agent.fetch_references_parallel(
topic,
keywords,
min_required=literature_candidate_target,
progress_callback=lambda: self._progress(9, "S2 + OpenAlex"),
)
if not agent.citation_map:
self._record_evidence_coverage_event(
state, [("retrieved literature pool", 0, 1)],
phase="retrieval_pool_empty", preview_status="limited_risk",
)
logging.warning(
"No directly validated literature was retrieved; continuing in evidence-limited mode without padding references."
)
state["literature_candidate_target"] = literature_candidate_target
if user_lit_context:
core_refs = agent.parse_user_core_literature(user_lit_context)
if core_refs:
refs_text = (
"【USER PROVIDED CORE LITERATURE】\n" + core_refs
+ "\n\n【RETRIEVED LITERATURE】\n" + refs_text
)
state.update({"refs_text": refs_text, "agent_state": agent.export_state()})
self._checkpoint("literature_ready", state, 11)
rag_engine = initialize_rag([value for value in (user_lit_context, refs_text) if value])
synthesis_result = str(state.get("synthesis_result") or "")
if not synthesis_result:
self._progress(12, "literature_synthesis")
prewrite_started = time.monotonic()
prewrite_calls = agent.ds_calls
if not str(refs_text or "").strip() and not str(user_lit_context or "").strip():
synthesis_result = (
"EVIDENCE-LIMITED CONTEXT: no directly validated external literature is currently available. "
"Continue with the frozen study evidence, cautious conceptual reasoning, and general scholarly knowledge, "
"but do not invent citations or bibliographic metadata and do not present general model knowledge as retrieved evidence."
)
else:
try:
synthesis_result = self._stream_text(
agent.stream_synthesize_literature(topic, refs_text, user_core_literature=user_lit_context)
)
except PipelineCancelledError:
raise
except Exception as exc:
synthesis_result = (
"LITERATURE-SYNTHESIS FALLBACK: retrieved literature remains available through the citation/RAG layer, "
"but the narrative synthesis provider step was unavailable. Continue from the frozen study evidence and "
"claim-specific retrieved sources; do not invent bibliographic facts, expected findings, or causal claims."
)
quality_diagnostics.append({
"scope": "delivery_first_literature_synthesis_fallback",
"name": "Literature Synthesis",
"status": "deterministic_fallback",
"reason": type(exc).__name__,
})
logging.warning(
"Delivery-first literature synthesis fallback activated | error=%s",
type(exc).__name__,
)
agent.record_prewrite_stage("literature_synthesis", prewrite_started, prewrite_calls)
synthesis_result = re.sub(r".*?", "", synthesis_result, flags=re.DOTALL).strip()
state.update({"synthesis_result": synthesis_result, "agent_state": agent.export_state()})
self._checkpoint("synthesis_ready", state, 13)
adaptive_structure = state.get("adaptive_structure")
if not isinstance(adaptive_structure, list) or not adaptive_structure:
self._progress(14, "research_logic")
prewrite_started = time.monotonic()
prewrite_calls = agent.ds_calls
research_logic_error = None
try:
research_logic_response = self._stream_text(
agent.stream_generate_research_logic(
topic, keywords, synthesis_result, refs_text,
data_evidence=planning_data_md, mode=mode,
hypothesis_contract=hypothesis_contract,
)
)
except PipelineCancelledError:
raise
except Exception as exc:
research_logic_error = exc
research_logic_response = ""
logging.warning(
"Delivery-first research-logic provider path unavailable; using deterministic planning fallback | error=%s",
type(exc).__name__,
)
finally:
agent.record_prewrite_stage("research_logic", prewrite_started, prewrite_calls)
research_logic_contract = re.sub(
r".*?", "", research_logic_response, flags=re.DOTALL
).strip()
logic_hypothesis_rows = []
if hypothesis_ledger and research_logic_contract:
research_logic_contract = self._normalize_hypothesis_ordinal_aliases(
research_logic_contract, hypothesis_ledger
)
logic_hypothesis_rows = self._hypothesis_contract_violations(
research_logic_contract, hypothesis_ledger, section_name="Research Logic Contract"
)
if not research_logic_contract or logic_hypothesis_rows:
fallback_reason = (
"provider_error" if research_logic_error is not None
else "empty_contract" if not research_logic_contract
else "hypothesis_identity_mismatch"
)
research_logic_contract = self._deterministic_research_logic_contract(
mode, topic, keywords, data_metadata, hypothesis_ledger=hypothesis_ledger
)
fallback_rows = (
self._hypothesis_contract_violations(
research_logic_contract, hypothesis_ledger, section_name="Research Logic Contract"
) if hypothesis_ledger else []
)
if fallback_rows:
# This is an internal deterministic invariant failure, not a
# model-quality rejection. Do not continue with a corrupted
# H-identity planning contract.
raise RuntimeError(
f"delivery_first_research_logic_fallback_invalid:{fallback_rows[:3]}"
)
quality_diagnostics.append({
"scope": "delivery_first_research_logic_fallback",
"name": "Research Logic Contract",
"status": "deterministic_fallback",
"reason": fallback_reason,
"prior_violations": logic_hypothesis_rows[:3],
})
logging.warning(
"Delivery-first Research Logic fallback activated | reason=%s prior_violations=%s",
fallback_reason, len(logic_hypothesis_rows),
)
agent.global_memory["research_logic_contract"] = research_logic_contract
self._progress(14, "working_front_matter")
working_front_error = None
try:
working_front = agent.generate_working_front_matter(
reader_topic_en, reader_keywords_prompt, research_logic_contract, synthesis_result,
data_evidence=planning_data_md, mode=mode,
hypothesis_contract=hypothesis_contract,
authoritative_keywords=self._user_authoritative_front_matter_keywords(data_metadata),
keyword_policy=self._resolved_front_matter_keyword_policy(data_metadata),
fallback_keywords=self._front_matter_keyword_fallback_terms(data_metadata),
)
except PipelineCancelledError:
raise
except Exception as exc:
working_front_error = exc
working_front = {}
logging.warning(
"Delivery-first Working Front Matter provider path unavailable; preparing deterministic fallback | error=%s",
type(exc).__name__,
)
working_abstract = str(working_front.get("abstract") or "").strip()
if hypothesis_ledger and working_abstract:
working_abstract = self._normalize_hypothesis_ordinal_aliases(
working_abstract, hypothesis_ledger
)
working_keywords = self._canonical_quantitative_front_matter_keywords(
mode, list(working_front.get("keywords") or []), data_metadata
)
if not working_abstract or not working_keywords:
fallback = self._deterministic_working_front_matter(
mode, reader_topic_en, reader_keywords_prompt, data_metadata, hypothesis_ledger=hypothesis_ledger
)
working_abstract = str(fallback.get("abstract") or "").strip()
working_keywords = list(fallback.get("keywords") or [])
quality_diagnostics.append({
"scope": "delivery_first_working_front_matter_fallback",
"name": "Working Abstract",
"status": "deterministic_fallback",
"reason": "provider_error" if working_front_error is not None else "missing_component",
})
logging.warning(
"Delivery-first Working Front Matter fallback activated | reason=%s",
"provider_error" if working_front_error is not None else "missing_component",
)
_, front_fact_rows = self._repair_validate_study_fact_contract(
mode, agent, quant_engine, working_abstract, data_metadata,
section_name="Working Abstract",
)
if hypothesis_ledger:
front_fact_rows.extend(self._hypothesis_contract_violations(
working_abstract, hypothesis_ledger, section_name="Working Abstract"
))
if front_fact_rows:
feedback = self._section_retry_feedback(front_fact_rows)
logging.warning(
"Working Abstract fact validation retry | mode=%s count=%s types=%s",
mode, len(front_fact_rows), sorted({str(row.get("type") or "") for row in front_fact_rows}),
)
try:
working_front = agent.generate_working_front_matter(
reader_topic_en, reader_keywords_prompt, research_logic_contract, synthesis_result,
data_evidence=planning_data_md, mode=mode, validation_feedback=feedback,
hypothesis_contract=hypothesis_contract,
authoritative_keywords=self._user_authoritative_front_matter_keywords(data_metadata),
keyword_policy=self._resolved_front_matter_keyword_policy(data_metadata),
fallback_keywords=self._front_matter_keyword_fallback_terms(data_metadata),
)
retry_abstract = str(working_front.get("abstract") or "").strip()
if hypothesis_ledger and retry_abstract:
retry_abstract = self._normalize_hypothesis_ordinal_aliases(
retry_abstract, hypothesis_ledger
)
retry_keywords = self._canonical_quantitative_front_matter_keywords(
mode, list(working_front.get("keywords") or []), data_metadata
)
if retry_abstract and retry_keywords:
working_abstract, working_keywords = retry_abstract, retry_keywords
except PipelineCancelledError:
raise
except Exception as exc:
logging.warning(
"Working Front Matter validation retry unavailable; continuing to deterministic/adaptive fallback | error=%s",
type(exc).__name__,
)
_, remaining_front_rows = self._repair_validate_study_fact_contract(
mode, agent, quant_engine, working_abstract, data_metadata, section_name="Working Abstract",
)
if hypothesis_ledger:
remaining_front_rows.extend(self._hypothesis_contract_violations(
working_abstract, hypothesis_ledger, section_name="Working Abstract"
))
if remaining_front_rows:
working_abstract, residual_front_rows = self._adaptive_repair_front_matter(
agent=agent, quant_engine=quant_engine, abstract_text=working_abstract,
violations=remaining_front_rows, mode=mode, data_metadata=data_metadata,
study_evidence=planning_data_md, hypothesis_ledger=hypothesis_ledger,
quality_notes=quality_notes, label="Working Abstract",
)
if residual_front_rows:
logging.warning("Adaptive Working Abstract accepted with residual diagnostics | count=%s", len(residual_front_rows))
quality_diagnostics.append({
"scope":"working_abstract_study_fact_integrity", "name":"Working Abstract",
"status":"section_retry_resolved", "count":len(front_fact_rows), "examples":front_fact_rows[:3],
})
if not working_abstract or not working_keywords:
fallback = self._deterministic_working_front_matter(
mode, reader_topic_en, reader_keywords_prompt, data_metadata, hypothesis_ledger=hypothesis_ledger
)
working_abstract = str(fallback.get("abstract") or "").strip()
working_keywords = list(fallback.get("keywords") or [])
quality_diagnostics.append({
"scope": "delivery_first_working_front_matter_fallback",
"name": "Working Abstract",
"status": "deterministic_fallback_after_validation",
"reason": "empty_after_repair",
})
logging.warning(
"Delivery-first Working Front Matter fallback activated after validation repair exhaustion."
)
agent.global_memory["working_abstract"] = working_abstract
agent.global_memory["working_abstract"] = working_abstract
self._progress(14, "outline_planning")
plan = agent.generate_outline(topic, keywords, mode=mode, data_metadata=data_metadata)
# User title is reader-facing authority. Outline planning may adapt
# structure but may not silently rename the study.
en_topic = str(reader_topic_en or source_topic or topic).strip()
structure = plan.get("structure", [])
full_draft_en = (
f"# {en_topic}\n\n## Abstract\n{working_abstract}\n\n"
f"## Keywords\n{', '.join(working_keywords)}\n\n"
)
# Publish the Working Abstract/Keywords immediately, before JIT and
# citation planning, so the user sees the manuscript anchor before
# Introduction while those longer planning stages continue.
self._publish_preview(full_draft_en, "working_front_matter", force=True)
front_budget = max(180, int(APP_CONFIG.get("word_budget", {}).get("front_matter_words", 220)))
body_target_words = max(1000, target_words - front_budget)
adaptive_structure = agent.build_generation_tasks(structure, body_target_words, safe_chunk_words)
for task in adaptive_structure:
task["is_first"] = bool(task.get("section_first"))
quote_invocation_plan = (
self._build_qualitative_quote_invocation_plan(adaptive_structure, data_metadata)
if mode == "asq" else {}
)
jit_texts = agent.prefetch_jit_literature(
topic, keywords, adaptive_structure,
quantitative_evidence_ledger=(
data_metadata.get("quantitative_evidence_ledger") if mode == "amj" else None
),
)
if jit_texts:
rag_engine._initialize_chunks([jit_texts])
self._progress(15, "citation_planning")
citation_minimums, citation_reserve_targets = self._adaptive_citation_requirements(
adaptive_structure, mode
)
citation_top_level_minimums, citation_top_level_reserve_targets = self._top_level_citation_requirements(
adaptive_structure, mode
)
citation_unit_section_map = self._citation_unit_section_map(adaptive_structure)
citation_plan_target = sum(citation_top_level_reserve_targets.values())
citation_plan = agent.build_claim_evidence_plan(
adaptive_structure, topic, synthesis_result, mode=mode,
target_unique=max(1, citation_plan_target),
)
# Early neutral risk signal: evaluate the high-relevance plan before
# reserve filling/JIT can make a raw candidate count look healthier
# than the claim-specific plan actually is. This is advisory only.
initial_top_level_counts = self._top_level_unique_tag_counts(
citation_plan, citation_unit_section_map
)
initial_shortfalls = [
(section_id, int(initial_top_level_counts.get(section_id, 0)), required)
for section_id, required in citation_top_level_minimums.items()
if int(initial_top_level_counts.get(section_id, 0)) < int(required)
]
if initial_shortfalls:
self._record_evidence_coverage_event(
state, initial_shortfalls, phase="initial_claim_specific_plan",
preview_status="limited_risk",
)
# Fill unit-specific reserve gaps before expensive drafting. Existing
# retrieved evidence is preferred; bounded contextual JIT is last resort.
planned_tags = {
str(item.get("tag") or "")
for items in citation_plan.values()
for item in items
if item.get("tag")
}
task_by_name = {
str(task.get("display_name") or ""): task
for task in adaptive_structure
if str(task.get("display_name") or "")
}
for display_name, reserve_target in citation_reserve_targets.items():
planned_items = citation_plan.setdefault(display_name, [])
unit_section_id = str(task_by_name.get(display_name, {}).get("section_id") or "")
if not self._quota_driven_citation_recovery_allowed(unit_section_id):
logging.info(
"Discussion pre-draft reserve recovery skipped: unit=%s direct_plan=%s policy=claim_driven",
display_name, len(planned_items),
)
continue
shortfall = max(0, reserve_target - len(planned_items))
if shortfall <= 0:
continue
section_context = self._citation_task_context(
task_by_name.get(display_name, {}), research_logic_contract, synthesis_result
)
recovery_items = agent.select_contextual_citation_recovery_items(
[], section_name=display_name,
section_id=str(task_by_name.get(display_name, {}).get("section_id") or ""),
section_context=section_context, needed=shortfall, excluded_tags=planned_tags,
)
for item in recovery_items:
tag = str(item.get("tag") or "")
if tag and tag not in planned_tags:
planned_items.append(item)
planned_tags.add(tag)
# Do not trigger external retrieval merely because a technical
# generation chunk is below a word-derived target. Fatal evidence
# authority belongs to the canonical top-level academic section.
top_level_counts = self._top_level_unique_tag_counts(citation_plan, citation_unit_section_map)
for section_id, required in citation_top_level_minimums.items():
deficit = max(0, int(required) - int(top_level_counts.get(section_id, 0)))
if deficit <= 0:
continue
if not self._quota_driven_citation_recovery_allowed(section_id):
logging.info(
"Discussion pre-draft JIT quota top-up skipped: section=%s direct_plan=%s required=%s policy=claim_driven",
section_id, int(top_level_counts.get(section_id, 0)), int(required),
)
continue
section_units = [
name for name, mapped in citation_unit_section_map.items() if mapped == section_id
]
if not section_units:
continue
primary_unit = section_units[0]
section_context = " ".join(
self._citation_task_context(task_by_name.get(name, {}), research_logic_contract, synthesis_result)
for name in section_units
)
jit_items = agent.fetch_contextual_jit_recovery_items(
topic=topic, section_name=section_id, section_id=section_id,
section_context=section_context, needed=deficit, excluded_tags=planned_tags,
)
planned_items = citation_plan.setdefault(primary_unit, [])
for item in jit_items:
tag = str(item.get("tag") or "")
if tag and tag not in planned_tags:
planned_items.append(item)
planned_tags.add(tag)
top_level_counts = self._top_level_unique_tag_counts(citation_plan, citation_unit_section_map)
shortfalls = [
(section_id, int(top_level_counts.get(section_id, 0)), required)
for section_id, required in citation_top_level_minimums.items()
if int(top_level_counts.get(section_id, 0)) < int(required)
]
if shortfalls:
self._record_evidence_coverage_event(
state, shortfalls, phase="bounded_retrieval_complete",
preview_status="limited_risk",
)
logging.warning(
"Top-level citation feasibility limited: minimum_total=%s reserve_target=%s planned_unique=%s sections=%s nonfatal=true",
sum(citation_top_level_minimums.values()), citation_plan_target, len(planned_tags), len(citation_top_level_minimums),
)
else:
logging.info(
"Top-level citation feasibility passed: minimum_total=%s reserve_target=%s planned_unique=%s sections=%s",
sum(citation_top_level_minimums.values()), citation_plan_target, len(planned_tags), len(citation_top_level_minimums),
)
logging.info(
"Top-level citation planning complete: minimum_total=%s reserve_target=%s planned_unique=%s sections=%s",
sum(citation_top_level_minimums.values()), citation_plan_target, len(planned_tags), len(citation_top_level_minimums),
)
canonical_concept_ledger = self._build_canonical_concept_ledger(
mode, data_metadata, keywords=keywords, adaptive_structure=adaptive_structure,
research_logic_contract=research_logic_contract,
)
state.update(
{
"adaptive_structure": adaptive_structure,
"quote_invocation_plan": quote_invocation_plan,
"citation_plan": citation_plan,
"citation_minimums": citation_minimums,
"citation_reserve_targets": citation_reserve_targets,
"citation_top_level_minimums": citation_top_level_minimums,
"citation_top_level_reserve_targets": citation_top_level_reserve_targets,
"citation_unit_section_map": citation_unit_section_map,
"citation_validated_tags_by_unit": {},
"citation_required_claim_counts_by_unit": {},
"section_integrity_seals": {},
"english_section_seal_version": ENGLISH_SECTION_SEAL_VERSION,
"research_logic_contract": research_logic_contract,
"working_abstract": working_abstract,
"working_keywords": working_keywords,
"front_matter_phase": "working",
"body_target_words": body_target_words,
"full_draft_en": full_draft_en,
"full_draft_translated": "",
"translation_sections": {},
"translation_glossary": {},
"canonical_concept_ledger": canonical_concept_ledger,
"completed_sections": [],
"context_tail": "",
"accumulated_deviation": 0.0,
"agent_state": agent.export_state(),
"outline_ready": True,
}
)
self._publish_preview(full_draft_en, "outline_ready", force=True)
self._checkpoint("outline_ready", state, 15)
else:
full_draft_en = str(state.get("full_draft_en") or "")
if str(data_metadata.get("source_type") or "") == "system_generated" and mode == "amj":
full_draft_en = self._replace_title(full_draft_en, reader_topic_en)
canonical_resume_keywords = self._canonical_quantitative_front_matter_keywords(
mode, [], data_metadata
)
full_draft_en = self._replace_keywords(full_draft_en, canonical_resume_keywords)
state["full_draft_en"] = full_draft_en
state["working_keywords"] = canonical_resume_keywords
if state.get("final_keywords") is not None:
state["final_keywords"] = canonical_resume_keywords
# JIT text itself is already represented in agent citation state;
# refs + user literature rebuild a safe RAG baseline on resume.
completed_sections = list(state.get("completed_sections") or [])
context_tail = str(state.get("context_tail") or "")
accumulated_deviation = float(state.get("accumulated_deviation") or 0.0)
research_logic_contract = str(
state.get("research_logic_contract") or agent.global_memory.get("research_logic_contract") or ""
)
resumed_logic_rows = []
if hypothesis_ledger and research_logic_contract:
research_logic_contract = self._normalize_hypothesis_ordinal_aliases(
research_logic_contract, hypothesis_ledger
)
resumed_logic_rows = self._hypothesis_contract_violations(
research_logic_contract, hypothesis_ledger, section_name="Research Logic Contract"
)
if not research_logic_contract or resumed_logic_rows:
research_logic_contract = self._deterministic_research_logic_contract(
mode, topic, keywords, data_metadata, hypothesis_ledger=hypothesis_ledger
)
fallback_rows = (
self._hypothesis_contract_violations(
research_logic_contract, hypothesis_ledger, section_name="Research Logic Contract"
) if hypothesis_ledger else []
)
if fallback_rows:
raise RuntimeError(
f"delivery_first_research_logic_fallback_invalid_on_resume:{fallback_rows[:3]}"
)
quality_diagnostics.append({
"scope": "delivery_first_research_logic_fallback",
"name": "Research Logic Contract",
"status": "deterministic_resume_fallback",
"reason": "missing_contract" if not str(state.get("research_logic_contract") or "").strip() else "hypothesis_identity_mismatch",
"prior_violations": resumed_logic_rows[:3],
})
state["research_logic_contract"] = research_logic_contract
state["quality_diagnostics"] = quality_diagnostics
logging.warning(
"Delivery-first Research Logic resume fallback activated | prior_violations=%s",
len(resumed_logic_rows),
)
agent.global_memory["research_logic_contract"] = research_logic_contract
citation_plan = state.get("citation_plan") if isinstance(state.get("citation_plan"), dict) else {}
citation_minimums = dict(state.get("citation_minimums") or {})
citation_reserve_targets = dict(state.get("citation_reserve_targets") or {})
citation_top_level_minimums = dict(state.get("citation_top_level_minimums") or {})
citation_top_level_reserve_targets = dict(state.get("citation_top_level_reserve_targets") or {})
citation_unit_section_map = dict(state.get("citation_unit_section_map") or {})
citation_required_claim_counts_by_unit = {
str(name): int(count or 0)
for name, count in dict(state.get("citation_required_claim_counts_by_unit") or {}).items()
}
citation_validated_tags_by_unit = {
str(name): list(tags or [])
for name, tags in dict(state.get("citation_validated_tags_by_unit") or {}).items()
}
section_integrity_seals = {
str(key): dict(record or {})
for key, record in dict(state.get("section_integrity_seals") or {}).items()
if isinstance(record, dict)
}
if not citation_minimums:
citation_minimums, citation_reserve_targets = self._adaptive_citation_requirements(
adaptive_structure, mode
)
if not citation_top_level_minimums:
citation_top_level_minimums, citation_top_level_reserve_targets = self._top_level_citation_requirements(
adaptive_structure, mode
)
if not citation_unit_section_map:
citation_unit_section_map = self._citation_unit_section_map(adaptive_structure)
translation_sections = dict(state.get('translation_sections') or {})
translation_glossary = dict(state.get('translation_glossary') or {})
if mode == "asq":
appendix_record = dict(translation_sections.get("appendix") or {})
if appendix_record and appendix_record.get("qualitative_appendix_binding_version") != qual_engine.TARGET_APPENDIX_BINDING_VERSION:
logging.info("Invalidating pre-R5 qualitative Appendix translation record; deterministic binding required.")
translation_sections.pop("appendix", None)
state["translation_sections"] = dict(translation_sections)
canonical_concept_ledger = state.get('canonical_concept_ledger')
if not isinstance(canonical_concept_ledger, dict) or not canonical_concept_ledger.get('items'):
canonical_concept_ledger = self._build_canonical_concept_ledger(
mode, data_metadata, keywords=keywords, adaptive_structure=adaptive_structure,
research_logic_contract=research_logic_contract,
)
state['canonical_concept_ledger'] = canonical_concept_ledger
working_abstract = str(state.get("working_abstract") or "").strip()
working_keywords = self._canonical_quantitative_front_matter_keywords(
mode, list(state.get("working_keywords") or []), data_metadata
)
if working_keywords:
state["working_keywords"] = working_keywords
if working_abstract and not state.get("english_finalized"):
if hypothesis_ledger:
working_abstract = self._normalize_hypothesis_ordinal_aliases(
working_abstract, hypothesis_ledger
)
_, resumed_front_rows = self._repair_validate_study_fact_contract(
mode, agent, quant_engine, working_abstract, data_metadata, section_name="Working Abstract",
)
if hypothesis_ledger:
resumed_front_rows.extend(self._hypothesis_contract_violations(
working_abstract, hypothesis_ledger, section_name="Working Abstract"
))
if resumed_front_rows:
feedback = self._section_retry_feedback(resumed_front_rows)
try:
working_front = agent.generate_working_front_matter(
reader_topic_en, reader_keywords_prompt, research_logic_contract, synthesis_result,
data_evidence=planning_data_md, mode=mode, validation_feedback=feedback,
hypothesis_contract=hypothesis_contract,
authoritative_keywords=self._user_authoritative_front_matter_keywords(data_metadata),
keyword_policy=self._resolved_front_matter_keyword_policy(data_metadata),
fallback_keywords=self._front_matter_keyword_fallback_terms(data_metadata),
)
candidate_abstract = str(working_front.get("abstract") or "").strip()
if hypothesis_ledger and candidate_abstract:
candidate_abstract = self._normalize_hypothesis_ordinal_aliases(
candidate_abstract, hypothesis_ledger
)
candidate_keywords = self._canonical_quantitative_front_matter_keywords(
mode, list(working_front.get("keywords") or []), data_metadata
)
except PipelineCancelledError:
raise
except Exception as exc:
logging.warning(
"Resumed Working Front Matter repair unavailable; using deterministic fallback | error=%s",
type(exc).__name__,
)
fallback = self._deterministic_working_front_matter(
mode, reader_topic_en, reader_keywords_prompt, data_metadata, hypothesis_ledger=hypothesis_ledger
)
candidate_abstract = str(fallback.get("abstract") or "").strip()
candidate_keywords = list(fallback.get("keywords") or [])
if not candidate_abstract or not candidate_keywords:
fallback = self._deterministic_working_front_matter(
mode, reader_topic_en, reader_keywords_prompt, data_metadata, hypothesis_ledger=hypothesis_ledger
)
candidate_abstract = str(fallback.get("abstract") or "").strip()
candidate_keywords = list(fallback.get("keywords") or [])
_, remaining = self._repair_validate_study_fact_contract(
mode, agent, quant_engine, candidate_abstract, data_metadata, section_name="Working Abstract",
)
if hypothesis_ledger:
remaining.extend(self._hypothesis_contract_violations(
candidate_abstract, hypothesis_ledger, section_name="Working Abstract"
))
if remaining:
candidate_abstract, residual_front_rows = self._adaptive_repair_front_matter(
agent=agent, quant_engine=quant_engine, abstract_text=candidate_abstract,
violations=remaining, mode=mode, data_metadata=data_metadata,
study_evidence=planning_data_md, hypothesis_ledger=hypothesis_ledger,
quality_notes=quality_notes, label="Working Abstract",
)
if residual_front_rows:
logging.warning("Adaptive resumed Working Abstract accepted with residual diagnostics | count=%s", len(residual_front_rows))
if not candidate_abstract:
fallback = self._deterministic_working_front_matter(
mode, reader_topic_en, reader_keywords_prompt, data_metadata, hypothesis_ledger=hypothesis_ledger
)
candidate_abstract = str(fallback.get("abstract") or "").strip()
candidate_keywords = list(fallback.get("keywords") or candidate_keywords)
working_abstract = candidate_abstract
working_keywords = candidate_keywords or working_keywords
state["working_abstract"] = working_abstract
state["working_keywords"] = working_keywords
if str(state.get("full_draft_en") or "").strip():
state["full_draft_en"] = replace_abstract(str(state.get("full_draft_en") or ""), working_abstract)
state["full_draft_en"] = self._replace_keywords(state["full_draft_en"], working_keywords)
full_draft_en = str(state["full_draft_en"] or "")
quality_diagnostics.append({
"scope":"working_abstract_study_fact_integrity", "name":"Working Abstract",
"status":"resume_section_retry_resolved", "count":len(resumed_front_rows), "examples":resumed_front_rows[:3],
})
state["quality_diagnostics"] = quality_diagnostics
self._checkpoint("working_front_matter_fact_retry_resolved", state, 15)
if working_abstract:
agent.global_memory["working_abstract"] = working_abstract
if self._preview_target_text == '':
self._preview_target_text = str(state.get('full_draft_translated') or '')
# S0-R1 compatibility migration: old completed English is never silently
# repaired. Unsealed sections are inspected only. Valid text receives a
# seal; invalid text is removed from the completed set so only that owning
# top-level section is regenerated by the normal Section Writer.
seal_migration = False
seal_text_changed = False
retry_section_keys = set()
for section_key, section_title in self._unique_top_level_sections(adaptive_structure):
if not self._section_is_complete(adaptive_structure, completed_sections, section_key):
continue
section_text = self._extract_top_level_section(full_draft_en, section_title)
if not section_text:
continue
existing_seal = dict(section_integrity_seals.get(section_key) or {})
if (
existing_seal.get("version") == ENGLISH_SECTION_SEAL_VERSION
and existing_seal.get("sha256") == self._section_integrity_hash(section_text)
):
continue
violations = self._study_fact_violations(
mode, agent, quant_engine, section_text, data_metadata,
section_name=f"Resume sealed section: {section_title}",
)
try:
agent.validate_enumeration_contract(
section_text, section_name=f"Resume sealed section: {section_title}"
)
except Exception as exc:
violations.append({
"type":"enumeration_contract_invalid",
"reason":str(exc)[:500], "preview":section_title,
})
ai_meta_rows = list(agent.inspect_authorial_ai_meta(section_text) or [])
violations.extend(ai_meta_rows)
if violations:
retry_section_keys.add(str(section_key))
translation_sections.pop(section_key, None)
section_integrity_seals.pop(section_key, None)
quality_diagnostics.append({
"scope":"english_section_seal_migration", "name":section_title,
"status":"local_section_retry_required", "count":len(violations),
"examples":violations[:3],
})
continue
section_integrity_seals[section_key] = self._build_section_integrity_seal(
section_key, section_title, section_text
)
seal_migration = True
if retry_section_keys:
# Remove only invalid unsealed top-level sections and their completed
# unit markers. Other completed English stays byte-stable.
for section_key, section_title in self._unique_top_level_sections(adaptive_structure):
if str(section_key) not in retry_section_keys:
continue
if self._extract_top_level_section(full_draft_en, section_title):
full_draft_en = self._remove_top_level_section(full_draft_en, section_title)
seal_text_changed = True
relevant_names = {
str(task.get('display_name') or '')
for task in adaptive_structure
if str(task.get('section_id') or task.get('section') or '') == str(section_key)
}
completed_sections = [name for name in completed_sections if name not in relevant_names]
seal_migration = True
if seal_migration:
# Rebuild mutable manuscript memory only from still-completed sealed English text.
for memory_key in ("hypotheses", "variables", "model_summary", "method_summary"):
agent.global_memory[memory_key] = ""
last_sealed = ""
for section_key, section_title in self._unique_top_level_sections(adaptive_structure):
if not self._section_is_complete(adaptive_structure, completed_sections, section_key):
continue
sealed_text = self._extract_top_level_section(full_draft_en, section_title)
if sealed_text:
agent._extract_memory_from_text(sealed_text, section_title)
last_sealed = sealed_text
if last_sealed:
context_tail = " ".join(last_sealed.split()[-200:])
state.update({
"completed_sections": completed_sections,
"full_draft_en": full_draft_en,
"translation_sections": translation_sections,
"section_integrity_seals": section_integrity_seals,
"english_section_seal_version": ENGLISH_SECTION_SEAL_VERSION,
"context_tail": context_tail,
"quality_diagnostics": quality_diagnostics,
"agent_state": agent.export_state(),
})
if seal_text_changed:
state["full_draft_translated"] = self._assemble_partial_translation(
translation_sections, adaptive_structure, include_front=True
)
self._checkpoint("english_section_seal_migrated", state, 15)
# Target-language work remains Executor-owned but no longer blocks the
# English writer. A dedicated single translation agent/worker keeps
# glossary order deterministic and avoids sharing mutable writer state.
translation_agent = SSciPipelineAgent(ds_key, s2_key)
translation_agent.set_model_policy_snapshot(plan_policy_snapshot)
if trusted_task_policy_snapshot is not None:
translation_agent.set_trusted_product_policy_snapshot(trusted_task_policy_snapshot)
self.translation_agent = translation_agent
translation_agent.set_execution_progress_hook(lambda: self._progress_activity())
translation_agent.set_cancel_checker(lambda: self.cancel_checker())
seeded_glossary = translation_agent.seed_canonical_concept_glossary(
canonical_concept_ledger, target_lang, translation_glossary
)
if seeded_glossary != translation_glossary:
translation_glossary = dict(seeded_glossary)
state['translation_glossary'] = translation_glossary
state['canonical_concept_ledger'] = canonical_concept_ledger
self._checkpoint('canonical_concept_ledger_ready', state, 15)
glossary_quality_version = str(
getattr(translation_agent, 'CANONICAL_GLOSSARY_QUALITY_VERSION', 'canonical-glossary-quality-v1')
or 'canonical-glossary-quality-v1'
)
if state.get('canonical_glossary_quality_version') != glossary_quality_version:
reviewed_glossary = translation_agent.validate_and_refine_canonical_concept_glossary(
canonical_concept_ledger, target_lang, translation_glossary
)
if reviewed_glossary != translation_glossary:
translation_glossary = dict(reviewed_glossary)
# A corrected canonical identity invalidates only target-language
# work. Sealed English remains authoritative and unchanged.
translation_sections = {}
state['translation_sections'] = {}
state['full_draft_translated'] = ''
state['translation_complete'] = False
state['package_validated'] = False
state['translation_glossary'] = translation_glossary
state['canonical_glossary_quality_version'] = glossary_quality_version
self._checkpoint('canonical_glossary_quality_ready', state, 15)
current_entity_ledger_version = str(
getattr(translation_agent, 'PROTECTED_ENTITY_LEDGER_VERSION', 'protected-entity-ledger-v1')
or 'protected-entity-ledger-v1'
)
stale_entity_units = [
key for key, record in translation_sections.items()
if isinstance(record, dict)
and record.get('translated_text')
and record.get('qa_status') in {'passed', 'passed_with_warning'}
and record.get('protected_entity_ledger_version') != current_entity_ledger_version
]
if stale_entity_units:
logging.warning(
'Protected Entity Ledger contract advanced; invalidating target-language cache only | units=%s version=%s',
stale_entity_units[:8], current_entity_ledger_version,
)
state['translation_complete'] = False
state['package_validated'] = False
state['full_draft_translated'] = ''
self._preview_target_text = ''
self._checkpoint('protected_entity_ledger_advanced', state, 15)
self._start_translation_worker(
translation_agent,
translation_sections=translation_sections,
translation_glossary=translation_glossary,
adaptive_structure=adaptive_structure,
target_lang=target_lang,
)
# Capture a dedicated abort Event, not the worker lifecycle stop Event.
# The worker is gracefully closed before the strict final translation
# gate, but the same translation agent remains valid for deferred/final
# units. Abortive shutdown still cancels any in-flight target call.
translation_abort_event = self._translation_worker.cancel_event
translation_agent.set_cancel_checker(
lambda: self.cancel_checker() or translation_abort_event.is_set()
)
# Backward-compatible resume path for a pre-R3.2.8.3 checkpoint that has
# an outline but no Working Abstract yet. Generate it once, checkpoint it,
# and reuse it thereafter; never regenerate an unchanged anchor on resume.
if not working_abstract and not state.get("english_finalized"):
try:
working_front = agent.generate_working_front_matter(
reader_topic_en, reader_keywords_prompt, research_logic_contract, synthesis_result,
data_evidence=planning_data_md, mode=mode,
hypothesis_contract=hypothesis_contract,
authoritative_keywords=self._user_authoritative_front_matter_keywords(data_metadata),
keyword_policy=self._resolved_front_matter_keyword_policy(data_metadata),
fallback_keywords=self._front_matter_keyword_fallback_terms(data_metadata),
)
except PipelineCancelledError:
raise
except Exception as exc:
logging.warning(
"Migrated Working Front Matter provider path unavailable; using deterministic fallback | error=%s",
type(exc).__name__,
)
working_front = self._deterministic_working_front_matter(
mode, reader_topic_en, reader_keywords_prompt, data_metadata, hypothesis_ledger=hypothesis_ledger
)
quality_diagnostics.append({
"scope": "delivery_first_working_front_matter_fallback",
"name": "Working Abstract",
"status": "deterministic_resume_migration_fallback",
"reason": "provider_error",
})
working_abstract = str(working_front.get("abstract") or "").strip()
if hypothesis_ledger and working_abstract:
working_abstract = self._normalize_hypothesis_ordinal_aliases(
working_abstract, hypothesis_ledger
)
working_keywords = self._canonical_quantitative_front_matter_keywords(
mode, list(working_front.get("keywords") or []), data_metadata
)
if not working_abstract or not working_keywords:
working_front = self._deterministic_working_front_matter(
mode, reader_topic_en, reader_keywords_prompt, data_metadata, hypothesis_ledger=hypothesis_ledger
)
working_abstract = str(working_front.get("abstract") or "").strip()
working_keywords = list(working_front.get("keywords") or [])
quality_diagnostics.append({
"scope": "delivery_first_working_front_matter_fallback",
"name": "Working Abstract",
"status": "deterministic_resume_migration_fallback",
"reason": "missing_component",
})
_, front_fact_rows = self._repair_validate_study_fact_contract(
mode, agent, quant_engine, working_abstract, data_metadata, section_name="Working Abstract",
)
if hypothesis_ledger:
front_fact_rows.extend(self._hypothesis_contract_violations(
working_abstract, hypothesis_ledger, section_name="Working Abstract"
))
if front_fact_rows:
feedback = self._section_retry_feedback(front_fact_rows)
try:
working_front = agent.generate_working_front_matter(
reader_topic_en, reader_keywords_prompt, research_logic_contract, synthesis_result,
data_evidence=planning_data_md, mode=mode, validation_feedback=feedback,
hypothesis_contract=hypothesis_contract,
authoritative_keywords=self._user_authoritative_front_matter_keywords(data_metadata),
keyword_policy=self._resolved_front_matter_keyword_policy(data_metadata),
fallback_keywords=self._front_matter_keyword_fallback_terms(data_metadata),
)
retry_abstract = str(working_front.get("abstract") or "").strip()
if hypothesis_ledger and retry_abstract:
retry_abstract = self._normalize_hypothesis_ordinal_aliases(
retry_abstract, hypothesis_ledger
)
retry_keywords = self._canonical_quantitative_front_matter_keywords(
mode, list(working_front.get("keywords") or []), data_metadata
)
if retry_abstract and retry_keywords:
working_abstract, working_keywords = retry_abstract, retry_keywords
except PipelineCancelledError:
raise
except Exception as exc:
logging.warning(
"Migrated Working Front Matter validation retry unavailable; preserving deterministic-safe candidate | error=%s",
type(exc).__name__,
)
_, remaining = self._repair_validate_study_fact_contract(
mode, agent, quant_engine, working_abstract, data_metadata, section_name="Working Abstract",
)
if hypothesis_ledger:
remaining.extend(self._hypothesis_contract_violations(
working_abstract, hypothesis_ledger, section_name="Working Abstract"
))
if remaining:
working_abstract, residual_front_rows = self._adaptive_repair_front_matter(
agent=agent, quant_engine=quant_engine, abstract_text=working_abstract,
violations=remaining, mode=mode, data_metadata=data_metadata,
study_evidence=planning_data_md, hypothesis_ledger=hypothesis_ledger,
quality_notes=quality_notes, label="Working Abstract",
)
if residual_front_rows:
logging.warning("Adaptive migrated Working Abstract accepted with residual diagnostics | count=%s", len(residual_front_rows))
quality_diagnostics.append({
"scope":"working_abstract_study_fact_integrity", "name":"Working Abstract",
"status":"section_retry_resolved", "count":len(front_fact_rows), "examples":front_fact_rows[:3],
})
if not working_abstract or not working_keywords:
working_front = self._deterministic_working_front_matter(
mode, reader_topic_en, reader_keywords_prompt, data_metadata, hypothesis_ledger=hypothesis_ledger
)
working_abstract = str(working_front.get("abstract") or "").strip()
working_keywords = list(working_front.get("keywords") or [])
logging.warning("Delivery-first migrated Working Front Matter fallback activated after repair exhaustion.")
agent.global_memory["working_abstract"] = working_abstract
full_draft_en = replace_abstract(full_draft_en, working_abstract)
full_draft_en = self._replace_keywords(full_draft_en, working_keywords)
state.update({
"working_abstract": working_abstract,
"working_keywords": working_keywords,
"front_matter_phase": "working",
"full_draft_en": full_draft_en,
"quality_diagnostics": quality_diagnostics,
"agent_state": agent.export_state(),
})
self._publish_preview(full_draft_en, "working_front_matter", force=True)
self._checkpoint("working_front_matter_ready", state, 15)
# Queue the QA-frozen Working front matter. Translation is allowed to
# overlap later English work, but only frozen English units may enter
# this Executor-owned ordered worker.
if working_abstract and not state.get("english_finalized"):
section_order_for_front = self._unique_top_level_sections(adaptive_structure)
if section_order_for_front:
first_section_title = section_order_for_front[0][1]
working_front_source = self._extract_front_matter(full_draft_en, first_section_title)
if not working_front_source and not completed_sections:
# Fresh tasks contain only Title/Abstract/Keywords here; the
# first body heading does not exist yet. Preserve legacy
# resume behavior when completed body sections are present.
working_front_source = str(full_draft_en or '').strip()
self._enqueue_translation_unit(
translation_agent,
unit_key='__front_matter__', unit_name='Working Front Matter',
source_text=working_front_source,
translation_sections=translation_sections,
english_commit_text=working_front_source,
)
# Resume/idempotency: completed English top-level sections whose current
# source-hash translation is absent/stale are queued exactly once.
for section_key, section_title in self._unique_top_level_sections(adaptive_structure):
if not self._section_is_complete(adaptive_structure, completed_sections, section_key):
continue
frozen_section = self._extract_top_level_section(full_draft_en, section_title)
self._enqueue_translation_unit(
translation_agent,
unit_key=section_key,
unit_name=section_title,
source_text=frozen_section,
translation_sections=translation_sections,
english_commit_text=self._english_prefix_through_top_level(full_draft_en, section_title),
)
translation_sections, translation_glossary = self._collect_translation_results(
translation_sections, translation_glossary, block=False
)
if not citation_plan:
self._progress(15, "citation_planning")
citation_plan = agent.build_claim_evidence_plan(
adaptive_structure, topic, synthesis_result, mode=mode,
target_unique=max(1, sum(citation_top_level_reserve_targets.values())),
)
top_level_counts = self._top_level_unique_tag_counts(citation_plan, citation_unit_section_map)
shortfalls = [
(section_id, int(top_level_counts.get(section_id, 0)), required)
for section_id, required in citation_top_level_minimums.items()
if int(top_level_counts.get(section_id, 0)) < int(required)
]
if shortfalls:
self._record_evidence_coverage_event(
state, shortfalls, phase="resume_claim_specific_plan",
preview_status="limited_risk",
)
state["citation_plan"] = citation_plan
state["citation_minimums"] = citation_minimums
state["citation_reserve_targets"] = citation_reserve_targets
state["citation_top_level_minimums"] = citation_top_level_minimums
state["citation_top_level_reserve_targets"] = citation_top_level_reserve_targets
state["citation_unit_section_map"] = citation_unit_section_map
state["citation_required_claim_counts_by_unit"] = citation_required_claim_counts_by_unit
state["citation_validated_tags_by_unit"] = citation_validated_tags_by_unit
quote_invocation_plan = (
dict(state.get("quote_invocation_plan") or {}) if mode == "asq" else {}
)
if mode == "asq":
if not quote_invocation_plan:
quote_invocation_plan = self._build_qualitative_quote_invocation_plan(
adaptive_structure, data_metadata
)
state["quote_invocation_plan"] = quote_invocation_plan
self._checkpoint("quote_plan_ready", state, 15)
self._validate_qualitative_quote_invocation_plan(
quote_invocation_plan, adaptive_structure, data_metadata
)
hypothesis_ledger = dict(data_metadata.get("hypothesis_ledger") or {}) if mode == "amj" else {}
total_steps = len(adaptive_structure)
for sec in adaptive_structure:
display_name = sec["display_name"]
if display_name in completed_sections:
continue
if self.cancel_checker():
raise RuntimeError("task_cancel_requested")
remaining = sum(1 for item in adaptive_structure if item["display_name"] not in completed_sections)
adjusted_target = self._bounded_target(
int(sec["target_words"]), 0.0, remaining, safe_chunk_words
)
sec_title = sec["section"]
subsection = sec.get("subsection")
section_id = str(sec.get("section_id") or "")
is_quarantined = self._section_forbids_external_citations(sec_title, section_id)
section_logical_node = self._section_logical_node(
mode, section_id, sec.get("subsection_id")
)
planned_items = [
item for item in list(citation_plan.get(display_name, []))
if agent.citation_source_role_eligible(
str(item.get("tag") or ""), section_id=section_id, section_name=display_name
)
]
citation_plan[display_name] = copy.deepcopy(planned_items)
assigned_tag_list = [str(item.get("tag") or "") for item in planned_items if item.get("tag")]
claim_plan_text = agent.format_claim_evidence_plan(planned_items)
section_role = str(
APP_CONFIG.get("section_reasoning_profiles", {}).get(mode, {}).get(section_id) or ""
)
mode_directive = ""
if section_id in {"theory", "theoretical_framing"}:
mode_directive = str(APP_CONFIG.get("hypothesis_development_directive") or "")
elif section_id in {"methodology", "methods"}:
mode_directive = str(APP_CONFIG.get("methodological_defense_directive") or "")
elif section_id == "discussion":
mode_directive = str(APP_CONFIG.get("discussion_depth_directive") or "")
elif section_id == "conclusion":
mode_directive = str(APP_CONFIG.get("conclusion_anchor_directive") or "")
if is_quarantined:
citation_rule = (
"NO EXTERNAL LITERATURE CITATIONS ALLOWED IN THIS UNIT. "
"Report only documented empirical/qualitative evidence and the interpretation permitted by the section role."
)
else:
citation_rule = (
"Use the citation-first claim-evidence plan below as bounded evidence options, not as a claim quota. "
"Do not create a new claim merely to consume an assigned citation. Use a source only when it directly supports a claim independently required by the research logic, section role, or already established manuscript argument; omit any item that would introduce another application domain or mutate the study's facts. "
"When a source is used, keep the claim no broader than the supplied evidence and attach exactly its assigned [CITE-XX] tag. "
"Never stack sources, substitute tags, or use literature as evidence for this study's own facts.\n"
f"\n{claim_plan_text}\n"
)
if not claim_plan_text.strip():
citation_rule += (
"\nEVIDENCE-LIMITED WRITING RULE: no directly validated source is assigned to this unit. "
"Continue the scholarly argument using the documented study evidence, cautious conceptual reasoning, and general scholarly knowledge. "
"Do not invent citations, authors, titles, DOI values, or bibliographic metadata; do not describe general model knowledge as if it were retrieved or directly validated literature. "
"Use appropriately qualified language for propositions, interpretations, and research opportunities."
)
if mode == "asq" and section_id not in {"findings", "appendix"}:
citation_rule += (
"\nASQ EVIDENCE-CHANNEL RULE: A sentence whose factual subject is this study's setting, participants, interviews, data sources, cases, actors, observations, events, quotations, analytic steps, or empirical findings must not carry an external [CITE-XX] tag. "
"Place any prior-literature or methodological-rationale claim in a separate sentence and cite only that external claim."
)
if section_id not in {"results", "findings", "appendix"}:
citation_rule += (
"\nCURRENT-STUDY PROVENANCE RULE: Do not attach external literature citations to this study's own research question, H1-Hn statement, sample/design fact, analytic choice, result, limitation specific to the available study evidence, or author positioning. "
"If prior literature provides a rationale, state that external rationale in a separate sentence and cite only that sentence."
)
if section_id == "discussion":
citation_rule += (
"\nDISCUSSION EVIDENCE-LIMITED CONTRACT: Distinguish prior-literature claims from study-owned interpretation. "
"Statements such as 'the present evidence suggests', 'these findings indicate', or 'the present results suggest' are interpretations of this study and must not borrow an external citation. "
"Do not create or broaden an external-literature claim merely to consume a citation or satisfy a citation count. "
"When claim-specific literature is sparse, develop the Discussion from documented study evidence, cautious conceptual reasoning, boundary conditions, and limitations. "
"If comparing with prior literature, put the directly supported prior-literature proposition in its own cited sentence and the study-owned interpretation in a separate uncited sentence."
)
if section_id not in {"results", "findings", "appendix"}:
citation_rule += (
"\nATOMIC LITERATURE CLAIM CONTRACT: Write one externally verifiable proposition per sentence. "
"Do not combine two literature-dependent findings, mechanisms, definitions, or outcome claims under one citation. "
"When two propositions require different sources, write two sentences and attach each sentence only to its own assigned source. "
"A current-study fact or H1-Hn/RQ statement must remain citation-free unless a separate prior-literature rationale is stated in its own sentence."
)
if bool(state.get("evidence_risk_detected")) and section_id not in {"results", "findings", "appendix"}:
citation_rule += (
"\nEVIDENCE-LIMITED CLAIM-DENSITY CONTRACT: Direct high-relevance literature coverage is limited. Treat assigned claim-evidence items as the ceiling for source-dependent empirical, definitional, prevalence, and field-gap claims in this unit. "
"Do not invent additional literature claims merely to make the section sound comprehensive. Prefer concise study positioning, cautious conceptual reasoning, and clearly qualified interpretations when no directly validated source is assigned."
)
memory_context = self._strip_unscoped_citation_identities(
agent._build_memory_context(sec_title, 2500)
)
memory_block = f"\n\n{memory_context}\n\n" if memory_context else ""
snippets = self._section_citation_capability_context(agent, planned_items)
if snippets:
memory_block += f"\n\n{snippets}\n\n"
heading_directive = "Do not output the section heading; it will be inserted by the system."
if subsection:
heading_directive += (
f" Do not output the subsection heading '{subsection}'; it will also be inserted by the system. "
"Use level-3 headings only when analytically necessary."
)
elif sec_title.lower() in {"introduction", "conclusion"}:
heading_directive += " Do not use additional headings."
else:
heading_directive += (
" Use level-2 or level-3 headings only when they clarify the argument; "
"never use level-4 headings or numbered headings."
)
if mode == "amj" and section_id == "results" and data_metadata.get("quantitative_table_manifest"):
table_rule = (
"AUTHORITATIVE TABLE CONTRACT: Do not create, reproduce, renumber, or alter any Markdown statistical table. "
"Required statistical tables are handled separately from narrative prose. "
f"You may discuss and refer only to the frozen table numbers {[int(row.get('number') or 0) for row in data_metadata.get('quantitative_table_manifest', {}).get('tables', []) if int(row.get('number') or 0) > 0]}; do not invent an additional table or change a table number."
)
elif sec["is_method_or_result"] and mode == "amj":
table_rule = "Present only study-supported statistical evidence; do not invent tables or numerical results."
else:
table_rule = ""
concepts = ", ".join(sec.get("concepts") or [])
theme_evidence = [item for item in (sec.get("theme_evidence") or []) if isinstance(item, dict)]
evidence_profiles = []
for item in theme_evidence:
stage = str(item.get("stage") or "Not specified").strip()
role = str(item.get("evidence_role") or "typical").strip()
case_label = str(item.get("case_label") or "Not specified").strip()
evidence_profiles.append(f"stage={stage}; case={case_label}; role={role}")
planned_quote_count = sum(
1 for record in quote_invocation_plan.values()
if str(record.get("render_task_key") or "") == self._qualitative_task_key(sec)
)
qualitative_rule = (
f"Assigned qualitative concepts: {concepts}. Assigned evidence profiles: {' | '.join(evidence_profiles)}. "
f"This unit has {planned_quote_count} assigned quotation slot(s). "
"Use the assigned evidence analytically, but output narrative prose only: do not reproduce any verbatim quotation and do not output any QUOTE-XX identifier. "
"Preserve stage/case/deviant meaning and use deviant evidence to qualify rather than erase the pattern."
if concepts or evidence_profiles else ""
)
scoped_data_md = user_data_md
if mode == "asq":
if section_id == "appendix":
# The appendix is the only non-owning unit allowed to see
# the full frozen evidence table. It is excluded from the
# body duplicate-quote gate by design.
scoped_data_md = user_data_md
elif section_id == "findings":
current_theme = str(sec.get("theme_title") or "").strip()
scoped_data_md = qual_engine.build_scoped_evidence_md(
data_metadata,
include_themes=True,
theme_titles=[current_theme] if current_theme else None,
assigned_evidence=theme_evidence,
include_quotes=bool(theme_evidence),
include_quote_ids=False,
)
elif section_id == "discussion":
scoped_data_md = qual_engine.build_scoped_evidence_md(
data_metadata,
include_themes=True,
)
else:
scoped_data_md = qual_engine.build_scoped_evidence_md(
data_metadata,
include_themes=False,
)
writer_base_evidence_contract = self._writer_visible_evidence_context(base_evidence_contract)
writer_scoped_data_md = self._writer_visible_evidence_context(scoped_data_md)
evidence_packet = (
f"{writer_base_evidence_contract}\n\n{writer_scoped_data_md}\n"
if writer_scoped_data_md else writer_base_evidence_contract
)
completed_label = ", ".join(completed_sections)
anti_repeat = (
f"Already completed: {completed_label}. Do not restate their background, gap, or contribution."
if completed_label else ""
)
if re.search(r"\(Part\s+2/2\)", display_name, flags=re.I):
anti_repeat += " Part 2/2 must extend and complement Part 1/2; do not recap its background, mechanism explanation, evidence, or contribution framing."
voice = str(APP_CONFIG.get("journal_voice_profiles", {}).get(mode) or "")
section_role = self._writer_visible_evidence_context(section_role)
mode_directive = self._writer_visible_evidence_context(mode_directive)
research_logic_for_writer = self._writer_visible_evidence_context(research_logic_contract)
working_abstract_for_writer = self._writer_visible_evidence_context(working_abstract)
sys_prompt = (
APP_CONFIG.get("sys_academic_writer", "") + "\n"
+ evidence_packet + "\n" + voice + "\n"
+ APP_CONFIG.get("deep_refinement_directive", "")
)
sys_prompt = self._writer_visible_evidence_context(sys_prompt)
word_budget = APP_CONFIG.get("word_budget", {})
min_ratio = float(word_budget.get("section_min_ratio", 0.92))
max_ratio = float(word_budget.get("section_max_ratio", 1.08))
prompt_context_tail = context_tail[-2500:]
if mode == "asq":
prompt_context_tail = self._redact_qualitative_quotes(prompt_context_tail, data_metadata)
if bool(state.get("evidence_risk_detected")):
writer_synthesis_context = (
"EVIDENCE-LIMITED SYNTHESIS BOUNDARY: direct claim-specific literature coverage is limited. "
"The claim-evidence plan is the only source-dependent manuscript authority for externally verifiable literature statements in this unit. "
"Do not convert broad synthesis themes, general model knowledge, or unassigned references into manuscript claims. "
"When no supported external claim is assigned, prefer concise study positioning, transparent limitations, and cautious conceptual reasoning without invented citation support."
)
else:
# Literature synthesis is planning context, not a citation capability.
# Raw identities are stripped; only the frozen Section plan below may
# expose a usable [CITE-XX] token to the writer.
writer_synthesis_context = self._strip_unscoped_citation_identities(synthesis_result)
user_prompt = f"""Write the assigned English manuscript unit.
Section: {sec_title}
Subsection: {subsection or 'None'}
Section reasoning role: {section_role}
Additional mode directive: {mode_directive}
Planning length: approximately {int(adjusted_target * min_ratio)}-{int(adjusted_target * max_ratio)} words. This is guidance, not a content quota. Prioritize Conciseness, Accuracy, and Rigor; never add filler or remove useful reasoning solely to hit the range.
{heading_directive}
{citation_rule}
{table_rule}
AUTHOR-FACING PURITY: Write only reader-facing scholarly prose. Do not mention internal orchestration, validation, state-management, evidence-management, or execution terminology. Describe the study, method, evidence, and limitations directly.
EVIDENCE-OWNERSHIP CONTRACT: Statements about this study's sample, measures, available or missing documentation, procedures, statistics, quotations, or findings are owned by the study evidence and must not borrow an external citation. General methodological principles may use only a directly supporting METHOD_AUTHORITY source. Author interpretation or limitation reasoning must not be presented as if an external source established this study-specific conclusion. If one sentence would combine a study-owned fact with an external methodological principle or author inference, split it into separate claim-specific sentences and attach the citation only to the directly supported external principle.
{("CAUSAL-STRENGTH CONTRACT: This cross-sectional observational design supports association/covariation language only. Do not state that one construct causes, leads to, drives, determines, produces, or causally mediates another. Explicit limitation language is valid and preferred when relevant, including 'restricted to associations, not causal effects', 'cannot establish causality', and 'the indirect association is not causal mediation identification'." if mode == "amj" else "")}
{self._hypothesis_contract_directive(hypothesis_ledger)}
{str(sec.get('required_content') or '')}
{qualitative_rule}
{anti_repeat}
{self._canonical_concept_writer_directive(canonical_concept_ledger)}
{research_logic_for_writer}
{working_abstract_for_writer}
ANCHOR PRECEDENCE: The STUDY_EVIDENCE and PROVISIONAL_RESEARCH_LOGIC override the Working Abstract if any tension appears. The Working Abstract is continuity guidance only and must never force a result.
{memory_block}
Literature synthesis: {writer_synthesis_context}
{writer_scoped_data_md}
Previous tail: {prompt_context_tail}
Output only final academic prose. Do not report the word count."""
if agent._count_tokens(sys_prompt) + agent._count_tokens(user_prompt) > 60000:
user_prompt = agent._truncate_context(user_prompt, 60000 - agent._count_tokens(sys_prompt))
current_index = len(completed_sections) + 1
pct = min(88, 15 + int((current_index / max(1, total_steps)) * 72))
self._progress(pct, f"writing:{current_index}:{total_steps}")
prefix = ""
if sec.get("section_first"):
prefix += f"# {sec_title}\n\n"
if sec.get("subsection_first") and subsection:
prefix += f"## {subsection}\n\n"
prior_retry_exhausted = state.get("section_retry_exhausted")
if (
isinstance(prior_retry_exhausted, dict)
and str(prior_retry_exhausted.get("section_name") or "") == display_name
):
# A2: content exhaustion is resumable. The checkpoint records the
# diagnostics, but it no longer creates a terminal lifecycle state.
logging.warning("Adaptive completion resumed exhausted section | unit=%s", display_name)
state.pop("section_retry_exhausted", None)
deterministic_appendix = mode == "asq" and section_id == "appendix"
# R4: one initial attempt plus bounded lineage-owned repair turns.
# Four is a hard safety cap, not a blanket four-attempt entitlement.
max_section_attempts = 1 if deterministic_appendix else 4
lineage_attempts = {}
# R6-R10 monotonic retry state. A defect that was repaired and absent
# on an intervening attempt may receive one regression-correction turn
# if a later sibling repair reintroduces it. The Section hard cap is
# unchanged, so this cannot create an unbounded retry loop.
seen_lineages = set()
cleared_lineages = set()
regression_attempts = {}
historical_retry_rows = {}
frozen_citation_state = self._snapshot_section_citation_state(
agent, citation_plan, display_name
)
attempt_feedback = ""
retry_excluded_tags = set()
en_resolved = ""
survived = set()
for section_attempt in range(1, max_section_attempts + 1):
# Every attempt starts from exactly the same evidence/citation packet.
# Rejected attempts may not leak used tags, JIT references, or audit
# decisions into the retry or any later Section.
if section_attempt > 1:
self._restore_section_citation_state(
agent, citation_plan, display_name, frozen_citation_state
)
planned_items = [
item for item in copy.deepcopy(list(citation_plan.get(display_name, []) or []))
if str(item.get("tag") or "") not in retry_excluded_tags
and agent.citation_source_role_eligible(
str(item.get("tag") or ""), section_id=section_id, section_name=display_name
)
]
citation_plan[display_name] = planned_items
attempt_assigned_tag_list = [
str(item.get("tag") or "") for item in planned_items if item.get("tag")
]
attempt_diagnostics = []
candidate_user_prompt = user_prompt
if retry_excluded_tags:
retry_claim_plan = agent.format_claim_evidence_plan(planned_items)
candidate_user_prompt = _regex_sub_literal(
r".*?",
f"\n{retry_claim_plan}\n",
candidate_user_prompt,
flags=re.S,
)
retry_capability_context = self._section_citation_capability_context(agent, planned_items)
candidate_user_prompt = _regex_sub_literal(
r"\n.*?\n",
(f"\n\n{retry_capability_context}\n\n"
if retry_capability_context else "\n"),
candidate_user_prompt,
flags=re.S,
)
if attempt_feedback:
excluded_text = ", ".join(sorted(retry_excluded_tags, key=agent._citation_tag_number)) or "None"
candidate_user_prompt += f"""
# Internal retry invariant: the same frozen evidence universe remains authoritative; this control phrase is not exposed to manuscript prose.
The previous attempt for this same manuscript unit was rejected by validation. Rewrite ONLY this same unit once. Keep the identical study evidence, research logic, and section role. Do not fetch, invent, or broaden evidence. Citation tags rejected for this claim context are ineligible on the retry and must not be reused or deterministically reinserted: {excluded_text}. Use another already-authorized eligible source only when it directly supports the rewritten claim, or narrow/qualify the claim so no unsupported citation is needed. Resolve every material issue below without changing the task or manuscript scope:
{attempt_feedback}
"""
if agent._count_tokens(sys_prompt) + agent._count_tokens(candidate_user_prompt) > 60000:
candidate_user_prompt = agent._truncate_context(
candidate_user_prompt, 60000 - agent._count_tokens(sys_prompt)
)
if deterministic_appendix:
# R6A-R4: Appendix is a deterministic reader-facing projection
# of the Evidence Ledger. It is never regenerated by the LLM.
en_raw = qual_engine.build_appendix_evidence_md(data_metadata)
else:
en_raw = self._stream_text(
agent.stream_deepseek(
sys_prompt, candidate_user_prompt, max_tokens=7000,
model_role="writing", thinking=False, allow_fallback=False,
logical_node=section_logical_node,
),
preview_prefix=full_draft_en + prefix,
preview_stage=(display_name if section_attempt == 1 else f"{display_name}:retry"),
)
en_raw = re.sub(r".*?", "", en_raw, flags=re.S).strip()
en_raw = re.sub(rf"(?im)^#\s*{re.escape(sec_title)}\s*\n+", "", en_raw).strip()
if subsection:
en_raw = re.sub(rf"(?im)^##\s*{re.escape(subsection)}\s*\n+", "", en_raw).strip()
en_raw = re.sub(r"(?m)^(#{2,3})\s+[\d\.\s一二三四五六七八九十、]+\s*(.*)", r"\1 \2", en_raw)
en_raw = re.sub(r"(?m)^#{4,}\s+(.*)", r"**\1**", en_raw)
en_raw = re.sub(r"(?m)^#\s+(.+)$", r"## \1", en_raw)
if mode == "amj" and hypothesis_ledger:
normalized_hypothesis_text = self._normalize_hypothesis_ordinal_aliases(
en_raw, hypothesis_ledger
)
if normalized_hypothesis_text != en_raw:
logging.info(
"Deterministic hypothesis ordinal alias normalization | unit=%s",
display_name,
)
en_raw = normalized_hypothesis_text
if mode == "asq":
# Model prose has no authority over quote presence/count.
en_raw = self._strip_model_controlled_qualitative_quotes(en_raw, data_metadata)
# C5.5 uses the same private-vocabulary registry for both
# Writer-facing evidence and reader-facing output. This
# deterministic phrasing cleanup cannot change study facts,
# citations, hypotheses, numbers, or quote ownership.
en_raw = self._writer_visible_evidence_context(en_raw)
# S0-R1-C4 phase-1 barrier: all writer-owned semantic validators
# inspect the same attempt before the single same-Section retry budget
# can be consumed. Citation semantics therefore run before the one
# unified validation decision instead of being short-circuited by a
# validator that happens to execute earlier.
# S0-R1-C2/C4: citation semantics are part of the same phase-1
# decision. The authoritative path never attaches a citation merely
# to satisfy section-level coverage; it audits the writer-owned
# claim/evidence binding exactly as written. Any validated coverage
# shortfall still flows to the existing claim-anchored recovery path.
# This rule is shared by ASQ and AMJ and keeps sealed English
# authoritative for ZH/RU/KK translation.
self._progress(pct, f"citation_audit:{current_index}:{total_steps}")
used_before = set(agent.used_citations)
allowed_section_citation_tags = (
set(attempt_assigned_tag_list) - set(retry_excluded_tags)
if not is_quarantined else set()
)
attempt_section_text = prefix + en_raw
en_resolved = agent.audit_and_resolve_citations(
attempt_section_text, section_name=display_name, section_id=section_id,
quarantine=is_quarantined, allowed_tags=allowed_section_citation_tags
)
# Capture immediately: later recovery may replace the agent's last-
# violation buffer. Only citation-owned rejected tags may enter the
# retry exclusion set.
citation_semantic_rows = copy.deepcopy(
list(getattr(agent, "_last_citation_semantic_violations", []) or [])
)
phase1_validation_rows = self._section_validation_bundle(
mode, agent, quant_engine, attempt_section_text, data_metadata,
section_id=section_id, section_name=display_name,
evidence_contract=evidence_packet,
include_critic=not deterministic_appendix,
citation_violations=citation_semantic_rows, hypothesis_ledger=hypothesis_ledger,
)
if phase1_validation_rows:
retry_excluded_tags.update(self._citation_violation_tags(citation_semantic_rows))
hard_cap_reached = section_attempt >= max_section_attempts
# Severity is evaluated as exhausted only when this defect
# lineage has already consumed its own repair entitlement.
preview_blocking, preview_advisory = self._partition_section_violations(
phase1_validation_rows, section_name=display_name, mode=mode, exhausted=False
)
current_lineages = {
self._section_violation_lineage(row) for row in preview_blocking
}
cleared_lineages.update(seen_lineages - current_lineages)
regressed_lineages = self._section_regressed_lineages(
current_lineages, cleared_lineages, regression_attempts
)
seen_lineages.update(current_lineages)
may_retry, exhausted_lineages, fresh_lineages = self._section_retry_decision(
preview_blocking, lineage_attempts, hard_cap_reached=hard_cap_reached
)
retry_lineages = set(fresh_lineages)
if regressed_lineages and not hard_cap_reached:
retry_lineages.update(regressed_lineages)
may_retry = True
if may_retry:
retry_rows = [
row for row in preview_blocking
if self._section_violation_lineage(row) in retry_lineages
]
for lineage in fresh_lineages:
lineage_attempts[lineage] = int(lineage_attempts.get(lineage, 0)) + 1
for lineage in regressed_lineages:
regression_attempts[lineage] = int(regression_attempts.get(lineage, 0)) + 1
for row in preview_blocking:
historical_retry_rows[self._section_violation_lineage(row)] = dict(row)
# Every later repair is constrained by all earlier resolved
# contracts, not only the newly visible lineage. This makes
# A+B -> C -> A+B repair interference explicitly non-monotonic
# and gives it one bounded correction turn.
attempt_feedback = self._section_retry_feedback(retry_rows)
historical_feedback = self._section_retry_feedback(
list(historical_retry_rows.values())
)
if historical_feedback:
attempt_feedback += (
"\nNONREGRESSION CONSTRAINTS — previously repaired lineages must remain resolved:\n"
+ historical_feedback
)
logging.warning(
"Unified Phase-1 Section lineage retry | unit=%s attempt=%s lineages=%s exhausted_siblings=%s regressed_lineages=%s count=%s types=%s",
display_name, section_attempt, sorted(retry_lineages), sorted(exhausted_lineages),
sorted(regressed_lineages), len(retry_rows),
sorted({str(row.get("type") or "") for row in retry_rows if isinstance(row, dict)}),
)
self._restore_section_citation_state(
agent, citation_plan, display_name, frozen_citation_state
)
continue
blocking_rows, advisory_rows = self._partition_section_violations_by_lineage(
phase1_validation_rows, lineage_attempts, section_name=display_name, mode=mode,
hard_cap_reached=hard_cap_reached
)
if advisory_rows:
self._record_section_advisories(
quality_notes, advisory_rows, section_name=display_name
)
logging.warning(
"Nonfatal section quality advisory | unit=%s count=%s types=%s",
display_name, len(advisory_rows),
sorted({str(row.get("type") or "") for row in advisory_rows if isinstance(row, dict)}),
)
if blocking_rows:
# A2: phase-1 exhaustion escalates to a provider-neutral
# semantic repair rather than terminating the manuscript.
adaptive_raw = agent.adaptive_repair_academic_unit(
attempt_section_text, violations=blocking_rows, section_id=section_id,
section_name=display_name, mode=mode, study_evidence=writer_scoped_data_md,
evidence_contract=evidence_packet, repair_scope="section",
)
adaptive_resolved = agent.audit_and_resolve_citations(
adaptive_raw, section_name=display_name, section_id=section_id,
quarantine=is_quarantined, allowed_tags=allowed_section_citation_tags,
)
adaptive_citation_rows = copy.deepcopy(
list(getattr(agent, "_last_citation_semantic_violations", []) or [])
)
adaptive_rows = self._section_validation_bundle(
mode, agent, quant_engine, adaptive_raw, data_metadata,
section_id=section_id, section_name=display_name,
evidence_contract=evidence_packet, include_critic=False,
citation_violations=adaptive_citation_rows, hypothesis_ledger=hypothesis_ledger,
)
adaptive_blocking, adaptive_advisory = self._partition_section_violations(
adaptive_rows, section_name=display_name, mode=mode, exhausted=True
)
if adaptive_advisory:
self._record_section_advisories(quality_notes, adaptive_advisory, section_name=display_name)
if adaptive_blocking:
# P3-A4: citation identity/scope authority is a hard
# pre-seal invariant. Adaptive completion may retain
# reader-facing advisories for unrelated quality
# defects, but it may never seal a Section that still
# contains an unauthorized/ambiguous citation identity
# or a citation outside the frozen unit whitelist.
persistent_citation_authority = self._persistent_citation_authority_rows(
adaptive_blocking
)
if persistent_citation_authority:
self._checkpoint_section_retry_exhausted(
state, section_id=section_id, section_name=display_name,
violations=persistent_citation_authority, pct=pct, attempts=section_attempt,
)
raise RuntimeError(
"citation_authority_repair_exhausted:"
+ str(display_name)
+ ":"
+ ",".join(sorted({
str((row or {}).get("type") or "citation_authority")
for row in persistent_citation_authority if isinstance(row, dict)
}))
)
self._record_adaptive_completion_residual(
quality_notes, adaptive_blocking, section_name=display_name, stage="phase1_semantic_repair"
)
en_resolved = adaptive_resolved
state.pop("section_retry_exhausted", None)
used_after = set(agent.used_citations)
survived = used_after - used_before
if (
attempt_assigned_tag_list
and not is_quarantined
and self._quota_driven_citation_recovery_allowed(section_id)
):
required_survival = min(
len(attempt_assigned_tag_list),
max(1, int(math.ceil(len(attempt_assigned_tag_list) * CITATION_SEMANTIC_SURVIVAL_RATIO))),
)
if len(survived) < required_survival:
missing_items = [
item for item in planned_items
if str(item.get("tag") or "") not in used_after
]
needed = max(0, required_survival - len(survived))
recovery_items = agent.select_contextual_citation_recovery_items(
missing_items, section_name=display_name, section_id=section_id,
section_context=en_resolved, needed=needed,
excluded_tags=used_after | retry_excluded_tags,
allowed_tags=set(attempt_assigned_tag_list),
)
# Evidence retrieval is permitted only on the first writer
# attempt. Attempt 2 is bound to the original frozen packet.
if section_attempt == 1 and len(recovery_items) < needed:
recovery_items.extend(
agent.fetch_contextual_jit_recovery_items(
topic=topic, section_name=display_name, section_id=section_id,
section_context=en_resolved, needed=needed - len(recovery_items),
excluded_tags=(
used_after | retry_excluded_tags
| {str(item.get("tag") or "") for item in recovery_items}
),
)
)
planned_tags_for_unit = {str(item.get("tag") or "") for item in planned_items}
for recovery_item in recovery_items[:needed]:
tag = str(recovery_item.get("tag") or "")
if tag and tag not in planned_tags_for_unit:
planned_items.append(recovery_item)
planned_tags_for_unit.add(tag)
citation_plan[display_name] = planned_items
# Recovery may attach evidence only to an already-written
# claim. It never creates supplemental prose.
self._progress(pct, f"citation_recovery:{current_index}:{total_steps}")
recovered = agent.recover_citations_on_existing_claims(
en_resolved,
recovery_items[:needed],
section_name=display_name, section_id=section_id,
needed=needed,
)
if recovered != en_resolved:
en_resolved = recovered
used_after = set(agent.used_citations)
survived = used_after - used_before
if mode == "asq" and section_id == "findings":
en_resolved = self._render_planned_qualitative_quotes(
en_resolved,
task=sec,
data_metadata=data_metadata,
quote_invocation_plan=quote_invocation_plan,
)
if mode == "amj" and data_metadata.get("quantitative_table_manifest"):
if section_id == "results":
en_resolved = self._apply_quantitative_table_contract(
en_resolved, data_metadata, display_name
)
else:
# System-generated AMJ tables have one authoritative
# manifest. Writer-created tables anywhere else are not
# permitted to create a fifth/competing artifact table.
en_resolved = self._strip_model_markdown_tables(en_resolved)
# S0-R1 phase 2: only local/idempotent structural normalizers may
# mutate the candidate before the final pure validation bundle.
en_resolved, enumeration_rows = agent.repair_and_validate_enumeration_contract(
en_resolved, section_name=display_name
)
if enumeration_rows:
logging.warning(
"Structural enumeration repaired | unit=%s count=%s details=%s",
display_name, len(enumeration_rows), enumeration_rows[:3],
)
attempt_diagnostics.append({
"scope":"structural_enumeration", "name":display_name,
"status":"deterministic_repair", "count":len(enumeration_rows),
"examples":enumeration_rows[:3],
})
en_resolved, duplicate_rows = self._remove_severe_cross_unit_duplicates(
en_resolved, full_draft_en
)
if duplicate_rows:
logging.warning(
"Exact cross-unit redundancy removed | unit=%s count=%s",
display_name, len(duplicate_rows),
)
attempt_diagnostics.append({
"scope":"manuscript_redundancy", "name":display_name,
"status":"exact_duplicate_removed", "count":len(duplicate_rows),
"examples":duplicate_rows[:3],
})
en_resolved, sentence_echo_rows = agent.remove_severe_cross_unit_sentence_echoes(
en_resolved, full_draft_en
)
if sentence_echo_rows:
logging.warning(
"Exact cross-unit sentence echo removed | unit=%s count=%s",
display_name, len(sentence_echo_rows),
)
attempt_diagnostics.append({
"scope":"manuscript_sentence_echo", "name":display_name,
"status":"exact_duplicate_removed", "count":len(sentence_echo_rows),
"examples":sentence_echo_rows[:3],
})
# Final Section validation is pure. Once it passes, the exact bytes
# are eligible for the Section Integrity Seal and may never be
# semantically rewritten by a later validator.
post_validation_rows = self._section_validation_bundle(
mode, agent, quant_engine, en_resolved, data_metadata,
section_id=section_id, section_name=display_name,
evidence_contract=evidence_packet, include_critic=False,
hypothesis_ledger=hypothesis_ledger,
)
if post_validation_rows:
hard_cap_reached = section_attempt >= max_section_attempts
preview_blocking, preview_advisory = self._partition_section_violations(
post_validation_rows, section_name=display_name, mode=mode, exhausted=False
)
may_retry, exhausted_lineages, fresh_lineages = self._section_retry_decision(
preview_blocking, lineage_attempts, hard_cap_reached=hard_cap_reached
)
if may_retry:
retry_rows = [
row for row in preview_blocking
if self._section_violation_lineage(row) in fresh_lineages
]
for lineage in fresh_lineages:
lineage_attempts[lineage] = int(lineage_attempts.get(lineage, 0)) + 1
attempt_feedback = self._section_retry_feedback(retry_rows)
logging.warning(
"Post-normalization Section lineage retry | unit=%s attempt=%s lineages=%s exhausted_siblings=%s count=%s types=%s",
display_name, section_attempt, sorted(fresh_lineages), sorted(exhausted_lineages), len(retry_rows),
sorted({str(row.get("type") or "") for row in retry_rows if isinstance(row, dict)}),
)
self._restore_section_citation_state(
agent, citation_plan, display_name, frozen_citation_state
)
continue
blocking_rows, advisory_rows = self._partition_section_violations_by_lineage(
post_validation_rows, lineage_attempts, section_name=display_name, mode=mode,
hard_cap_reached=hard_cap_reached
)
if advisory_rows:
self._record_section_advisories(
quality_notes, advisory_rows, section_name=display_name
)
logging.warning(
"Nonfatal post-normalization quality advisory | unit=%s count=%s types=%s",
display_name, len(advisory_rows),
sorted({str(row.get("type") or "") for row in advisory_rows if isinstance(row, dict)}),
)
if blocking_rows:
en_resolved, residual_rows = self._adaptive_repair_section_after_exhaustion(
agent=agent, quant_engine=quant_engine, text=en_resolved,
violations=blocking_rows, mode=mode, section_id=section_id,
section_name=display_name, data_metadata=data_metadata,
study_evidence=writer_scoped_data_md, evidence_contract=evidence_packet,
hypothesis_ledger=hypothesis_ledger, task=sec,
quote_invocation_plan=quote_invocation_plan, quality_notes=quality_notes,
allowed_citation_tags=allowed_section_citation_tags,
quarantine=is_quarantined,
)
persistent_citation_authority = self._persistent_citation_authority_rows(
residual_rows
)
if persistent_citation_authority:
self._checkpoint_section_retry_exhausted(
state, section_id=section_id, section_name=display_name,
violations=persistent_citation_authority, pct=pct, attempts=section_attempt,
)
raise RuntimeError(
"citation_authority_repair_exhausted:"
+ str(display_name)
+ ":"
+ ",".join(sorted({
str((row or {}).get("type") or "citation_authority")
for row in persistent_citation_authority if isinstance(row, dict)
}))
)
logging.warning(
"Adaptive completion accepted section | unit=%s residual=%s",
display_name, len(residual_rows),
)
state.pop("section_retry_exhausted", None)
quality_diagnostics.extend(attempt_diagnostics)
assigned_tag_list = list(attempt_assigned_tag_list)
state.pop("section_retry_exhausted", None)
break
# R7-R1: heading structure is orchestrator-owned, not model-owned.
# Any adaptive repair may replace the accepted prose bytes and drop
# or duplicate the system-inserted H1/H2. Restore only those exact
# structural anchors after the final model-owned repair and before
# the unit is appended to the immutable English source / Section Seal.
normalized_en_resolved = self._normalize_orchestrator_owned_headings(en_resolved, sec)
if normalized_en_resolved != en_resolved:
logging.info("Deterministic English heading normalization | unit=%s", display_name)
quality_diagnostics.append({
"scope": "english_structural_ownership",
"name": display_name,
"status": "deterministic_heading_normalization",
})
en_resolved = normalized_en_resolved
# Recompute citation survival/claim demand from the exact text that
# will be sealed, not from an earlier pre-cleanup intermediate.
if not is_quarantined and section_id in self._citation_eligible_section_ids(mode):
final_survived = set()
for tag in set(survived or set()):
marker = str((agent.citation_map.get(tag) or {}).get("in_text") or "").strip()
if marker and marker in en_resolved:
final_survived.add(tag)
survived = final_survived
citation_required_claim_counts_by_unit[display_name] = agent.count_external_literature_claims(
en_resolved, section_name=display_name
)
if assigned_tag_list:
citation_validated_tags_by_unit[display_name] = sorted(
survived, key=agent._citation_tag_number
)
logging.info(
"Top-level citation accumulation: unit=%s section=%s validated=%s external_claims=%s",
display_name, citation_unit_section_map.get(display_name, section_id),
len(survived), citation_required_claim_counts_by_unit[display_name],
)
full_draft_en += en_resolved + "\n\n"
if assigned_tag_list:
logging.info(
"Citation-first survival: section=%s planned=%s validated_new=%s",
display_name, len(assigned_tag_list), len(survived),
)
self._publish_preview(full_draft_en, display_name, force=True)
# Only sealed/reconciled prose may feed subsequent manuscript memory.
agent._extract_memory_from_text(en_resolved, sec_title)
context_tail = " ".join(en_resolved.split()[-200:])
actual_words = self._english_count_for_target(en_resolved)
if not bool(sec.get("exclude_from_word_budget")):
accumulated_deviation += actual_words - adjusted_target
self._upsert_quality_diagnostic(
quality_diagnostics, self._section_word_diagnostic(display_name, actual_words, adjusted_target)
)
completed_sections.append(display_name)
section_key = str(sec.get('section_id') or sec_title)
if self._section_is_complete(adaptive_structure, completed_sections, section_key):
frozen_section = self._extract_top_level_section(full_draft_en, sec_title)
if not frozen_section:
raise RuntimeError(f"english_section_seal_source_missing:{sec_title}")
# C5: one bounded publication-quality evidence opportunity is
# allowed before the English top-level Section is sealed and
# before translation begins. It never writes new prose and may
# attach only citations that survive the same semantic auditor.
if section_key in citation_top_level_minimums:
top_units = [
unit for unit, owner in citation_unit_section_map.items()
if str(owner) == str(section_key)
]
top_validated = set()
top_claims = 0
for unit in top_units:
top_validated.update(citation_validated_tags_by_unit.get(unit) or [])
top_claims += int(citation_required_claim_counts_by_unit.get(unit, 0) or 0)
top_required = min(
int(citation_top_level_minimums.get(section_key, 0) or 0),
max(0, top_claims),
) if top_claims else 0
if len(top_validated) < top_required:
completed_section, added_tags = self._bounded_top_level_evidence_completion(
agent, topic=topic, section_id=section_key, section_name=sec_title,
section_text=frozen_section, required=top_required,
existing_tags=top_validated, excluded_tags=set(agent.used_citations),
)
if added_tags and completed_section != frozen_section:
full_draft_en = self._replace_top_level_section(
full_draft_en, sec_title, completed_section
)
frozen_section = completed_section
merged_tags = set(citation_validated_tags_by_unit.get(display_name) or [])
merged_tags.update(added_tags)
citation_validated_tags_by_unit[display_name] = sorted(
merged_tags, key=agent._citation_tag_number
)
logging.info(
"Top-level evidence completion sealed | section=%s added=%s",
sec_title, len(added_tags),
)
section_integrity_seals[section_key] = self._build_section_integrity_seal(
section_key, sec_title, frozen_section
)
translation_source = frozen_section
if (
mode == "amj"
and str(section_key).strip().casefold() == "results"
and data_metadata.get("quantitative_table_manifest")
):
# Table values are manifest-owned and must never be handed to the
# translation model as mutable prose. The target tables are rebound
# deterministically after Results translation QA.
translation_source = quant_engine._strip_target_quantitative_markdown_tables(
frozen_section
)
if mode == "asq" and str(section_key).strip().casefold() == "appendix":
# R5: the qualitative Appendix is not model-owned Markdown.
# It is translated/bound at the strict final gate from typed cells.
pass
else:
self._enqueue_translation_unit(
translation_agent,
unit_key=section_key, unit_name=sec_title, source_text=translation_source,
translation_sections=translation_sections,
english_commit_text=full_draft_en,
)
# Collect only already-finished target jobs. This call never waits,
# so English Section N+1 can start while target Section N runs.
translation_sections, translation_glossary = self._collect_translation_results(
translation_sections, translation_glossary, block=False
)
partial_translation = self._assemble_partial_translation(translation_sections, adaptive_structure, include_front=True)
state.update(
{
"completed_sections": completed_sections,
"full_draft_en": full_draft_en,
"full_draft_translated": partial_translation,
"translation_sections": translation_sections,
"translation_glossary": translation_glossary,
"context_tail": context_tail,
"accumulated_deviation": accumulated_deviation,
"quality_diagnostics": quality_diagnostics,
"quality_notes": quality_notes,
"citation_plan": citation_plan,
"citation_minimums": citation_minimums,
"citation_reserve_targets": citation_reserve_targets,
"citation_top_level_minimums": citation_top_level_minimums,
"citation_top_level_reserve_targets": citation_top_level_reserve_targets,
"citation_unit_section_map": citation_unit_section_map,
"citation_required_claim_counts_by_unit": citation_required_claim_counts_by_unit,
"citation_validated_tags_by_unit": citation_validated_tags_by_unit,
"section_integrity_seals": section_integrity_seals,
"english_section_seal_version": ENGLISH_SECTION_SEAL_VERSION,
"agent_state": agent.export_state(),
"data_metadata": data_metadata,
}
)
self._checkpoint(
f"section_{len(completed_sections):03d}", state, pct,
display_node=f"writing_complete:{len(completed_sections)}:{total_steps}",
)
if not state.get("english_finalized"):
self._progress(91, "english_validation")
full_draft_en = _clean_trailing_references(full_draft_en, ["Reference", "References", "Reference List"])
section_order_for_fact = self._unique_top_level_sections(adaptive_structure)
if not section_order_for_fact:
raise RuntimeError("final_study_fact_section_plan_missing")
# Final English validation is validation-only. Every mutable repair
# was completed before the top-level section received its integrity
# seal; finalization must never repair the whole manuscript again.
for section_key, section_title in section_order_for_fact:
sealed_section = self._extract_top_level_section(full_draft_en, section_title)
if not sealed_section:
raise RuntimeError(f"final_english_section_missing:{section_title}")
seal = dict(section_integrity_seals.get(section_key) or {})
self._validate_section_integrity_seal(seal, sealed_section, section_title)
final_fact_rows = self._study_fact_violations(
mode, agent, quant_engine, sealed_section, data_metadata, section_name=section_title
)
if final_fact_rows:
self._record_adaptive_completion_residual(
quality_notes, final_fact_rows, section_name=section_title, stage="final_sealed_fact_review"
)
# A2: final validators diagnose residual content; they do not
# destroy an otherwise complete manuscript after adaptive repair.
try:
agent.validate_enumeration_contract(
sealed_section, section_name=f"Final sealed section: {section_title}"
)
except Exception as exc:
self._record_adaptive_completion_residual(
quality_notes, [{"type":"enumeration_contract_invalid","reason":str(exc)[:400]}],
section_name=section_title, stage="final_enumeration_review"
)
internal_rows = self._author_facing_internal_workflow_violations(sealed_section)
if internal_rows:
self._record_adaptive_completion_residual(
quality_notes, internal_rows, section_name=section_title, stage="final_reader_facing_review"
)
if mode == "amj" and hypothesis_ledger:
hypothesis_rows = self._hypothesis_contract_violations(
full_draft_en, hypothesis_ledger, section_name="Full Manuscript", require_all=True
)
if hypothesis_rows:
reconciled, reconciliation_notes = self._deterministic_hypothesis_reconcile(
full_draft_en, hypothesis_ledger
)
reconciled_rows = self._hypothesis_contract_violations(
reconciled, hypothesis_ledger, section_name="Full Manuscript", require_all=True
)
if reconciliation_notes:
self._record_adaptive_completion_residual(
quality_notes, reconciliation_notes, section_name="Full Manuscript",
stage="delivery_first_hypothesis_reconciliation"
)
if reconciled_rows:
self._raise_if_fatal_completion_residual(
reconciled_rows, section_name="Full Manuscript", mode=mode,
error_code="final_hypothesis_contract_unresolved"
)
else:
full_draft_en = reconciled
changed_sections = self._refresh_section_integrity_seals_after_delivery_repair(
full_draft_en, section_order_for_fact, section_integrity_seals
)
if changed_sections:
requeued = self._requeue_resealed_translation_sections(
changed_sections=changed_sections, mode=mode, quant_engine=quant_engine,
data_metadata=data_metadata, translation_agent=translation_agent,
translation_sections=translation_sections, english_commit_text=full_draft_en,
)
self._record_adaptive_completion_residual(
quality_notes, [{
"type": "delivery_first_section_resealed",
"count": len(changed_sections),
"requeued_translations": requeued,
"reason": "Deterministic H-identity repair changed sealed English bytes; section seals and target translation source authority were refreshed before finalization.",
}], section_name="Full Manuscript", stage="delivery_first_hypothesis_reseal"
)
self._record_adaptive_completion_residual(
quality_notes, hypothesis_rows, section_name="Full Manuscript", stage="final_hypothesis_review"
)
# Validate the completed body while the Working Abstract is still in
# place. Final front matter must not be generated until all body
# sections, citation cleanup, and any Discussion rebalance are frozen.
agent.validate_manuscript_structure(full_draft_en, adaptive_structure, translated=False)
if mode == "asq":
try:
self._validate_qualitative_quote_rendering(
full_draft_en, quote_invocation_plan, data_metadata
)
except Exception as exc:
self._record_adaptive_completion_residual(
quality_notes, [{"type":"qualitative_quote_rendering","reason":str(exc)[:500]}],
section_name="Findings", stage="final_quote_review"
)
repeated_quotes = agent.find_repeated_qualitative_quotes(full_draft_en, data_metadata)
if repeated_quotes:
self._record_adaptive_completion_residual(
quality_notes, [{"type":"qualitative_quote_repetition","reason":str(repeated_quotes[:5])}],
section_name="Findings", stage="final_quote_review"
)
# R4-R3: the body is already citation-resolved at this point. Never
# rescan it for raw CITE tags; that destroys a valid citation ledger.
final_citation_ledger = agent.finalize_resolved_citation_ledger(full_draft_en)
references = agent.build_final_reference_lines(final_citation_ledger)
if not references:
self._record_evidence_coverage_event(
state, [("final validated references", 0, 1)],
phase="final_validated_references_empty", preview_status="limited",
)
logging.warning(
"Final validated reference list is empty; completing in evidence-limited mode without unrelated or invented references."
)
validated_by_top_level = {}
claims_by_top_level = {}
for unit_name, section_id in citation_unit_section_map.items():
if section_id not in citation_top_level_minimums:
continue
validated_by_top_level.setdefault(section_id, set()).update(
citation_validated_tags_by_unit.get(unit_name) or []
)
claims_by_top_level[section_id] = claims_by_top_level.get(section_id, 0) + int(
citation_required_claim_counts_by_unit.get(unit_name, 0) or 0
)
method_section_id = 'methods' if mode == 'asq' else 'methodology'
method_validated_tags = set(validated_by_top_level.get(method_section_id) or []) & set(final_citation_ledger)
method_authority_tags = {
tag for tag in method_validated_tags
if 'method_authority' in set((agent.citation_map.get(tag) or {}).get('evidence_lanes') or [])
}
method_external_claims = max(0, int(claims_by_top_level.get(method_section_id, 0) or 0))
final_shortfalls = []
for section_id, planned_minimum in citation_top_level_minimums.items():
validated_tags = set(validated_by_top_level.get(section_id) or [])
external_claims = max(0, int(claims_by_top_level.get(section_id, 0)))
# Evidence authority is need-based, not a word-count quota. The
# top-level planning minimum is therefore capped by the number of
# externally-oriented claims that actually survived drafting.
required = min(int(planned_minimum), external_claims) if external_claims else 0
if len(validated_tags) < required:
final_shortfalls.append((section_id, len(validated_tags), required))
final_reference_count = len(references)
if final_shortfalls or final_reference_count == 0:
if final_shortfalls:
self._record_evidence_coverage_event(
state, final_shortfalls, phase="final_claim_evidence_gate",
preview_status="limited",
)
evidence_status = "limited"
logging.warning(
"Top-level claim-evidence final Gate completed with evidence warning: validated_references=%s sections=%s shortfalls=%s",
final_reference_count, len(citation_top_level_minimums), final_shortfalls,
)
else:
evidence_status = "adequate"
self._preview_evidence_status = "adequate"
self._write_preview_payload(force=True)
logging.info(
"Top-level claim-evidence final Gate passed: validated_references=%s sections=%s",
final_reference_count, len(citation_top_level_minimums),
)
evidence_coverage_summary = self._evidence_coverage_summary(
status=evidence_status,
early_risk=bool(state.get("evidence_risk_detected")),
final_reference_count=final_reference_count,
minimums=citation_top_level_minimums,
validated_by_top_level=validated_by_top_level,
claims_by_top_level=claims_by_top_level,
events=state.get("evidence_coverage_events") or self._evidence_coverage_events,
method_authority_count=len(method_authority_tags),
method_external_claims=method_external_claims,
)
state["evidence_status"] = evidence_status
state["evidence_coverage_summary"] = evidence_coverage_summary
full_draft_en = full_draft_en.rstrip() + "\n\n# References\n" + "\n\n".join(references)
actual_words = self._english_count_for_target(full_draft_en)
self._upsert_quality_diagnostic(
quality_diagnostics, self._validate_target_length(full_draft_en, target_words)
)
body_only, _reference_tail = agent.validate_reference_tail(
full_draft_en, minimum_references=final_reference_count
)
agent.validate_manuscript_structure(body_only, adaptive_structure, translated=False)
agent.enforce_resolved_claim_specific_citations(body_only)
agent.validate_final_citation_bijection(body_only, _reference_tail, final_citation_ledger)
# Body is now frozen. Generate both Final Abstract and Final Keywords
# from the completed manuscript/evidence, then replace the provisional
# front matter deterministically. No body rebalance may occur afterward.
self._progress(92, "final_front_matter")
try:
final_front = agent.generate_final_front_matter(
reader_topic_en, reader_keywords_prompt, body_only, data_evidence=user_data_md, mode=mode,
hypothesis_contract=hypothesis_contract,
authoritative_keywords=self._user_authoritative_front_matter_keywords(data_metadata),
keyword_policy=self._resolved_front_matter_keyword_policy(data_metadata),
fallback_keywords=self._front_matter_keyword_fallback_terms(data_metadata),
)
except Exception as exc:
logging.warning(
"Delivery-first Final Front Matter fallback | mode=%s error=%s", mode, type(exc).__name__
)
final_front = {
"abstract": self._deterministic_final_abstract(
mode, reader_topic_en or source_topic or topic, data_metadata, hypothesis_ledger=hypothesis_ledger
),
"keywords": self._canonical_quantitative_front_matter_keywords(
mode, list(reader_keywords_en), data_metadata
) or self._front_matter_keyword_fallback_terms(data_metadata),
}
self._record_adaptive_completion_residual(
quality_notes, [{
"type": "final_front_matter_provider_fallback",
"reason": f"Final Front Matter model/validation failed ({type(exc).__name__}); deterministic frozen-authority fallback was used."
}], section_name="Final Abstract", stage="delivery_first_front_matter_fallback"
)
final_abstract = str(final_front.get("abstract") or "").strip()
if hypothesis_ledger:
final_abstract = self._normalize_hypothesis_ordinal_aliases(
final_abstract, hypothesis_ledger
)
final_keywords = self._canonical_quantitative_front_matter_keywords(
mode, list(final_front.get("keywords") or []), data_metadata
)
if not final_abstract or not final_keywords:
final_abstract = final_abstract or self._deterministic_final_abstract(
mode, reader_topic_en or source_topic or topic, data_metadata, hypothesis_ledger=hypothesis_ledger
)
final_keywords = final_keywords or self._canonical_quantitative_front_matter_keywords(
mode, list(reader_keywords_en), data_metadata
) or self._front_matter_keyword_fallback_terms(data_metadata)
self._record_adaptive_completion_residual(
quality_notes, [{
"type": "final_front_matter_missing_fallback",
"reason": "Missing Final Front Matter component was reconstructed deterministically from frozen title/keyword/finding authority."
}], section_name="Final Abstract", stage="delivery_first_front_matter_fallback"
)
final_front_fact_rows = self._study_fact_violations(
mode, agent, quant_engine, final_abstract, data_metadata,
section_name="Final Abstract",
)
if hypothesis_ledger:
final_front_fact_rows.extend(self._hypothesis_contract_violations(
final_abstract, hypothesis_ledger, section_name="Final Abstract"
))
final_front_fact_rows.extend(self._final_abstract_finding_violations(
mode, final_abstract, data_metadata, hypothesis_ledger=hypothesis_ledger
))
if final_front_fact_rows:
feedback = self._section_retry_feedback(final_front_fact_rows)
logging.warning(
"Final Abstract pure-validation retry | mode=%s count=%s types=%s",
mode, len(final_front_fact_rows),
sorted({str(row.get("type") or "") for row in final_front_fact_rows if isinstance(row, dict)}),
)
try:
final_front = agent.generate_final_front_matter(
reader_topic_en, reader_keywords_prompt, body_only, data_evidence=user_data_md, mode=mode,
validation_feedback=feedback, hypothesis_contract=hypothesis_contract,
authoritative_keywords=self._user_authoritative_front_matter_keywords(data_metadata),
keyword_policy=self._resolved_front_matter_keyword_policy(data_metadata),
fallback_keywords=self._front_matter_keyword_fallback_terms(data_metadata),
)
except Exception as exc:
logging.warning(
"Delivery-first Final Front Matter retry fallback | mode=%s error=%s", mode, type(exc).__name__
)
final_front = {
"abstract": self._deterministic_final_abstract(
mode, reader_topic_en or source_topic or topic, data_metadata, hypothesis_ledger=hypothesis_ledger
),
"keywords": final_keywords or self._canonical_quantitative_front_matter_keywords(
mode, list(reader_keywords_en), data_metadata
) or self._front_matter_keyword_fallback_terms(data_metadata),
}
final_abstract = str(final_front.get("abstract") or "").strip()
if hypothesis_ledger:
final_abstract = self._normalize_hypothesis_ordinal_aliases(
final_abstract, hypothesis_ledger
)
final_keywords = self._canonical_quantitative_front_matter_keywords(
mode, list(final_front.get("keywords") or []), data_metadata
)
if not final_abstract or not final_keywords:
final_abstract = final_abstract or self._deterministic_final_abstract(
mode, reader_topic_en or source_topic or topic, data_metadata, hypothesis_ledger=hypothesis_ledger
)
final_keywords = final_keywords or self._canonical_quantitative_front_matter_keywords(
mode, list(reader_keywords_en), data_metadata
) or self._front_matter_keyword_fallback_terms(data_metadata)
self._record_adaptive_completion_residual(
quality_notes, [{
"type": "final_front_matter_retry_exhausted_fallback",
"reason": "Final Front Matter remained incomplete after retry; deterministic frozen-authority fallback was used."
}], section_name="Final Abstract", stage="delivery_first_front_matter_fallback"
)
remaining_final_fact_rows = self._study_fact_violations(
mode, agent, quant_engine, final_abstract, data_metadata,
section_name="Final Abstract",
)
if hypothesis_ledger:
remaining_final_fact_rows.extend(self._hypothesis_contract_violations(
final_abstract, hypothesis_ledger, section_name="Final Abstract"
))
remaining_final_fact_rows.extend(self._final_abstract_finding_violations(
mode, final_abstract, data_metadata, hypothesis_ledger=hypothesis_ledger
))
if remaining_final_fact_rows:
resilient_abstract, resilience_notes = self._final_abstract_resilience_repair(
mode, final_abstract, remaining_final_fact_rows, data_metadata,
hypothesis_ledger=hypothesis_ledger
)
if resilient_abstract != final_abstract:
final_abstract = resilient_abstract
remaining_final_fact_rows = self._study_fact_violations(
mode, agent, quant_engine, final_abstract, data_metadata,
section_name="Final Abstract",
)
if hypothesis_ledger:
remaining_final_fact_rows.extend(self._hypothesis_contract_violations(
final_abstract, hypothesis_ledger, section_name="Final Abstract"
))
remaining_final_fact_rows.extend(self._final_abstract_finding_violations(
mode, final_abstract, data_metadata, hypothesis_ledger=hypothesis_ledger
))
if resilience_notes:
self._record_adaptive_completion_residual(
quality_notes, resilience_notes, section_name="Final Abstract",
stage="final_abstract_resilience_recovery"
)
if remaining_final_fact_rows:
final_abstract, residual_final_rows = self._adaptive_repair_front_matter(
agent=agent, quant_engine=quant_engine, abstract_text=final_abstract,
violations=remaining_final_fact_rows, mode=mode, data_metadata=data_metadata,
study_evidence=user_data_md, hypothesis_ledger=hypothesis_ledger,
quality_notes=quality_notes, label="Final Abstract",
)
if residual_final_rows:
deterministic_abstract = self._deterministic_final_abstract(
mode, reader_topic_en or source_topic or topic, data_metadata, hypothesis_ledger=hypothesis_ledger
)
hard_rows = self._study_fact_violations(
mode, agent, quant_engine, deterministic_abstract, data_metadata, section_name="Final Abstract"
)
if hypothesis_ledger:
hard_rows.extend(self._hypothesis_contract_violations(
deterministic_abstract, hypothesis_ledger, section_name="Final Abstract"
))
if hard_rows:
self._raise_if_fatal_completion_residual(
hard_rows, section_name="Final Abstract", mode=mode,
error_code="final_abstract_integrity_unresolved"
)
final_abstract = deterministic_abstract
self._record_adaptive_completion_residual(
quality_notes, residual_final_rows, section_name="Final Abstract",
stage="delivery_first_final_abstract_fallback"
)
logging.warning(
"Delivery-first Final Abstract deterministic fallback accepted | residual_diagnostics=%s",
len(residual_final_rows)
)
quality_diagnostics.append({
"scope":"final_abstract_study_fact_integrity", "name":"Final Abstract",
"status":"front_matter_retry_resolved", "count":len(final_front_fact_rows),
"examples":final_front_fact_rows[:3],
})
state.pop("final_abstract_retry_exhausted", None)
final_title = str(reader_topic_en or source_topic or topic or "").strip()
if mode == "amj" and str(data_metadata.get("source_type") or "") == "system_generated":
semantic_contract = dict(data_metadata.get("front_matter_contract") or {})
reconciled_front = agent.reconcile_quantitative_reader_front_matter(
topic=source_topic, keywords=source_keywords, contract=semantic_contract,
manuscript=body_only, reader_title=reader_topic_en, reader_keywords=reader_keywords_en,
)
candidate_title = re.sub(r"\s+", " ", str(reconciled_front.get("title") or "")).strip()
candidate_keywords = self._canonical_quantitative_front_matter_keywords(
mode, list(reconciled_front.get("keywords") or []), data_metadata
)
if candidate_title:
final_title = candidate_title
if candidate_keywords:
# R7-R2C-R6B: the resolved policy owns keyword identity/count
# semantics. No local numeric cardinality branch is permitted.
final_keywords = candidate_keywords
else:
logging.warning(
"Adaptive final keyword reconciliation returned no usable keywords; preserving validated Final Front Matter keywords."
)
full_draft_en = replace_abstract(full_draft_en, final_abstract)
full_draft_en = self._replace_title(full_draft_en, final_title)
full_draft_en = self._replace_keywords(full_draft_en, final_keywords)
agent.validate_reader_facing_english_projection(
{"title": final_title, "keywords": final_keywords},
expected_keyword_count=(len(reader_keywords_en) if reader_keywords_en else None),
)
final_abstract, language_rebuilt = self._delivery_first_final_abstract_language_guard(
mode=mode, agent=agent, quant_engine=quant_engine, final_abstract=final_abstract,
reader_topic_en=reader_topic_en, final_title=final_title, data_metadata=data_metadata,
hypothesis_ledger=hypothesis_ledger, quality_notes=quality_notes,
)
if language_rebuilt:
full_draft_en = replace_abstract(full_draft_en, final_abstract)
if mode == "amj" and hypothesis_ledger:
post_front_hypothesis_rows = self._hypothesis_contract_violations(
full_draft_en, hypothesis_ledger, section_name="Full Manuscript", require_all=True
)
if post_front_hypothesis_rows:
reconciled, reconciliation_notes = self._deterministic_hypothesis_reconcile(
full_draft_en, hypothesis_ledger
)
remaining_post_rows = self._hypothesis_contract_violations(
reconciled, hypothesis_ledger, section_name="Full Manuscript", require_all=True
)
if reconciliation_notes:
self._record_adaptive_completion_residual(
quality_notes, reconciliation_notes, section_name="Final Abstract",
stage="delivery_first_post_front_hypothesis_reconciliation"
)
if remaining_post_rows:
self._raise_if_fatal_completion_residual(
remaining_post_rows, section_name="Full Manuscript", mode=mode,
error_code="post_front_matter_hypothesis_contract_unresolved"
)
else:
full_draft_en = reconciled
changed_sections = self._refresh_section_integrity_seals_after_delivery_repair(
full_draft_en, section_order_for_fact, section_integrity_seals
)
if changed_sections:
requeued = self._requeue_resealed_translation_sections(
changed_sections=changed_sections, mode=mode, quant_engine=quant_engine,
data_metadata=data_metadata, translation_agent=translation_agent,
translation_sections=translation_sections, english_commit_text=full_draft_en,
)
self._record_adaptive_completion_residual(
quality_notes, [{
"type": "delivery_first_section_resealed",
"count": len(changed_sections),
"requeued_translations": requeued,
"reason": "Post-front-matter H-identity repair changed sealed English bytes; section seals and target translation source authority were refreshed before checkpointing.",
}], section_name="Full Manuscript", stage="delivery_first_post_front_reseal"
)
self._record_adaptive_completion_residual(
quality_notes, post_front_hypothesis_rows, section_name="Final Abstract",
stage="post_front_matter_hypothesis_review"
)
body_only, _reference_tail = agent.validate_reference_tail(
full_draft_en, minimum_references=final_reference_count
)
agent.validate_manuscript_structure(body_only, adaptive_structure, translated=False)
agent.enforce_resolved_claim_specific_citations(body_only)
post_front_ledger = agent.finalize_resolved_citation_ledger(body_only)
if set(post_front_ledger) != set(final_citation_ledger):
logging.warning(
"Adaptive finalization resealed citation ledger after Final Front Matter | before=%s after=%s",
len(final_citation_ledger), len(post_front_ledger),
)
# Final Front Matter is allowed to add/remove an already-authoritative
# citation. Rebuild References deterministically from the post-front
# resolved body instead of discarding an otherwise complete manuscript.
# Orphan/ambiguous citations still fail closed inside ledger finalization.
final_citation_ledger = list(post_front_ledger)
references = agent.build_final_reference_lines(final_citation_ledger)
final_reference_count = len(references)
full_draft_en = body_only.rstrip() + "\n\n# References\n" + "\n\n".join(references)
coverage_summary = dict(state.get("evidence_coverage_summary") or {})
if coverage_summary:
coverage_summary["final_reference_count"] = final_reference_count
state["evidence_coverage_summary"] = coverage_summary
body_only, _reference_tail = agent.validate_reference_tail(
full_draft_en, minimum_references=final_reference_count
)
agent.validate_final_citation_bijection(body_only, _reference_tail, final_citation_ledger)
self._upsert_quality_diagnostic(
quality_diagnostics, self._validate_target_length(full_draft_en, target_words)
)
# Final front matter has a different source hash from the Working
# front matter. Retranslate only __front_matter__; unchanged body
# translations are retained and reused.
section_order_for_front = self._unique_top_level_sections(adaptive_structure)
if section_order_for_front:
final_front_source = self._extract_front_matter(body_only, section_order_for_front[0][1])
self._enqueue_translation_unit(
translation_agent,
unit_key='__front_matter__', unit_name='Final Front Matter',
source_text=final_front_source,
translation_sections=translation_sections,
english_commit_text=full_draft_en,
)
translation_sections, translation_glossary = self._collect_translation_results(
translation_sections, translation_glossary, block=False
)
partial_target = self._assemble_partial_translation(
translation_sections, adaptive_structure, include_front=True
)
self._publish_preview(full_draft_en, "english_complete", force=True, complete=True)
state.update(
{
"full_draft_en": full_draft_en,
"full_draft_translated": partial_target,
"translation_sections": translation_sections,
"translation_glossary": translation_glossary,
"final_abstract": final_abstract,
"final_title": final_title,
"final_keywords": final_keywords,
"front_matter_phase": "final",
"final_reference_count": final_reference_count,
"final_citation_ledger": list(final_citation_ledger),
"final_citation_ledger_version": FINAL_CITATION_LEDGER_VERSION,
"evidence_status": str(state.get("evidence_status") or "adequate"),
"evidence_risk_detected": bool(state.get("evidence_risk_detected")),
"evidence_coverage_events": list(state.get("evidence_coverage_events") or []),
"evidence_coverage_summary": dict(state.get("evidence_coverage_summary") or {}),
"citation_plan": citation_plan,
"citation_minimums": citation_minimums,
"citation_reserve_targets": citation_reserve_targets,
"citation_top_level_minimums": citation_top_level_minimums,
"citation_top_level_reserve_targets": citation_top_level_reserve_targets,
"citation_unit_section_map": citation_unit_section_map,
"citation_required_claim_counts_by_unit": citation_required_claim_counts_by_unit,
"citation_validated_tags_by_unit": citation_validated_tags_by_unit,
"section_integrity_seals": section_integrity_seals,
"english_section_seal_version": ENGLISH_SECTION_SEAL_VERSION,
"quality_diagnostics": quality_diagnostics,
"english_finalized": True,
"translation_complete": False,
"agent_state": agent.export_state(),
}
)
self._checkpoint("english_complete", state, 93)
else:
full_draft_en = str(state.get("full_draft_en") or "")
full_draft_translated = str(state.get("full_draft_translated") or "")
translation_sections = dict(state.get('translation_sections') or translation_sections or {})
translation_glossary = dict(state.get('translation_glossary') or translation_glossary or {})
if not state.get("translation_complete") or not full_draft_translated:
self._progress(94, "translation")
# English is fully frozen. Wait for all previously queued section
# translations before the strict final gate; checkpoint ownership
# remains on this main execution thread.
translation_sections, translation_glossary = self._collect_translation_results(
translation_sections, translation_glossary, block=True
)
self._shutdown_translation_worker(wait=True)
body_only, reference_tail = agent.validate_reference_tail(
full_draft_en, minimum_references=max(0, int(state.get("final_reference_count") or 0))
)
section_order = self._unique_top_level_sections(adaptive_structure)
if not section_order:
raise RuntimeError('translation_section_plan_missing')
front_matter = self._extract_front_matter(body_only, section_order[0][1])
final_units = [('__front_matter__', 'Front Matter', front_matter)]
for key, title in section_order:
final_units.append((key, title, self._extract_top_level_section(body_only, title)))
for index, (unit_key, unit_name, source_text) in enumerate(final_units, start=1):
translation_source = source_text
if (
mode == "amj"
and str(unit_key).strip().casefold() == "results"
and data_metadata.get("quantitative_table_manifest")
):
translation_source = quant_engine._strip_target_quantitative_markdown_tables(
source_text
)
if mode == "asq" and str(unit_key).strip().casefold() == "appendix":
translation_sections, translation_glossary = self._translate_qualitative_appendix_unit(
translation_agent, qual_engine, unit_key=unit_key, source_text=source_text,
target_lang=target_lang, translation_sections=translation_sections,
translation_glossary=translation_glossary, data_metadata=data_metadata,
)
else:
translation_sections, translation_glossary = self._translate_frozen_unit(
translation_agent, unit_key=unit_key, unit_name=unit_name, source_text=translation_source,
target_lang=target_lang, translation_sections=translation_sections,
translation_glossary=translation_glossary, adaptive_structure=adaptive_structure,
include_front=True, defer_failures=False,
english_commit_text=full_draft_en, final_gate=True,
)
# R2: quantitative tables are manifest-owned artifacts, not translation-model output.
# Rebind them only after the frozen Results translation has passed its language/semantic QA.
# Re-running this block is idempotent because the engine first removes any prior/model table block.
if mode == "amj" and str(unit_key).strip().casefold() == "results":
table_manifest = dict(data_metadata.get("quantitative_table_manifest") or {})
current_result_translation = dict(translation_sections.get(unit_key) or {})
if (
table_manifest
and current_result_translation.get("qa_status") in {"passed", "passed_with_warning"}
and current_result_translation.get("translated_text")
):
current_result_translation["translated_text"] = quant_engine.bind_target_language_quantitative_tables(
current_result_translation.get("translated_text"),
table_manifest, target_lang, translation_glossary,
)
current_result_translation["quantitative_table_binding_version"] = (
quant_engine.TARGET_TABLE_BINDING_VERSION
)
translation_sections[unit_key] = current_result_translation
partial = self._assemble_partial_translation(
translation_sections, adaptive_structure, include_front=True
)
state.update({
'full_draft_en': full_draft_en,
'full_draft_translated': partial,
'translation_sections': translation_sections,
'translation_glossary': translation_glossary,
'agent_state': agent.export_state(),
})
pct = min(96, 94 + int(index * 2 / max(1, len(final_units))))
current_translation = dict(translation_sections.get(unit_key) or {})
blocking = (
current_translation.get('qa_status') not in {'passed', 'passed_with_warning'}
or not current_translation.get('translated_text')
)
if blocking:
state['translation_blocking_unit'] = str(unit_key)
state['translation_blocking_unit_name'] = str(unit_name)
state['translation_blocking_error_code'] = str(current_translation.get('error_code') or 'TRANSLATION_PENDING')
self._checkpoint(
f'translation_{unit_key}_repair_pending', state, pct,
display_node=f'translation_repair_pending:{index}:{len(final_units)}',
)
if mode == "asq" and str(unit_key).strip().casefold() == "appendix":
# The Appendix is a manifest-owned typed artifact. Generic
# whole-unit translation must never bypass its row/cell/Quote
# identity contract. _translate_qualitative_appendix_unit
# already exhausted bounded typed recovery above.
raise RuntimeError(
'qualitative_appendix_typed_recovery_exhausted:'
+ str(state.get('translation_blocking_error_code') or 'TRANSLATION_PENDING')
)
# A2: strict claim QA is diagnostic/recovery authority, not a
# manuscript kill switch. Escalate the same frozen source to
# a whole-unit provider-neutral fallback and keep going.
if mode == "asq":
fallback, translation_sections, translation_glossary = (
self._adaptive_translate_qualitative_unit_with_quotes(
translation_agent, source_text=translation_source,
target_lang=target_lang, section_name=unit_name,
translation_sections=translation_sections,
translation_glossary=translation_glossary,
)
)
else:
fallback = translation_agent.adaptive_translate_fallback(
translation_source, target_lang, section_name=unit_name, glossary=translation_glossary
)
current_translation.update({
'translated_text': str(fallback.get('text') or '').strip(),
'qa_status': str(fallback.get('qa_status') or 'passed_with_warning'),
'diagnostics': list(current_translation.get('diagnostics') or []) + list(fallback.get('diagnostics') or []),
'error_code': '',
'adaptive_completion_fallback': True,
})
translation_sections[unit_key] = current_translation
if mode == "amj" and str(unit_key).strip().casefold() == "results":
table_manifest = dict(data_metadata.get("quantitative_table_manifest") or {})
if table_manifest and current_translation.get('translated_text'):
current_translation['translated_text'] = quant_engine.bind_target_language_quantitative_tables(
current_translation.get('translated_text'), table_manifest, target_lang, translation_glossary
)
translation_sections[unit_key] = current_translation
self._record_adaptive_completion_residual(
quality_notes, [{'type':'translation_strict_gate_fallback','reason':str(state.get('translation_blocking_error_code') or '')}],
section_name=unit_name, stage='translation_fallback'
)
state.pop('translation_blocking_unit', None)
state.pop('translation_blocking_unit_name', None)
state.pop('translation_blocking_error_code', None)
self._checkpoint(
f'translation_{unit_key}', state, pct,
display_node=f'translation_final_gate:{index}:{len(final_units)}',
)
unresolved_units = [
key for key, _name, _source in final_units
if (translation_sections.get(key) or {}).get('qa_status') not in {'passed', 'passed_with_warning'}
or not (translation_sections.get(key) or {}).get('translated_text')
]
if unresolved_units:
# A2 should normally resolve these inside the per-unit fallback.
# Keep the manuscript moving if a stale checkpoint status remains
# but usable translated text exists; empty target text is still an
# infrastructure/completion failure and cannot produce a document.
empty_units = [key for key in unresolved_units if not (translation_sections.get(key) or {}).get('translated_text')]
if empty_units:
raise RuntimeError('translation_output_missing:' + ','.join(empty_units[:8]))
for key in unresolved_units:
translation_sections[key]['qa_status'] = 'passed_with_warning'
full_draft_translated = self._assemble_partial_translation(
translation_sections, adaptive_structure, include_front=True
).rstrip()
full_draft_translated += (
f"\n\n# {agent._localized_reference_heading(target_lang)}\n" + reference_tail.strip()
)
full_draft_translated = _normalize_target_manuscript_typography(
full_draft_translated, target_lang
)
full_draft_translated, intent_front_diagnostics = self._rebind_target_user_intent_front_matter(
full_draft_translated, target_lang, data_metadata, translation_glossary
)
for diagnostic in intent_front_diagnostics:
self._record_adaptive_completion_residual(
quality_notes, [diagnostic], section_name='Target-language front matter',
stage='user_research_intent_translation_recovery'
)
if mode == "asq":
appendix_manifest = qual_engine.build_appendix_manifest(data_metadata)
try:
qual_engine.validate_bound_target_appendix(
full_draft_translated, appendix_manifest, target_lang
)
except Exception as exc:
self._record_adaptive_completion_residual(
quality_notes, [{'type':'target_qualitative_appendix_review','reason':str(exc)[:500]}],
section_name='Appendix', stage='translation_final_review'
)
# One deterministic self-heal is allowed when the typed
# Appendix record itself is valid. No model is called and no
# row/cell/Quote identity can change during this rebind.
appendix_record = dict(translation_sections.get('appendix') or {})
canonical_appendix = _normalize_target_manuscript_typography(
str(appendix_record.get('translated_text') or '').strip(), target_lang
)
try:
qual_engine.validate_bound_target_appendix(
canonical_appendix, appendix_manifest, target_lang
)
full_draft_translated = self._rebind_target_qualitative_appendix_once(
full_draft_translated, canonical_appendix,
agent._localized_reference_heading(target_lang),
)
qual_engine.validate_bound_target_appendix(
full_draft_translated, appendix_manifest, target_lang
)
self._record_adaptive_completion_residual(
quality_notes,
[{'type':'target_qualitative_appendix_deterministic_rebind','reason':type(exc).__name__}],
section_name='Appendix', stage='translation_final_recovery'
)
except Exception as recovery_exc:
# A structurally invalid/missing typed Appendix is a real
# completion failure, not a cosmetic translation warning.
raise RuntimeError('target_qualitative_appendix_final_validation_failed') from recovery_exc
try:
agent.validate_translation_alignment(full_draft_en, full_draft_translated)
except Exception as exc:
self._record_adaptive_completion_residual(
quality_notes, [{'type':'translation_final_alignment_review','reason':str(exc)[:500]}],
section_name='Target-language manuscript', stage='translation_final_review'
)
try:
agent.validate_reference_preservation(full_draft_en, full_draft_translated, target_lang)
except Exception as exc:
logging.warning(
"Adaptive target References rebind | language=%s reason=%s",
target_lang, str(exc)[:300],
)
full_draft_translated = agent.rebind_target_reference_tail(
full_draft_translated, reference_tail, target_lang
)
# References are identity-bearing academic data. Recovery is
# deterministic and provider-free; if this still fails, package
# generation must stop rather than ship a corrupted bibliography.
agent.validate_reference_preservation(
full_draft_en, full_draft_translated, target_lang
)
self._record_adaptive_completion_residual(
quality_notes, [{'type':'target_reference_tail_rebound','reason':str(exc)[:500]}],
section_name='Target-language manuscript', stage='translation_final_recovery'
)
self._publish_target_preview(
full_draft_translated, 'translation_complete', force=True, complete=True
)
state.update(
{
"full_draft_translated": full_draft_translated,
"translation_sections": translation_sections,
"translation_glossary": translation_glossary,
"translation_complete": True,
"agent_state": agent.export_state(),
}
)
self._checkpoint("translation_complete", state, 96)
normalized_target = _normalize_target_manuscript_typography(
full_draft_translated, target_lang
)
if normalized_target != full_draft_translated:
full_draft_translated = normalized_target
state["full_draft_translated"] = full_draft_translated
existing_output = str(state.get("output_filename") or "").strip()
if state.get("package_validated") and existing_output:
existing_path = self.output_dir / os.path.basename(existing_output)
if existing_path.is_file() and existing_path.stat().st_size > 0:
self._progress(99, "package_validated")
result = {"output_filename": existing_path.name, "output_path": str(existing_path)}
if str(state.get("evidence_status") or "") == "limited":
result.update({
"quality_status": "evidence_limited",
"submission_readiness": "review_required",
"safe_error_code": "W_EVIDENCE_COVERAGE_LIMITED",
"safe_error_detail": self._evidence_warning_detail(),
})
return result
self._progress(97, "packaging")
bj_time = datetime.now(timezone(timedelta(hours=8))).strftime("%Y%m%d_%H%M")
target_file_language = (
"Chinese" if "chinese" in target_lang.casefold() or "中文" in target_lang
else "Russian" if "russian" in target_lang.casefold() or "рус" in target_lang.casefold()
else "Kazakh" if "kazakh" in target_lang.casefold() or "қазақ" in target_lang.casefold()
else re.sub(r"[^A-Za-z]+", "", output_suffix) or "Target"
)
package = build_validated_package(
english_markdown=full_draft_en,
translated_markdown=full_draft_translated,
english_filename="Paper_English.docx",
translated_filename=f"Paper_{target_file_language}.docx",
data_metadata=data_metadata,
mode=mode,
citation_audit=agent.citation_audit_log,
literature_metrics=agent.get_source_metrics(),
quality_diagnostics=quality_diagnostics,
quality_notes=quality_notes,
organize_appendix=True,
evidence_coverage=state.get("evidence_coverage_summary") or {
"status": str(state.get("evidence_status") or "adequate"),
"early_risk_detected": bool(state.get("evidence_risk_detected")),
"final_reference_count": int(state.get("final_reference_count") or 0),
"sections": [],
"events": list(state.get("evidence_coverage_events") or []),
},
)
self._progress(98, "package_validation")
output_filename = f"SmartPaper_{bj_time}.zip"
pending = self.output_dir / (".pending_" + output_filename)
pending.write_bytes(package.zip_bytes)
final = self.output_dir / output_filename
os.replace(pending, final)
state.update(
{
"output_filename": output_filename,
"package_validated": True,
"evidence_status": str(state.get("evidence_status") or "adequate"),
"evidence_coverage_summary": dict(state.get("evidence_coverage_summary") or {}),
"quality_notes": quality_notes,
"agent_state": agent.export_state(),
}
)
self._checkpoint("package_validated", state, 99, display_node="package_validated")
result = {"output_filename": output_filename, "output_path": str(final)}
if str(state.get("evidence_status") or "") == "limited":
result.update({
"quality_status": "evidence_limited",
"submission_readiness": "review_required",
"safe_error_code": "W_EVIDENCE_COVERAGE_LIMITED",
"safe_error_detail": self._evidence_warning_detail(),
})
return result
def build_recovery_artifact(self, error: Exception) -> dict[str, str]:
"""Best-effort failed/refunded download when English authority is already frozen.
This method never changes task success semantics. The Executor may expose
the returned ZIP only on a FAILED task after quota release.
"""
if "cancel" in str(error or "").casefold():
return {}
latest_path = self.task_dir / "checkpoints" / "latest.json"
try:
payload = json.loads(latest_path.read_text(encoding="utf-8"))
except Exception:
return {}
if not isinstance(payload, dict) or payload.get("request_hash") != self.request_hash:
return {}
if not bool(payload.get("english_finalized")):
return {}
english = str(payload.get("full_draft_en") or "").strip()
if len(english) < 1500:
return {}
translated = str(payload.get("full_draft_translated") or "").strip()
target_lang = str(self.request.get("target_lang") or "").strip()
output_suffix = str(self.request.get("output_suffix") or "Target").strip()
target_file_language = (
"Chinese" if "chinese" in target_lang.casefold() or "中文" in target_lang
else "Russian" if "russian" in target_lang.casefold() or "рус" in target_lang.casefold()
else "Kazakh" if "kazakh" in target_lang.casefold() or "қазақ" in target_lang.casefold()
else re.sub(r"[^A-Za-z]+", "", output_suffix) or "Target"
)
try:
package = build_recovery_package(
english_markdown=english,
translated_markdown=translated,
english_filename="Paper_English.docx",
translated_filename=f"Paper_{target_file_language}_Partial.docx",
error_detail=str(error or "")[:1000],
)
except Exception as recovery_exc:
logging.warning(
"Delivery-first recovery package unavailable | error=%s original=%s",
type(recovery_exc).__name__, type(error).__name__,
)
return {}
bj_time = datetime.now(timezone(timedelta(hours=8))).strftime("%Y%m%d_%H%M")
output_filename = f"SmartPaper_Recovery_{bj_time}.zip"
pending = self.output_dir / (".pending_" + output_filename)
pending.write_bytes(package.zip_bytes)
final = self.output_dir / output_filename
os.replace(pending, final)
return {"output_filename": output_filename, "output_path": str(final)}
def run(self) -> dict[str, Any]:
try:
return self._run_impl()
finally:
self._shutdown_translation_worker(wait=False)
def _progress_activity(self) -> None:
# Activity remains Executor-owned. We also snapshot actual request
# counters so the next Executor heartbeat can persist live UI metrics.
ds_calls, s2_calls, oa_calls = self._compute_request_counts()
try:
self.progress_callback(
activity_only=True,
ds_calls=ds_calls,
s2_calls=s2_calls,
oa_calls=oa_calls,
)
except TypeError:
pass