# -*- coding: utf-8 -*-
"""R3-Free headless SmartPaper generation runner.
The runner contains no Streamlit calls. It is safe to execute in a subprocess
whose lifetime is independent from Streamlit ScriptRunner reruns and browser
connections. Checkpoints are task-scoped, immutable-by-sequence JSON snapshots.
"""
from __future__ import annotations
import copy
import hashlib
import inspect
import json
import logging
import math
import os
import queue
import re
import threading
import time
from concurrent.futures import Future
from datetime import datetime, timezone, timedelta
from pathlib import Path
from typing import Callable, Any
from config import APP_CONFIG, AMJ_PROMPTS, ASQ_PROMPTS
from core.checkpoint_store import TaskCheckpointStore
from core.meta_extractor import extract_golden_metadata
from core.model_gateway import resolve_plan_policy_snapshot
from core.pipeline_agent import SSciPipelineAgent, PipelineCancelledError
from core.production_guard import build_validated_package, replace_abstract
from core.qualitative_engine import get_qualitative_engine
from core.quantitative_engine import get_quantitative_engine
from core.rag_engine import initialize_rag
from core.study_design_contract import (
build_user_research_intent,
keyword_policy_from_contract,
apply_keyword_policy,
parse_keyword_terms,
)
from utils.file_parser import extract_text_from_files
ProgressCallback = Callable[..., None]
CITATION_SEMANTIC_SURVIVAL_RATIO = 0.78
EVIDENCE_COMPLETION_MAX_PER_TOP_LEVEL = 4
CITATION_ROLE_RULES = {
"introduction": (3, 180),
"theory": (3, 180),
"theoretical_framing": (3, 180),
"methodology": (2, 280),
"methods": (2, 280),
"discussion": (2, 220),
}
TRANSLATION_CONTRACT_VERSION = 'hierarchical-multilingual-v9'
FINAL_CITATION_LEDGER_VERSION = 'resolved-final-ledger-v1'
ENGLISH_SECTION_SEAL_VERSION = 'english-section-integrity-seal-v1'
# S0-R1-C5.5: one shared private-vocabulary registry is consumed by both
# Writer-facing sanitization and final reader-facing validation. Keeping a
# single registry prevents the historical drift where a phrase was allowed
# into the Writer prompt/output but rejected later by the purity Gate.
AUTHOR_FACING_PRIVATE_VOCABULARY = (
(r"\bfrozen\s+research\s+evidence\s+packet\b", "study evidence"),
(r"\bfrozen\s+evidence\s+packet\b", "study evidence"),
(r"\bevidence\s+packet\b", "study evidence"),
(r"\b(?:study\s+)?evidence\s+ledger\b", "study evidence record"),
(r"\bfrozen\s+(?:research\s+)?evidence\b", "study evidence"),
(r"\bfrozen\s+study\s+evidence\b", "study evidence"),
(r"\bfrozen\s+(?:research\s+|study\s+)?(facts?|numbers?|design|setting|sample|measures?|results?|themes?|quotations?|procedures?)\b", r"documented \1"),
(r"\b(?:study|result)-owned\s+facts?\b", "study-specific facts"),
(r"\bfrozen\s+task-level\s+concepts?\b", "canonical research concepts"),
(r"\bfrozen\s+theme\s+summary\b", "theme summary"),
(r"\bfrozen\s+thematic\s+structure\b", "thematic structure"),
(r"\bfrozen\s+statistical\s+evidence\b", "study statistical evidence"),
(r"\bfrozen\s+measurement\s+design\b", "documented measurement design"),
(r"\bfrozen\s+association\s+model\b", "specified association model"),
(r"\bfrozen\s+bootstrap\s+and\s+collinearity\s+checks\b", "documented bootstrap and collinearity checks"),
)
def _clean_trailing_references(text, keywords=("reference", "references", "reference list", "参考文献")):
lines = str(text or "").split("\n")
for i in range(len(lines) - 1, max(-1, len(lines) - 100), -1):
if any(re.match(rf"^##?\s*{kw}", lines[i], re.IGNORECASE) for kw in keywords):
return "\n".join(lines[:i]).strip()
return str(text or "").strip()
def _clean_markdown_headings(text):
return re.sub(r"(?m)^#+\s+(.*)$", r"**\1**", str(text or ""))
def _canonical_request_hash(request_data: dict[str, Any]) -> str:
material = dict(request_data or {})
material.pop("request_hash", None)
encoded = json.dumps(material, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
return hashlib.sha256(encoded).hexdigest()
def _normalize_target_manuscript_typography(text: str, target_lang: str) -> str:
"""Apply only deterministic language-local typography after semantic QA.
R14-TQ2-PQ1-R1 fixes English-style spaces around Chinese full-width
punctuation without touching Russian/Kazakh spacing, references, semantic
content, or code-fenced material. This runs only after the target manuscript
has passed translation QA; it is presentation normalization, not translation.
"""
value = str(text or "")
language = str(target_lang or "")
if "chinese" not in language.casefold() and "中文" not in language:
return value
value = value.replace("\r\n", "\n").replace("\r", "\n")
# The English References tail is immutable data, not target-language prose.
# Chinese punctuation normalization therefore stops before References.
reference_match = re.search(r"(?m)^#\s+参考文献\s*$", value)
frozen_reference_tail = ""
if reference_match:
frozen_reference_tail = value[reference_match.start():]
value = value[:reference_match.start()]
lines = value.split("\n")
normalized = []
in_fence = False
for line in lines:
stripped = line.lstrip()
if stripped.startswith("```"):
in_fence = not in_fence
normalized.append(line)
continue
if in_fence:
normalized.append(line)
continue
current = re.sub(r"[ \t]+([，。！？；：])", r"\1", line)
current = re.sub(r"([，。！？；：])[ \t]+(?=\S)", r"\1", current)
normalized.append(current)
normalized_body = "\n".join(normalized)
return normalized_body + frozen_reference_tail
class _OrderedTranslationWorker:
"""One daemon worker owned by the headless Executor process.
It intentionally has no checkpoint authority. Results are returned through
Future objects and are checkpointed only by GenerationRunner's main thread.
"""
def __init__(self, handler):
self._handler = handler
self._queue = queue.Queue()
self._stop = threading.Event()
# Lifecycle closure and task cancellation are deliberately distinct.
# A graceful drain/close before the strict final translation gate must
# not poison the translation agent with E_TASK_CANCELLED.
self._cancel = threading.Event()
self._thread = threading.Thread(
target=self._run,
name="smartpaper-translation-worker",
daemon=True,
)
self._thread.start()
@property
def stop_event(self):
return self._stop
@property
def cancel_event(self):
return self._cancel
def submit(self, job: dict[str, Any]) -> Future:
future = Future()
self._queue.put((future, dict(job)))
return future
def _run(self):
while True:
item = self._queue.get()
try:
if item is None:
return
future, job = item
if not future.set_running_or_notify_cancel():
continue
try:
result = self._handler(job)
except BaseException as error:
future.set_exception(error)
else:
future.set_result(result)
finally:
self._queue.task_done()
def close(self, *, wait: bool = False) -> None:
if self._stop.is_set():
if wait and self._thread.is_alive():
self._thread.join(timeout=2.0)
return
# wait=True is the normal final-gate path after all queued work has
# been collected. Only an abortive/non-waiting shutdown signals the
# translation agent to cancel an in-flight model call.
if not wait:
self._cancel.set()
self._stop.set()
while True:
try:
item = self._queue.get_nowait()
except queue.Empty:
break
try:
if item is not None:
future, _job = item
future.cancel()
finally:
self._queue.task_done()
self._queue.put(None)
if wait and self._thread.is_alive():
self._thread.join(timeout=2.0)
class GenerationRunner:
def __init__(
self,
*,
task_dir: str | os.PathLike,
request_data: dict[str, Any],
execution_epoch: int,
executor_instance_id: str = "",
confirmed_checkpoint_seq: int | None = None,
confirmed_checkpoint_stage: str = "",
trusted_task_policy_snapshot: dict[str, Any] | None = None,
progress_callback: ProgressCallback | None = None,
cancel_checker: Callable[[], bool] | None = None,
):
self.task_dir = Path(task_dir)
self.request = dict(request_data)
self.execution_epoch = int(execution_epoch)
self.executor_instance_id = str(executor_instance_id or "")
self.confirmed_checkpoint_seq = (
None if confirmed_checkpoint_seq is None else max(0, int(confirmed_checkpoint_seq))
)
self.confirmed_checkpoint_stage = str(confirmed_checkpoint_stage or "")[:120]
self.trusted_task_policy_snapshot = copy.deepcopy(
dict(trusted_task_policy_snapshot or {})
)
self.progress_callback = progress_callback or (lambda **_: None)
self.cancel_checker = cancel_checker or (lambda: False)
expected_hash = str(self.request.get("request_hash") or "")
actual_hash = _canonical_request_hash(self.request)
if not expected_hash or expected_hash != actual_hash:
raise RuntimeError("request_hash_mismatch")
self.request_hash = expected_hash
self.checkpoints = TaskCheckpointStore(
self.task_dir,
self.request_hash,
self.execution_epoch,
task_id=str(self.request.get("task_id") or "") if self.executor_instance_id else "",
executor_instance_id=self.executor_instance_id,
confirmed_checkpoint_seq=self.confirmed_checkpoint_seq,
confirmed_checkpoint_stage=self.confirmed_checkpoint_stage,
)
self.output_dir = self.task_dir / "outputs"
self.output_dir.mkdir(parents=True, exist_ok=True)
self.agent = None
self.translation_agent = None
self._preview_lock = threading.RLock()
preview_token = str(self.request.get("preview_token") or "").strip()
self.preview_token = preview_token if re.fullmatch(r"[A-Za-z0-9_-]{32,128}", preview_token) else ""
self.preview_path = None
self._last_preview_write = 0.0
self._preview_english_text = ""
self._preview_english_stage = ""
self._preview_english_complete = False
self._preview_target_text = ""
self._preview_target_stage = "translation_waiting"
self._preview_target_complete = False
self._preview_target_english_commit_text = ""
self._translation_worker = None
self._translation_worker_sections = {}
self._translation_worker_glossary = {}
self._translation_worker_jobs = []
self._translation_worker_collected = 0
self._translation_worker_pending = set()
self._translation_worker_context = {}
# Display-only observability. These counters never decide task success,
# checkpoint authority, quota, or translation reuse.
self._translation_progress_sections = {}
self._translation_progress_queued = set()
self._translation_progress_inflight = set()
self._translation_progress_total = 0
self._qualitative_quote_sources = {}
# R14-TQ2-PQ1-R3-R1: evidence coverage is a quality signal, not a
# manuscript-completeness authority. It is persisted in checkpoints and
# exposed to the browser only as a stable status code.
self._preview_evidence_status = "normal"
self._evidence_coverage_events = []
if self.preview_token:
preview_dir = Path(__file__).resolve().parents[1] / "static" / "previews"
preview_dir.mkdir(parents=True, exist_ok=True)
self.preview_path = preview_dir / f"{self.preview_token}.json"
# Best-effort cleanup of stale display-only previews; task/checkpoint
# authority is unaffected by this cache.
cutoff = time.time() - 24 * 3600
for old_preview in preview_dir.glob("*.json"):
try:
if old_preview.stat().st_mtime < cutoff:
old_preview.unlink()
except OSError:
pass
def _compute_request_counts(self) -> tuple[int, int, int]:
"""Return UI metrics: DeepSeek requests and eligible literature results."""
agent = self.agent
if agent is None:
return 0, 0, 0
metrics = getattr(agent, "source_metrics", {}) or {}
s2_metric = metrics.get("s2", {}) if isinstance(metrics, dict) else {}
oa_metric = metrics.get("oa", {}) if isinstance(metrics, dict) else {}
translator_calls = int(getattr(getattr(self, "translation_agent", None), "ds_calls", 0) or 0)
return (
int(getattr(agent, "ds_calls", 0) or 0) + translator_calls,
int(s2_metric.get("eligible_result_count", 0) or 0),
int(oa_metric.get("eligible_result_count", 0) or 0),
)
def _progress(self, pct: int, node: str, *, checkpoint_seq=None, checkpoint_stage=None) -> None:
ds_calls, s2_calls, oa_calls = self._compute_request_counts()
self.progress_callback(
progress_pct=max(0, min(100, int(pct))),
current_node=str(node or "")[:240],
checkpoint_seq=checkpoint_seq,
checkpoint_stage=checkpoint_stage,
ds_calls=ds_calls,
s2_calls=s2_calls,
oa_calls=oa_calls,
)
def _checkpoint(self, stage: str, state: dict[str, Any], pct: int, *, display_node: str | None = None) -> None:
seq, normalized_stage = self.checkpoints.save(stage, state)
self._progress(
pct, display_node or normalized_stage,
checkpoint_seq=seq, checkpoint_stage=normalized_stage,
)
@staticmethod
def _preview_visible_text(value: str) -> str:
text = str(value or "")
# Never expose hidden reasoning. Raw CITE placeholders are also hidden
# from the user because they have not passed semantic citation QA yet;
# the authoritative manuscript buffer remains untouched.
text = re.sub(r"(?is).*?(?:|$)", "", text)
text = re.sub(r"\s*\[?CITE-\d+\]?\s*", " ", text, flags=re.I)
text = re.sub(r"[ \t]{2,}", " ", text)
return text.strip()
@staticmethod
def _build_qualitative_quote_ownership(tasks, data_metadata):
"""Freeze one and only one Findings owner for every qualitative quote."""
evidence_by_tag = {
str(item.get("tag") or "").strip(): dict(item)
for item in (data_metadata or {}).get("evidence", [])
if isinstance(item, dict) and str(item.get("tag") or "").strip()
}
owners = {}
for task in tasks or []:
if str(task.get("section_id") or "") != "findings":
continue
owner = str(task.get("display_name") or "").strip()
for item in task.get("theme_evidence") or []:
if not isinstance(item, dict):
continue
tag = str(item.get("tag") or "").strip()
if not tag:
continue
if tag not in evidence_by_tag:
raise RuntimeError(f"Qualitative quotation ownership references unknown tag: {tag}")
if tag in owners:
raise RuntimeError(
f"Qualitative quotation has multiple Findings owners: {tag} -> {owners[tag]!r}, {owner!r}"
)
owners[tag] = owner
missing = sorted(set(evidence_by_tag) - set(owners))
if missing:
raise RuntimeError(f"Qualitative quotations lack a Findings owner: {missing[:8]}")
return owners
@staticmethod
def _qualitative_task_key(task):
task = task or {}
explicit = str(task.get("task_key") or "").strip()
if explicit:
return explicit
section_id = str(task.get("section_id") or task.get("section") or "section").strip()
stable_unit = str(task.get("theme_id") or task.get("subsection_id") or section_id).strip()
part_index = int(task.get("part_index") or 0) + 1
return f"{section_id}:{stable_unit}:part:{part_index:02d}"
@classmethod
def _build_qualitative_quote_invocation_plan(cls, tasks, data_metadata):
"""Freeze one deterministic body invocation for every retained quote.
Logical ownership is a stable Theme object. Runtime Part chunks only
select where the already-owned quote is rendered; they never become the
quote owner and can never change cardinality.
"""
metadata = data_metadata or {}
evidence_by_tag = {
str(item.get("tag") or "").strip(): dict(item)
for item in metadata.get("evidence", [])
if isinstance(item, dict) and str(item.get("tag") or "").strip()
}
theme_id_by_title = {}
for index, theme in enumerate(metadata.get("themes") or [], start=1):
if not isinstance(theme, dict):
continue
title = re.sub(r"\s+", " ", str(theme.get("title") or "")).strip().casefold()
if title:
theme_id_by_title[title] = str(theme.get("theme_id") or f"THEME-{index:02d}").strip()
plan = {}
task_slots = {}
for task in tasks or []:
if str(task.get("section_id") or "") != "findings":
continue
task_key = cls._qualitative_task_key(task)
task_theme_id = str(task.get("theme_id") or "").strip()
task_theme_title = str(task.get("theme_title") or task.get("subsection") or "").strip()
for item in task.get("theme_evidence") or []:
if not isinstance(item, dict):
continue
tag = str(item.get("tag") or "").strip()
if not tag:
continue
evidence = evidence_by_tag.get(tag)
if evidence is None:
raise RuntimeError(f"Qualitative quote plan references unknown evidence: {tag}")
if tag in plan:
raise RuntimeError(f"Qualitative quote plan has duplicate render assignment: {tag}")
owner_theme_id = str(
task_theme_id
or evidence.get("theme_id")
or theme_id_by_title.get(
re.sub(r"\s+", " ", str(evidence.get("theme") or task_theme_title)).strip().casefold()
)
or ""
).strip()
if not owner_theme_id:
raise RuntimeError(f"Qualitative quote lacks a stable theme owner: {tag}")
quote = str(evidence.get("quote") or "").strip()
if not quote:
raise RuntimeError(f"Qualitative quote is empty: {tag}")
quote_hash = str(evidence.get("quote_hash") or "").strip() or hashlib.sha256(
re.sub(r"\s+", " ", quote).strip().encode("utf-8")
).hexdigest()
slot_ordinal = task_slots.get(task_key, 0) + 1
task_slots[task_key] = slot_ordinal
plan[tag] = {
"quote_id": tag,
"quote_hash": quote_hash,
"owner_theme_id": owner_theme_id,
"owner_theme_title": str(evidence.get("theme") or task_theme_title).strip(),
"render_task_key": task_key,
"render_part_index": int(task.get("part_index") or 0),
"render_part_count": int(task.get("part_count") or 1),
"slot_ordinal": slot_ordinal,
"cardinality": 1,
}
missing = sorted(set(evidence_by_tag) - set(plan))
extra = sorted(set(plan) - set(evidence_by_tag))
if missing or extra:
raise RuntimeError(
f"Qualitative quote invocation plan mismatch: missing={missing[:8]} extra={extra[:8]}"
)
cls._validate_qualitative_quote_invocation_plan(plan, tasks, metadata)
return plan
@classmethod
def _validate_qualitative_quote_invocation_plan(cls, plan, tasks, data_metadata):
metadata = data_metadata or {}
evidence_by_tag = {
str(item.get("tag") or "").strip(): dict(item)
for item in metadata.get("evidence", [])
if isinstance(item, dict) and str(item.get("tag") or "").strip()
}
task_by_key = {cls._qualitative_task_key(task): task for task in tasks or []}
if set(plan or {}) != set(evidence_by_tag):
raise RuntimeError("Qualitative quote invocation plan does not match the frozen evidence ledger.")
for tag, record in (plan or {}).items():
if int(record.get("cardinality") or 0) != 1:
raise RuntimeError(f"Qualitative quote cardinality must equal one: {tag}")
owner_theme_id = str(record.get("owner_theme_id") or "").strip()
if not owner_theme_id or "part " in owner_theme_id.casefold():
raise RuntimeError(f"Qualitative quote owner is not a stable Theme object: {tag}")
task_key = str(record.get("render_task_key") or "").strip()
task = task_by_key.get(task_key)
if not task or str(task.get("section_id") or "") != "findings":
raise RuntimeError(f"Qualitative quote render task is invalid: {tag} -> {task_key!r}")
task_theme_id = str(task.get("theme_id") or "").strip()
if task_theme_id and task_theme_id != owner_theme_id:
raise RuntimeError(f"Qualitative quote render task crossed theme ownership: {tag}")
evidence = evidence_by_tag[tag]
quote = str(evidence.get("quote") or "").strip()
expected_hash = str(evidence.get("quote_hash") or "").strip() or hashlib.sha256(
re.sub(r"\s+", " ", quote).strip().encode("utf-8")
).hexdigest()
if str(record.get("quote_hash") or "") != expected_hash:
raise RuntimeError(f"Qualitative quote hash drifted after planning: {tag}")
return True
@staticmethod
def _strip_model_controlled_qualitative_quotes(text, data_metadata):
"""Remove model-emitted quote controls/literals before Executor rendering.
This is a containment boundary, not the quote control plane: quotation
presence and cardinality come exclusively from the frozen invocation
plan. The model's attempted tags/literals therefore have zero authority.
"""
value = str(text or "")
value = re.sub(r"(?)", block)
]
if not anchor_indices:
anchor_indices = [len(blocks) - 1] if blocks else []
insertions = {}
for index, record in enumerate(scheduled):
tag = str(record.get("quote_id") or "").strip()
quote = str((evidence_by_tag.get(tag) or {}).get("quote") or "").strip()
if not quote:
raise RuntimeError(f"Planned qualitative quotation is unavailable at render time: {tag}")
if anchor_indices:
anchor_pos = min(
len(anchor_indices) - 1,
max(0, int(((index + 1) * len(anchor_indices)) / (len(scheduled) + 1))),
)
block_index = anchor_indices[anchor_pos]
insertions.setdefault(block_index, []).append(f'"{quote}"')
else:
blocks.append(f'"{quote}"')
rendered_blocks = []
for index, block in enumerate(blocks):
rendered_blocks.append(block)
rendered_blocks.extend(insertions.get(index, []))
rendered = "\n\n".join(rendered_blocks).strip()
for record in scheduled:
tag = str(record.get("quote_id") or "").strip()
quote = str((evidence_by_tag.get(tag) or {}).get("quote") or "").strip()
if rendered.count(quote) != 1:
raise RuntimeError(f"Qualitative deterministic quote rendering failed: {tag}")
return rendered
@staticmethod
def _validate_qualitative_quote_rendering(text, quote_invocation_plan, data_metadata):
body_text = re.split(
r"(?m)^#\s+Appendix(?:[:\s].*)?$", str(text or ""), maxsplit=1
)[0]
unresolved = sorted(set(re.findall(r"\bQUOTE-\d+\b", body_text, flags=re.I)))
if unresolved:
raise RuntimeError(f"Unresolved qualitative quote controls in body: {unresolved[:8]}")
normalized_body = re.sub(r"\s+", " ", body_text).casefold()
evidence_by_tag = {
str(item.get("tag") or "").strip(): dict(item)
for item in (data_metadata or {}).get("evidence", [])
if isinstance(item, dict) and str(item.get("tag") or "").strip()
}
for tag, record in (quote_invocation_plan or {}).items():
quote = re.sub(r"\s+", " ", str((evidence_by_tag.get(tag) or {}).get("quote") or "")).strip().casefold()
expected = int(record.get("cardinality") or 0)
actual = normalized_body.count(quote) if quote else 0
if actual != expected:
raise RuntimeError(
f"Qualitative quote invocation invariant failed: {tag} actual={actual} planned={expected}"
)
return True
@staticmethod
def _redact_qualitative_quotes(text, data_metadata):
"""Hide prior verbatim quote text from cross-unit continuity context."""
value = str(text or "")
quotes = sorted(
{
str(item.get("quote") or "").strip()
for item in (data_metadata or {}).get("evidence", [])
if isinstance(item, dict) and str(item.get("quote") or "").strip()
},
key=len,
reverse=True,
)
for quote in quotes:
value = value.replace(f'"{quote}"', "[prior qualitative quotation omitted]")
value = value.replace(quote, "[prior qualitative quotation omitted]")
return value
@classmethod
def _bind_owned_qualitative_quotes(
cls,
text,
*,
display_name,
data_metadata,
quote_ownership,
prior_manuscript="",
):
"""Bind only the current Findings unit's owned quotation tags.
Other manuscript units are not allowed to contain frozen quote text or
quote tags. A direct verbatim reproduction of an owned quote is
normalized back to its tag before deterministic binding, preserving the
frozen wording without letting model prose create a second copy.
"""
value = str(text or "")
evidence_by_tag = {
str(item.get("tag") or "").strip(): dict(item)
for item in (data_metadata or {}).get("evidence", [])
if isinstance(item, dict) and str(item.get("tag") or "").strip()
}
allowed = {
tag for tag, owner in (quote_ownership or {}).items()
if str(owner) == str(display_name)
}
raw_tags = set(re.findall(r"\bQUOTE-\d+\b", value))
unauthorized_tags = sorted(raw_tags - allowed)
if unauthorized_tags:
raise RuntimeError(
f"Qualitative quotation tag used outside owning Findings unit {display_name!r}: {unauthorized_tags[:8]}"
)
# Quote text itself is also fenced. Owned verbatim text is converted
# back to a tag and then bound once; unowned verbatim text fails early.
for tag, item in evidence_by_tag.items():
quote = str(item.get("quote") or "").strip()
if not quote or quote not in value:
continue
if tag not in allowed:
raise RuntimeError(
f"Qualitative quotation text used outside owning Findings unit: {tag} in {display_name!r}"
)
value = value.replace(f'"{quote}"', tag)
value = value.replace(quote, tag)
# A quote already present in frozen prior prose cannot be rebound later,
# even after resume. This makes ownership idempotent without new state.
normalized_prior = re.sub(r"\s+", " ", str(prior_manuscript or "")).casefold()
for tag in sorted(allowed):
quote = str((evidence_by_tag.get(tag) or {}).get("quote") or "").strip()
if quote and re.sub(r"\s+", " ", quote).casefold() in normalized_prior:
raise RuntimeError(f"Qualitative quotation already used before owning unit: {tag}")
for tag in sorted(allowed):
tag_pattern = rf"(? 1:
raise RuntimeError(
f"Qualitative quotation repeated within owning Findings unit: {tag} count={count}"
)
if count == 1:
quote = str((evidence_by_tag.get(tag) or {}).get("quote") or "").strip()
value = re.sub(
tag_pattern,
lambda _match, frozen_quote=quote: f'"{frozen_quote}"',
value,
count=1,
)
# No unresolved quote tag may escape into citation QA or final output.
unresolved = sorted(set(re.findall(r"\bQUOTE-\d+\b", value)))
if unresolved:
raise RuntimeError(f"Unresolved qualitative quotation tags: {unresolved[:8]}")
return value
@staticmethod
def _qualitative_quote_source_map(data_metadata):
return {
str(item.get("tag") or "").strip(): str(item.get("quote") or "").strip()
for item in (data_metadata or {}).get("evidence", [])
if isinstance(item, dict)
and str(item.get("tag") or "").strip()
and str(item.get("quote") or "").strip()
}
@staticmethod
def _qualitative_quote_ledger_identity_ok(
record, *, target_lang: str, request_hash: str, source_quote_hashes: dict,
task_id: str = "", translation_contract_version: str = TRANSLATION_CONTRACT_VERSION,
) -> bool:
"""Type-safe identity gate for task-level target-language Quote reuse.
Missing scalar identity fields remain backward-compatible with pre-R4
checkpoints. Present fields must be strings and match exactly. Quote
hashes must be a mapping and match the current frozen English evidence.
execution_epoch is deliberately not part of logical identity so a newer
epoch can reuse the same QA-passed target rendering for the same task.
"""
if not isinstance(record, dict) or not isinstance(source_quote_hashes, dict):
return False
def scalar_matches(stored, current: str) -> bool:
if stored is None or stored == "":
return True
return isinstance(stored, str) and stored == str(current or "")
stored_hashes = record.get("source_quote_hashes")
if stored_hashes is None or stored_hashes == "":
hashes_match = True
else:
hashes_match = isinstance(stored_hashes, dict) and stored_hashes == source_quote_hashes
return (
scalar_matches(record.get("task_id"), task_id)
and scalar_matches(record.get("target_lang"), target_lang)
and scalar_matches(record.get("request_hash"), request_hash)
and scalar_matches(record.get("translation_contract_version"), translation_contract_version)
and hashes_match
)
@classmethod
def _prepare_qualitative_translation_source(cls, source, quote_sources, quote_ledger):
"""Replace every frozen qualitative quote with a typed semantic slot.
QUOTE identity is never inferred from translated narrative surface text.
Missing target objects are classified as pending and translated through
the dedicated typed-object path before the surrounding narrative runs.
"""
value = str(source or "")
pending = []
reused = []
for tag, quote in sorted((quote_sources or {}).items()):
wrapped = f'"{quote}"'
tagged = f'{wrapped} [{tag}]'
if tagged in value:
value = value.replace(tagged, f"[{tag}]")
(reused if tag in (quote_ledger or {}) else pending).append(tag)
continue
if wrapped not in value:
continue
value = value.replace(wrapped, f"[{tag}]")
(reused if tag in (quote_ledger or {}) else pending).append(tag)
return value, pending, reused
@classmethod
def _finalize_qualitative_translation_quotes(cls, translated, pending, reused, quote_ledger):
value = str(translated or "")
ledger = dict(quote_ledger or {})
ordered_tags = list(dict.fromkeys(list(pending or []) + list(reused or [])))
for tag in ordered_tags:
rendered_quote = str(ledger.get(tag) or "").strip()
if not rendered_quote:
raise RuntimeError(f"Qualitative quote translation ledger missing target text: {tag}")
pattern = rf"(? dict[str, int]:
"""Return non-authoritative logical target-unit progress for the UI."""
mapping = dict(getattr(self, '_translation_progress_sections', {}) or {})
context = getattr(self, '_translation_worker_context', {}) or {}
tasks = context.get('adaptive_structure') or [] if isinstance(context, dict) else []
logical_keys = []
if tasks:
logical_keys = ['__front_matter__'] + [
key for key, _title in self._unique_top_level_sections(tasks)
]
logical_keys = list(dict.fromkeys(key for key in logical_keys if key))
total = int(getattr(self, '_translation_progress_total', 0) or len(logical_keys))
if total <= 0 and logical_keys:
total = len(logical_keys)
qa_passed = 0
repair_pending = 0
inflight = set(getattr(self, '_translation_progress_inflight', set()) or set())
for key in logical_keys:
record = mapping.get(key) or {}
status = str(record.get('qa_status') or '')
if key not in inflight and status in {'passed', 'passed_with_warning'} and record.get('translated_text'):
qa_passed += 1
elif status == 'repair_pending':
repair_pending += 1
queued = len(set(getattr(self, '_translation_progress_queued', set()) or set()).intersection(logical_keys))
in_flight = len(inflight.intersection(logical_keys))
return {
'qa_passed': min(qa_passed, total) if total else qa_passed,
'repair_pending': min(repair_pending, total) if total else repair_pending,
'queued': min(queued, total) if total else queued,
'in_flight': min(in_flight, total) if total else in_flight,
'total': max(0, total),
}
def _write_preview_payload(self, *, force: bool = False) -> None:
with self._preview_lock:
if self.preview_path is None:
return
now = time.monotonic()
if not force and now - self._last_preview_write < 0.45:
return
payload = {
"task_id": str(self.request.get("task_id") or ""),
"request_hash": self.request_hash,
"execution_epoch": self.execution_epoch,
"stage": str(self._preview_english_stage or "")[:240],
"text": self._preview_visible_text(self._preview_english_text),
"complete": bool(self._preview_english_complete),
"target_stage": str(self._preview_target_stage or "")[:240],
"target_text": self._preview_visible_text(self._preview_target_text),
"target_complete": bool(self._preview_target_complete),
"target_english_commit_text": self._preview_visible_text(
self._preview_target_english_commit_text
),
"target_progress": self._translation_progress_snapshot(),
"evidence_status": str(self._preview_evidence_status or "normal")[:40],
"updated_at": datetime.now(timezone.utc).isoformat(),
}
temporary = self.preview_path.with_suffix(".json.tmp")
temporary.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
os.replace(temporary, self.preview_path)
self._last_preview_write = now
def _publish_preview(self, text: str, stage: str, *, force: bool = False, complete: bool = False) -> None:
with self._preview_lock:
self._preview_english_text = str(text or "")
self._preview_english_stage = str(stage or "")
self._preview_english_complete = bool(complete)
self._write_preview_payload(force=force)
def _publish_target_preview(
self, text: str, stage: str, *, force: bool = True, complete: bool = False,
english_commit_text: str | None = None,
) -> None:
with self._preview_lock:
self._preview_target_text = str(text or "")
self._preview_target_stage = str(stage or "")
self._preview_target_complete = bool(complete)
if english_commit_text is not None:
self._preview_target_english_commit_text = str(english_commit_text or "")
self._write_preview_payload(force=force)
def _stream_text(self, generator, *, preview_prefix: str | None = None, preview_stage: str = "") -> str:
chunks = []
for chunk in generator:
value = str(chunk or "")
if not value:
continue
chunks.append(value)
if preview_prefix is not None:
self._publish_preview(preview_prefix + "".join(chunks), preview_stage, force=False)
text = "".join(chunks)
if preview_prefix is not None:
self._publish_preview(preview_prefix + text, preview_stage, force=True)
return text
@staticmethod
def _text_hash(value: str) -> str:
normalized = re.sub(r'\s+', ' ', str(value or '')).strip()
return hashlib.sha256(normalized.encode('utf-8')).hexdigest()
@staticmethod
def _unique_top_level_sections(tasks):
output = []
seen = set()
for task in tasks or []:
key = str(task.get('section_id') or task.get('section') or '').strip()
title = str(task.get('section') or '').strip()
if key and title and key not in seen:
seen.add(key); output.append((key, title))
return output
@staticmethod
def _extract_top_level_section(manuscript: str, title: str) -> str:
value = str(manuscript or '')
match = re.search(
rf'(?ms)^#\s+{re.escape(str(title))}\s*$\n(.*?)(?=^#\s+[^#].*?$|\Z)', value
)
if not match:
return ''
return f"# {title}\n" + match.group(1).strip()
@staticmethod
def _normalize_orchestrator_owned_headings(text: str, task: dict) -> str:
"""Restore immutable H1/H2 structure after any model-owned repair path.
Writer and repair models own prose only. The orchestrator owns the
expected top-level and subsection headings described by the generation
task. This deterministic normalizer removes only exact markdown
heading lines for those owned titles, then restores the canonical
heading(s) required for this unit. All non-heading text is preserved.
The operation is intentionally idempotent: N(N(x)) == N(x).
"""
value = str(text or "")
task = dict(task or {})
section_title = str(task.get("section") or "").strip()
subsection_title = str(task.get("subsection") or "").strip()
section_first = bool(task.get("section_first")) and bool(section_title)
subsection_first = bool(task.get("subsection_first")) and bool(subsection_title)
owned_titles = [title for title in (section_title, subsection_title) if title]
if not owned_titles:
return value
patterns = [
re.compile(rf"^[ \\t]*#{{1,6}}[ \\t]+{re.escape(title)}[ \\t]*$")
for title in owned_titles
]
kept_lines = []
for line in value.splitlines(keepends=True):
visible = line.rstrip("\r\n")
if any(pattern.fullmatch(visible) for pattern in patterns):
continue
kept_lines.append(line)
body = "".join(kept_lines).lstrip("\r\n")
canonical = []
if section_first:
canonical.append(f"# {section_title}")
if subsection_first:
canonical.append(f"## {subsection_title}")
if not canonical:
return body
prefix = "\n\n".join(canonical)
return prefix if not body else prefix + "\n\n" + body
@staticmethod
def _english_prefix_through_top_level(manuscript: str, title: str) -> str:
value = str(manuscript or '')
match = re.search(
rf'(?ms)^#\s+{re.escape(str(title))}\s*$\n.*?(?=^#\s+[^#].*?$|\Z)', value
)
return value[:match.end()].strip() if match else value.strip()
@staticmethod
def _extract_front_matter(manuscript: str, first_section_title: str) -> str:
value = str(manuscript or '')
match = re.search(rf'(?m)^#\s+{re.escape(str(first_section_title))}\s*$', value)
return value[:match.start()].strip() if match else ''
@staticmethod
def _extract_body_from_first_section(manuscript: str, first_section_title: str) -> str:
value = str(manuscript or '')
match = re.search(rf'(?m)^#\s+{re.escape(str(first_section_title))}\s*$', value)
return value[match.start():].strip() if match else value.strip()
@staticmethod
def _repair_validate_study_fact_contract(
mode: str, agent, quant_engine, text: str, data_metadata: dict, *, section_name: str
):
"""S0-R1 compatibility gateway: inspect only and never mutate prose."""
value = str(text or '').strip()
diagnostics = []
if str(mode or '') == 'asq':
diagnostics = agent.inspect_qualitative_study_fact_contract(
value, data_metadata, section_name=section_name
)
elif str(mode or '') == 'amj':
diagnostics = quant_engine.inspect_quantitative_study_fact_contract(
value, data_metadata, section_name=section_name
)
diagnostics = list(diagnostics or [])
scope_inspector = getattr(agent, "inspect_study_design_scope_contract", None)
if callable(scope_inspector):
diagnostics.extend(scope_inspector(value, data_metadata, section_name=section_name) or [])
return value, diagnostics
@staticmethod
def _study_fact_violations(mode: str, agent, quant_engine, text: str, data_metadata: dict, *, section_name: str):
rows = []
if str(mode or '') == 'asq':
rows.extend(agent.inspect_qualitative_study_fact_contract(text, data_metadata, section_name=section_name) or [])
elif str(mode or '') == 'amj':
rows.extend(quant_engine.inspect_quantitative_study_fact_contract(text, data_metadata, section_name=section_name) or [])
scope_inspector = getattr(agent, "inspect_study_design_scope_contract", None)
if callable(scope_inspector):
rows.extend(scope_inspector(text, data_metadata, section_name=section_name) or [])
return list(rows)
@staticmethod
def _section_retry_feedback(violations, *, limit=8) -> str:
rows = []
for item in list(violations or [])[:max(1, int(limit))]:
if not isinstance(item, dict):
rows.append(str(item)[:500])
continue
kind = str(item.get('type') or 'validation_issue')
detail = str(item.get('repair_instruction') or item.get('reason') or item.get('preview') or '')
if kind == 'causal_language_violation' and not detail:
detail = (
"Rewrite only the causal-strength wording. Preserve the study fact and analytical meaning, "
"but state association/covariation rather than causal identification. Explicit limitation "
"phrases such as 'restricted to associations, not causal effects' and 'cannot establish "
"causality' are valid and should not be weakened further."
)
if kind == 'citation_evidence_ownership_mismatch':
detail = (
"Split the sentence by evidence ownership. Keep the study-owned fact as a separate statement grounded only in the study evidence and without an external citation. "
"State any methodological principle separately and attach only one directly supporting METHOD_AUTHORITY citation. Keep author inference separate, cautious, and uncited unless a source directly supports that exact inference."
)
actor = str(item.get('actor') or '')
expected = item.get('expected')
observed = item.get('observed')
parts = [kind]
if actor:
parts.append(f'actor={actor}')
if observed not in (None, ''):
parts.append(f'observed={observed}')
if expected not in (None, ''):
parts.append(f'expected={expected}')
if detail:
parts.append(detail[:360])
rows.append(' | '.join(parts))
return '\n'.join(f'- {row}' for row in rows)
@staticmethod
def _section_forbids_external_citations(section_title: str = "", section_id: str = "") -> bool:
"""One authority for manuscript units that must remain citation-free.
P3-A4 closes a historic split where Results/Findings were declared
citation-free in the Writer prompt but did not enter the same quarantine
validation path as Conclusion/Appendix-style units.
"""
title = str(section_title or '').casefold()
sid = str(section_id or '').strip().casefold()
return (
any(word in title for word in ("abstract", "keyword", "conclusion", "references", "appendix"))
or sid in {"results", "findings"}
)
@staticmethod
def _section_violation_severity(row, *, section_name: str = "", mode: str = "", exhausted: bool = False) -> str:
"""Map one validation issue to REPAIRABLE/FATAL/ADVISORY.
Before the one local retry, all substantive correctness issues remain
repairable. After retry exhaustion only issues that can change study
facts, core inference, hypothesis identity, core evidence support, or
citation integrity may terminate the manuscript. Minor reader-facing
style/background defects become advisory notes instead of destroying an
otherwise complete paper.
"""
item = dict(row or {}) if isinstance(row, dict) else {"type": "unknown_validation_issue"}
kind = str(item.get("type") or "unknown_validation_issue")
section = str(section_name or "")
if not exhausted:
if kind == "top_journal_critic":
return "ADVISORY"
return "REPAIRABLE"
always_fatal = {
"causal_language_violation",
"sample_size_mismatch", "sample_count_mismatch", "scale_points_mismatch",
"construct_role_mismatch", "quantitative_metric_mismatch", "unfrozen_p_value",
"unsupported_quantitative_method", "actor_data_source_relation_mismatch",
"unsupported_study_procedure", "enumeration_contract_invalid",
"hypothesis_contract_missing_id", "hypothesis_contract_ordinal_alias",
"hypothesis_contract_pairing_mismatch", "hypothesis_contract_unknown_id",
"citation_outside_frozen_section_whitelist",
"citation_resolved_marker_outside_frozen_section_whitelist",
"citation_unknown_tag", "citation_resolved_marker_without_authority",
"citation_resolved_marker_ambiguous", "citation_noncanonical_resolved_marker",
"citation_forbidden_in_quarantined_unit",
"authorial_ai_meta_trace",
"study_design_scope_mismatch", "unsupported_focal_construct", "unsupported_relation_claim",
"unsupported_moderation_claim", "unsupported_mediation_claim", "unsupported_temporal_scope",
"unsupported_design_tradition", "unsupported_unit_of_analysis", "unsupported_data_source_claim",
"unsupported_process_claim",
}
if kind in always_fatal:
return "FATAL"
if kind in {"author_facing_internal_workflow_language", "top_journal_critic"}:
return "ADVISORY"
if kind in {"citation_claim_split_required", "citation_unsupported_claim", "citation_support_unverified", "citation_evidence_ownership_mismatch"}:
core_citation_section = bool(re.search(
r"(?i)(?:Theoretical Mechanisms|Hypothesis Development|Methodology|Results?|Findings?)",
section,
))
return "FATAL" if core_citation_section else "ADVISORY"
# Unknown validation types remain fail-closed. This keeps C5.5 from
# silently downgrading a future integrity contract merely because it is
# new to the severity registry.
return "FATAL"
@staticmethod
def _section_violation_lineage(row) -> str:
"""Return a stable repair lineage for one section validation defect.
R4 assigns the bounded rewrite entitlement to the defect lineage rather
than to the whole Section. Closely related validators share a lineage
so spelling variants of the same integrity problem cannot manufacture
fresh retries.
"""
item = dict(row or {}) if isinstance(row, dict) else {"type": "unknown_validation_issue"}
kind = str(item.get("type") or "unknown_validation_issue")
groups = {
"citation_scope": {
"citation_outside_frozen_section_whitelist",
"citation_resolved_marker_outside_frozen_section_whitelist",
},
"citation_authority": {
"citation_unknown_tag",
"citation_resolved_marker_without_authority",
"citation_resolved_marker_ambiguous",
"citation_noncanonical_resolved_marker",
"citation_forbidden_in_quarantined_unit",
},
"citation_ownership": {"citation_evidence_ownership_mismatch"},
"citation_support": {"citation_unsupported_claim", "citation_support_unverified"},
"citation_compound": {"citation_claim_split_required"},
"causal_scope": {"causal_language_violation"},
"quant_method": {"unsupported_quantitative_method", "quantitative_metric_mismatch", "unfrozen_p_value"},
"study_fact": {"sample_size_mismatch", "sample_count_mismatch", "scale_points_mismatch",
"construct_role_mismatch", "actor_data_source_relation_mismatch", "unsupported_study_procedure"},
"enumeration": {"enumeration_contract_invalid", "enumeration_contract_mismatch"},
"hypothesis": {"hypothesis_contract_missing_id", "hypothesis_contract_ordinal_alias",
"hypothesis_contract_pairing_mismatch", "hypothesis_contract_unknown_id"},
"workflow_trace": {"authorial_ai_meta_trace", "author_facing_internal_workflow_language"},
"study_design_scope": {
"study_design_scope_mismatch", "unsupported_focal_construct", "unsupported_relation_claim",
"unsupported_moderation_claim", "unsupported_mediation_claim", "unsupported_temporal_scope",
"unsupported_design_tradition", "unsupported_unit_of_analysis", "unsupported_data_source_claim",
"unsupported_process_claim",
},
}
for lineage, kinds in groups.items():
if kind in kinds:
return lineage
return kind
@staticmethod
def _strip_unscoped_citation_identities(value):
"""Remove raw citation capability tokens from non-authoritative context."""
return re.sub(r"\[CITE-\d+\]", "", str(value or ""), flags=re.I)
@staticmethod
def _section_citation_capability_context(agent, planned_items, max_chars=12000):
"""Expose only citation identities already frozen for this manuscript unit.
Retrieval visibility is not citation authority. R6-R10 therefore builds
writer-facing targeted context from the unit's frozen claim-evidence plan
instead of the global RAG pool, which may contain METHOD_AUTHORITY tags
owned by another unit.
"""
rows = []
seen = set()
for item in list(planned_items or []):
if not isinstance(item, dict):
continue
tag = str(item.get("tag") or "").strip()
if not tag or tag in seen:
continue
metadata = dict(getattr(agent, "citation_map", {}).get(tag) or {})
evidence = str(item.get("evidence") or metadata.get("evidence") or "").strip()
claim = str(item.get("claim") or "").strip()
if not evidence:
continue
seen.add(tag)
row = f"{tag} AUTHORIZED CLAIM: {claim}\nEVIDENCE: {evidence}" if claim else f"{tag} EVIDENCE: {evidence}"
rows.append(row)
value = "\n\n--- Authorized Section Evidence ---\n".join(rows)
return value[:max(0, int(max_chars))]
@staticmethod
def _section_regressed_lineages(current_lineages, cleared_lineages, regression_attempts):
"""Return previously-cleared lineages reintroduced by a later sibling repair."""
current = {str(value) for value in (current_lineages or set()) if str(value)}
cleared = {str(value) for value in (cleared_lineages or set()) if str(value)}
attempts = dict(regression_attempts or {})
return {name for name in current & cleared if int(attempts.get(name, 0)) < 1}
@classmethod
def _section_retry_decision(cls, rows, lineage_attempts, *, hard_cap_reached: bool = False):
"""Return (may_retry, exhausted_lineages, new_lineages).
Each lineage receives one rewrite entitlement. A lineage that survives
its own rewrite remains fail-closed. A genuinely new lineage discovered
only after a previous repair receives one bounded chance, subject to the
Section hard cap.
"""
lineages = {cls._section_violation_lineage(row) for row in (rows or [])}
exhausted = {name for name in lineages if int(lineage_attempts.get(name, 0)) >= 1}
fresh = lineages - exhausted
# R6-R3 fairness closure: an exhausted lineage may not steal the bounded
# repair entitlement of a genuinely fresh lineage discovered in the same
# validation round. The Section hard cap remains the global stop.
return bool(fresh) and not hard_cap_reached, exhausted, fresh
@classmethod
def _partition_section_violations(cls, rows, *, section_name: str = "", mode: str = "", exhausted: bool = False):
blocking, advisory = [], []
for row in list(rows or []):
severity = cls._section_violation_severity(
row, section_name=section_name, mode=mode, exhausted=exhausted
)
if severity == "ADVISORY":
advisory.append(row)
else:
blocking.append(row)
return blocking, advisory
@classmethod
def _partition_section_violations_by_lineage(
cls, rows, lineage_attempts, *, section_name: str = "", mode: str = "",
hard_cap_reached: bool = False,
):
"""Classify each defect using only its own repair-exhaustion state.
R6-R3 prevents a previously exhausted sibling lineage from globally
changing the severity of a newly discovered defect. When the Section
hard cap is reached every lineage is evaluated as exhausted.
"""
blocking, advisory = [], []
for row in list(rows or []):
lineage = cls._section_violation_lineage(row)
lineage_exhausted = bool(hard_cap_reached) or int(lineage_attempts.get(lineage, 0)) >= 1
severity = cls._section_violation_severity(
row, section_name=section_name, mode=mode, exhausted=lineage_exhausted
)
if severity == "ADVISORY":
advisory.append(row)
else:
blocking.append(row)
return blocking, advisory
@classmethod
def _persistent_citation_authority_rows(cls, rows) -> list[dict]:
"""Return citation identity/scope defects that may never cross a Section seal.
P3-A4 intentionally keeps this narrower than general validation failure.
Citation support/style advisories retain their existing adaptive-completion
behavior; only source identity and unit-scope authority stay fail-closed.
"""
return [
row for row in list(rows or [])
if isinstance(row, dict)
and cls._section_violation_lineage(row) in {"citation_scope", "citation_authority"}
]
@staticmethod
def _record_section_advisories(quality_notes: list, rows, *, section_name: str = "") -> None:
"""Append reader-facing, nonfatal notes without exposing control vocabulary."""
for row in list(rows or []):
if not isinstance(row, dict):
continue
kind = str(row.get("type") or "quality_advisory")
if kind == "citation_claim_split_required":
detail = "A non-core literature sentence may combine more than one externally verifiable proposition."
action = "Before formal submission, consider splitting that sentence and checking the citation attached to each proposition."
elif kind in {"citation_unsupported_claim", "citation_support_unverified"}:
detail = "A non-core literature statement could not be fully verified against the available retrieved evidence."
action = "Review the statement and its supporting literature before formal submission; the manuscript was completed because the issue does not alter the study's core results or design."
elif kind == "author_facing_internal_workflow_language":
detail = "A minor reader-facing wording issue remained after automatic revision."
action = "Review the indicated passage for ordinary academic phrasing before formal submission."
elif kind == "authorial_ai_meta_trace":
detail = "A minor author-facing wording pattern may benefit from manual stylistic review."
action = "Review the indicated passage and remove any wording that does not read as ordinary scholarly prose."
elif kind == "top_journal_critic":
detail = str(row.get("reason") or "An optional journal-style refinement was identified.")[:500]
action = "This is a non-blocking style suggestion; revise only if it improves conciseness, accuracy, or rigor."
else:
detail = "A nonfatal quality issue was recorded for author review."
action = "Review the indicated section before formal submission if additional polishing is desired."
note = {
"type": "section_quality_advisory",
"severity": "ADVISORY",
"section": str(section_name or "Manuscript"),
"issue_type": kind,
"detail": detail,
"user_action": action,
}
signature = (note["section"], note["issue_type"], note["detail"])
if any(
(str(existing.get("section") or ""), str(existing.get("issue_type") or ""), str(existing.get("detail") or "")) == signature
for existing in quality_notes if isinstance(existing, dict)
):
continue
quality_notes.append(note)
@staticmethod
def _record_adaptive_completion_residual(quality_notes: list, rows, *, section_name: str, stage: str) -> None:
kinds = sorted({str((row or {}).get("type") or "validation_issue") for row in list(rows or []) if isinstance(row, dict)})
note = {
"type": "adaptive_completion_residual",
"severity": "ADVISORY",
"section": str(section_name or "Manuscript"),
"issue_type": ",".join(kinds[:8]) or "validation_issue",
"detail": (
"SmartPaper completed the manuscript after bounded adaptive repair. "
"A conservative validator still reported a residual issue, so it is recorded for optional author review instead of terminating generation."
),
"user_action": "Review this passage before formal submission if additional refinement is desired.",
"stage": str(stage or "adaptive_completion"),
}
sig = (note["section"], note["issue_type"], note["stage"])
if not any(
(str(x.get("section") or ""), str(x.get("issue_type") or ""), str(x.get("stage") or "")) == sig
for x in quality_notes if isinstance(x, dict)
):
quality_notes.append(note)
def _adaptive_repair_section_after_exhaustion(
self, *, agent, quant_engine, text: str, violations, mode: str, section_id: str,
section_name: str, data_metadata: dict, study_evidence: str, evidence_contract: str,
hypothesis_ledger: dict, task: dict, quote_invocation_plan: dict, quality_notes: list,
allowed_citation_tags=None, quarantine=False,
):
"""A2 last-mile semantic repair after lineage retry exhaustion.
Content defects are recoverable by default. The model gets the locked
study/evidence context, then deterministic validators re-check the exact
returned unit. If conservative diagnostics still disagree after bounded
repair, the best complete unit is retained and the residual is recorded
in Appendix quality notes rather than killing the whole paper.
"""
current = str(text or '').strip()
remaining = list(violations or [])
passes = max(1, int((APP_CONFIG.get("adaptive_completion") or {}).get("section_repair_passes", 2)))
scopes = ("paragraph", "section", "section")
for idx in range(passes):
if self.cancel_checker():
raise RuntimeError("task_cancel_requested")
repaired = agent.adaptive_repair_academic_unit(
current, violations=remaining, section_id=section_id, section_name=section_name,
mode=mode, study_evidence=study_evidence, evidence_contract=evidence_contract,
repair_scope=scopes[min(idx, len(scopes) - 1)],
)
if not str(repaired or '').strip():
repaired = current
if mode == "amj" and hypothesis_ledger:
repaired = self._normalize_hypothesis_ordinal_aliases(repaired, hypothesis_ledger)
if mode == "asq" and str(section_id or '').lower() == "findings":
repaired = self._strip_model_controlled_qualitative_quotes(repaired, data_metadata)
repaired = self._render_planned_qualitative_quotes(
repaired, task=task, data_metadata=data_metadata, quote_invocation_plan=quote_invocation_plan,
)
if mode == "amj" and data_metadata.get("quantitative_table_manifest"):
if str(section_id or '').lower() == "results":
repaired = self._apply_quantitative_table_contract(repaired, data_metadata, section_name)
else:
repaired = self._strip_model_markdown_tables(repaired)
repaired, _enum_rows = agent.repair_and_validate_enumeration_contract(
repaired, section_name=section_name
)
# P3-A4: every model-owned last-mile rewrite must re-enter the same
# citation authority + claim-specific semantic Gate before it can be
# considered sealable. Otherwise an unrelated adaptive repair could
# reintroduce a pre-resolved/unknown/wrong-scope citation after the
# earlier Section citation audit had already passed.
repaired = agent.audit_and_resolve_citations(
repaired, section_name=section_name, section_id=section_id,
quarantine=quarantine, allowed_tags=allowed_citation_tags,
)
adaptive_citation_rows = copy.deepcopy(
list(getattr(agent, "_last_citation_semantic_violations", []) or [])
)
rows = self._section_validation_bundle(
mode, agent, quant_engine, repaired, data_metadata, section_id=section_id,
section_name=section_name, evidence_contract=evidence_contract, include_critic=False,
citation_violations=adaptive_citation_rows, hypothesis_ledger=hypothesis_ledger,
)
blocking, advisory = self._partition_section_violations(
rows, section_name=section_name, mode=mode, exhausted=True
)
if advisory:
self._record_section_advisories(quality_notes, advisory, section_name=section_name)
current = repaired
remaining = blocking
if not blocking:
return current, []
if remaining:
self._record_adaptive_completion_residual(
quality_notes, remaining, section_name=section_name, stage="section_repair_exhausted"
)
return current, remaining
def _adaptive_repair_front_matter(
self, *, agent, quant_engine, abstract_text: str, violations, mode: str,
data_metadata: dict, study_evidence: str, hypothesis_ledger: dict, quality_notes: list,
label: str,
):
current = str(abstract_text or '').strip()
remaining = list(violations or [])
passes = max(1, int((APP_CONFIG.get("adaptive_completion") or {}).get("front_matter_repair_passes", 2)))
for idx in range(passes):
repaired = agent.adaptive_repair_academic_unit(
current, violations=remaining, section_id="abstract", section_name=label, mode=mode,
study_evidence=study_evidence, evidence_contract="",
repair_scope="paragraph" if idx == 0 else "section",
)
repaired = str(repaired or current).strip()
if hypothesis_ledger:
repaired = self._normalize_hypothesis_ordinal_aliases(repaired, hypothesis_ledger)
rows = self._study_fact_violations(
mode, agent, quant_engine, repaired, data_metadata, section_name=label
)
if hypothesis_ledger:
rows.extend(self._hypothesis_contract_violations(
repaired, hypothesis_ledger, section_name=label
))
current, remaining = repaired, rows
if not remaining:
return current, []
if remaining:
self._record_adaptive_completion_residual(
quality_notes, remaining, section_name=label, stage="front_matter_repair_exhausted"
)
return current, remaining
@staticmethod
def _citation_violation_tags(rows) -> set:
"""Extract only citation tags owned by the rejected claim sentence.
S0-R1-C1 uses this set as a bounded retry exclusion. Reasons may mention
comparison sources, so only explicit tag fields and rejected sentences are
inspected; this avoids excluding unrelated frozen evidence.
"""
tags = set()
for row in rows or []:
if not isinstance(row, dict):
continue
owned = ' '.join(str(row.get(key) or '') for key in ('tag', 'citation', 'sentence'))
for raw in re.findall(r'\[?CITE-\d+\]?', owned, flags=re.I):
token = raw.upper().strip('[]')
tags.add(f'[{token}]')
return tags
@staticmethod
def _author_facing_internal_workflow_violations(text: str) -> list[dict]:
"""Detect reader-facing leakage using the same registry as sanitization."""
value = str(text or "")
rows = []
for pattern, _replacement in AUTHOR_FACING_PRIVATE_VOCABULARY:
for match in re.finditer(pattern, value, flags=re.I):
start = max(0, match.start() - 80)
end = min(len(value), match.end() + 120)
preview = re.sub(r"\s+", " ", value[start:end]).strip()
rows.append({
"type": "author_facing_internal_workflow_language",
"preview": preview[:320],
"reason": "Private generation-control vocabulary surfaced in reader-facing manuscript prose.",
"repair_instruction": (
"Rewrite the sentence as ordinary academic prose describing the study, evidence, method, or limitation directly."
),
})
if len(rows) >= 6:
return rows
return rows
@staticmethod
def _writer_visible_evidence_context(text: str) -> str:
"""Return a reader-safe Writer view using the shared private registry."""
value = str(text or "")
# Formatting-only labels are sanitized before the phrase registry.
value = re.sub(r"(?im)^(#{1,6}\s+)Frozen\s+", r"\1", value)
value = re.sub(r"(?im)^\|\s*Field\s*\|\s*Frozen value\s*\|$", "| Field | Study value |", value)
for pattern, replacement in AUTHOR_FACING_PRIVATE_VOCABULARY:
value = re.sub(pattern, replacement, value, flags=re.I)
value = re.sub(r"(?im);\s*the Executor\b[^\n]*", ".", value)
value = re.sub(r"(?im)^\s*the Executor\b[^\n]*$", "", value)
return value
@staticmethod
def _hypothesis_contract_directive(ledger: dict) -> str:
record = dict(ledger or {})
items = [item for item in (record.get("items") or []) if isinstance(item, dict)]
if not items:
return ""
lines = []
for item in items:
hid = str(item.get("hypothesis_id") or "").strip()
predictor = str(item.get("predictor") or "").strip()
outcome = str(item.get("outcome") or "").strip()
moderator = str(item.get("moderator") or "").strip()
relation = str(item.get("relation") or "association").strip().casefold()
if hid and predictor and outcome:
if relation == "moderation" and moderator:
lines.append(f"{hid}: {moderator} moderates the association between {predictor} and {outcome}.")
else:
lines.append(f"{hid}: {predictor} is associated with {outcome}.")
if not lines:
return ""
indirect = dict(record.get("indirect_analysis") or {})
indirect_line = ""
if indirect:
p = str(indirect.get("predictor") or "").strip()
m = str(indirect.get("mediator") or "").strip()
o = str(indirect.get("outcome") or "").strip()
if p and m and o:
indirect_line = (
f"The estimated indirect association {p} -> {m} -> {o} is an analysis target, not a separately numbered hypothesis."
)
rendered = " ".join(lines + ([indirect_line] if indirect_line else []))
return (
"IMMUTABLE HYPOTHESIS CONTRACT: " + rendered
+ " Use H1/H2/H3-style identifiers exactly as listed whenever hypotheses are defined or referenced. "
"Do not renumber, reorder, create an additional hypothesis, replace an H-label with 'first/second/third hypothesis', "
"or change the construct pairing to fit observed results. The contract is result-independent."
)
@staticmethod
def _hypothesis_memory_text(ledger: dict) -> str:
"""Render the sole AMJ H1-Hn memory from the frozen ledger."""
lines = []
for item in (dict(ledger or {}).get("items") or []):
if not isinstance(item, dict):
continue
hid = str(item.get("hypothesis_id") or "").strip().upper()
predictor = re.sub(r"\s+", " ", str(item.get("predictor") or "")).strip()
outcome = re.sub(r"\s+", " ", str(item.get("outcome") or "")).strip()
moderator = str(item.get("moderator") or "").strip()
relation = str(item.get("relation") or "association").strip().casefold()
if hid and predictor and outcome:
if relation == "moderation" and moderator:
lines.append(f"{hid}: {moderator} moderates the association between {predictor} and {outcome}.")
else:
lines.append(f"{hid}: {predictor} is associated with {outcome}.")
return "\n".join(lines)
@staticmethod
def _normalize_hypothesis_ordinal_aliases(text: str, ledger: dict) -> str:
"""Map safe ordinal aliases to frozen H-identifiers without semantic rewrite."""
value = str(text or "")
items = [item for item in (dict(ledger or {}).get("items") or []) if isinstance(item, dict)]
if not items:
return value
words = ["first", "second", "third", "fourth", "fifth", "sixth"]
for index, item in enumerate(items[: len(words)]):
hid = str(item.get("hypothesis_id") or "").strip().upper()
if not hid:
continue
word = words[index]
value = re.sub(rf"\b(?:the\s+)?{word}\s+hypothesis\b", hid, value, flags=re.I)
return value
@classmethod
def _hypothesis_contract_violations(
cls, text: str, ledger: dict, *, section_name: str = "", require_all: bool = False
) -> list[dict]:
record = dict(ledger or {})
items = [item for item in (record.get("items") or []) if isinstance(item, dict)]
if not items:
return []
value = str(text or "")
expected = {
str(item.get("hypothesis_id") or "").upper(): item
for item in items if str(item.get("hypothesis_id") or "").strip()
}
if not expected:
return []
rows = []
observed = {f"H{n}" for n in re.findall(r"\bH(\d{1,2})\b", value, flags=re.I)}
unknown = sorted(observed - set(expected))
if unknown:
rows.append({
"type": "hypothesis_contract_unknown_id",
"observed": unknown,
"expected": sorted(expected),
"reason": "The manuscript introduced a hypothesis identifier outside the frozen result-independent hypothesis ledger.",
"repair_instruction": "Use only the frozen H-identifiers and keep the indirect association unnumbered unless the ledger explicitly numbers it.",
})
ordinal = re.search(r"\b(?:first|second|third|fourth|fifth|sixth)\s+hypothesis\b", value, flags=re.I)
if ordinal:
rows.append({
"type": "hypothesis_contract_ordinal_alias",
"preview": re.sub(r"\s+", " ", value[max(0, ordinal.start()-80):ordinal.end()+120]).strip()[:320],
"reason": "Ordinal hypothesis aliases can silently reorder the canonical H1-Hn identity.",
"repair_instruction": "Refer to hypotheses only by their frozen H1-Hn identifiers.",
})
all_constructs = {
re.sub(r"\s+", " ", str(item.get(key) or "")).strip()
for item in items for key in ("predictor", "outcome", "moderator")
if str(item.get(key) or "").strip()
}
for start, end, sentence in SSciPipelineAgent._sentence_spans(value):
ids = list(dict.fromkeys(
f"H{n}" for n in re.findall(r"\bH(\d{1,2})\b", sentence, flags=re.I)
))
# Collective references such as "H1-H3 are summarized below"
# are not definitions and must not be mistaken for remapping.
if len(ids) != 1:
continue
hid = ids[0]
item = expected.get(hid)
if not item:
continue
predictor = re.sub(r"\s+", " ", str(item.get("predictor") or "")).strip()
outcome = re.sub(r"\s+", " ", str(item.get("outcome") or "")).strip()
moderator = re.sub(r"\s+", " ", str(item.get("moderator") or "")).strip()
lower_sentence = re.sub(r"\s+", " ", sentence).casefold()
mentioned_constructs = {
term for term in all_constructs if term.casefold() in lower_sentence
}
# Pairing identity is global, not limited to the theory-definition
# sections. Require at least two named constructs so a bare reference
# such as "H1 was supported" cannot be misclassified as remapping.
expected_terms = [term for term in (predictor, outcome, moderator) if term]
if len(mentioned_constructs) >= 2 and predictor and outcome and any(
term.casefold() not in lower_sentence for term in expected_terms
):
rows.append({
"type": "hypothesis_contract_pairing_mismatch",
"id": hid,
"sentence": re.sub(r"\s+", " ", sentence).strip()[:420],
"expected": f"{predictor} -> {outcome}",
"reason": "A canonical hypothesis label was attached to a different construct pairing.",
"repair_instruction": f"Define {hid} only for {predictor} and {outcome}, using association language.",
})
if require_all:
missing = sorted(set(expected) - observed)
if missing:
rows.append({
"type": "hypothesis_contract_missing_id",
"observed": sorted(observed),
"expected": sorted(expected),
"reason": "The completed hypothesis architecture omitted one or more frozen hypothesis identifiers.",
"repair_instruction": "Preserve every canonical H1-Hn identity from theory through the integrated model and downstream reporting.",
})
return rows[:8]
@staticmethod
def _strip_model_markdown_tables(text: str) -> str:
"""Remove writer-created Markdown tables before deterministic AMJ table binding."""
lines = str(text or "").splitlines()
kept = []
index = 0
while index < len(lines):
line = lines[index]
if re.match(r"^\s*(?:#{1,6}\s*)?Table\s+\d+\s*[:.\-]", line, flags=re.I):
index += 1
while index < len(lines) and not lines[index].strip():
index += 1
if index < len(lines) and lines[index].lstrip().startswith("|"):
while index < len(lines) and lines[index].lstrip().startswith("|"):
index += 1
if index < len(lines) and re.match(r"^\s*\*?Note\s*:", lines[index], flags=re.I):
index += 1
continue
if line.lstrip().startswith("|"):
start = index
while index < len(lines) and lines[index].lstrip().startswith("|"):
index += 1
block = lines[start:index]
if len(block) >= 2 and re.match(r"^\s*\|?\s*:?-{3,}", block[1]):
continue
kept.extend(block)
continue
kept.append(line)
index += 1
return re.sub(r"\n{3,}", "\n\n", "\n".join(kept)).strip()
@classmethod
def _apply_quantitative_table_contract(cls, text: str, data_metadata: dict, display_name: str) -> str:
manifest = dict((data_metadata or {}).get("quantitative_table_manifest") or {})
tables = [item for item in (manifest.get("tables") or []) if isinstance(item, dict)]
if not tables:
return str(text or "")
value = cls._strip_model_markdown_tables(text)
assigned = []
for item in tables:
target = str(item.get("target_subsection") or "").strip()
if target and target.casefold() in str(display_name or "").casefold():
assigned.append(item)
if not assigned:
return value
blocks = []
for item in sorted(assigned, key=lambda row: int(row.get("number") or 0)):
number = int(item.get("number") or 0)
title = str(item.get("title") or "").strip()
markdown = str(item.get("markdown") or "").strip()
note = str(item.get("note") or "").strip()
if not number or not title or not markdown:
raise RuntimeError(f"quantitative_table_manifest_invalid:{display_name}")
block = f"Table {number}: {title}\n{markdown}"
if note:
block += f"\n\n*Note: {note}*"
blocks.append(block)
return (value.rstrip() + "\n\n" + "\n\n".join(blocks)).strip()
@staticmethod
def _snapshot_section_citation_state(agent, citation_plan: dict, unit_name: str) -> dict:
"""Snapshot only Section-owned citation state so a rejected attempt cannot leak forward."""
return {
"used_citations": set(getattr(agent, "used_citations", set()) or set()),
"citation_audit_log": copy.deepcopy(list(getattr(agent, "citation_audit_log", []) or [])),
"citation_map": copy.deepcopy(dict(getattr(agent, "citation_map", {}) or {})),
"planned_items": copy.deepcopy(list(dict(citation_plan or {}).get(unit_name, []) or [])),
}
@staticmethod
def _restore_section_citation_state(agent, citation_plan: dict, unit_name: str, snapshot: dict) -> None:
"""Restore citation ownership after an invalid Section attempt without rewinding API counters."""
snap = dict(snapshot or {})
lock = getattr(agent, "lock", None)
if lock is None:
agent.used_citations = set(snap.get("used_citations") or set())
agent.citation_audit_log = copy.deepcopy(list(snap.get("citation_audit_log") or []))
agent.citation_map = copy.deepcopy(dict(snap.get("citation_map") or {}))
else:
with lock:
agent.used_citations = set(snap.get("used_citations") or set())
agent.citation_audit_log = copy.deepcopy(list(snap.get("citation_audit_log") or []))
agent.citation_map = copy.deepcopy(dict(snap.get("citation_map") or {}))
citation_plan[unit_name] = copy.deepcopy(list(snap.get("planned_items") or []))
setattr(agent, "_last_citation_semantic_violations", [])
@classmethod
def _section_validation_bundle(
cls, mode: str, agent, quant_engine, text: str, data_metadata: dict, *,
section_id: str, section_name: str, evidence_contract: str = "", include_critic: bool = True,
citation_violations=None, hypothesis_ledger=None,
) -> list[dict]:
"""Collect typed S0-R1 validation feedback without mutating manuscript prose."""
value = str(text or "")
violations = []
if include_critic:
verdict = agent.critique_section(
value, section_id=section_id, section_name=section_name, mode=mode,
evidence_contract=evidence_contract,
)
if isinstance(verdict, dict) and verdict.get("pass") is False:
violations.append({
"type": "top_journal_critic",
"reason": str(verdict.get("critical_issue") or "")[:800],
"repair_instruction": str(verdict.get("repair_instruction") or "")[:1200],
})
violations.extend(list(agent.inspect_authorial_ai_meta(value) or []))
violations.extend(cls._study_fact_violations(
mode, agent, quant_engine, value, data_metadata, section_name=section_name
))
violations.extend(cls._author_facing_internal_workflow_violations(value))
if str(mode or "").lower() == "amj":
violations.extend(cls._hypothesis_contract_violations(
value, hypothesis_ledger or {}, section_name=section_name,
require_all=bool(re.search(r"Integrated Research Model", str(section_name or ""), flags=re.I)),
))
violations.extend(copy.deepcopy(list(citation_violations or [])))
return violations
def _checkpoint_section_retry_exhausted(
self, state: dict, *, section_id: str, section_name: str, violations, pct: int, attempts: int = 2
) -> None:
typed = copy.deepcopy(list(violations or []))
state["section_retry_exhausted"] = {
"section_id": str(section_id or ""),
"section_name": str(section_name or ""),
"attempts": max(1, int(attempts or 1)),
"violations": typed[:12],
"semantic_integrity_corruption": False,
}
self._checkpoint(
"section_retry_exhausted", state, pct,
display_node=f"section_retry_exhausted:{section_name}",
)
@staticmethod
def _section_integrity_hash(value: str) -> str:
exact = str(value or "").replace("\r\n", "\n").replace("\r", "\n").strip()
return hashlib.sha256(exact.encode("utf-8")).hexdigest()
@classmethod
def _build_section_integrity_seal(cls, section_key: str, section_title: str, section_text: str) -> dict:
return {
"version": ENGLISH_SECTION_SEAL_VERSION,
"section_key": str(section_key or ""),
"section_title": str(section_title or ""),
"sha256": cls._section_integrity_hash(section_text),
}
@staticmethod
def _replace_top_level_section(manuscript: str, title: str, replacement: str) -> str:
value = str(manuscript or "")
pattern = re.compile(
rf'(?ms)^#\s+{re.escape(str(title))}\s*$\n.*?(?=^#\s+[^#].*?$|\Z)'
)
matches = list(pattern.finditer(value))
if len(matches) != 1:
raise RuntimeError(f"top_level_section_replace_count_invalid:{title}:{len(matches)}")
match = matches[0]
rendered = str(replacement or "").strip()
return (value[:match.start()] + rendered + "\n\n" + value[match.end():].lstrip("\n")).strip() + "\n\n"
@staticmethod
def _remove_top_level_section(manuscript: str, title: str) -> str:
value = str(manuscript or "")
pattern = re.compile(rf'(?ms)^#\s+{re.escape(str(title))}\s*$\n.*?(?=^#\s+[^#].*?$|\Z)')
matches = list(pattern.finditer(value))
if len(matches) != 1:
raise RuntimeError(f"top_level_section_remove_count_invalid:{title}:{len(matches)}")
match = matches[0]
return (value[:match.start()] + value[match.end():].lstrip("\n")).strip() + "\n\n"
@classmethod
def _validate_section_integrity_seal(cls, seal: dict, section_text: str, section_title: str) -> None:
record = dict(seal or {})
if record.get("version") != ENGLISH_SECTION_SEAL_VERSION:
raise RuntimeError(f"english_section_integrity_seal_version_invalid:{section_title}")
expected = str(record.get("sha256") or "")
actual = cls._section_integrity_hash(section_text)
if not expected or expected != actual:
raise RuntimeError(f"english_section_integrity_seal_mismatch:{section_title}")
@staticmethod
def _apply_quantitative_front_matter_contract(mode: str, topic: str, keywords: str, data_metadata: dict):
"""Preserve original research seeds through retrieval and reasoning.
Reader-facing AMJ title/keywords are reconciled only after the empirical
scope and completed manuscript are frozen.
"""
return str(topic or "").strip(), str(keywords or "").strip()
@staticmethod
def _resolved_front_matter_keyword_policy(data_metadata: dict):
"""Resolve one policy object for Working/Final/retry/resume/fallback paths."""
meta = dict(data_metadata or {})
contract = dict(meta.get("study_design_contract") or meta.get("front_matter_contract") or {})
if not contract and isinstance(meta.get("user_research_intent"), dict):
contract = {"user_research_intent": dict(meta.get("user_research_intent") or {})}
return keyword_policy_from_contract(contract)
@staticmethod
def _front_matter_keyword_fallback_terms(data_metadata: dict):
"""Derive recoverable generated keywords only from frozen study-design authority."""
meta = dict(data_metadata or {})
contract = dict(meta.get("study_design_contract") or {})
values = []
quant = dict(contract.get("quantitative") or {})
for row in quant.get("constructs") or []:
if isinstance(row, dict):
label = re.sub(r"\s+", " ", str(row.get("label") or "")).strip()
if label:
values.append(label)
qual = dict(contract.get("qualitative") or {})
if qual:
for candidate in (
qual.get("phenomenon"),
qual.get("design_tradition"),
):
value = re.sub(r"\s+", " ", str(candidate or "")).strip()
if value:
values.append(value)
for theme in meta.get("themes") or []:
if isinstance(theme, dict):
value = re.sub(r"\s+", " ", str(theme.get("title") or "")).strip()
if value:
values.append(value)
return parse_keyword_terms(values, explicit_user=False)
@staticmethod
def _user_authoritative_front_matter_keywords(data_metadata: dict):
"""Return explicit user keywords only when the resolved policy says USER_EXPLICIT."""
meta = dict(data_metadata or {})
contract = dict(meta.get("study_design_contract") or meta.get("front_matter_contract") or {})
intent = dict(contract.get("user_research_intent") or meta.get("user_research_intent") or {})
policy = keyword_policy_from_contract(contract)
if str(policy.get("authority_source") or "").upper() != "USER_EXPLICIT":
return []
return apply_keyword_policy(
intent.get("source_keywords") or [],
policy,
authoritative=intent.get("source_keywords") or [],
)
@staticmethod
def _canonical_quantitative_front_matter_keywords(mode: str, candidate_keywords, data_metadata: dict):
"""Apply one keyword policy across quantitative/qualitative and all product tiers."""
meta = dict(data_metadata or {})
contract = dict(meta.get("study_design_contract") or meta.get("front_matter_contract") or {})
intent = dict(contract.get("user_research_intent") or meta.get("user_research_intent") or {})
policy = keyword_policy_from_contract(contract)
authoritative = list(intent.get("source_keywords") or [])
values = apply_keyword_policy(
candidate_keywords,
policy,
authoritative=authoritative if str(policy.get("authority_source") or "").upper() == "USER_EXPLICIT" else None,
)
return values
@staticmethod
def _replace_title(manuscript: str, title: str) -> str:
value = str(manuscript or "")
rendered = re.sub(r"\s+", " ", str(title or "")).strip()
if not rendered:
return value
updated, count = re.subn(r"(?m)^#\s+.+$", f"# {rendered}", value, count=1)
if count == 1:
return updated
logging.warning("Manuscript title heading missing during final reconciliation; reconstructing canonical H1 instead of failing generation.")
return f"# {rendered}\n\n" + value.lstrip("\n")
@staticmethod
def _replace_keywords(manuscript: str, keywords) -> str:
value = str(manuscript or '')
if isinstance(keywords, (list, tuple)):
rendered = ', '.join(str(item).strip() for item in keywords if str(item).strip())
else:
rendered = str(keywords or '').strip()
pattern = re.compile(r'(?ms)(^##\s+Keywords\s*$\n)(.*?)(?=^#\s+[^#].*?$|\Z)')
matches = list(pattern.finditer(value))
if len(matches) == 1:
match = matches[0]
return value[:match.start()] + match.group(1) + rendered + '\n\n' + value[match.end():].lstrip('\n')
logging.warning("Keywords heading missing or ambiguous during final reconciliation; rebuilding front-matter keyword block without aborting generation | count=%s", len(matches))
abstract_pattern = re.compile(r'(?ms)(^##\s+Abstract\s*$\n.*?)(?=^#\s+[^#].*?$|\Z)')
abstract_match = abstract_pattern.search(value)
block = f"## Keywords\n{rendered}\n\n"
if abstract_match:
return value[:abstract_match.end()].rstrip() + "\n\n" + block + value[abstract_match.end():].lstrip("\n")
title_match = re.search(r'(?m)^#\s+.+$', value)
if title_match:
return value[:title_match.end()] + "\n\n" + block + value[title_match.end():].lstrip("\n")
return block + value.lstrip("\n")
@staticmethod
def _target_keyword_heading_aliases(target_lang: str) -> list[str]:
code = str(target_lang or '').strip().casefold().replace('_', '-')
if code.startswith('zh'):
return ['关键词', '关键字']
if code.startswith('ru'):
return ['Ключевые слова']
if code.startswith('kk') or code.startswith('kz'):
return ['Түйін сөздер', 'Кілт сөздер']
return ['Keywords']
@classmethod
def _rebind_target_user_intent_front_matter(cls, manuscript: str, target_lang: str, data_metadata: dict, glossary: dict):
"""Preserve user keyword identity and explicit relation type in target front matter."""
value = str(manuscript or '')
meta = dict(data_metadata or {})
contract = dict(meta.get('study_design_contract') or {})
intent = dict(contract.get('user_research_intent') or meta.get('user_research_intent') or {})
source_keywords = list(intent.get('source_keywords') or contract.get('source_keywords') or [])
mapping = dict(glossary or {})
diagnostics = []
if source_keywords:
target_keywords = [re.sub(r'\s+', ' ', str(mapping.get(src) or '')).strip() for src in source_keywords]
if all(target_keywords):
aliases = cls._target_keyword_heading_aliases(target_lang)
heading_re = re.compile(r'(?mi)^##\s+(?:' + '|'.join(re.escape(x) for x in aliases) + r')\s*$')
match = heading_re.search(value)
if match:
next_heading = re.search(r'(?m)^#{1,3}\s+', value[match.end():])
end = match.end() + (next_heading.start() if next_heading else len(value[match.end():]))
rendered = ', '.join(target_keywords)
value = value[:match.end()] + '\n' + rendered + '\n\n' + value[end:].lstrip('\n')
diagnostics.append({'type':'target_user_keywords_rebound','count':len(target_keywords)})
explicit = set(intent.get('explicit_relation_types') or [])
if explicit:
title_match = re.search(r'(?m)^#\s+(.+?)\s*$', value)
if title_match:
title = title_match.group(1).strip()
target_intent = build_user_research_intent(title, '', mode=str(contract.get('mode') or ''))
missing = explicit - set(target_intent.get('explicit_relation_types') or [])
if missing:
quant = dict(contract.get('quantitative') or {})
construct_rows = [row for row in (quant.get('constructs') or []) if isinstance(row, dict)]
role_label = {}
for role in ('mediator', 'moderator'):
values = [str(row.get('label') or '') for row in construct_rows if role in (row.get('roles') or []) and row.get('label')]
if len(values) == 1:
role_label[role] = re.sub(r'\s+', ' ', str(mapping.get(values[0]) or values[0])).strip()
code = str(target_lang or '').strip().casefold().replace('_', '-')
suffixes = []
if 'moderation' in missing:
label = role_label.get('moderator') or ''
if code.startswith('zh'):
suffixes.append(f'调节变量：{label}' if label else '调节作用')
elif code.startswith('ru'):
suffixes.append(f'модератор: {label}' if label else 'модерация')
elif code.startswith('kk') or code.startswith('kz'):
suffixes.append(f'модератор: {label}' if label else 'модераторлық рөл')
else:
suffixes.append(f'moderator: {label}' if label else 'moderation')
if missing.intersection({'mediation', 'serial_mediation', 'parallel_mediation'}):
label = role_label.get('mediator') or ''
if code.startswith('zh'):
suffixes.append(f'中介变量：{label}' if label else '中介作用')
elif code.startswith('ru'):
suffixes.append(f'медиатор: {label}' if label else 'медиация')
elif code.startswith('kk') or code.startswith('kz'):
suffixes.append(f'медиатор: {label}' if label else 'медиаторлық рөл')
else:
suffixes.append(f'mediator: {label}' if label else 'mediation')
if suffixes:
repaired_title = title.rstrip(' .;:') + ' (' + '; '.join(suffixes) + ')'
value = value[:title_match.start(1)] + repaired_title + value[title_match.end(1):]
diagnostics.append({'type':'target_user_relation_title_rebound','missing':sorted(missing)})
return value, diagnostics
@staticmethod
def _section_is_complete(tasks, completed_display_names, section_key: str) -> bool:
relevant = [
str(task.get('display_name') or '') for task in (tasks or [])
if str(task.get('section_id') or task.get('section') or '') == str(section_key)
]
done = set(str(item) for item in (completed_display_names or []))
return bool(relevant) and all(name in done for name in relevant)
@staticmethod
def _glossary_version(glossary) -> str:
encoded = json.dumps(dict(glossary or {}), ensure_ascii=False, sort_keys=True).encode('utf-8')
return hashlib.sha256(encoded).hexdigest()[:16]
@staticmethod
def _build_canonical_concept_ledger(
mode: str, data_metadata: dict, *, keywords='', adaptive_structure=None, research_logic_contract=''
) -> dict:
"""Freeze task-level research concept identity before body translation.
This ledger is intentionally conservative: quantitative construct names
come directly from frozen metadata; qualitative concept/theme labels come
from the Evidence Ledger/outline. Lower-case hyphenated concepts used in
the frozen research logic are added because the legacy Title-Case glossary
discovery cannot see them. No semantic synonym merging is performed.
"""
items = []
index_by_key = {}
def add(value, kind, policy='FLEXIBLE_TERM'):
term = re.sub(r'\s+', ' ', str(value or '')).strip(' \t\r\n,;:.')
if not term or len(term) < 3 or len(term) > 100:
return
words = re.findall(r"[A-Za-zÀ-ÖØ-öø-ÿА-Яа-яЁёӘәҒғҚқҢңӨөҰұҮүҺһІі][\w’'\-]*", term, flags=re.UNICODE)
if not words or len(words) > 10:
return
key = re.sub(r'[–—-]+', '-', term).casefold()
normalized_policy = 'STRICT_CONSTRUCT' if str(policy or '').upper() == 'STRICT_CONSTRUCT' else 'FLEXIBLE_TERM'
if key in index_by_key:
row = items[index_by_key[key]]
if normalized_policy == 'STRICT_CONSTRUCT' and row.get('policy') != 'STRICT_CONSTRUCT':
row['policy'] = 'STRICT_CONSTRUCT'
row['kind'] = str(kind or row.get('kind') or 'concept')
return
index_by_key[key] = len(items)
items.append({'source': term, 'kind': str(kind or 'concept'), 'policy': normalized_policy})
meta = dict(data_metadata or {})
contract = dict(meta.get('study_design_contract') or {})
user_intent = dict(contract.get('user_research_intent') or meta.get('user_research_intent') or {})
if str(mode or '') == 'amj':
construct_items = [row for row in (meta.get('construct_items') or []) if isinstance(row, dict)]
if construct_items:
for row in construct_items:
roles = list(row.get('roles') or [])
role_label = '_'.join(str(role) for role in roles if str(role)) or 'construct'
add(row.get('label'), f'quantitative_{role_label}', 'STRICT_CONSTRUCT')
else:
constructs = meta.get('constructs') if isinstance(meta.get('constructs'), dict) else {}
for role in ('predictor', 'mediator', 'moderator', 'outcome', 'control', 'covariate'):
add(constructs.get(role), f'quantitative_{role}', 'STRICT_CONSTRUCT')
else:
for theme in (meta.get('themes') or []):
if not isinstance(theme, dict):
continue
add(theme.get('title'), 'qualitative_theme', 'STRICT_CONSTRUCT')
for concept in (theme.get('concepts') or []):
add(concept, 'qualitative_concept', 'STRICT_CONSTRUCT')
for row in (meta.get('evidence') or []):
if isinstance(row, dict):
add(row.get('concept'), 'qualitative_concept', 'STRICT_CONSTRUCT')
for task in (adaptive_structure or []):
if not isinstance(task, dict):
continue
for concept in (task.get('concepts') or []):
add(concept, 'outline_concept', 'FLEXIBLE_TERM')
# R7-R2C-R5: user keywords are research-intent identities, not expendable
# planning seeds. Freeze them for multilingual inheritance while still
# distinguishing them from empirical construct roles.
intent_keywords = list(user_intent.get('source_keywords') or [])
if not intent_keywords:
intent_keywords = parse_keyword_terms(keywords, explicit_user=True)
for seed in intent_keywords:
if 1 <= len(str(seed).split()) <= 10:
add(seed, 'user_intent_keyword', 'STRICT_CONSTRUCT')
# Capture explicit lower-case hyphenated academic expressions only when
# they recur or carry a research-concept cue. This closes the
# context-awareness/context-aware blind spot without freezing generic
# methodological adjectives such as cross-sectional by default.
hyphenated = re.findall(r'(?= 2 or parts.intersection(concept_cues):
add(term, 'logic_hyphenated_concept', 'FLEXIBLE_TERM')
return {
'version': 'concept-ledger-v2',
'items': items,
'explicit_relation_types': list(user_intent.get('explicit_relation_types') or []),
'source_topic': str(user_intent.get('source_topic') or contract.get('source_topic') or ''),
'source_keywords': list(user_intent.get('source_keywords') or contract.get('source_keywords') or []),
}
@staticmethod
def _canonical_concept_writer_directive(ledger: dict) -> str:
terms = [str(item.get('source') or '').strip() for item in (dict(ledger or {}).get('items') or []) if isinstance(item, dict)]
terms = [term for term in terms if term]
if not terms:
return ''
rendered = '; '.join(terms[:40])
relation_types = [str(item).strip() for item in (dict(ledger or {}).get('explicit_relation_types') or []) if str(item).strip()]
relation_directive = ''
if relation_types:
relation_directive = (
' USER RESEARCH INTENT: Preserve these explicitly requested study relations throughout drafting and target-language translation: '
+ ', '.join(relation_types)
+ '. Do not downgrade a mediator/moderator or other explicit relation into a generic contextual term merely because direct prior-literature support is sparse. Prior-literature claims still require real citation evidence; novel study logic may be stated cautiously as author inference without inventing a source.'
)
return (
'CANONICAL CONCEPT IDENTITY: The following research labels are frozen task-level concepts: '
+ rendered
+ '. When referring to the same construct or concept, preserve its canonical English label; '
'do not silently replace it with a near-synonym that changes concept identity. Ordinary grammatical '
'articles, possessives, and surrounding syntax remain free. Distinct listed labels are not synonyms unless the evidence explicitly says so.'
+ relation_directive
)
@staticmethod
def _translation_ledger_hash(agent, source: str) -> str:
method = getattr(agent, 'translation_fact_ledger_hash', None)
if callable(method):
return str(method(source) or '')
# Test doubles / legacy adapters still get deterministic invalidation.
return hashlib.sha256(str(source or '').encode('utf-8')).hexdigest()
def _translation_identity_bundle(self, agent, source: str) -> dict[str, str]:
"""Return one canonical identity domain for translation reuse.
The frozen English source remains the manuscript authority, while the
Translation Compiler may consume a canonicalized representation (for
example qualitative quote literals become QUOTE semantic slots). Cache
identity must therefore persist both hashes and derive the private Fact
Ledger only from the exact compiled source sent to the translator.
"""
raw_source = str(source or '').strip()
compiled_source, _pending, _reused = self._prepare_qualitative_translation_source(
raw_source, getattr(self, '_qualitative_quote_sources', {}), {}
)
compiled_source = str(compiled_source or '').strip()
return {
'source_hash': self._text_hash(raw_source),
'compiled_source': compiled_source,
'compiled_source_hash': self._text_hash(compiled_source),
'fact_ledger_hash': self._translation_ledger_hash(agent, compiled_source),
}
@staticmethod
def _assemble_partial_translation(translation_sections, adaptive_structure, *, include_front=False) -> str:
mapping = translation_sections if isinstance(translation_sections, dict) else {}
parts = []
if include_front:
record = mapping.get('__front_matter__') or {}
if record.get('qa_status') in {'passed', 'passed_with_warning'} and record.get('translated_text'):
parts.append(str(record['translated_text']).strip())
for key, _title in GenerationRunner._unique_top_level_sections(adaptive_structure):
record = mapping.get(key) or {}
if record.get('qa_status') in {'passed', 'passed_with_warning'} and record.get('translated_text'):
parts.append(str(record['translated_text']).strip())
return '\n\n'.join(part for part in parts if part).strip()
def _start_translation_worker(
self, agent, *, translation_sections, translation_glossary,
adaptive_structure, target_lang,
) -> None:
if self._translation_worker is not None:
return
self.translation_agent = agent
self._translation_worker_sections = dict(translation_sections or {})
self._translation_worker_glossary = dict(translation_glossary or {})
self._translation_worker_context = {
'adaptive_structure': list(adaptive_structure or []),
'target_lang': str(target_lang or ''),
}
logical_keys = ['__front_matter__'] + [
key for key, _title in self._unique_top_level_sections(adaptive_structure)
]
logical_keys = list(dict.fromkeys(key for key in logical_keys if key))
self._translation_progress_total = len(logical_keys)
self._translation_progress_sections = dict(translation_sections or {})
self._translation_progress_queued = {
key for key in logical_keys if key in self._translation_progress_sections
}
self._translation_progress_inflight = set()
self._translation_worker = _OrderedTranslationWorker(self._run_translation_job)
self._write_preview_payload(force=True)
def _run_translation_job(self, job: dict[str, Any]) -> dict[str, Any]:
if self.translation_agent is None:
raise RuntimeError('translation_worker_agent_missing')
context = dict(self._translation_worker_context or {})
sections, glossary = self._translate_frozen_unit(
self.translation_agent,
unit_key=str(job.get('unit_key') or ''),
unit_name=str(job.get('unit_name') or ''),
source_text=str(job.get('source_text') or ''),
target_lang=str(context.get('target_lang') or ''),
translation_sections=self._translation_worker_sections,
translation_glossary=self._translation_worker_glossary,
adaptive_structure=context.get('adaptive_structure') or [],
include_front=True,
defer_failures=True,
english_commit_text=str(job.get('english_commit_text') or ''),
)
self._translation_worker_sections = dict(sections or {})
self._translation_worker_glossary = dict(glossary or {})
return {
'unit_key': str(job.get('unit_key') or ''),
'source_hash': str(job.get('source_hash') or ''),
'translation_sections': dict(self._translation_worker_sections),
'translation_glossary': dict(self._translation_worker_glossary),
}
def _enqueue_translation_unit(
self, agent, *, unit_key: str, unit_name: str, source_text: str,
translation_sections: dict, english_commit_text: str,
) -> bool:
source = str(source_text or '').strip()
if not source or self._translation_worker is None:
return False
identity = self._translation_identity_bundle(agent, source)
source_hash = identity['source_hash']
compiled_source_hash = identity['compiled_source_hash']
fact_ledger_hash = identity['fact_ledger_hash']
entity_ledger_method = getattr(agent, 'translation_protected_entity_ledger_hash', None)
protected_entity_ledger_hash = str(
entity_ledger_method(identity['compiled_source'], self._translation_worker_glossary)
if callable(entity_ledger_method) else ''
)
protected_entity_ledger_version = str(
getattr(agent, 'PROTECTED_ENTITY_LEDGER_VERSION', 'protected-entity-ledger-v1')
or 'protected-entity-ledger-v1'
)
existing = dict((translation_sections or {}).get(unit_key) or {})
if not isinstance(getattr(self, '_translation_progress_queued', None), set):
self._translation_progress_queued = set()
if not isinstance(getattr(self, '_translation_progress_inflight', None), set):
self._translation_progress_inflight = set()
# Do not replace the worker-owned display snapshot with a possibly stale
# main-thread copy here; collection or the worker completion path updates
# _translation_progress_sections monotonically.
self._translation_progress_queued.add(str(unit_key))
if (
existing.get('qa_status') in {'passed', 'passed_with_warning'}
and existing.get('source_hash') == source_hash
and existing.get('compiled_source_hash') == compiled_source_hash
and existing.get('target_lang') == str(self._translation_worker_context.get('target_lang') or '')
and existing.get('translation_contract_version') == TRANSLATION_CONTRACT_VERSION
and existing.get('fact_ledger_hash') == fact_ledger_hash
and existing.get('protected_entity_ledger_version') == protected_entity_ledger_version
and existing.get('protected_entity_ledger_hash') == protected_entity_ledger_hash
and existing.get('translated_text')
):
return False
if existing and existing.get('translated_text'):
reasons = []
if existing.get('source_hash') != source_hash: reasons.append('english_source_hash')
if existing.get('compiled_source_hash') != compiled_source_hash: reasons.append('compiled_source_hash')
if existing.get('fact_ledger_hash') != fact_ledger_hash: reasons.append('fact_ledger_hash')
if existing.get('protected_entity_ledger_version') != protected_entity_ledger_version: reasons.append('protected_entity_ledger_version')
if existing.get('protected_entity_ledger_hash') != protected_entity_ledger_hash: reasons.append('protected_entity_ledger_hash')
if existing.get('translation_contract_version') != TRANSLATION_CONTRACT_VERSION: reasons.append('contract_version')
if existing.get('target_lang') != str(self._translation_worker_context.get('target_lang') or ''): reasons.append('target_lang')
if existing.get('qa_status') not in {'passed', 'passed_with_warning'}: reasons.append('qa_status')
logging.info('Translation reuse miss | unit=%s phase=enqueue reasons=%s', unit_name, ','.join(reasons) or 'unknown')
pending_key = (str(unit_key), source_hash, compiled_source_hash, fact_ledger_hash, protected_entity_ledger_version, protected_entity_ledger_hash)
if pending_key in self._translation_worker_pending:
return False
self._translation_progress_inflight.add(str(unit_key))
future = self._translation_worker.submit({
'unit_key': str(unit_key),
'unit_name': str(unit_name),
'source_text': source,
'source_hash': source_hash,
'compiled_source_hash': compiled_source_hash,
'fact_ledger_hash': fact_ledger_hash,
'english_commit_text': str(english_commit_text or ''),
})
self._translation_worker_pending.add(pending_key)
self._translation_worker_jobs.append({
'pending_key': pending_key,
'future': future,
})
self._write_preview_payload(force=True)
return True
def _collect_translation_results(self, translation_sections, translation_glossary, *, block=False):
sections = dict(translation_sections or {})
glossary = dict(translation_glossary or {})
while self._translation_worker_collected < len(self._translation_worker_jobs):
item = self._translation_worker_jobs[self._translation_worker_collected]
future = item['future']
if not block and not future.done():
break
result = future.result()
sections = dict(result.get('translation_sections') or sections)
glossary = dict(result.get('translation_glossary') or glossary)
self._translation_progress_sections = dict(sections)
self._translation_worker_pending.discard(item['pending_key'])
self._translation_worker_collected += 1
return sections, glossary
def _shutdown_translation_worker(self, *, wait=False) -> None:
worker = self._translation_worker
if worker is None:
return
worker.close(wait=bool(wait))
self._translation_worker = None
def _translate_frozen_unit(
self, agent, *, unit_key: str, unit_name: str, source_text: str, target_lang: str,
translation_sections: dict, translation_glossary: dict, adaptive_structure, include_front=False,
defer_failures=True, english_commit_text: str | None = None, final_gate: bool = False,
):
source = str(source_text or '').strip()
if not source:
return translation_sections, translation_glossary
identity = self._translation_identity_bundle(agent, source)
source_hash = identity['source_hash']
compiled_source_hash = identity['compiled_source_hash']
canonical_translation_source = identity['compiled_source']
contract_version = TRANSLATION_CONTRACT_VERSION
fact_ledger_hash = identity['fact_ledger_hash']
entity_ledger_method = getattr(agent, 'translation_protected_entity_ledger_hash', None)
protected_entity_ledger_hash = str(
entity_ledger_method(canonical_translation_source, translation_glossary)
if callable(entity_ledger_method) else ''
)
protected_entity_ledger_version = str(
getattr(agent, 'PROTECTED_ENTITY_LEDGER_VERSION', 'protected-entity-ledger-v1') or 'protected-entity-ledger-v1'
)
existing = dict((translation_sections or {}).get(unit_key) or {})
if not isinstance(getattr(self, '_translation_progress_queued', None), set):
self._translation_progress_queued = set()
if not isinstance(getattr(self, '_translation_progress_inflight', None), set):
self._translation_progress_inflight = set()
self._translation_progress_sections = dict(translation_sections or {})
self._translation_progress_queued.add(str(unit_key))
if (
existing.get('qa_status') in {'passed', 'passed_with_warning'}
and existing.get('source_hash') == source_hash
and existing.get('compiled_source_hash') == compiled_source_hash
and existing.get('target_lang') == target_lang
and existing.get('translation_contract_version') == contract_version
and existing.get('fact_ledger_hash') == fact_ledger_hash
and existing.get('protected_entity_ledger_version') == protected_entity_ledger_version
and existing.get('protected_entity_ledger_hash') == protected_entity_ledger_hash
and existing.get('translated_text')
):
self._translation_progress_inflight.discard(str(unit_key))
return translation_sections, translation_glossary
if existing and existing.get('translated_text'):
reasons = []
if existing.get('source_hash') != source_hash: reasons.append('english_source_hash')
if existing.get('compiled_source_hash') != compiled_source_hash: reasons.append('compiled_source_hash')
if existing.get('fact_ledger_hash') != fact_ledger_hash: reasons.append('fact_ledger_hash')
if existing.get('protected_entity_ledger_version') != protected_entity_ledger_version: reasons.append('protected_entity_ledger_version')
if existing.get('protected_entity_ledger_hash') != protected_entity_ledger_hash: reasons.append('protected_entity_ledger_hash')
if existing.get('translation_contract_version') != contract_version: reasons.append('contract_version')
if existing.get('target_lang') != target_lang: reasons.append('target_lang')
if existing.get('qa_status') not in {'passed', 'passed_with_warning'}: reasons.append('qa_status')
logging.info('Translation reuse miss | unit=%s phase=final_or_direct reasons=%s', unit_name, ','.join(reasons) or 'unknown')
self._translation_progress_inflight.add(str(unit_key))
partial = self._assemble_partial_translation(translation_sections, adaptive_structure, include_front=include_front)
initial_target_stage = f'translation_final_gate:{unit_name}' if final_gate else f'translating:{unit_name}'
self._publish_target_preview(
partial, initial_target_stage, force=True, complete=False,
english_commit_text=english_commit_text,
)
updated_sections = dict(translation_sections or {})
raw_quote_ledger_record = updated_sections.get("__quote_ledger__")
if raw_quote_ledger_record is not None and not isinstance(raw_quote_ledger_record, dict):
logging.warning(
"Qualitative quote translation ledger ignored malformed record type=%s",
type(raw_quote_ledger_record).__name__,
)
quote_ledger_record = (
dict(raw_quote_ledger_record) if isinstance(raw_quote_ledger_record, dict) else {}
)
current_quote_hashes = {
tag: hashlib.sha256(re.sub(r"\s+", " ", quote).strip().encode("utf-8")).hexdigest()
for tag, quote in getattr(self, "_qualitative_quote_sources", {}).items()
}
current_request_hash = str(getattr(self, "request_hash", "") or "")
current_task_id = str(getattr(getattr(self, "task_dir", None), "name", "") or "")
ledger_identity_ok = self._qualitative_quote_ledger_identity_ok(
quote_ledger_record,
task_id=current_task_id,
target_lang=target_lang,
request_hash=current_request_hash,
translation_contract_version=TRANSLATION_CONTRACT_VERSION,
source_quote_hashes=current_quote_hashes,
)
stored_translations = quote_ledger_record.get("translations")
quote_ledger = (
dict(stored_translations)
if ledger_identity_ok and isinstance(stored_translations, dict)
else {}
)
translation_source, pending_quote_tags, reused_quote_tags = self._prepare_qualitative_translation_source(
source, getattr(self, "_qualitative_quote_sources", {}), quote_ledger
)
translation_source = str(translation_source or '').strip()
if translation_source != canonical_translation_source:
raise RuntimeError('Canonical translation source changed with Quote Ledger state.')
def publish_translation_stage(stage):
value = str(stage or '').strip().casefold()
if value == 'qa':
preview_stage = f'translation_qa:{unit_name}'
elif value == 'repairing':
preview_stage = f'translation_repairing:{unit_name}'
elif value == 'adjudicating':
preview_stage = f'translation_adjudicating:{unit_name}'
elif final_gate:
preview_stage = f'translation_final_gate:{unit_name}'
else:
preview_stage = f'translating:{unit_name}'
self._publish_target_preview(
partial, preview_stage, force=True, complete=False,
english_commit_text=english_commit_text,
)
try:
if pending_quote_tags:
quote_translate_method = getattr(agent, 'translate_qualitative_quote_objects', None)
if not callable(quote_translate_method):
raise RuntimeError('Typed qualitative quote translation entry point is unavailable.')
quote_objects = {
tag: getattr(self, '_qualitative_quote_sources', {}).get(tag, '')
for tag in pending_quote_tags
}
quote_result = quote_translate_method(
quote_objects,
target_lang,
glossary=translation_glossary,
max_attempts=2,
stage_callback=publish_translation_stage,
)
returned_quotes = (quote_result or {}).get('translations')
if not isinstance(returned_quotes, dict) or set(returned_quotes) != set(pending_quote_tags):
raise RuntimeError(
'Typed qualitative quote translation identity mismatch: '
f'expected={sorted(pending_quote_tags)} got={sorted(returned_quotes) if isinstance(returned_quotes, dict) else []}'
)
for tag in pending_quote_tags:
rendered_quote = str(returned_quotes.get(tag) or '').strip()
if not rendered_quote:
raise RuntimeError(f'Typed qualitative quote translation is empty: {tag}')
quote_ledger[tag] = rendered_quote
returned_glossary = (quote_result or {}).get('glossary')
if isinstance(returned_glossary, dict):
translation_glossary = dict(returned_glossary)
updated_sections['__quote_ledger__'] = {
'task_id': getattr(getattr(self, "task_dir", None), "name", ""),
'request_hash': current_request_hash,
'writer_execution_epoch': int(getattr(self, "execution_epoch", 0) or 0),
'target_lang': target_lang,
'translation_contract_version': TRANSLATION_CONTRACT_VERSION,
'source_quote_hashes': current_quote_hashes,
'translations': dict(quote_ledger),
'updated_at': datetime.now(timezone.utc).isoformat(),
}
translate_method = agent.translate_frozen_section
call_kwargs = {
'glossary': translation_glossary, 'section_name': unit_name, 'max_attempts': 2,
}
try:
parameters = list(inspect.signature(translate_method).parameters.values())
supports_stage_callback = any(
item.name == 'stage_callback' or item.kind == inspect.Parameter.VAR_KEYWORD
for item in parameters
)
supports_claim_states = any(
item.name == 'claim_states' or item.kind == inspect.Parameter.VAR_KEYWORD
for item in parameters
)
except (TypeError, ValueError):
supports_stage_callback = False
supports_claim_states = False
if supports_stage_callback:
call_kwargs['stage_callback'] = publish_translation_stage
if supports_claim_states and isinstance(existing.get('claim_states'), dict):
call_kwargs['claim_states'] = dict(existing.get('claim_states') or {})
result = translate_method(translation_source, target_lang, **call_kwargs)
translated_text = str(result.get('text') or '').strip()
if pending_quote_tags or reused_quote_tags:
translated_text, quote_ledger = self._finalize_qualitative_translation_quotes(
translated_text, pending_quote_tags, reused_quote_tags, quote_ledger
)
agent.validate_translation_alignment(source, translated_text)
result = dict(result)
result['text'] = translated_text
updated_sections['__quote_ledger__'] = {
'task_id': getattr(getattr(self, "task_dir", None), "name", ""),
'request_hash': current_request_hash,
'writer_execution_epoch': int(getattr(self, "execution_epoch", 0) or 0),
'target_lang': target_lang,
'translation_contract_version': TRANSLATION_CONTRACT_VERSION,
'source_quote_hashes': current_quote_hashes,
'translations': dict(quote_ledger),
'updated_at': datetime.now(timezone.utc).isoformat(),
}
except PipelineCancelledError:
raise
except Exception as error:
# Persist repair state for both intermediate and strict final gates.
# The caller owns checkpoint authority and decides whether to stop.
code_method = getattr(agent, 'translation_error_code', None)
error_code = str(code_method(error) if callable(code_method) else 'TRANSLATION_PENDING')
updated_sections[unit_key] = {
'source_hash': source_hash,
'compiled_source_hash': compiled_source_hash,
'target_lang': target_lang,
'glossary_version': self._glossary_version(translation_glossary),
'translation_contract_version': contract_version,
'fact_ledger_hash': fact_ledger_hash,
'protected_entity_ledger_version': protected_entity_ledger_version,
'protected_entity_ledger_hash': protected_entity_ledger_hash,
'translated_text': '',
'qa_status': 'repair_pending',
'claim_states': dict(getattr(agent, '_last_translation_claim_states', {}) or {}),
'claim_state_version': 'translation-claim-state-v2',
'error_code': error_code,
'attempts': int(existing.get('attempts') or 0) + 1,
'deferred_at': datetime.now(timezone.utc).isoformat(),
}
logging.warning(
'Translation repair pending | unit=%s final_gate=%s code=%s error=%s',
unit_name, bool(final_gate), error_code, type(error).__name__,
)
self._translation_progress_sections = dict(updated_sections)
self._translation_progress_inflight.discard(str(unit_key))
self._publish_target_preview(
partial, f'translation_repair_pending:{unit_name}', force=True, complete=False,
english_commit_text=english_commit_text,
)
return updated_sections, translation_glossary
updated_glossary = dict(result.get('glossary') or translation_glossary or {})
result_fact_ledger_hash = str(result.get('fact_ledger_hash') or fact_ledger_hash)
if result_fact_ledger_hash != fact_ledger_hash:
raise RuntimeError('Translation result Fact Ledger identity differs from canonical compiled source.')
updated_sections[unit_key] = {
'source_hash': source_hash,
'compiled_source_hash': compiled_source_hash,
'target_lang': target_lang,
'glossary_version': self._glossary_version(updated_glossary),
'translation_contract_version': str(result.get('translation_contract_version') or contract_version),
'fact_ledger_hash': result_fact_ledger_hash,
'protected_entity_ledger_version': str(result.get('protected_entity_ledger_version') or protected_entity_ledger_version),
'protected_entity_ledger_hash': str(result.get('protected_entity_ledger_hash') or protected_entity_ledger_hash),
'translated_text': str(result.get('text') or '').strip(),
'qa_status': str(result.get('qa_status') or 'passed'),
'language_adapter': str(result.get('language_adapter') or ''),
'diagnostics': list(result.get('diagnostics') or [])[-20:],
'claim_states': dict(result.get('claim_states') or {}),
'claim_state_version': str(result.get('claim_state_version') or 'translation-claim-state-v2'),
'attempts': int(result.get('attempts') or 1),
'completed_at': datetime.now(timezone.utc).isoformat(),
}
partial = self._assemble_partial_translation(updated_sections, adaptive_structure, include_front=include_front)
self._translation_progress_sections = dict(updated_sections)
self._translation_progress_inflight.discard(str(unit_key))
self._publish_target_preview(
partial, f'translation_ready:{unit_name}', force=True, complete=False,
english_commit_text=english_commit_text,
)
return updated_sections, updated_glossary
def _translate_qualitative_appendix_unit(
self, agent, qual_engine, *, unit_key: str, source_text: str, target_lang: str,
translation_sections: dict, translation_glossary: dict, data_metadata: dict,
):
"""Translate Appendix cells, then bind the six-column table deterministically."""
source = str(source_text or '').strip()
manifest = qual_engine.build_appendix_manifest(data_metadata)
if not qual_engine._validate_appendix_manifest(manifest):
raise RuntimeError('qualitative_appendix_manifest_invalid')
manifest_hash = str(manifest.get('manifest_hash') or '')
identity = self._translation_identity_bundle(agent, source)
entity_hash_method = getattr(agent, 'translation_protected_entity_ledger_hash', None)
protected_entity_ledger_hash = str(
entity_hash_method(identity['compiled_source'], translation_glossary)
if callable(entity_hash_method) else ''
)
protected_entity_ledger_version = str(
getattr(agent, 'PROTECTED_ENTITY_LEDGER_VERSION', 'protected-entity-ledger-v1')
or 'protected-entity-ledger-v1'
)
sections = dict(translation_sections or {})
existing = dict(sections.get(unit_key) or {})
if (
existing.get('qa_status') in {'passed', 'passed_with_warning'}
and existing.get('source_hash') == identity['source_hash']
and existing.get('target_lang') == target_lang
and existing.get('qualitative_appendix_binding_version') == qual_engine.TARGET_APPENDIX_BINDING_VERSION
and existing.get('qualitative_appendix_manifest_hash') == manifest_hash
and existing.get('translated_text')
):
qual_engine.validate_bound_target_appendix(existing['translated_text'], manifest, target_lang)
return sections, translation_glossary
quote_record = sections.get('__quote_ledger__')
quote_map = dict((quote_record or {}).get('translations') or {}) if isinstance(quote_record, dict) else {}
required_quotes = {str(row.get('quote_id') or '') for row in manifest.get('rows') or []}
if any(not str(quote_map.get(tag) or '').strip() for tag in required_quotes):
missing = sorted(tag for tag in required_quotes if not str(quote_map.get(tag) or '').strip())
raise RuntimeError(f'qualitative_appendix_quote_ledger_incomplete:{missing[:6]}')
translate_cells = getattr(agent, 'translate_qualitative_appendix_cells', None)
if not callable(translate_cells):
raise RuntimeError('Typed qualitative Appendix cell translation entry point is unavailable.')
try:
prebound_cells, model_cells = qual_engine.appendix_translation_plan(
manifest, translation_glossary
)
if model_cells:
result = translate_cells(
model_cells, target_lang,
glossary=translation_glossary, max_attempts=2,
claim_states=(existing.get('appendix_cell_claim_states') if isinstance(existing.get('appendix_cell_claim_states'), dict) else None),
)
model_translations = dict((result or {}).get('translations') or {})
updated_glossary = dict((result or {}).get('glossary') or translation_glossary or {})
else:
result = {
'qa_status': 'passed', 'attempts': 1, 'claim_states': {},
'appendix_cell_contract_version': 'typed-qualitative-appendix-cells-v2',
}
model_translations = {}
updated_glossary = dict(translation_glossary or {})
cell_translations = {**dict(prebound_cells), **dict(model_translations)}
expected_cell_ids = set(qual_engine.appendix_translation_cells(manifest))
if set(cell_translations) != expected_cell_ids:
raise RuntimeError('qualitative_appendix_target_cell_binding_incomplete')
translated_text = qual_engine.bind_target_language_appendix(
manifest, target_lang, cell_translations, quote_map
)
qual_engine.validate_bound_target_appendix(translated_text, manifest, target_lang)
except PipelineCancelledError:
raise
except Exception as error:
code_method = getattr(agent, 'translation_error_code', None)
error_code = str(code_method(error) if callable(code_method) else 'TRANSLATION_PENDING')
sections[unit_key] = {
'source_hash': identity['source_hash'],
'compiled_source_hash': identity['compiled_source_hash'],
'target_lang': target_lang,
'translation_contract_version': TRANSLATION_CONTRACT_VERSION,
'fact_ledger_hash': identity['fact_ledger_hash'],
'protected_entity_ledger_version': protected_entity_ledger_version,
'protected_entity_ledger_hash': protected_entity_ledger_hash,
'translated_text': '',
'qa_status': 'repair_pending',
'attempts': int(existing.get('attempts') or 0) + 1,
'error_code': error_code,
'qualitative_appendix_binding_version': qual_engine.TARGET_APPENDIX_BINDING_VERSION,
'qualitative_appendix_manifest_hash': manifest_hash,
'appendix_cell_claim_states': dict(getattr(agent, '_last_translation_claim_states', {}) or {}),
'updated_at': datetime.now(timezone.utc).isoformat(),
}
logging.warning(
'Qualitative Appendix typed translation repair pending | unit=%s code=%s error=%s',
unit_key, error_code, type(error).__name__,
)
return sections, translation_glossary
sections[unit_key] = {
'source_hash': identity['source_hash'],
'compiled_source_hash': identity['compiled_source_hash'],
'target_lang': target_lang,
'glossary_version': self._glossary_version(updated_glossary),
'translation_contract_version': TRANSLATION_CONTRACT_VERSION,
'fact_ledger_hash': identity['fact_ledger_hash'],
'protected_entity_ledger_version': protected_entity_ledger_version,
'protected_entity_ledger_hash': protected_entity_ledger_hash,
'translated_text': translated_text,
'qa_status': str((result or {}).get('qa_status') or 'passed'),
'attempts': int((result or {}).get('attempts') or 1),
'qualitative_appendix_binding_version': qual_engine.TARGET_APPENDIX_BINDING_VERSION,
'qualitative_appendix_manifest_hash': manifest_hash,
'appendix_cell_contract_version': str((result or {}).get('appendix_cell_contract_version') or ''),
'appendix_cell_claim_states': dict((result or {}).get('claim_states') or {}),
'appendix_prebound_label_count': len(prebound_cells),
'updated_at': datetime.now(timezone.utc).isoformat(),
}
return sections, updated_glossary
@staticmethod
def _bounded_target(base_target: int, accumulated_deviation: float, remaining_including_current: int, safe_chunk: int) -> int:
base = max(80, int(base_target))
correction = -float(accumulated_deviation) / max(1, int(remaining_including_current))
lower = max(80, int(round(base * 0.75)))
upper = min(int(safe_chunk * 1.5), max(lower, int(round(base * 1.25))))
return max(lower, min(upper, int(round(base + correction))))
@staticmethod
def _english_count_for_target(manuscript: str) -> int:
body = re.split(r"(?m)^#\s+References\s*$", str(manuscript or ""), maxsplit=1)[0]
body = re.split(r"(?m)^#\s+Appendix: Data Structure and Evidence\s*$", body, maxsplit=1)[0]
# Markdown punctuation is ignored; contractions/hyphenated terms stay one token.
return len(re.findall(r"\b[A-Za-z0-9]+(?:[-'][A-Za-z0-9]+)*\b", body))
def _validate_target_length(self, manuscript: str, target_words: int) -> dict[str, Any]:
"""Return a nonfatal total-length quality diagnostic.
The user-selected target remains a planning input, not a completion gate.
Completeness is validated separately by structure, evidence, citation,
translation, DOCX, and ZIP checks.
"""
quality = APP_CONFIG.get("manuscript_validation", {})
minimum = int(target_words * float(quality.get("minimum_total_ratio", 0.97)))
maximum = int(target_words * float(quality.get("maximum_total_ratio", 1.03)))
short_notice = float(quality.get("total_short_notice_ratio", 0.70))
topic_review = float(quality.get("total_topic_review_ratio", 0.50))
actual = self._english_count_for_target(manuscript)
ratio = (actual / max(1, int(target_words)))
status = "normal"
recommendation = ""
if ratio < topic_review:
status = "very_short"
recommendation = (
"First verify that generation completed normally and no section/evidence is missing. "
"If the manuscript is complete, consider broadening the research scope or selecting a new research topic."
)
elif ratio < short_notice:
status = "short"
recommendation = (
"If the manuscript is structurally complete, consider broadening the theoretical scope, context, "
"research question, or research topic."
)
elif actual < minimum:
status = "slightly_short"
elif actual > maximum:
status = "long"
if status != "normal":
logging.warning(
"Manuscript length diagnostic | actual=%s | target=%s | ratio=%.3f | status=%s | nonfatal=true",
actual, target_words, ratio, status,
)
return {
"scope": "manuscript", "name": "English manuscript", "actual_words": actual,
"target_words": int(target_words), "ratio": round(ratio, 4), "status": status,
"recommendation": recommendation, "nonfatal": True,
}
@staticmethod
def _upsert_quality_diagnostic(items: list[dict[str, Any]], diagnostic: dict[str, Any]) -> None:
key = (str(diagnostic.get("scope") or ""), str(diagnostic.get("name") or ""))
items[:] = [
item for item in items
if (str(item.get("scope") or ""), str(item.get("name") or "")) != key
]
items.append(dict(diagnostic))
def _section_word_diagnostic(self, name: str, actual_words: int, target_words: int) -> dict[str, Any]:
validation = APP_CONFIG.get("manuscript_validation", {})
budget = APP_CONFIG.get("word_budget", {})
target = max(1, int(target_words))
actual = max(0, int(actual_words))
ratio = actual / target
short_notice = float(validation.get("section_short_notice_ratio", 0.60))
topic_review = float(validation.get("section_topic_review_ratio", 0.40))
preferred_min = float(budget.get("section_min_ratio", 0.92))
preferred_max = float(budget.get("section_max_ratio", 1.08))
status = "normal"
recommendation = ""
if ratio < topic_review:
status = "very_short"
recommendation = (
"First verify that this section completed normally and that evidence was available. "
"If it is complete, consider broadening the research scope or selecting a new research topic."
)
elif ratio < short_notice:
status = "short"
recommendation = (
"If the section is complete, consider broadening the theoretical scope, context, research question, "
"or research topic."
)
elif ratio < preferred_min:
status = "slightly_short"
elif ratio > preferred_max:
status = "long"
if status != "normal":
logging.warning(
"Section length diagnostic | section=%s | actual=%s | target=%s | ratio=%.3f | status=%s | nonfatal=true",
name, actual, target, ratio, status,
)
return {
"scope": "section", "name": str(name), "actual_words": actual, "target_words": target,
"ratio": round(ratio, 4), "status": status, "recommendation": recommendation, "nonfatal": True,
}
@staticmethod
def _citation_eligible_section_ids(mode: str) -> set[str]:
return (
{"introduction", "theoretical_framing", "methods", "discussion"}
if str(mode).lower() == "asq"
else {"introduction", "theory", "methodology", "discussion"}
)
@staticmethod
def _quota_driven_citation_recovery_allowed(section_id: str) -> bool:
"""Discussion evidence is claim-driven, never padded to a word-count target.
R6-R5 keeps claim-specific planner assignments, but disables reserve/JIT
top-ups whose only trigger is a numeric citation target. After drafting,
the existing bounded top-level completion may still attach a directly
validated source to an actual external claim.
"""
return str(section_id or '').strip().lower() != 'discussion'
# Backward-readable name used by R6-R5 replay documentation.
_pre_draft_citation_recovery_allowed = _quota_driven_citation_recovery_allowed
@classmethod
def _adaptive_citation_requirements(cls, tasks, mode: str):
"""Derive evidence demand and semantic-survival reserve from manuscript units."""
allowed = cls._citation_eligible_section_ids(mode)
minimums = {}
reserve_targets = {}
for task in tasks or []:
section_id = str(task.get("section_id") or "")
display_name = str(task.get("display_name") or "").strip()
if section_id not in allowed or not display_name:
continue
base, words_per_source = CITATION_ROLE_RULES.get(section_id, (2, 240))
target_words = max(1, int(task.get("target_words") or 1))
minimum = max(base, int(math.ceil(target_words / max(1, words_per_source))))
reserve = max(
minimum,
int(math.ceil(minimum / max(0.01, CITATION_SEMANTIC_SURVIVAL_RATIO))),
)
minimums[display_name] = minimum
reserve_targets[display_name] = reserve
return minimums, reserve_targets
@classmethod
def _top_level_citation_requirements(cls, tasks, mode: str):
"""Aggregate citation authority by academic section, never by technical chunk.
``Introduction (Part 1/2)`` and ``Introduction (Part 2/2)`` are generation
chunks, not separate scholarly evidence authorities. Planning may remain
unit-local, but fatal literature sufficiency is evaluated on the canonical
top-level academic section.
"""
allowed = cls._citation_eligible_section_ids(mode)
words_by_section = {}
for task in tasks or []:
section_id = str(task.get('section_id') or '')
if section_id not in allowed:
continue
words_by_section[section_id] = words_by_section.get(section_id, 0) + max(
1, int(task.get('target_words') or 1)
)
minimums = {}
reserve_targets = {}
for section_id, target_words in words_by_section.items():
base, words_per_source = CITATION_ROLE_RULES.get(section_id, (2, 240))
minimum = max(base, int(math.ceil(target_words / max(1, words_per_source))))
reserve = max(
minimum,
int(math.ceil(minimum / max(0.01, CITATION_SEMANTIC_SURVIVAL_RATIO))),
)
minimums[section_id] = minimum
reserve_targets[section_id] = reserve
return minimums, reserve_targets
@staticmethod
def _citation_unit_section_map(tasks):
return {
str(task.get('display_name') or ''): str(task.get('section_id') or '')
for task in (tasks or []) if str(task.get('display_name') or '').strip()
}
@staticmethod
def _top_level_unique_tag_counts(citation_plan, unit_section_map):
tags_by_section = {}
for unit_name, items in dict(citation_plan or {}).items():
section_id = str(unit_section_map.get(str(unit_name)) or '')
if not section_id:
continue
bucket = tags_by_section.setdefault(section_id, set())
for item in items or []:
tag = str((item or {}).get('tag') or '')
if tag:
bucket.add(tag)
return {section_id: len(tags) for section_id, tags in tags_by_section.items()}
@staticmethod
def _adaptive_literature_candidate_target(target_words: int, agent) -> int:
"""Adaptive retrieval aspiration; never a final reference-count quota."""
dynamic = max(24, int(math.ceil(max(1, int(target_words or 1)) / 90.0)))
configured_cap = max(1, int(getattr(agent, "lit_target_papers", dynamic) or dynamic))
return min(configured_cap, dynamic)
@staticmethod
def _citation_task_context(task, research_logic_contract="", synthesis_result="") -> str:
parts = [
str(task.get("display_name") or ""),
str(task.get("required_content") or ""),
" ".join(str(value) for value in (task.get("concepts") or []) if str(value).strip()),
str(research_logic_contract or "")[:1600],
str(synthesis_result or "")[:1600],
]
return re.sub(r"\s+", " ", " ".join(part for part in parts if part)).strip()
@staticmethod
def _literature_insufficient_error(shortfalls) -> RuntimeError:
details = "; ".join(
f"{name}={actual}/{required}"
for name, actual, required in (shortfalls or [])[:8]
)
return RuntimeError(
"E_LITERATURE_INSUFFICIENT: high-relevance claim-specific literature is insufficient "
f"for the planned manuscript units ({details}). Please revise the research topic or "
"keywords and try again; unrelated references will not be used to satisfy a quota."
)
@staticmethod
def _evidence_warning_detail() -> str:
return (
"Some externally oriented claims have limited direct high-relevance literature coverage. "
"This may reflect a newer or specialized topic, dispersed literature, search/keyword limits, "
"or limited prior evidence; this status does not determine the novelty, importance, or overall "
"research value of the topic. Unrelated sources were not used to satisfy a citation count."
)
def _record_evidence_coverage_event(self, state, shortfalls, *, phase: str, preview_status: str) -> None:
"""Persist a neutral evidence-coverage signal without stopping generation.
The event records only deterministic coverage counts. It never infers why
literature is sparse and never evaluates topic value. Duplicate events
are collapsed across resume/replay.
"""
rows = []
for name, actual, required in shortfalls or []:
rows.append({
"section": str(name or ""),
"actual": max(0, int(actual or 0)),
"required": max(0, int(required or 0)),
})
if not rows:
return
signature = (str(phase or ""), tuple((r["section"], r["actual"], r["required"]) for r in rows))
existing = list(state.get("evidence_coverage_events") or self._evidence_coverage_events or [])
seen = {
(str(item.get("phase") or ""), tuple(
(str(row.get("section") or ""), int(row.get("actual") or 0), int(row.get("required") or 0))
for row in (item.get("shortfalls") or []) if isinstance(row, dict)
))
for item in existing if isinstance(item, dict)
}
if signature not in seen:
existing.append({
"phase": str(phase or ""),
"shortfalls": rows,
"observed_at": datetime.now(timezone.utc).isoformat(),
})
self._evidence_coverage_events = existing
state["evidence_coverage_events"] = existing
state["evidence_risk_detected"] = True
self._preview_evidence_status = str(preview_status or "limited_risk")
self._write_preview_payload(force=True)
logging.warning(
"Evidence coverage warning | phase=%s | shortfalls=%s | nonfatal=true",
phase, rows,
)
@staticmethod
def _evidence_coverage_summary(*, status, early_risk, final_reference_count, minimums, validated_by_top_level, claims_by_top_level, events, method_authority_count=0, method_external_claims=0):
sections = []
for section_id, planned_minimum in dict(minimums or {}).items():
validated = len(set((validated_by_top_level or {}).get(section_id) or []))
external_claims = max(0, int((claims_by_top_level or {}).get(section_id, 0) or 0))
required = min(int(planned_minimum), external_claims) if external_claims else 0
sections.append({
"section": str(section_id),
"validated_direct_citations": validated,
"evidence_requirement": required,
"external_claims": external_claims,
"planning_minimum": int(planned_minimum),
})
normalized_status = str(status or "adequate")
return {
"status": normalized_status,
"submission_readiness": "review_required" if normalized_status == "limited" else "quality_gate_passed",
"early_risk_detected": bool(early_risk),
"final_reference_count": max(0, int(final_reference_count or 0)),
"sections": sections,
"methodology_authority": {
"validated_method_authority_citations": max(0, int(method_authority_count or 0)),
"methodology_external_claims": max(0, int(method_external_claims or 0)),
},
"events": list(events or []),
}
def _bounded_top_level_evidence_completion(
self, agent, *, topic: str, section_id: str, section_name: str, section_text: str,
required: int, existing_tags, excluded_tags=None,
) -> tuple[str, list[str]]:
"""Give one completed top-level section a single bounded evidence top-up.
This is not a prose-generation path. It issues at most one contextual JIT
retrieval and may attach only semantically validated citations to claims
that already exist. If no directly supporting source is found, the prose
remains unchanged and the manuscript later reports evidence-limited quality.
"""
value = str(section_text or "")
current = {str(tag or "").strip() for tag in (existing_tags or []) if str(tag or "").strip()}
needed = max(0, int(required or 0) - len(current))
needed = min(EVIDENCE_COMPLETION_MAX_PER_TOP_LEVEL, needed)
if needed <= 0 or not value.strip():
return value, []
before = set(getattr(agent, "used_citations", set()) or set())
items = agent.fetch_contextual_jit_recovery_items(
topic=topic,
section_name=section_name,
section_id=section_id,
section_context=value,
needed=needed,
excluded_tags=(before | set(excluded_tags or set())),
)
if not items:
logging.info(
"Bounded evidence completion found no eligible source | section=%s needed=%s",
section_name, needed,
)
return value, []
recovered = agent.recover_citations_on_existing_claims(
value, items, section_name=section_name, section_id=section_id, needed=needed
)
after = set(getattr(agent, "used_citations", set()) or set())
added = sorted(after - before, key=agent._citation_tag_number)
if recovered == value or not added:
logging.info(
"Bounded evidence completion made no validated attachment | section=%s candidates=%s",
section_name, len(items),
)
return value, []
logging.info(
"Bounded evidence completion accepted | section=%s added=%s required=%s",
section_name, len(added), required,
)
return recovered, added
@staticmethod
def _paragraph_token_set(value):
text = re.sub(r'\[CITE-\d+\]', ' ', str(value or ''), flags=re.I)
return set(re.findall(r'[A-Za-z][A-Za-z0-9_-]{2,}', text.lower()))
@classmethod
def _remove_severe_cross_unit_duplicates(cls, candidate, prior_text):
"""Drop only very high-confidence cross-unit duplicate paragraphs."""
previous = [p.strip() for p in re.split(r'\n\s*\n', str(prior_text or '')) if len(p.split()) >= 55 and not p.lstrip().startswith('#')]
kept, diagnostics = [], []
for paragraph in re.split(r'(\n\s*\n)', str(candidate or '')):
if re.fullmatch(r'\n\s*\n', paragraph or ''):
kept.append(paragraph); continue
value = str(paragraph or '').strip()
if len(value.split()) < 55 or value.startswith('#'):
kept.append(paragraph); continue
tokens = cls._paragraph_token_set(value)
citations = set(re.findall(r'\[CITE-\d+\]', value, flags=re.I))
numerics = set(re.findall(r'(? list[str]:
output = []
for item in items or []:
path = (self.task_dir / str(item)).resolve()
if self.task_dir.resolve() not in path.parents:
raise RuntimeError("input_path_escape")
if path.is_file():
output.append(str(path))
return output
def _validate_trusted_task_policy_for_execution(self, mode: str, target_words: int) -> dict[str, Any] | None:
snapshot = copy.deepcopy(self.trusted_task_policy_snapshot)
if not snapshot:
return None
resolved = snapshot.get("resolved_policy")
if not isinstance(resolved, dict):
raise RuntimeError("trusted_task_policy_snapshot_invalid")
task_id = str(snapshot.get("task_id") or "").strip()
request_task_id = str(self.request.get("task_id") or "").strip()
if not task_id or task_id != request_task_id:
raise RuntimeError("trusted_task_policy_task_mismatch")
tier = str(snapshot.get("product_tier") or "").strip().upper()
entitlement_class = str(snapshot.get("entitlement_class") or "").strip().upper()
if tier not in {"STANDARD", "FLAGSHIP"}:
raise RuntimeError("trusted_task_policy_product_tier_invalid")
if entitlement_class not in {"STANDARD_TRIAL", "STANDARD_PAID", "FLAGSHIP_PAID"}:
raise RuntimeError("trusted_task_policy_entitlement_invalid")
if tier == "STANDARD" and entitlement_class == "FLAGSHIP_PAID":
raise RuntimeError("trusted_task_policy_entitlement_mismatch")
if tier == "FLAGSHIP" and entitlement_class != "FLAGSHIP_PAID":
raise RuntimeError("trusted_task_policy_entitlement_mismatch")
if str(resolved.get("product_tier") or "").strip().upper() != tier:
raise RuntimeError("trusted_task_policy_product_tier_mismatch")
if str(resolved.get("snapshot_schema_version") or "") != str(snapshot.get("snapshot_schema_version") or ""):
raise RuntimeError("trusted_task_policy_schema_mismatch")
for field in ("product_policy_version", "word_count_policy_version", "model_policy_version", "cost_policy_version"):
if str(resolved.get(field) or "") != str(snapshot.get(field) or ""):
raise RuntimeError("trusted_task_policy_version_mismatch")
for field in ("billing_policy_version", "trial_policy_version"):
resolved_value = str(resolved.get(field) or "").strip() or None
snapshot_value = str(snapshot.get(field) or "").strip() or None
if resolved_value != snapshot_value:
raise RuntimeError("trusted_task_policy_version_mismatch")
if str(resolved.get("manuscript_mode") or "").strip().lower() != mode:
raise RuntimeError("trusted_task_policy_mode_mismatch")
minimum = int(snapshot.get("min_body_words") or 0)
maximum = int(snapshot.get("max_body_words") or 0)
if minimum <= 0 or maximum < minimum or not minimum <= target_words <= maximum:
raise RuntimeError("trusted_task_policy_word_range_mismatch")
word_count = resolved.get("word_count") if isinstance(resolved.get("word_count"), dict) else {}
if (
int(word_count.get("min_body_words") or 0) != minimum
or int(word_count.get("max_body_words") or 0) != maximum
or str(word_count.get("counting_contract_version") or "") != str(snapshot.get("counting_contract_version") or "")
):
raise RuntimeError("trusted_task_policy_word_policy_mismatch")
envelope = {
"task_id": task_id,
"snapshot_schema_version": str(snapshot.get("snapshot_schema_version") or ""),
"product_tier": str(snapshot.get("product_tier") or ""),
"product_policy_version": str(snapshot.get("product_policy_version") or ""),
"word_count_policy_version": str(snapshot.get("word_count_policy_version") or ""),
"model_policy_version": str(snapshot.get("model_policy_version") or ""),
"cost_policy_version": str(snapshot.get("cost_policy_version") or ""),
"billing_policy_version": str(snapshot.get("billing_policy_version") or "").strip() or None,
"trial_policy_version": str(snapshot.get("trial_policy_version") or "").strip() or None,
"entitlement_class": str(snapshot.get("entitlement_class") or "").strip() or None,
"min_body_words": minimum,
"max_body_words": maximum,
"counting_contract_version": str(snapshot.get("counting_contract_version") or ""),
"resolved_policy": resolved,
}
observed_sha = hashlib.sha256(
json.dumps(envelope, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
).hexdigest()
if observed_sha != str(snapshot.get("snapshot_sha256") or "").strip().lower():
raise RuntimeError("trusted_task_policy_snapshot_sha_mismatch")
model = resolved.get("model") if isinstance(resolved.get("model"), dict) else {}
routes = model.get("routes") if isinstance(model.get("routes"), dict) else {}
if not routes:
raise RuntimeError("trusted_task_policy_model_routes_missing")
if str(snapshot.get("model_policy_version") or "") not in {
"model-standard-v2", "model-flagship-v2"
}:
# Preserve the dormant R3A v1 fail-closed contract; only v2 may
# cross the new trusted logical-node execution seam.
raise RuntimeError("trusted_task_policy_model_route_contract_not_ready")
if str(model.get("provider_catalog_version") or "") != "deepseek-v4-concrete-v1":
raise RuntimeError("trusted_task_policy_provider_catalog_invalid")
return snapshot
@staticmethod
def _section_logical_node(mode: str, section_id: str, subsection_id: str | None = None) -> str | None:
"""Map academic topology to a tier-neutral writer logical node."""
mode = str(mode or "").strip().lower()
section_id = str(section_id or "").strip().lower()
subsection_id = str(subsection_id or "").strip().lower()
if mode == "amj":
if section_id in {"introduction", "theory"}:
return "core_section_writing"
if section_id == "methodology":
return "methods_writing"
if section_id == "results":
return "quantitative_results_writing"
if section_id == "discussion":
if subsection_id in {"practical_implications", "limitations"}:
return "ordinary_section_writing"
return "core_section_writing"
if section_id == "conclusion":
return "ordinary_section_writing"
elif mode == "asq":
if section_id in {"introduction", "theoretical_framing"}:
return "core_section_writing"
if section_id == "methods":
return "methods_writing"
if section_id == "findings":
if subsection_id == "emergent_model":
return "core_section_writing"
return "evidence_rich_findings_narration"
if section_id == "discussion":
if subsection_id in {"boundaries", "practical_implications"}:
return "ordinary_section_writing"
return "core_section_writing"
if section_id == "conclusion":
return "ordinary_section_writing"
if section_id == "appendix":
return None
raise RuntimeError(f"logical_node_mapping_missing:{mode}:{section_id}:{subsection_id}")
def _run_impl(self) -> dict[str, Any]:
req = self.request
mode = str(req.get("mode") or "").lower()
if mode not in {"amj", "asq"}:
raise RuntimeError("invalid_generation_mode")
target_words = int(req.get("target_words") or 0)
if not APP_CONFIG["target_words_min"] <= target_words <= APP_CONFIG["target_words_max"]:
raise RuntimeError("target_words_out_of_range")
trusted_task_policy_snapshot = self._validate_trusted_task_policy_for_execution(
mode, target_words
)
# A2: provider credentials are adapter-owned. DeepSeek remains the
# current default adapter, but the runner no longer hard-codes it as the
# only legal model provider.
ds_key = str(os.environ.get("DEEPSEEK_API_KEY") or "").strip()
s2_key = str(os.environ.get("S2_API_KEY") or "").strip()
plan_policy_snapshot = resolve_plan_policy_snapshot(APP_CONFIG, req)
if mode == "amj":
APP_CONFIG.update(AMJ_PROMPTS)
else:
APP_CONFIG.update(ASQ_PROMPTS)
agent = SSciPipelineAgent(ds_key, s2_key)
self.agent = agent
agent.set_model_policy_snapshot(plan_policy_snapshot)
if trusted_task_policy_snapshot is not None:
agent.set_trusted_product_policy_snapshot(trusted_task_policy_snapshot)
agent.set_cancel_checker(self.cancel_checker)
# DeepSeek/search activity marks the executor as genuinely alive; the
# heartbeat thread decides separately when to renew D1's lease.
agent.set_execution_progress_hook(lambda: self._progress_activity())
safe_chunk_words = int(agent.chunk_words_default)
quant_engine = get_quantitative_engine()
qual_engine = get_qualitative_engine(agent)
if self.confirmed_checkpoint_seq is None:
# Compatibility path for isolated unit helpers that do not represent
# a claimed production executor. Production always supplies Worker
# checkpoint metadata through task_executor.
state = self.checkpoints.load_latest() or {}
else:
state = self.checkpoints.load_confirmed() or {}
quality_diagnostics = list(state.get("quality_diagnostics") or [])
quality_notes = list(state.get("quality_notes") or [])
state.setdefault("plan_policy_snapshot", copy.deepcopy(plan_policy_snapshot))
if trusted_task_policy_snapshot is not None:
state.setdefault(
"trusted_task_policy_snapshot",
copy.deepcopy(trusted_task_policy_snapshot),
)
self._evidence_coverage_events = list(state.get("evidence_coverage_events") or [])
if str(state.get("evidence_status") or "") == "limited":
self._preview_evidence_status = "limited"
elif bool(state.get("evidence_risk_detected")):
self._preview_evidence_status = "limited_risk"
if state:
self._preview_english_text = str(state.get('full_draft_en') or '')
self._preview_target_text = str(state.get('full_draft_translated') or '')
logging.info(
"[SP_EXECUTOR] resume checkpoint seq=%s stage=%s",
state.get("checkpoint_seq"), state.get("checkpoint_stage"),
)
topic = str(req.get("topic") or "").strip()
keywords = str(req.get("keywords") or "").strip()
target_lang = str(req.get("target_lang") or "").strip()
output_suffix = str(req.get("output_suffix") or "Target").strip()
interface_language = str(req.get("interface_language") or "zh").strip()
reference_paths = self._resolve_paths(req.get("reference_files") or [])
data_paths = self._resolve_paths(req.get("data_files") or [])
interview_paths = self._resolve_paths(req.get("interview_files") or [])
base_evidence_contract = str(APP_CONFIG.get("evidence_integrity_contract") or "")
# Restore agent state before continuing any dependent step.
if isinstance(state.get("agent_state"), dict):
agent.load_state(state["agent_state"])
if state.get("english_finalized") and state.get("final_citation_ledger_version") != FINAL_CITATION_LEDGER_VERSION:
# R4-R3: an R4-R2 checkpoint may contain a correct resolved body but
# an empty References tail caused by rescanning already-resolved CITE
# tags. Re-enter only finalization/translation/package gates; completed
# English units and the same task/reservation remain authoritative.
logging.warning("Final citation ledger contract advanced; re-running finalization from frozen completed units.")
state["english_finalized"] = False
state["translation_complete"] = False
state["package_validated"] = False
if state.get("english_finalized") and state.get("english_section_seal_version") != ENGLISH_SECTION_SEAL_VERSION:
# R2 compatibility: upgrade old completed English sections locally.
# No writing-model call, new task, or new quota reservation is allowed.
logging.warning("English Section Integrity Seal contract advanced; re-validating frozen completed sections locally.")
state["english_finalized"] = False
state["translation_complete"] = False
state["package_validated"] = False
user_lit_context = str(state.get("user_lit_context") or "")
user_data_md = str(state.get("user_data_md") or "")
data_metadata = dict(state.get("data_metadata") or {})
if not state.get("evidence_ready"):
self._progress(3, "inputs_and_evidence")
if reference_paths:
user_lit_context = extract_text_from_files(reference_paths[:20])
all_paper_texts = [extract_text_from_files([path]) for path in reference_paths[:20]]
agent.global_memory["core_golden_metadata"] = extract_golden_metadata(agent, all_paper_texts)
if mode == "amj":
data_file = data_paths[0] if data_paths else None
_, user_data_md, _, data_metadata = quant_engine.resolve_empirical_data(
data_file, interface_language, topic=topic, keywords=keywords, agent=agent
)
else:
raw_interviews = extract_text_from_files(interview_paths[:1]) if interview_paths else ""
prewrite_started = time.monotonic()
prewrite_calls = agent.ds_calls
user_data_md, _, data_metadata = qual_engine.resolve_qualitative_data(topic, keywords, raw_interviews)
agent.record_prewrite_stage("qualitative_packet", prewrite_started, prewrite_calls)
state.update(
{
"topic": topic,
"keywords": keywords,
"target_lang": target_lang,
"target_words": target_words,
"mode": mode,
"interface_language": interface_language,
"user_lit_context": user_lit_context,
"user_data_md": user_data_md,
"data_metadata": data_metadata,
"agent_state": agent.export_state(),
"evidence_ready": True,
}
)
self._checkpoint("evidence_ready", state, 6)
if mode == "amj":
current_quant_ledger = dict(data_metadata.get("quantitative_evidence_ledger") or {})
expected_quant_version = str(getattr(quant_engine, "QUANTITATIVE_EVIDENCE_LEDGER_VERSION", ""))
if current_quant_ledger.get("version") != expected_quant_version:
data_metadata["quantitative_evidence_ledger"] = quant_engine.build_quantitative_evidence_ledger(
data_metadata, user_data_md
)
state["data_metadata"] = data_metadata
logging.warning(
"Quantitative Evidence Ledger contract advanced; rebuilt locally version=%s",
expected_quant_version,
)
self._checkpoint("quantitative_evidence_ledger_upgraded", state, 6)
if str(data_metadata.get("source_type") or "") == "system_generated":
table_manifest = dict(data_metadata.get("quantitative_table_manifest") or {})
expected_table_version = str(getattr(quant_engine, "QUANTITATIVE_TABLE_MANIFEST_VERSION", ""))
if table_manifest.get("version") != expected_table_version:
rebuilt_tables = quant_engine.build_table_manifest_from_evidence_markdown(user_data_md)
if not rebuilt_tables:
raise RuntimeError("quantitative_table_manifest_rebuild_failed")
data_metadata["quantitative_table_manifest"] = rebuilt_tables
logging.warning(
"Quantitative table manifest contract advanced; rebuilt locally version=%s",
expected_table_version,
)
hypothesis_ledger = dict(data_metadata.get("hypothesis_ledger") or {})
expected_hypothesis_version = str(getattr(quant_engine, "HYPOTHESIS_LEDGER_VERSION", ""))
if hypothesis_ledger.get("version") != expected_hypothesis_version:
study_contract = dict(data_metadata.get("study_design_contract") or {})
if study_contract:
rebuilt_hypotheses = quant_engine.build_hypothesis_ledger(study_contract)
else:
constructs = dict(data_metadata.get("constructs") or {})
rebuilt_hypotheses = quant_engine.build_hypothesis_ledger([
constructs.get("predictor"), constructs.get("mediator"), constructs.get("outcome")
])
if not rebuilt_hypotheses:
raise RuntimeError("quantitative_hypothesis_ledger_rebuild_failed")
data_metadata["hypothesis_ledger"] = rebuilt_hypotheses
logging.warning(
"Quantitative hypothesis ledger contract advanced; rebuilt locally version=%s",
expected_hypothesis_version,
)
front_contract = dict(data_metadata.get("front_matter_contract") or {})
expected_front_version = str(getattr(quant_engine, "FRONT_MATTER_CONTRACT_VERSION", ""))
if front_contract.get("version") != expected_front_version:
study_contract = dict(data_metadata.get("study_design_contract") or {})
if study_contract:
rebuilt_front = quant_engine.build_front_matter_contract(topic, keywords, study_contract)
else:
constructs = dict(data_metadata.get("constructs") or {})
rebuilt_front = quant_engine.build_front_matter_contract(
topic, keywords, [
constructs.get("predictor"), constructs.get("mediator"), constructs.get("outcome")
]
)
if rebuilt_front:
data_metadata["front_matter_contract"] = rebuilt_front
else:
logging.warning("Quantitative front-matter semantic contract could not be rebuilt; continuing with existing research seeds and degraded final reconciliation.")
prior_translation_sections = dict(state.get("translation_sections") or {})
prior_translation_sections.pop("__front_matter__", None)
state["translation_sections"] = prior_translation_sections
state["translation_complete"] = False
state["package_validated"] = False
logging.warning(
"Quantitative front-matter contract advanced; rebuilt locally version=%s",
expected_front_version,
)
state["data_metadata"] = data_metadata
source_topic, source_keywords = topic, keywords
topic, keywords = self._apply_quantitative_front_matter_contract(
mode, topic, keywords, data_metadata
)
state.pop("effective_topic", None)
state.pop("effective_keywords", None)
hypothesis_ledger = dict(data_metadata.get("hypothesis_ledger") or {}) if mode == "amj" else {}
hypothesis_contract = self._hypothesis_contract_directive(hypothesis_ledger)
if hypothesis_ledger:
# C5.1 single authority: legacy prose extraction may not redefine AMJ
# H1-Hn once the result-independent ledger exists. Downstream memory is
# populated deterministically from that ledger and survives checkpoints.
agent.global_memory["hypotheses"] = self._hypothesis_memory_text(hypothesis_ledger)
agent.global_memory["hypothesis_contract_locked"] = True
self._qualitative_quote_sources = (
self._qualitative_quote_source_map(data_metadata) if mode == "asq" else {}
)
# ASQ planning never needs access to verbatim quotations. Keep the
# research-logic/front-matter stage on a quote-free internal-evidence
# summary; verbatim text is disclosed later only to its owning Findings
# unit (or the appendix).
planning_data_md = (
qual_engine.build_scoped_evidence_md(data_metadata, include_themes=True)
if mode == "asq"
else user_data_md
)
refs_text = str(state.get("refs_text") or "")
if not refs_text:
self._progress(8, "S2 + OpenAlex")
literature_candidate_target = self._adaptive_literature_candidate_target(target_words, agent)
refs_text = agent.fetch_references_parallel(
topic,
keywords,
min_required=literature_candidate_target,
progress_callback=lambda: self._progress(9, "S2 + OpenAlex"),
)
if not agent.citation_map:
self._record_evidence_coverage_event(
state, [("retrieved literature pool", 0, 1)],
phase="retrieval_pool_empty", preview_status="limited_risk",
)
logging.warning(
"No directly validated literature was retrieved; continuing in evidence-limited mode without padding references."
)
state["literature_candidate_target"] = literature_candidate_target
if user_lit_context:
core_refs = agent.parse_user_core_literature(user_lit_context)
if core_refs:
refs_text = (
"【USER PROVIDED CORE LITERATURE】\n" + core_refs
+ "\n\n【RETRIEVED LITERATURE】\n" + refs_text
)
state.update({"refs_text": refs_text, "agent_state": agent.export_state()})
self._checkpoint("literature_ready", state, 11)
rag_engine = initialize_rag([value for value in (user_lit_context, refs_text) if value])
synthesis_result = str(state.get("synthesis_result") or "")
if not synthesis_result:
self._progress(12, "literature_synthesis")
prewrite_started = time.monotonic()
prewrite_calls = agent.ds_calls
if not str(refs_text or "").strip() and not str(user_lit_context or "").strip():
synthesis_result = (
"EVIDENCE-LIMITED CONTEXT: no directly validated external literature is currently available. "
"Continue with the frozen study evidence, cautious conceptual reasoning, and general scholarly knowledge, "
"but do not invent citations or bibliographic metadata and do not present general model knowledge as retrieved evidence."
)
else:
synthesis_result = self._stream_text(
agent.stream_synthesize_literature(topic, refs_text, user_core_literature=user_lit_context)
)
agent.record_prewrite_stage("literature_synthesis", prewrite_started, prewrite_calls)
synthesis_result = re.sub(r".*?", "", synthesis_result, flags=re.DOTALL).strip()
state.update({"synthesis_result": synthesis_result, "agent_state": agent.export_state()})
self._checkpoint("synthesis_ready", state, 13)
adaptive_structure = state.get("adaptive_structure")
if not isinstance(adaptive_structure, list) or not adaptive_structure:
self._progress(14, "research_logic")
prewrite_started = time.monotonic()
prewrite_calls = agent.ds_calls
research_logic_response = self._stream_text(
agent.stream_generate_research_logic(
topic, keywords, synthesis_result, refs_text,
data_evidence=planning_data_md, mode=mode,
hypothesis_contract=hypothesis_contract,
)
)
agent.record_prewrite_stage("research_logic", prewrite_started, prewrite_calls)
research_logic_contract = re.sub(
r".*?", "", research_logic_response, flags=re.DOTALL
).strip()
if hypothesis_ledger:
research_logic_contract = self._normalize_hypothesis_ordinal_aliases(
research_logic_contract, hypothesis_ledger
)
logic_hypothesis_rows = self._hypothesis_contract_violations(
research_logic_contract, hypothesis_ledger, section_name="Research Logic Contract"
)
if logic_hypothesis_rows:
raise RuntimeError(
f"research_logic_hypothesis_contract_invalid:{logic_hypothesis_rows[:3]}"
)
if not research_logic_contract:
raise RuntimeError("research_logic_contract_missing")
agent.global_memory["research_logic_contract"] = research_logic_contract
self._progress(14, "working_front_matter")
working_front = agent.generate_working_front_matter(
topic, keywords, research_logic_contract, synthesis_result,
data_evidence=planning_data_md, mode=mode,
hypothesis_contract=hypothesis_contract,
authoritative_keywords=self._user_authoritative_front_matter_keywords(data_metadata),
keyword_policy=self._resolved_front_matter_keyword_policy(data_metadata),
fallback_keywords=self._front_matter_keyword_fallback_terms(data_metadata),
)
working_abstract = str(working_front.get("abstract") or "").strip()
if hypothesis_ledger:
working_abstract = self._normalize_hypothesis_ordinal_aliases(
working_abstract, hypothesis_ledger
)
working_keywords = self._canonical_quantitative_front_matter_keywords(
mode, list(working_front.get("keywords") or []), data_metadata
)
if not working_abstract or not working_keywords:
raise RuntimeError("working_front_matter_missing")
_, front_fact_rows = self._repair_validate_study_fact_contract(
mode, agent, quant_engine, working_abstract, data_metadata,
section_name="Working Abstract",
)
if hypothesis_ledger:
front_fact_rows.extend(self._hypothesis_contract_violations(
working_abstract, hypothesis_ledger, section_name="Working Abstract"
))
if front_fact_rows:
feedback = self._section_retry_feedback(front_fact_rows)
logging.warning(
"Working Abstract fact validation retry | mode=%s count=%s types=%s",
mode, len(front_fact_rows), sorted({str(row.get("type") or "") for row in front_fact_rows}),
)
working_front = agent.generate_working_front_matter(
topic, keywords, research_logic_contract, synthesis_result,
data_evidence=planning_data_md, mode=mode, validation_feedback=feedback,
hypothesis_contract=hypothesis_contract,
authoritative_keywords=self._user_authoritative_front_matter_keywords(data_metadata),
keyword_policy=self._resolved_front_matter_keyword_policy(data_metadata),
fallback_keywords=self._front_matter_keyword_fallback_terms(data_metadata),
)
working_abstract = str(working_front.get("abstract") or "").strip()
if hypothesis_ledger:
working_abstract = self._normalize_hypothesis_ordinal_aliases(
working_abstract, hypothesis_ledger
)
working_keywords = self._canonical_quantitative_front_matter_keywords(
mode, list(working_front.get("keywords") or []), data_metadata
)
_, remaining_front_rows = self._repair_validate_study_fact_contract(
mode, agent, quant_engine, working_abstract, data_metadata, section_name="Working Abstract",
)
if hypothesis_ledger:
remaining_front_rows.extend(self._hypothesis_contract_violations(
working_abstract, hypothesis_ledger, section_name="Working Abstract"
))
if remaining_front_rows:
working_abstract, residual_front_rows = self._adaptive_repair_front_matter(
agent=agent, quant_engine=quant_engine, abstract_text=working_abstract,
violations=remaining_front_rows, mode=mode, data_metadata=data_metadata,
study_evidence=planning_data_md, hypothesis_ledger=hypothesis_ledger,
quality_notes=quality_notes, label="Working Abstract",
)
if residual_front_rows:
logging.warning("Adaptive Working Abstract accepted with residual diagnostics | count=%s", len(residual_front_rows))
quality_diagnostics.append({
"scope":"working_abstract_study_fact_integrity", "name":"Working Abstract",
"status":"section_retry_resolved", "count":len(front_fact_rows), "examples":front_fact_rows[:3],
})
if not working_abstract:
raise RuntimeError("working_front_matter_empty_after_validation")
agent.global_memory["working_abstract"] = working_abstract
self._progress(14, "outline_planning")
plan = agent.generate_outline(topic, keywords, mode=mode, data_metadata=data_metadata)
# User title is reader-facing authority. Outline planning may adapt
# structure but may not silently rename the study.
en_topic = str(source_topic or topic).strip()
structure = plan.get("structure", [])
full_draft_en = (
f"# {en_topic}\n\n## Abstract\n{working_abstract}\n\n"
f"## Keywords\n{', '.join(working_keywords)}\n\n"
)
# Publish the Working Abstract/Keywords immediately, before JIT and
# citation planning, so the user sees the manuscript anchor before
# Introduction while those longer planning stages continue.
self._publish_preview(full_draft_en, "working_front_matter", force=True)
front_budget = max(180, int(APP_CONFIG.get("word_budget", {}).get("front_matter_words", 220)))
body_target_words = max(1000, target_words - front_budget)
adaptive_structure = agent.build_generation_tasks(structure, body_target_words, safe_chunk_words)
for task in adaptive_structure:
task["is_first"] = bool(task.get("section_first"))
quote_invocation_plan = (
self._build_qualitative_quote_invocation_plan(adaptive_structure, data_metadata)
if mode == "asq" else {}
)
jit_texts = agent.prefetch_jit_literature(
topic, keywords, adaptive_structure,
quantitative_evidence_ledger=(
data_metadata.get("quantitative_evidence_ledger") if mode == "amj" else None
),
)
if jit_texts:
rag_engine._initialize_chunks([jit_texts])
self._progress(15, "citation_planning")
citation_minimums, citation_reserve_targets = self._adaptive_citation_requirements(
adaptive_structure, mode
)
citation_top_level_minimums, citation_top_level_reserve_targets = self._top_level_citation_requirements(
adaptive_structure, mode
)
citation_unit_section_map = self._citation_unit_section_map(adaptive_structure)
citation_plan_target = sum(citation_top_level_reserve_targets.values())
citation_plan = agent.build_claim_evidence_plan(
adaptive_structure, topic, synthesis_result, mode=mode,
target_unique=max(1, citation_plan_target),
)
# Early neutral risk signal: evaluate the high-relevance plan before
# reserve filling/JIT can make a raw candidate count look healthier
# than the claim-specific plan actually is. This is advisory only.
initial_top_level_counts = self._top_level_unique_tag_counts(
citation_plan, citation_unit_section_map
)
initial_shortfalls = [
(section_id, int(initial_top_level_counts.get(section_id, 0)), required)
for section_id, required in citation_top_level_minimums.items()
if int(initial_top_level_counts.get(section_id, 0)) < int(required)
]
if initial_shortfalls:
self._record_evidence_coverage_event(
state, initial_shortfalls, phase="initial_claim_specific_plan",
preview_status="limited_risk",
)
# Fill unit-specific reserve gaps before expensive drafting. Existing
# retrieved evidence is preferred; bounded contextual JIT is last resort.
planned_tags = {
str(item.get("tag") or "")
for items in citation_plan.values()
for item in items
if item.get("tag")
}
task_by_name = {
str(task.get("display_name") or ""): task
for task in adaptive_structure
if str(task.get("display_name") or "")
}
for display_name, reserve_target in citation_reserve_targets.items():
planned_items = citation_plan.setdefault(display_name, [])
unit_section_id = str(task_by_name.get(display_name, {}).get("section_id") or "")
if not self._quota_driven_citation_recovery_allowed(unit_section_id):
logging.info(
"Discussion pre-draft reserve recovery skipped: unit=%s direct_plan=%s policy=claim_driven",
display_name, len(planned_items),
)
continue
shortfall = max(0, reserve_target - len(planned_items))
if shortfall <= 0:
continue
section_context = self._citation_task_context(
task_by_name.get(display_name, {}), research_logic_contract, synthesis_result
)
recovery_items = agent.select_contextual_citation_recovery_items(
[], section_name=display_name,
section_id=str(task_by_name.get(display_name, {}).get("section_id") or ""),
section_context=section_context, needed=shortfall, excluded_tags=planned_tags,
)
for item in recovery_items:
tag = str(item.get("tag") or "")
if tag and tag not in planned_tags:
planned_items.append(item)
planned_tags.add(tag)
# Do not trigger external retrieval merely because a technical
# generation chunk is below a word-derived target. Fatal evidence
# authority belongs to the canonical top-level academic section.
top_level_counts = self._top_level_unique_tag_counts(citation_plan, citation_unit_section_map)
for section_id, required in citation_top_level_minimums.items():
deficit = max(0, int(required) - int(top_level_counts.get(section_id, 0)))
if deficit <= 0:
continue
if not self._quota_driven_citation_recovery_allowed(section_id):
logging.info(
"Discussion pre-draft JIT quota top-up skipped: section=%s direct_plan=%s required=%s policy=claim_driven",
section_id, int(top_level_counts.get(section_id, 0)), int(required),
)
continue
section_units = [
name for name, mapped in citation_unit_section_map.items() if mapped == section_id
]
if not section_units:
continue
primary_unit = section_units[0]
section_context = " ".join(
self._citation_task_context(task_by_name.get(name, {}), research_logic_contract, synthesis_result)
for name in section_units
)
jit_items = agent.fetch_contextual_jit_recovery_items(
topic=topic, section_name=section_id, section_id=section_id,
section_context=section_context, needed=deficit, excluded_tags=planned_tags,
)
planned_items = citation_plan.setdefault(primary_unit, [])
for item in jit_items:
tag = str(item.get("tag") or "")
if tag and tag not in planned_tags:
planned_items.append(item)
planned_tags.add(tag)
top_level_counts = self._top_level_unique_tag_counts(citation_plan, citation_unit_section_map)
shortfalls = [
(section_id, int(top_level_counts.get(section_id, 0)), required)
for section_id, required in citation_top_level_minimums.items()
if int(top_level_counts.get(section_id, 0)) < int(required)
]
if shortfalls:
self._record_evidence_coverage_event(
state, shortfalls, phase="bounded_retrieval_complete",
preview_status="limited_risk",
)
logging.warning(
"Top-level citation feasibility limited: minimum_total=%s reserve_target=%s planned_unique=%s sections=%s nonfatal=true",
sum(citation_top_level_minimums.values()), citation_plan_target, len(planned_tags), len(citation_top_level_minimums),
)
else:
logging.info(
"Top-level citation feasibility passed: minimum_total=%s reserve_target=%s planned_unique=%s sections=%s",
sum(citation_top_level_minimums.values()), citation_plan_target, len(planned_tags), len(citation_top_level_minimums),
)
logging.info(
"Top-level citation planning complete: minimum_total=%s reserve_target=%s planned_unique=%s sections=%s",
sum(citation_top_level_minimums.values()), citation_plan_target, len(planned_tags), len(citation_top_level_minimums),
)
canonical_concept_ledger = self._build_canonical_concept_ledger(
mode, data_metadata, keywords=keywords, adaptive_structure=adaptive_structure,
research_logic_contract=research_logic_contract,
)
state.update(
{
"adaptive_structure": adaptive_structure,
"quote_invocation_plan": quote_invocation_plan,
"citation_plan": citation_plan,
"citation_minimums": citation_minimums,
"citation_reserve_targets": citation_reserve_targets,
"citation_top_level_minimums": citation_top_level_minimums,
"citation_top_level_reserve_targets": citation_top_level_reserve_targets,
"citation_unit_section_map": citation_unit_section_map,
"citation_validated_tags_by_unit": {},
"citation_required_claim_counts_by_unit": {},
"section_integrity_seals": {},
"english_section_seal_version": ENGLISH_SECTION_SEAL_VERSION,
"research_logic_contract": research_logic_contract,
"working_abstract": working_abstract,
"working_keywords": working_keywords,
"front_matter_phase": "working",
"body_target_words": body_target_words,
"full_draft_en": full_draft_en,
"full_draft_translated": "",
"translation_sections": {},
"translation_glossary": {},
"canonical_concept_ledger": canonical_concept_ledger,
"completed_sections": [],
"context_tail": "",
"accumulated_deviation": 0.0,
"agent_state": agent.export_state(),
"outline_ready": True,
}
)
self._publish_preview(full_draft_en, "outline_ready", force=True)
self._checkpoint("outline_ready", state, 15)
else:
full_draft_en = str(state.get("full_draft_en") or "")
if str(data_metadata.get("source_type") or "") == "system_generated" and mode == "amj":
full_draft_en = self._replace_title(full_draft_en, topic)
canonical_resume_keywords = self._canonical_quantitative_front_matter_keywords(
mode, [], data_metadata
)
full_draft_en = self._replace_keywords(full_draft_en, canonical_resume_keywords)
state["full_draft_en"] = full_draft_en
state["working_keywords"] = canonical_resume_keywords
if state.get("final_keywords") is not None:
state["final_keywords"] = canonical_resume_keywords
# JIT text itself is already represented in agent citation state;
# refs + user literature rebuild a safe RAG baseline on resume.
completed_sections = list(state.get("completed_sections") or [])
context_tail = str(state.get("context_tail") or "")
accumulated_deviation = float(state.get("accumulated_deviation") or 0.0)
research_logic_contract = str(
state.get("research_logic_contract") or agent.global_memory.get("research_logic_contract") or ""
)
if hypothesis_ledger and research_logic_contract:
research_logic_contract = self._normalize_hypothesis_ordinal_aliases(
research_logic_contract, hypothesis_ledger
)
resumed_logic_rows = self._hypothesis_contract_violations(
research_logic_contract, hypothesis_ledger, section_name="Research Logic Contract"
)
if resumed_logic_rows:
raise RuntimeError(
f"research_logic_hypothesis_contract_invalid_on_resume:{resumed_logic_rows[:3]}"
)
agent.global_memory["research_logic_contract"] = research_logic_contract
citation_plan = state.get("citation_plan") if isinstance(state.get("citation_plan"), dict) else {}
citation_minimums = dict(state.get("citation_minimums") or {})
citation_reserve_targets = dict(state.get("citation_reserve_targets") or {})
citation_top_level_minimums = dict(state.get("citation_top_level_minimums") or {})
citation_top_level_reserve_targets = dict(state.get("citation_top_level_reserve_targets") or {})
citation_unit_section_map = dict(state.get("citation_unit_section_map") or {})
citation_required_claim_counts_by_unit = {
str(name): int(count or 0)
for name, count in dict(state.get("citation_required_claim_counts_by_unit") or {}).items()
}
citation_validated_tags_by_unit = {
str(name): list(tags or [])
for name, tags in dict(state.get("citation_validated_tags_by_unit") or {}).items()
}
section_integrity_seals = {
str(key): dict(record or {})
for key, record in dict(state.get("section_integrity_seals") or {}).items()
if isinstance(record, dict)
}
if not citation_minimums:
citation_minimums, citation_reserve_targets = self._adaptive_citation_requirements(
adaptive_structure, mode
)
if not citation_top_level_minimums:
citation_top_level_minimums, citation_top_level_reserve_targets = self._top_level_citation_requirements(
adaptive_structure, mode
)
if not citation_unit_section_map:
citation_unit_section_map = self._citation_unit_section_map(adaptive_structure)
translation_sections = dict(state.get('translation_sections') or {})
translation_glossary = dict(state.get('translation_glossary') or {})
if mode == "asq":
appendix_record = dict(translation_sections.get("appendix") or {})
if appendix_record and appendix_record.get("qualitative_appendix_binding_version") != qual_engine.TARGET_APPENDIX_BINDING_VERSION:
logging.info("Invalidating pre-R5 qualitative Appendix translation record; deterministic binding required.")
translation_sections.pop("appendix", None)
state["translation_sections"] = dict(translation_sections)
canonical_concept_ledger = state.get('canonical_concept_ledger')
if not isinstance(canonical_concept_ledger, dict) or not canonical_concept_ledger.get('items'):
canonical_concept_ledger = self._build_canonical_concept_ledger(
mode, data_metadata, keywords=keywords, adaptive_structure=adaptive_structure,
research_logic_contract=research_logic_contract,
)
state['canonical_concept_ledger'] = canonical_concept_ledger
working_abstract = str(state.get("working_abstract") or "").strip()
working_keywords = self._canonical_quantitative_front_matter_keywords(
mode, list(state.get("working_keywords") or []), data_metadata
)
if working_keywords:
state["working_keywords"] = working_keywords
if working_abstract and not state.get("english_finalized"):
if hypothesis_ledger:
working_abstract = self._normalize_hypothesis_ordinal_aliases(
working_abstract, hypothesis_ledger
)
_, resumed_front_rows = self._repair_validate_study_fact_contract(
mode, agent, quant_engine, working_abstract, data_metadata, section_name="Working Abstract",
)
if hypothesis_ledger:
resumed_front_rows.extend(self._hypothesis_contract_violations(
working_abstract, hypothesis_ledger, section_name="Working Abstract"
))
if resumed_front_rows:
feedback = self._section_retry_feedback(resumed_front_rows)
working_front = agent.generate_working_front_matter(
topic, keywords, research_logic_contract, synthesis_result,
data_evidence=planning_data_md, mode=mode, validation_feedback=feedback,
hypothesis_contract=hypothesis_contract,
authoritative_keywords=self._user_authoritative_front_matter_keywords(data_metadata),
keyword_policy=self._resolved_front_matter_keyword_policy(data_metadata),
fallback_keywords=self._front_matter_keyword_fallback_terms(data_metadata),
)
candidate_abstract = str(working_front.get("abstract") or "").strip()
if hypothesis_ledger:
candidate_abstract = self._normalize_hypothesis_ordinal_aliases(
candidate_abstract, hypothesis_ledger
)
candidate_keywords = self._canonical_quantitative_front_matter_keywords(
mode, list(working_front.get("keywords") or []), data_metadata
)
_, remaining = self._repair_validate_study_fact_contract(
mode, agent, quant_engine, candidate_abstract, data_metadata, section_name="Working Abstract",
)
if hypothesis_ledger:
remaining.extend(self._hypothesis_contract_violations(
candidate_abstract, hypothesis_ledger, section_name="Working Abstract"
))
if remaining:
candidate_abstract, residual_front_rows = self._adaptive_repair_front_matter(
agent=agent, quant_engine=quant_engine, abstract_text=candidate_abstract,
violations=remaining, mode=mode, data_metadata=data_metadata,
study_evidence=planning_data_md, hypothesis_ledger=hypothesis_ledger,
quality_notes=quality_notes, label="Working Abstract",
)
if residual_front_rows:
logging.warning("Adaptive resumed Working Abstract accepted with residual diagnostics | count=%s", len(residual_front_rows))
working_abstract = candidate_abstract
working_keywords = candidate_keywords or working_keywords
state["working_abstract"] = working_abstract
state["working_keywords"] = working_keywords
if str(state.get("full_draft_en") or "").strip():
state["full_draft_en"] = replace_abstract(str(state.get("full_draft_en") or ""), working_abstract)
state["full_draft_en"] = self._replace_keywords(state["full_draft_en"], working_keywords)
full_draft_en = str(state["full_draft_en"] or "")
quality_diagnostics.append({
"scope":"working_abstract_study_fact_integrity", "name":"Working Abstract",
"status":"resume_section_retry_resolved", "count":len(resumed_front_rows), "examples":resumed_front_rows[:3],
})
state["quality_diagnostics"] = quality_diagnostics
self._checkpoint("working_front_matter_fact_retry_resolved", state, 15)
if working_abstract:
agent.global_memory["working_abstract"] = working_abstract
if self._preview_target_text == '':
self._preview_target_text = str(state.get('full_draft_translated') or '')
# S0-R1 compatibility migration: old completed English is never silently
# repaired. Unsealed sections are inspected only. Valid text receives a
# seal; invalid text is removed from the completed set so only that owning
# top-level section is regenerated by the normal Section Writer.
seal_migration = False
seal_text_changed = False
retry_section_keys = set()
for section_key, section_title in self._unique_top_level_sections(adaptive_structure):
if not self._section_is_complete(adaptive_structure, completed_sections, section_key):
continue
section_text = self._extract_top_level_section(full_draft_en, section_title)
if not section_text:
continue
existing_seal = dict(section_integrity_seals.get(section_key) or {})
if (
existing_seal.get("version") == ENGLISH_SECTION_SEAL_VERSION
and existing_seal.get("sha256") == self._section_integrity_hash(section_text)
):
continue
violations = self._study_fact_violations(
mode, agent, quant_engine, section_text, data_metadata,
section_name=f"Resume sealed section: {section_title}",
)
try:
agent.validate_enumeration_contract(
section_text, section_name=f"Resume sealed section: {section_title}"
)
except Exception as exc:
violations.append({
"type":"enumeration_contract_invalid",
"reason":str(exc)[:500], "preview":section_title,
})
ai_meta_rows = list(agent.inspect_authorial_ai_meta(section_text) or [])
violations.extend(ai_meta_rows)
if violations:
retry_section_keys.add(str(section_key))
translation_sections.pop(section_key, None)
section_integrity_seals.pop(section_key, None)
quality_diagnostics.append({
"scope":"english_section_seal_migration", "name":section_title,
"status":"local_section_retry_required", "count":len(violations),
"examples":violations[:3],
})
continue
section_integrity_seals[section_key] = self._build_section_integrity_seal(
section_key, section_title, section_text
)
seal_migration = True
if retry_section_keys:
# Remove only invalid unsealed top-level sections and their completed
# unit markers. Other completed English stays byte-stable.
for section_key, section_title in self._unique_top_level_sections(adaptive_structure):
if str(section_key) not in retry_section_keys:
continue
if self._extract_top_level_section(full_draft_en, section_title):
full_draft_en = self._remove_top_level_section(full_draft_en, section_title)
seal_text_changed = True
relevant_names = {
str(task.get('display_name') or '')
for task in adaptive_structure
if str(task.get('section_id') or task.get('section') or '') == str(section_key)
}
completed_sections = [name for name in completed_sections if name not in relevant_names]
seal_migration = True
if seal_migration:
# Rebuild mutable manuscript memory only from still-completed sealed English text.
for memory_key in ("hypotheses", "variables", "model_summary", "method_summary"):
agent.global_memory[memory_key] = ""
last_sealed = ""
for section_key, section_title in self._unique_top_level_sections(adaptive_structure):
if not self._section_is_complete(adaptive_structure, completed_sections, section_key):
continue
sealed_text = self._extract_top_level_section(full_draft_en, section_title)
if sealed_text:
agent._extract_memory_from_text(sealed_text, section_title)
last_sealed = sealed_text
if last_sealed:
context_tail = " ".join(last_sealed.split()[-200:])
state.update({
"completed_sections": completed_sections,
"full_draft_en": full_draft_en,
"translation_sections": translation_sections,
"section_integrity_seals": section_integrity_seals,
"english_section_seal_version": ENGLISH_SECTION_SEAL_VERSION,
"context_tail": context_tail,
"quality_diagnostics": quality_diagnostics,
"agent_state": agent.export_state(),
})
if seal_text_changed:
state["full_draft_translated"] = self._assemble_partial_translation(
translation_sections, adaptive_structure, include_front=True
)
self._checkpoint("english_section_seal_migrated", state, 15)
# Target-language work remains Executor-owned but no longer blocks the
# English writer. A dedicated single translation agent/worker keeps
# glossary order deterministic and avoids sharing mutable writer state.
translation_agent = SSciPipelineAgent(ds_key, s2_key)
translation_agent.set_model_policy_snapshot(plan_policy_snapshot)
if trusted_task_policy_snapshot is not None:
translation_agent.set_trusted_product_policy_snapshot(trusted_task_policy_snapshot)
self.translation_agent = translation_agent
translation_agent.set_execution_progress_hook(lambda: self._progress_activity())
translation_agent.set_cancel_checker(lambda: self.cancel_checker())
seeded_glossary = translation_agent.seed_canonical_concept_glossary(
canonical_concept_ledger, target_lang, translation_glossary
)
if seeded_glossary != translation_glossary:
translation_glossary = dict(seeded_glossary)
state['translation_glossary'] = translation_glossary
state['canonical_concept_ledger'] = canonical_concept_ledger
self._checkpoint('canonical_concept_ledger_ready', state, 15)
glossary_quality_version = str(
getattr(translation_agent, 'CANONICAL_GLOSSARY_QUALITY_VERSION', 'canonical-glossary-quality-v1')
or 'canonical-glossary-quality-v1'
)
if state.get('canonical_glossary_quality_version') != glossary_quality_version:
reviewed_glossary = translation_agent.validate_and_refine_canonical_concept_glossary(
canonical_concept_ledger, target_lang, translation_glossary
)
if reviewed_glossary != translation_glossary:
translation_glossary = dict(reviewed_glossary)
# A corrected canonical identity invalidates only target-language
# work. Sealed English remains authoritative and unchanged.
translation_sections = {}
state['translation_sections'] = {}
state['full_draft_translated'] = ''
state['translation_complete'] = False
state['package_validated'] = False
state['translation_glossary'] = translation_glossary
state['canonical_glossary_quality_version'] = glossary_quality_version
self._checkpoint('canonical_glossary_quality_ready', state, 15)
current_entity_ledger_version = str(
getattr(translation_agent, 'PROTECTED_ENTITY_LEDGER_VERSION', 'protected-entity-ledger-v1')
or 'protected-entity-ledger-v1'
)
stale_entity_units = [
key for key, record in translation_sections.items()
if isinstance(record, dict)
and record.get('translated_text')
and record.get('qa_status') in {'passed', 'passed_with_warning'}
and record.get('protected_entity_ledger_version') != current_entity_ledger_version
]
if stale_entity_units:
logging.warning(
'Protected Entity Ledger contract advanced; invalidating target-language cache only | units=%s version=%s',
stale_entity_units[:8], current_entity_ledger_version,
)
state['translation_complete'] = False
state['package_validated'] = False
state['full_draft_translated'] = ''
self._preview_target_text = ''
self._checkpoint('protected_entity_ledger_advanced', state, 15)
self._start_translation_worker(
translation_agent,
translation_sections=translation_sections,
translation_glossary=translation_glossary,
adaptive_structure=adaptive_structure,
target_lang=target_lang,
)
# Capture a dedicated abort Event, not the worker lifecycle stop Event.
# The worker is gracefully closed before the strict final translation
# gate, but the same translation agent remains valid for deferred/final
# units. Abortive shutdown still cancels any in-flight target call.
translation_abort_event = self._translation_worker.cancel_event
translation_agent.set_cancel_checker(
lambda: self.cancel_checker() or translation_abort_event.is_set()
)
# Backward-compatible resume path for a pre-R3.2.8.3 checkpoint that has
# an outline but no Working Abstract yet. Generate it once, checkpoint it,
# and reuse it thereafter; never regenerate an unchanged anchor on resume.
if not working_abstract and not state.get("english_finalized"):
working_front = agent.generate_working_front_matter(
topic, keywords, research_logic_contract, synthesis_result,
data_evidence=planning_data_md, mode=mode,
hypothesis_contract=hypothesis_contract,
authoritative_keywords=self._user_authoritative_front_matter_keywords(data_metadata),
keyword_policy=self._resolved_front_matter_keyword_policy(data_metadata),
fallback_keywords=self._front_matter_keyword_fallback_terms(data_metadata),
)
working_abstract = str(working_front.get("abstract") or "").strip()
if hypothesis_ledger:
working_abstract = self._normalize_hypothesis_ordinal_aliases(
working_abstract, hypothesis_ledger
)
working_keywords = self._canonical_quantitative_front_matter_keywords(
mode, list(working_front.get("keywords") or []), data_metadata
)
if not working_abstract or not working_keywords:
raise RuntimeError("working_front_matter_missing")
_, front_fact_rows = self._repair_validate_study_fact_contract(
mode, agent, quant_engine, working_abstract, data_metadata, section_name="Working Abstract",
)
if hypothesis_ledger:
front_fact_rows.extend(self._hypothesis_contract_violations(
working_abstract, hypothesis_ledger, section_name="Working Abstract"
))
if front_fact_rows:
feedback = self._section_retry_feedback(front_fact_rows)
working_front = agent.generate_working_front_matter(
topic, keywords, research_logic_contract, synthesis_result,
data_evidence=planning_data_md, mode=mode, validation_feedback=feedback,
hypothesis_contract=hypothesis_contract,
authoritative_keywords=self._user_authoritative_front_matter_keywords(data_metadata),
keyword_policy=self._resolved_front_matter_keyword_policy(data_metadata),
fallback_keywords=self._front_matter_keyword_fallback_terms(data_metadata),
)
working_abstract = str(working_front.get("abstract") or "").strip()
if hypothesis_ledger:
working_abstract = self._normalize_hypothesis_ordinal_aliases(
working_abstract, hypothesis_ledger
)
working_keywords = self._canonical_quantitative_front_matter_keywords(
mode, list(working_front.get("keywords") or []), data_metadata
)
_, remaining = self._repair_validate_study_fact_contract(
mode, agent, quant_engine, working_abstract, data_metadata, section_name="Working Abstract",
)
if hypothesis_ledger:
remaining.extend(self._hypothesis_contract_violations(
working_abstract, hypothesis_ledger, section_name="Working Abstract"
))
if remaining:
working_abstract, residual_front_rows = self._adaptive_repair_front_matter(
agent=agent, quant_engine=quant_engine, abstract_text=working_abstract,
violations=remaining, mode=mode, data_metadata=data_metadata,
study_evidence=planning_data_md, hypothesis_ledger=hypothesis_ledger,
quality_notes=quality_notes, label="Working Abstract",
)
if residual_front_rows:
logging.warning("Adaptive migrated Working Abstract accepted with residual diagnostics | count=%s", len(residual_front_rows))
quality_diagnostics.append({
"scope":"working_abstract_study_fact_integrity", "name":"Working Abstract",
"status":"section_retry_resolved", "count":len(front_fact_rows), "examples":front_fact_rows[:3],
})
if not working_abstract:
raise RuntimeError("working_front_matter_empty_after_validation")
agent.global_memory["working_abstract"] = working_abstract
full_draft_en = replace_abstract(full_draft_en, working_abstract)
full_draft_en = self._replace_keywords(full_draft_en, working_keywords)
state.update({
"working_abstract": working_abstract,
"working_keywords": working_keywords,
"front_matter_phase": "working",
"full_draft_en": full_draft_en,
"agent_state": agent.export_state(),
})
self._publish_preview(full_draft_en, "working_front_matter", force=True)
self._checkpoint("working_front_matter_ready", state, 15)
# Queue the QA-frozen Working front matter. Translation is allowed to
# overlap later English work, but only frozen English units may enter
# this Executor-owned ordered worker.
if working_abstract and not state.get("english_finalized"):
section_order_for_front = self._unique_top_level_sections(adaptive_structure)
if section_order_for_front:
first_section_title = section_order_for_front[0][1]
working_front_source = self._extract_front_matter(full_draft_en, first_section_title)
if not working_front_source and not completed_sections:
# Fresh tasks contain only Title/Abstract/Keywords here; the
# first body heading does not exist yet. Preserve legacy
# resume behavior when completed body sections are present.
working_front_source = str(full_draft_en or '').strip()
self._enqueue_translation_unit(
translation_agent,
unit_key='__front_matter__', unit_name='Working Front Matter',
source_text=working_front_source,
translation_sections=translation_sections,
english_commit_text=working_front_source,
)
# Resume/idempotency: completed English top-level sections whose current
# source-hash translation is absent/stale are queued exactly once.
for section_key, section_title in self._unique_top_level_sections(adaptive_structure):
if not self._section_is_complete(adaptive_structure, completed_sections, section_key):
continue
frozen_section = self._extract_top_level_section(full_draft_en, section_title)
self._enqueue_translation_unit(
translation_agent,
unit_key=section_key,
unit_name=section_title,
source_text=frozen_section,
translation_sections=translation_sections,
english_commit_text=self._english_prefix_through_top_level(full_draft_en, section_title),
)
translation_sections, translation_glossary = self._collect_translation_results(
translation_sections, translation_glossary, block=False
)
if not citation_plan:
self._progress(15, "citation_planning")
citation_plan = agent.build_claim_evidence_plan(
adaptive_structure, topic, synthesis_result, mode=mode,
target_unique=max(1, sum(citation_top_level_reserve_targets.values())),
)
top_level_counts = self._top_level_unique_tag_counts(citation_plan, citation_unit_section_map)
shortfalls = [
(section_id, int(top_level_counts.get(section_id, 0)), required)
for section_id, required in citation_top_level_minimums.items()
if int(top_level_counts.get(section_id, 0)) < int(required)
]
if shortfalls:
self._record_evidence_coverage_event(
state, shortfalls, phase="resume_claim_specific_plan",
preview_status="limited_risk",
)
state["citation_plan"] = citation_plan
state["citation_minimums"] = citation_minimums
state["citation_reserve_targets"] = citation_reserve_targets
state["citation_top_level_minimums"] = citation_top_level_minimums
state["citation_top_level_reserve_targets"] = citation_top_level_reserve_targets
state["citation_unit_section_map"] = citation_unit_section_map
state["citation_required_claim_counts_by_unit"] = citation_required_claim_counts_by_unit
state["citation_validated_tags_by_unit"] = citation_validated_tags_by_unit
quote_invocation_plan = (
dict(state.get("quote_invocation_plan") or {}) if mode == "asq" else {}
)
if mode == "asq":
if not quote_invocation_plan:
quote_invocation_plan = self._build_qualitative_quote_invocation_plan(
adaptive_structure, data_metadata
)
state["quote_invocation_plan"] = quote_invocation_plan
self._checkpoint("quote_plan_ready", state, 15)
self._validate_qualitative_quote_invocation_plan(
quote_invocation_plan, adaptive_structure, data_metadata
)
hypothesis_ledger = dict(data_metadata.get("hypothesis_ledger") or {}) if mode == "amj" else {}
total_steps = len(adaptive_structure)
for sec in adaptive_structure:
display_name = sec["display_name"]
if display_name in completed_sections:
continue
if self.cancel_checker():
raise RuntimeError("task_cancel_requested")
remaining = sum(1 for item in adaptive_structure if item["display_name"] not in completed_sections)
adjusted_target = self._bounded_target(
int(sec["target_words"]), 0.0, remaining, safe_chunk_words
)
sec_title = sec["section"]
subsection = sec.get("subsection")
section_id = str(sec.get("section_id") or "")
is_quarantined = self._section_forbids_external_citations(sec_title, section_id)
section_logical_node = self._section_logical_node(
mode, section_id, sec.get("subsection_id")
)
planned_items = [
item for item in list(citation_plan.get(display_name, []))
if agent.citation_source_role_eligible(
str(item.get("tag") or ""), section_id=section_id, section_name=display_name
)
]
citation_plan[display_name] = copy.deepcopy(planned_items)
assigned_tag_list = [str(item.get("tag") or "") for item in planned_items if item.get("tag")]
claim_plan_text = agent.format_claim_evidence_plan(planned_items)
section_role = str(
APP_CONFIG.get("section_reasoning_profiles", {}).get(mode, {}).get(section_id) or ""
)
mode_directive = ""
if section_id in {"theory", "theoretical_framing"}:
mode_directive = str(APP_CONFIG.get("hypothesis_development_directive") or "")
elif section_id in {"methodology", "methods"}:
mode_directive = str(APP_CONFIG.get("methodological_defense_directive") or "")
elif section_id == "discussion":
mode_directive = str(APP_CONFIG.get("discussion_depth_directive") or "")
elif section_id == "conclusion":
mode_directive = str(APP_CONFIG.get("conclusion_anchor_directive") or "")
if is_quarantined:
citation_rule = (
"NO EXTERNAL LITERATURE CITATIONS ALLOWED IN THIS UNIT. "
"Report only documented empirical/qualitative evidence and the interpretation permitted by the section role."
)
else:
citation_rule = (
"Use the citation-first claim-evidence plan below as bounded evidence options, not as a claim quota. "
"Do not create a new claim merely to consume an assigned citation. Use a source only when it directly supports a claim independently required by the research logic, section role, or already established manuscript argument; omit any item that would introduce another application domain or mutate the study's facts. "
"When a source is used, keep the claim no broader than the supplied evidence and attach exactly its assigned [CITE-XX] tag. "
"Never stack sources, substitute tags, or use literature as evidence for this study's own facts.\n"
f"\n{claim_plan_text}\n"
)
if not claim_plan_text.strip():
citation_rule += (
"\nEVIDENCE-LIMITED WRITING RULE: no directly validated source is assigned to this unit. "
"Continue the scholarly argument using the documented study evidence, cautious conceptual reasoning, and general scholarly knowledge. "
"Do not invent citations, authors, titles, DOI values, or bibliographic metadata; do not describe general model knowledge as if it were retrieved or directly validated literature. "
"Use appropriately qualified language for propositions, interpretations, and research opportunities."
)
if mode == "asq" and section_id not in {"findings", "appendix"}:
citation_rule += (
"\nASQ EVIDENCE-CHANNEL RULE: A sentence whose factual subject is this study's setting, participants, interviews, data sources, cases, actors, observations, events, quotations, analytic steps, or empirical findings must not carry an external [CITE-XX] tag. "
"Place any prior-literature or methodological-rationale claim in a separate sentence and cite only that external claim."
)
if section_id not in {"results", "findings", "appendix"}:
citation_rule += (
"\nCURRENT-STUDY PROVENANCE RULE: Do not attach external literature citations to this study's own research question, H1-Hn statement, sample/design fact, analytic choice, result, limitation specific to the available study evidence, or author positioning. "
"If prior literature provides a rationale, state that external rationale in a separate sentence and cite only that sentence."
)
if section_id == "discussion":
citation_rule += (
"\nDISCUSSION EVIDENCE-LIMITED CONTRACT: Distinguish prior-literature claims from study-owned interpretation. "
"Statements such as 'the present evidence suggests', 'these findings indicate', or 'the present results suggest' are interpretations of this study and must not borrow an external citation. "
"Do not create or broaden an external-literature claim merely to consume a citation or satisfy a citation count. "
"When claim-specific literature is sparse, develop the Discussion from documented study evidence, cautious conceptual reasoning, boundary conditions, and limitations. "
"If comparing with prior literature, put the directly supported prior-literature proposition in its own cited sentence and the study-owned interpretation in a separate uncited sentence."
)
if section_id not in {"results", "findings", "appendix"}:
citation_rule += (
"\nATOMIC LITERATURE CLAIM CONTRACT: Write one externally verifiable proposition per sentence. "
"Do not combine two literature-dependent findings, mechanisms, definitions, or outcome claims under one citation. "
"When two propositions require different sources, write two sentences and attach each sentence only to its own assigned source. "
"A current-study fact or H1-Hn/RQ statement must remain citation-free unless a separate prior-literature rationale is stated in its own sentence."
)
if bool(state.get("evidence_risk_detected")) and section_id not in {"results", "findings", "appendix"}:
citation_rule += (
"\nEVIDENCE-LIMITED CLAIM-DENSITY CONTRACT: Direct high-relevance literature coverage is limited. Treat assigned claim-evidence items as the ceiling for source-dependent empirical, definitional, prevalence, and field-gap claims in this unit. "
"Do not invent additional literature claims merely to make the section sound comprehensive. Prefer concise study positioning, cautious conceptual reasoning, and clearly qualified interpretations when no directly validated source is assigned."
)
memory_context = self._strip_unscoped_citation_identities(
agent._build_memory_context(sec_title, 2500)
)
memory_block = f"\n\n{memory_context}\n\n" if memory_context else ""
snippets = self._section_citation_capability_context(agent, planned_items)
if snippets:
memory_block += f"\n\n{snippets}\n\n"
heading_directive = "Do not output the section heading; it will be inserted by the system."
if subsection:
heading_directive += (
f" Do not output the subsection heading '{subsection}'; it will also be inserted by the system. "
"Use level-3 headings only when analytically necessary."
)
elif sec_title.lower() in {"introduction", "conclusion"}:
heading_directive += " Do not use additional headings."
else:
heading_directive += (
" Use level-2 or level-3 headings only when they clarify the argument; "
"never use level-4 headings or numbered headings."
)
if mode == "amj" and section_id == "results" and data_metadata.get("quantitative_table_manifest"):
table_rule = (
"AUTHORITATIVE TABLE CONTRACT: Do not create, reproduce, renumber, or alter any Markdown statistical table. "
"Required statistical tables are handled separately from narrative prose. "
f"You may discuss and refer only to the frozen table numbers {[int(row.get('number') or 0) for row in data_metadata.get('quantitative_table_manifest', {}).get('tables', []) if int(row.get('number') or 0) > 0]}; do not invent an additional table or change a table number."
)
elif sec["is_method_or_result"] and mode == "amj":
table_rule = "Present only study-supported statistical evidence; do not invent tables or numerical results."
else:
table_rule = ""
concepts = ", ".join(sec.get("concepts") or [])
theme_evidence = [item for item in (sec.get("theme_evidence") or []) if isinstance(item, dict)]
evidence_profiles = []
for item in theme_evidence:
stage = str(item.get("stage") or "Not specified").strip()
role = str(item.get("evidence_role") or "typical").strip()
case_label = str(item.get("case_label") or "Not specified").strip()
evidence_profiles.append(f"stage={stage}; case={case_label}; role={role}")
planned_quote_count = sum(
1 for record in quote_invocation_plan.values()
if str(record.get("render_task_key") or "") == self._qualitative_task_key(sec)
)
qualitative_rule = (
f"Assigned qualitative concepts: {concepts}. Assigned evidence profiles: {' | '.join(evidence_profiles)}. "
f"This unit has {planned_quote_count} assigned quotation slot(s). "
"Use the assigned evidence analytically, but output narrative prose only: do not reproduce any verbatim quotation and do not output any QUOTE-XX identifier. "
"Preserve stage/case/deviant meaning and use deviant evidence to qualify rather than erase the pattern."
if concepts or evidence_profiles else ""
)
scoped_data_md = user_data_md
if mode == "asq":
if section_id == "appendix":
# The appendix is the only non-owning unit allowed to see
# the full frozen evidence table. It is excluded from the
# body duplicate-quote gate by design.
scoped_data_md = user_data_md
elif section_id == "findings":
current_theme = str(sec.get("theme_title") or "").strip()
scoped_data_md = qual_engine.build_scoped_evidence_md(
data_metadata,
include_themes=True,
theme_titles=[current_theme] if current_theme else None,
assigned_evidence=theme_evidence,
include_quotes=bool(theme_evidence),
include_quote_ids=False,
)
elif section_id == "discussion":
scoped_data_md = qual_engine.build_scoped_evidence_md(
data_metadata,
include_themes=True,
)
else:
scoped_data_md = qual_engine.build_scoped_evidence_md(
data_metadata,
include_themes=False,
)
writer_base_evidence_contract = self._writer_visible_evidence_context(base_evidence_contract)
writer_scoped_data_md = self._writer_visible_evidence_context(scoped_data_md)
evidence_packet = (
f"{writer_base_evidence_contract}\n\n{writer_scoped_data_md}\n"
if writer_scoped_data_md else writer_base_evidence_contract
)
completed_label = ", ".join(completed_sections)
anti_repeat = (
f"Already completed: {completed_label}. Do not restate their background, gap, or contribution."
if completed_label else ""
)
if re.search(r"\(Part\s+2/2\)", display_name, flags=re.I):
anti_repeat += " Part 2/2 must extend and complement Part 1/2; do not recap its background, mechanism explanation, evidence, or contribution framing."
voice = str(APP_CONFIG.get("journal_voice_profiles", {}).get(mode) or "")
section_role = self._writer_visible_evidence_context(section_role)
mode_directive = self._writer_visible_evidence_context(mode_directive)
research_logic_for_writer = self._writer_visible_evidence_context(research_logic_contract)
working_abstract_for_writer = self._writer_visible_evidence_context(working_abstract)
sys_prompt = (
APP_CONFIG.get("sys_academic_writer", "") + "\n"
+ evidence_packet + "\n" + voice + "\n"
+ APP_CONFIG.get("deep_refinement_directive", "")
)
sys_prompt = self._writer_visible_evidence_context(sys_prompt)
word_budget = APP_CONFIG.get("word_budget", {})
min_ratio = float(word_budget.get("section_min_ratio", 0.92))
max_ratio = float(word_budget.get("section_max_ratio", 1.08))
prompt_context_tail = context_tail[-2500:]
if mode == "asq":
prompt_context_tail = self._redact_qualitative_quotes(prompt_context_tail, data_metadata)
if bool(state.get("evidence_risk_detected")):
writer_synthesis_context = (
"EVIDENCE-LIMITED SYNTHESIS BOUNDARY: direct claim-specific literature coverage is limited. "
"The claim-evidence plan is the only source-dependent manuscript authority for externally verifiable literature statements in this unit. "
"Do not convert broad synthesis themes, general model knowledge, or unassigned references into manuscript claims. "
"When no supported external claim is assigned, prefer concise study positioning, transparent limitations, and cautious conceptual reasoning without invented citation support."
)
else:
# Literature synthesis is planning context, not a citation capability.
# Raw identities are stripped; only the frozen Section plan below may
# expose a usable [CITE-XX] token to the writer.
writer_synthesis_context = self._strip_unscoped_citation_identities(synthesis_result)
user_prompt = f"""Write the assigned English manuscript unit.
Section: {sec_title}
Subsection: {subsection or 'None'}
Section reasoning role: {section_role}
Additional mode directive: {mode_directive}
Planning length: approximately {int(adjusted_target * min_ratio)}-{int(adjusted_target * max_ratio)} words. This is guidance, not a content quota. Prioritize Conciseness, Accuracy, and Rigor; never add filler or remove useful reasoning solely to hit the range.
{heading_directive}
{citation_rule}
{table_rule}
AUTHOR-FACING PURITY: Write only reader-facing scholarly prose. Do not mention internal orchestration, validation, state-management, evidence-management, or execution terminology. Describe the study, method, evidence, and limitations directly.
EVIDENCE-OWNERSHIP CONTRACT: Statements about this study's sample, measures, available or missing documentation, procedures, statistics, quotations, or findings are owned by the study evidence and must not borrow an external citation. General methodological principles may use only a directly supporting METHOD_AUTHORITY source. Author interpretation or limitation reasoning must not be presented as if an external source established this study-specific conclusion. If one sentence would combine a study-owned fact with an external methodological principle or author inference, split it into separate claim-specific sentences and attach the citation only to the directly supported external principle.
{("CAUSAL-STRENGTH CONTRACT: This cross-sectional observational design supports association/covariation language only. Do not state that one construct causes, leads to, drives, determines, produces, or causally mediates another. Explicit limitation language is valid and preferred when relevant, including 'restricted to associations, not causal effects', 'cannot establish causality', and 'the indirect association is not causal mediation identification'." if mode == "amj" else "")}
{self._hypothesis_contract_directive(hypothesis_ledger)}
{str(sec.get('required_content') or '')}
{qualitative_rule}
{anti_repeat}
{self._canonical_concept_writer_directive(canonical_concept_ledger)}
{research_logic_for_writer}
{working_abstract_for_writer}
ANCHOR PRECEDENCE: The STUDY_EVIDENCE and PROVISIONAL_RESEARCH_LOGIC override the Working Abstract if any tension appears. The Working Abstract is continuity guidance only and must never force a result.
{memory_block}
Literature synthesis: {writer_synthesis_context}
{writer_scoped_data_md}
Previous tail: {prompt_context_tail}
Output only final academic prose. Do not report the word count."""
if agent._count_tokens(sys_prompt) + agent._count_tokens(user_prompt) > 60000:
user_prompt = agent._truncate_context(user_prompt, 60000 - agent._count_tokens(sys_prompt))
current_index = len(completed_sections) + 1
pct = min(88, 15 + int((current_index / max(1, total_steps)) * 72))
self._progress(pct, f"writing:{current_index}:{total_steps}")
prefix = ""
if sec.get("section_first"):
prefix += f"# {sec_title}\n\n"
if sec.get("subsection_first") and subsection:
prefix += f"## {subsection}\n\n"
prior_retry_exhausted = state.get("section_retry_exhausted")
if (
isinstance(prior_retry_exhausted, dict)
and str(prior_retry_exhausted.get("section_name") or "") == display_name
):
# A2: content exhaustion is resumable. The checkpoint records the
# diagnostics, but it no longer creates a terminal lifecycle state.
logging.warning("Adaptive completion resumed exhausted section | unit=%s", display_name)
state.pop("section_retry_exhausted", None)
deterministic_appendix = mode == "asq" and section_id == "appendix"
# R4: one initial attempt plus bounded lineage-owned repair turns.
# Four is a hard safety cap, not a blanket four-attempt entitlement.
max_section_attempts = 1 if deterministic_appendix else 4
lineage_attempts = {}
# R6-R10 monotonic retry state. A defect that was repaired and absent
# on an intervening attempt may receive one regression-correction turn
# if a later sibling repair reintroduces it. The Section hard cap is
# unchanged, so this cannot create an unbounded retry loop.
seen_lineages = set()
cleared_lineages = set()
regression_attempts = {}
historical_retry_rows = {}
frozen_citation_state = self._snapshot_section_citation_state(
agent, citation_plan, display_name
)
attempt_feedback = ""
retry_excluded_tags = set()
en_resolved = ""
survived = set()
for section_attempt in range(1, max_section_attempts + 1):
# Every attempt starts from exactly the same evidence/citation packet.
# Rejected attempts may not leak used tags, JIT references, or audit
# decisions into the retry or any later Section.
if section_attempt > 1:
self._restore_section_citation_state(
agent, citation_plan, display_name, frozen_citation_state
)
planned_items = [
item for item in copy.deepcopy(list(citation_plan.get(display_name, []) or []))
if str(item.get("tag") or "") not in retry_excluded_tags
and agent.citation_source_role_eligible(
str(item.get("tag") or ""), section_id=section_id, section_name=display_name
)
]
citation_plan[display_name] = planned_items
attempt_assigned_tag_list = [
str(item.get("tag") or "") for item in planned_items if item.get("tag")
]
attempt_diagnostics = []
candidate_user_prompt = user_prompt
if retry_excluded_tags:
retry_claim_plan = agent.format_claim_evidence_plan(planned_items)
candidate_user_prompt = re.sub(
r".*?",
f"\n{retry_claim_plan}\n",
candidate_user_prompt, flags=re.S,
)
retry_capability_context = self._section_citation_capability_context(agent, planned_items)
candidate_user_prompt = re.sub(
r"\n.*?\n",
(f"\n\n{retry_capability_context}\n\n"
if retry_capability_context else "\n"),
candidate_user_prompt, flags=re.S,
)
if attempt_feedback:
excluded_text = ", ".join(sorted(retry_excluded_tags, key=agent._citation_tag_number)) or "None"
candidate_user_prompt += f"""
# Internal retry invariant: the same frozen evidence universe remains authoritative; this control phrase is not exposed to manuscript prose.
The previous attempt for this same manuscript unit was rejected by validation. Rewrite ONLY this same unit once. Keep the identical study evidence, research logic, and section role. Do not fetch, invent, or broaden evidence. Citation tags rejected for this claim context are ineligible on the retry and must not be reused or deterministically reinserted: {excluded_text}. Use another already-authorized eligible source only when it directly supports the rewritten claim, or narrow/qualify the claim so no unsupported citation is needed. Resolve every material issue below without changing the task or manuscript scope:
{attempt_feedback}
"""
if agent._count_tokens(sys_prompt) + agent._count_tokens(candidate_user_prompt) > 60000:
candidate_user_prompt = agent._truncate_context(
candidate_user_prompt, 60000 - agent._count_tokens(sys_prompt)
)
if deterministic_appendix:
# R6A-R4: Appendix is a deterministic reader-facing projection
# of the Evidence Ledger. It is never regenerated by the LLM.
en_raw = qual_engine.build_appendix_evidence_md(data_metadata)
else:
en_raw = self._stream_text(
agent.stream_deepseek(
sys_prompt, candidate_user_prompt, max_tokens=7000,
model_role="writing", thinking=False, allow_fallback=False,
logical_node=section_logical_node,
),
preview_prefix=full_draft_en + prefix,
preview_stage=(display_name if section_attempt == 1 else f"{display_name}:retry"),
)
en_raw = re.sub(r".*?", "", en_raw, flags=re.S).strip()
en_raw = re.sub(rf"(?im)^#\s*{re.escape(sec_title)}\s*\n+", "", en_raw).strip()
if subsection:
en_raw = re.sub(rf"(?im)^##\s*{re.escape(subsection)}\s*\n+", "", en_raw).strip()
en_raw = re.sub(r"(?m)^(#{2,3})\s+[\d\.\s一二三四五六七八九十、]+\s*(.*)", r"\1 \2", en_raw)
en_raw = re.sub(r"(?m)^#{4,}\s+(.*)", r"**\1**", en_raw)
en_raw = re.sub(r"(?m)^#\s+(.+)$", r"## \1", en_raw)
if mode == "amj" and hypothesis_ledger:
normalized_hypothesis_text = self._normalize_hypothesis_ordinal_aliases(
en_raw, hypothesis_ledger
)
if normalized_hypothesis_text != en_raw:
logging.info(
"Deterministic hypothesis ordinal alias normalization | unit=%s",
display_name,
)
en_raw = normalized_hypothesis_text
if mode == "asq":
# Model prose has no authority over quote presence/count.
en_raw = self._strip_model_controlled_qualitative_quotes(en_raw, data_metadata)
# C5.5 uses the same private-vocabulary registry for both
# Writer-facing evidence and reader-facing output. This
# deterministic phrasing cleanup cannot change study facts,
# citations, hypotheses, numbers, or quote ownership.
en_raw = self._writer_visible_evidence_context(en_raw)
# S0-R1-C4 phase-1 barrier: all writer-owned semantic validators
# inspect the same attempt before the single same-Section retry budget
# can be consumed. Citation semantics therefore run before the one
# unified validation decision instead of being short-circuited by a
# validator that happens to execute earlier.
# S0-R1-C2/C4: citation semantics are part of the same phase-1
# decision. The authoritative path never attaches a citation merely
# to satisfy section-level coverage; it audits the writer-owned
# claim/evidence binding exactly as written. Any validated coverage
# shortfall still flows to the existing claim-anchored recovery path.
# This rule is shared by ASQ and AMJ and keeps sealed English
# authoritative for ZH/RU/KK translation.
self._progress(pct, f"citation_audit:{current_index}:{total_steps}")
used_before = set(agent.used_citations)
allowed_section_citation_tags = (
set(attempt_assigned_tag_list) - set(retry_excluded_tags)
if not is_quarantined else set()
)
attempt_section_text = prefix + en_raw
en_resolved = agent.audit_and_resolve_citations(
attempt_section_text, section_name=display_name, section_id=section_id,
quarantine=is_quarantined, allowed_tags=allowed_section_citation_tags
)
# Capture immediately: later recovery may replace the agent's last-
# violation buffer. Only citation-owned rejected tags may enter the
# retry exclusion set.
citation_semantic_rows = copy.deepcopy(
list(getattr(agent, "_last_citation_semantic_violations", []) or [])
)
phase1_validation_rows = self._section_validation_bundle(
mode, agent, quant_engine, attempt_section_text, data_metadata,
section_id=section_id, section_name=display_name,
evidence_contract=evidence_packet,
include_critic=not deterministic_appendix,
citation_violations=citation_semantic_rows, hypothesis_ledger=hypothesis_ledger,
)
if phase1_validation_rows:
retry_excluded_tags.update(self._citation_violation_tags(citation_semantic_rows))
hard_cap_reached = section_attempt >= max_section_attempts
# Severity is evaluated as exhausted only when this defect
# lineage has already consumed its own repair entitlement.
preview_blocking, preview_advisory = self._partition_section_violations(
phase1_validation_rows, section_name=display_name, mode=mode, exhausted=False
)
current_lineages = {
self._section_violation_lineage(row) for row in preview_blocking
}
cleared_lineages.update(seen_lineages - current_lineages)
regressed_lineages = self._section_regressed_lineages(
current_lineages, cleared_lineages, regression_attempts
)
seen_lineages.update(current_lineages)
may_retry, exhausted_lineages, fresh_lineages = self._section_retry_decision(
preview_blocking, lineage_attempts, hard_cap_reached=hard_cap_reached
)
retry_lineages = set(fresh_lineages)
if regressed_lineages and not hard_cap_reached:
retry_lineages.update(regressed_lineages)
may_retry = True
if may_retry:
retry_rows = [
row for row in preview_blocking
if self._section_violation_lineage(row) in retry_lineages
]
for lineage in fresh_lineages:
lineage_attempts[lineage] = int(lineage_attempts.get(lineage, 0)) + 1
for lineage in regressed_lineages:
regression_attempts[lineage] = int(regression_attempts.get(lineage, 0)) + 1
for row in preview_blocking:
historical_retry_rows[self._section_violation_lineage(row)] = dict(row)
# Every later repair is constrained by all earlier resolved
# contracts, not only the newly visible lineage. This makes
# A+B -> C -> A+B repair interference explicitly non-monotonic
# and gives it one bounded correction turn.
attempt_feedback = self._section_retry_feedback(retry_rows)
historical_feedback = self._section_retry_feedback(
list(historical_retry_rows.values())
)
if historical_feedback:
attempt_feedback += (
"\nNONREGRESSION CONSTRAINTS — previously repaired lineages must remain resolved:\n"
+ historical_feedback
)
logging.warning(
"Unified Phase-1 Section lineage retry | unit=%s attempt=%s lineages=%s exhausted_siblings=%s regressed_lineages=%s count=%s types=%s",
display_name, section_attempt, sorted(retry_lineages), sorted(exhausted_lineages),
sorted(regressed_lineages), len(retry_rows),
sorted({str(row.get("type") or "") for row in retry_rows if isinstance(row, dict)}),
)
self._restore_section_citation_state(
agent, citation_plan, display_name, frozen_citation_state
)
continue
blocking_rows, advisory_rows = self._partition_section_violations_by_lineage(
phase1_validation_rows, lineage_attempts, section_name=display_name, mode=mode,
hard_cap_reached=hard_cap_reached
)
if advisory_rows:
self._record_section_advisories(
quality_notes, advisory_rows, section_name=display_name
)
logging.warning(
"Nonfatal section quality advisory | unit=%s count=%s types=%s",
display_name, len(advisory_rows),
sorted({str(row.get("type") or "") for row in advisory_rows if isinstance(row, dict)}),
)
if blocking_rows:
# A2: phase-1 exhaustion escalates to a provider-neutral
# semantic repair rather than terminating the manuscript.
adaptive_raw = agent.adaptive_repair_academic_unit(
attempt_section_text, violations=blocking_rows, section_id=section_id,
section_name=display_name, mode=mode, study_evidence=writer_scoped_data_md,
evidence_contract=evidence_packet, repair_scope="section",
)
adaptive_resolved = agent.audit_and_resolve_citations(
adaptive_raw, section_name=display_name, section_id=section_id,
quarantine=is_quarantined, allowed_tags=allowed_section_citation_tags,
)
adaptive_citation_rows = copy.deepcopy(
list(getattr(agent, "_last_citation_semantic_violations", []) or [])
)
adaptive_rows = self._section_validation_bundle(
mode, agent, quant_engine, adaptive_raw, data_metadata,
section_id=section_id, section_name=display_name,
evidence_contract=evidence_packet, include_critic=False,
citation_violations=adaptive_citation_rows, hypothesis_ledger=hypothesis_ledger,
)
adaptive_blocking, adaptive_advisory = self._partition_section_violations(
adaptive_rows, section_name=display_name, mode=mode, exhausted=True
)
if adaptive_advisory:
self._record_section_advisories(quality_notes, adaptive_advisory, section_name=display_name)
if adaptive_blocking:
# P3-A4: citation identity/scope authority is a hard
# pre-seal invariant. Adaptive completion may retain
# reader-facing advisories for unrelated quality
# defects, but it may never seal a Section that still
# contains an unauthorized/ambiguous citation identity
# or a citation outside the frozen unit whitelist.
persistent_citation_authority = self._persistent_citation_authority_rows(
adaptive_blocking
)
if persistent_citation_authority:
self._checkpoint_section_retry_exhausted(
state, section_id=section_id, section_name=display_name,
violations=persistent_citation_authority, pct=pct, attempts=section_attempt,
)
raise RuntimeError(
"citation_authority_repair_exhausted:"
+ str(display_name)
+ ":"
+ ",".join(sorted({
str((row or {}).get("type") or "citation_authority")
for row in persistent_citation_authority if isinstance(row, dict)
}))
)
self._record_adaptive_completion_residual(
quality_notes, adaptive_blocking, section_name=display_name, stage="phase1_semantic_repair"
)
en_resolved = adaptive_resolved
state.pop("section_retry_exhausted", None)
used_after = set(agent.used_citations)
survived = used_after - used_before
if (
attempt_assigned_tag_list
and not is_quarantined
and self._quota_driven_citation_recovery_allowed(section_id)
):
required_survival = min(
len(attempt_assigned_tag_list),
max(1, int(math.ceil(len(attempt_assigned_tag_list) * CITATION_SEMANTIC_SURVIVAL_RATIO))),
)
if len(survived) < required_survival:
missing_items = [
item for item in planned_items
if str(item.get("tag") or "") not in used_after
]
needed = max(0, required_survival - len(survived))
recovery_items = agent.select_contextual_citation_recovery_items(
missing_items, section_name=display_name, section_id=section_id,
section_context=en_resolved, needed=needed,
excluded_tags=used_after | retry_excluded_tags,
allowed_tags=set(attempt_assigned_tag_list),
)
# Evidence retrieval is permitted only on the first writer
# attempt. Attempt 2 is bound to the original frozen packet.
if section_attempt == 1 and len(recovery_items) < needed:
recovery_items.extend(
agent.fetch_contextual_jit_recovery_items(
topic=topic, section_name=display_name, section_id=section_id,
section_context=en_resolved, needed=needed - len(recovery_items),
excluded_tags=(
used_after | retry_excluded_tags
| {str(item.get("tag") or "") for item in recovery_items}
),
)
)
planned_tags_for_unit = {str(item.get("tag") or "") for item in planned_items}
for recovery_item in recovery_items[:needed]:
tag = str(recovery_item.get("tag") or "")
if tag and tag not in planned_tags_for_unit:
planned_items.append(recovery_item)
planned_tags_for_unit.add(tag)
citation_plan[display_name] = planned_items
# Recovery may attach evidence only to an already-written
# claim. It never creates supplemental prose.
self._progress(pct, f"citation_recovery:{current_index}:{total_steps}")
recovered = agent.recover_citations_on_existing_claims(
en_resolved,
recovery_items[:needed],
section_name=display_name, section_id=section_id,
needed=needed,
)
if recovered != en_resolved:
en_resolved = recovered
used_after = set(agent.used_citations)
survived = used_after - used_before
if mode == "asq" and section_id == "findings":
en_resolved = self._render_planned_qualitative_quotes(
en_resolved,
task=sec,
data_metadata=data_metadata,
quote_invocation_plan=quote_invocation_plan,
)
if mode == "amj" and data_metadata.get("quantitative_table_manifest"):
if section_id == "results":
en_resolved = self._apply_quantitative_table_contract(
en_resolved, data_metadata, display_name
)
else:
# System-generated AMJ tables have one authoritative
# manifest. Writer-created tables anywhere else are not
# permitted to create a fifth/competing artifact table.
en_resolved = self._strip_model_markdown_tables(en_resolved)
# S0-R1 phase 2: only local/idempotent structural normalizers may
# mutate the candidate before the final pure validation bundle.
en_resolved, enumeration_rows = agent.repair_and_validate_enumeration_contract(
en_resolved, section_name=display_name
)
if enumeration_rows:
logging.warning(
"Structural enumeration repaired | unit=%s count=%s details=%s",
display_name, len(enumeration_rows), enumeration_rows[:3],
)
attempt_diagnostics.append({
"scope":"structural_enumeration", "name":display_name,
"status":"deterministic_repair", "count":len(enumeration_rows),
"examples":enumeration_rows[:3],
})
en_resolved, duplicate_rows = self._remove_severe_cross_unit_duplicates(
en_resolved, full_draft_en
)
if duplicate_rows:
logging.warning(
"Exact cross-unit redundancy removed | unit=%s count=%s",
display_name, len(duplicate_rows),
)
attempt_diagnostics.append({
"scope":"manuscript_redundancy", "name":display_name,
"status":"exact_duplicate_removed", "count":len(duplicate_rows),
"examples":duplicate_rows[:3],
})
en_resolved, sentence_echo_rows = agent.remove_severe_cross_unit_sentence_echoes(
en_resolved, full_draft_en
)
if sentence_echo_rows:
logging.warning(
"Exact cross-unit sentence echo removed | unit=%s count=%s",
display_name, len(sentence_echo_rows),
)
attempt_diagnostics.append({
"scope":"manuscript_sentence_echo", "name":display_name,
"status":"exact_duplicate_removed", "count":len(sentence_echo_rows),
"examples":sentence_echo_rows[:3],
})
# Final Section validation is pure. Once it passes, the exact bytes
# are eligible for the Section Integrity Seal and may never be
# semantically rewritten by a later validator.
post_validation_rows = self._section_validation_bundle(
mode, agent, quant_engine, en_resolved, data_metadata,
section_id=section_id, section_name=display_name,
evidence_contract=evidence_packet, include_critic=False,
hypothesis_ledger=hypothesis_ledger,
)
if post_validation_rows:
hard_cap_reached = section_attempt >= max_section_attempts
preview_blocking, preview_advisory = self._partition_section_violations(
post_validation_rows, section_name=display_name, mode=mode, exhausted=False
)
may_retry, exhausted_lineages, fresh_lineages = self._section_retry_decision(
preview_blocking, lineage_attempts, hard_cap_reached=hard_cap_reached
)
if may_retry:
retry_rows = [
row for row in preview_blocking
if self._section_violation_lineage(row) in fresh_lineages
]
for lineage in fresh_lineages:
lineage_attempts[lineage] = int(lineage_attempts.get(lineage, 0)) + 1
attempt_feedback = self._section_retry_feedback(retry_rows)
logging.warning(
"Post-normalization Section lineage retry | unit=%s attempt=%s lineages=%s exhausted_siblings=%s count=%s types=%s",
display_name, section_attempt, sorted(fresh_lineages), sorted(exhausted_lineages), len(retry_rows),
sorted({str(row.get("type") or "") for row in retry_rows if isinstance(row, dict)}),
)
self._restore_section_citation_state(
agent, citation_plan, display_name, frozen_citation_state
)
continue
blocking_rows, advisory_rows = self._partition_section_violations_by_lineage(
post_validation_rows, lineage_attempts, section_name=display_name, mode=mode,
hard_cap_reached=hard_cap_reached
)
if advisory_rows:
self._record_section_advisories(
quality_notes, advisory_rows, section_name=display_name
)
logging.warning(
"Nonfatal post-normalization quality advisory | unit=%s count=%s types=%s",
display_name, len(advisory_rows),
sorted({str(row.get("type") or "") for row in advisory_rows if isinstance(row, dict)}),
)
if blocking_rows:
en_resolved, residual_rows = self._adaptive_repair_section_after_exhaustion(
agent=agent, quant_engine=quant_engine, text=en_resolved,
violations=blocking_rows, mode=mode, section_id=section_id,
section_name=display_name, data_metadata=data_metadata,
study_evidence=writer_scoped_data_md, evidence_contract=evidence_packet,
hypothesis_ledger=hypothesis_ledger, task=sec,
quote_invocation_plan=quote_invocation_plan, quality_notes=quality_notes,
allowed_citation_tags=allowed_section_citation_tags,
quarantine=is_quarantined,
)
persistent_citation_authority = self._persistent_citation_authority_rows(
residual_rows
)
if persistent_citation_authority:
self._checkpoint_section_retry_exhausted(
state, section_id=section_id, section_name=display_name,
violations=persistent_citation_authority, pct=pct, attempts=section_attempt,
)
raise RuntimeError(
"citation_authority_repair_exhausted:"
+ str(display_name)
+ ":"
+ ",".join(sorted({
str((row or {}).get("type") or "citation_authority")
for row in persistent_citation_authority if isinstance(row, dict)
}))
)
logging.warning(
"Adaptive completion accepted section | unit=%s residual=%s",
display_name, len(residual_rows),
)
state.pop("section_retry_exhausted", None)
quality_diagnostics.extend(attempt_diagnostics)
assigned_tag_list = list(attempt_assigned_tag_list)
state.pop("section_retry_exhausted", None)
break
# R7-R1: heading structure is orchestrator-owned, not model-owned.
# Any adaptive repair may replace the accepted prose bytes and drop
# or duplicate the system-inserted H1/H2. Restore only those exact
# structural anchors after the final model-owned repair and before
# the unit is appended to the immutable English source / Section Seal.
normalized_en_resolved = self._normalize_orchestrator_owned_headings(en_resolved, sec)
if normalized_en_resolved != en_resolved:
logging.info("Deterministic English heading normalization | unit=%s", display_name)
quality_diagnostics.append({
"scope": "english_structural_ownership",
"name": display_name,
"status": "deterministic_heading_normalization",
})
en_resolved = normalized_en_resolved
# Recompute citation survival/claim demand from the exact text that
# will be sealed, not from an earlier pre-cleanup intermediate.
if not is_quarantined and section_id in self._citation_eligible_section_ids(mode):
final_survived = set()
for tag in set(survived or set()):
marker = str((agent.citation_map.get(tag) or {}).get("in_text") or "").strip()
if marker and marker in en_resolved:
final_survived.add(tag)
survived = final_survived
citation_required_claim_counts_by_unit[display_name] = agent.count_external_literature_claims(
en_resolved, section_name=display_name
)
if assigned_tag_list:
citation_validated_tags_by_unit[display_name] = sorted(
survived, key=agent._citation_tag_number
)
logging.info(
"Top-level citation accumulation: unit=%s section=%s validated=%s external_claims=%s",
display_name, citation_unit_section_map.get(display_name, section_id),
len(survived), citation_required_claim_counts_by_unit[display_name],
)
full_draft_en += en_resolved + "\n\n"
if assigned_tag_list:
logging.info(
"Citation-first survival: section=%s planned=%s validated_new=%s",
display_name, len(assigned_tag_list), len(survived),
)
self._publish_preview(full_draft_en, display_name, force=True)
# Only sealed/reconciled prose may feed subsequent manuscript memory.
agent._extract_memory_from_text(en_resolved, sec_title)
context_tail = " ".join(en_resolved.split()[-200:])
actual_words = self._english_count_for_target(en_resolved)
if not bool(sec.get("exclude_from_word_budget")):
accumulated_deviation += actual_words - adjusted_target
self._upsert_quality_diagnostic(
quality_diagnostics, self._section_word_diagnostic(display_name, actual_words, adjusted_target)
)
completed_sections.append(display_name)
section_key = str(sec.get('section_id') or sec_title)
if self._section_is_complete(adaptive_structure, completed_sections, section_key):
frozen_section = self._extract_top_level_section(full_draft_en, sec_title)
if not frozen_section:
raise RuntimeError(f"english_section_seal_source_missing:{sec_title}")
# C5: one bounded publication-quality evidence opportunity is
# allowed before the English top-level Section is sealed and
# before translation begins. It never writes new prose and may
# attach only citations that survive the same semantic auditor.
if section_key in citation_top_level_minimums:
top_units = [
unit for unit, owner in citation_unit_section_map.items()
if str(owner) == str(section_key)
]
top_validated = set()
top_claims = 0
for unit in top_units:
top_validated.update(citation_validated_tags_by_unit.get(unit) or [])
top_claims += int(citation_required_claim_counts_by_unit.get(unit, 0) or 0)
top_required = min(
int(citation_top_level_minimums.get(section_key, 0) or 0),
max(0, top_claims),
) if top_claims else 0
if len(top_validated) < top_required:
completed_section, added_tags = self._bounded_top_level_evidence_completion(
agent, topic=topic, section_id=section_key, section_name=sec_title,
section_text=frozen_section, required=top_required,
existing_tags=top_validated, excluded_tags=set(agent.used_citations),
)
if added_tags and completed_section != frozen_section:
full_draft_en = self._replace_top_level_section(
full_draft_en, sec_title, completed_section
)
frozen_section = completed_section
merged_tags = set(citation_validated_tags_by_unit.get(display_name) or [])
merged_tags.update(added_tags)
citation_validated_tags_by_unit[display_name] = sorted(
merged_tags, key=agent._citation_tag_number
)
logging.info(
"Top-level evidence completion sealed | section=%s added=%s",
sec_title, len(added_tags),
)
section_integrity_seals[section_key] = self._build_section_integrity_seal(
section_key, sec_title, frozen_section
)
translation_source = frozen_section
if (
mode == "amj"
and str(section_key).strip().casefold() == "results"
and data_metadata.get("quantitative_table_manifest")
):
# Table values are manifest-owned and must never be handed to the
# translation model as mutable prose. The target tables are rebound
# deterministically after Results translation QA.
translation_source = quant_engine._strip_target_quantitative_markdown_tables(
frozen_section
)
if mode == "asq" and str(section_key).strip().casefold() == "appendix":
# R5: the qualitative Appendix is not model-owned Markdown.
# It is translated/bound at the strict final gate from typed cells.
pass
else:
self._enqueue_translation_unit(
translation_agent,
unit_key=section_key, unit_name=sec_title, source_text=translation_source,
translation_sections=translation_sections,
english_commit_text=full_draft_en,
)
# Collect only already-finished target jobs. This call never waits,
# so English Section N+1 can start while target Section N runs.
translation_sections, translation_glossary = self._collect_translation_results(
translation_sections, translation_glossary, block=False
)
partial_translation = self._assemble_partial_translation(translation_sections, adaptive_structure, include_front=True)
state.update(
{
"completed_sections": completed_sections,
"full_draft_en": full_draft_en,
"full_draft_translated": partial_translation,
"translation_sections": translation_sections,
"translation_glossary": translation_glossary,
"context_tail": context_tail,
"accumulated_deviation": accumulated_deviation,
"quality_diagnostics": quality_diagnostics,
"quality_notes": quality_notes,
"citation_plan": citation_plan,
"citation_minimums": citation_minimums,
"citation_reserve_targets": citation_reserve_targets,
"citation_top_level_minimums": citation_top_level_minimums,
"citation_top_level_reserve_targets": citation_top_level_reserve_targets,
"citation_unit_section_map": citation_unit_section_map,
"citation_required_claim_counts_by_unit": citation_required_claim_counts_by_unit,
"citation_validated_tags_by_unit": citation_validated_tags_by_unit,
"section_integrity_seals": section_integrity_seals,
"english_section_seal_version": ENGLISH_SECTION_SEAL_VERSION,
"agent_state": agent.export_state(),
"data_metadata": data_metadata,
}
)
self._checkpoint(
f"section_{len(completed_sections):03d}", state, pct,
display_node=f"writing_complete:{len(completed_sections)}:{total_steps}",
)
if not state.get("english_finalized"):
self._progress(91, "english_validation")
full_draft_en = _clean_trailing_references(full_draft_en, ["Reference", "References", "Reference List"])
section_order_for_fact = self._unique_top_level_sections(adaptive_structure)
if not section_order_for_fact:
raise RuntimeError("final_study_fact_section_plan_missing")
# Final English validation is validation-only. Every mutable repair
# was completed before the top-level section received its integrity
# seal; finalization must never repair the whole manuscript again.
for section_key, section_title in section_order_for_fact:
sealed_section = self._extract_top_level_section(full_draft_en, section_title)
if not sealed_section:
raise RuntimeError(f"final_english_section_missing:{section_title}")
seal = dict(section_integrity_seals.get(section_key) or {})
self._validate_section_integrity_seal(seal, sealed_section, section_title)
final_fact_rows = self._study_fact_violations(
mode, agent, quant_engine, sealed_section, data_metadata, section_name=section_title
)
if final_fact_rows:
self._record_adaptive_completion_residual(
quality_notes, final_fact_rows, section_name=section_title, stage="final_sealed_fact_review"
)
# A2: final validators diagnose residual content; they do not
# destroy an otherwise complete manuscript after adaptive repair.
try:
agent.validate_enumeration_contract(
sealed_section, section_name=f"Final sealed section: {section_title}"
)
except Exception as exc:
self._record_adaptive_completion_residual(
quality_notes, [{"type":"enumeration_contract_invalid","reason":str(exc)[:400]}],
section_name=section_title, stage="final_enumeration_review"
)
internal_rows = self._author_facing_internal_workflow_violations(sealed_section)
if internal_rows:
self._record_adaptive_completion_residual(
quality_notes, internal_rows, section_name=section_title, stage="final_reader_facing_review"
)
if mode == "amj" and hypothesis_ledger:
hypothesis_rows = self._hypothesis_contract_violations(
full_draft_en, hypothesis_ledger, section_name="Full Manuscript", require_all=True
)
if hypothesis_rows:
self._record_adaptive_completion_residual(
quality_notes, hypothesis_rows, section_name="Full Manuscript", stage="final_hypothesis_review"
)
# Validate the completed body while the Working Abstract is still in
# place. Final front matter must not be generated until all body
# sections, citation cleanup, and any Discussion rebalance are frozen.
agent.validate_manuscript_structure(full_draft_en, adaptive_structure, translated=False)
if mode == "asq":
try:
self._validate_qualitative_quote_rendering(
full_draft_en, quote_invocation_plan, data_metadata
)
except Exception as exc:
self._record_adaptive_completion_residual(
quality_notes, [{"type":"qualitative_quote_rendering","reason":str(exc)[:500]}],
section_name="Findings", stage="final_quote_review"
)
repeated_quotes = agent.find_repeated_qualitative_quotes(full_draft_en, data_metadata)
if repeated_quotes:
self._record_adaptive_completion_residual(
quality_notes, [{"type":"qualitative_quote_repetition","reason":str(repeated_quotes[:5])}],
section_name="Findings", stage="final_quote_review"
)
# R4-R3: the body is already citation-resolved at this point. Never
# rescan it for raw CITE tags; that destroys a valid citation ledger.
final_citation_ledger = agent.finalize_resolved_citation_ledger(full_draft_en)
references = agent.build_final_reference_lines(final_citation_ledger)
if not references:
self._record_evidence_coverage_event(
state, [("final validated references", 0, 1)],
phase="final_validated_references_empty", preview_status="limited",
)
logging.warning(
"Final validated reference list is empty; completing in evidence-limited mode without unrelated or invented references."
)
validated_by_top_level = {}
claims_by_top_level = {}
for unit_name, section_id in citation_unit_section_map.items():
if section_id not in citation_top_level_minimums:
continue
validated_by_top_level.setdefault(section_id, set()).update(
citation_validated_tags_by_unit.get(unit_name) or []
)
claims_by_top_level[section_id] = claims_by_top_level.get(section_id, 0) + int(
citation_required_claim_counts_by_unit.get(unit_name, 0) or 0
)
method_section_id = 'methods' if mode == 'asq' else 'methodology'
method_validated_tags = set(validated_by_top_level.get(method_section_id) or []) & set(final_citation_ledger)
method_authority_tags = {
tag for tag in method_validated_tags
if 'method_authority' in set((agent.citation_map.get(tag) or {}).get('evidence_lanes') or [])
}
method_external_claims = max(0, int(claims_by_top_level.get(method_section_id, 0) or 0))
final_shortfalls = []
for section_id, planned_minimum in citation_top_level_minimums.items():
validated_tags = set(validated_by_top_level.get(section_id) or [])
external_claims = max(0, int(claims_by_top_level.get(section_id, 0)))
# Evidence authority is need-based, not a word-count quota. The
# top-level planning minimum is therefore capped by the number of
# externally-oriented claims that actually survived drafting.
required = min(int(planned_minimum), external_claims) if external_claims else 0
if len(validated_tags) < required:
final_shortfalls.append((section_id, len(validated_tags), required))
final_reference_count = len(references)
if final_shortfalls or final_reference_count == 0:
if final_shortfalls:
self._record_evidence_coverage_event(
state, final_shortfalls, phase="final_claim_evidence_gate",
preview_status="limited",
)
evidence_status = "limited"
logging.warning(
"Top-level claim-evidence final Gate completed with evidence warning: validated_references=%s sections=%s shortfalls=%s",
final_reference_count, len(citation_top_level_minimums), final_shortfalls,
)
else:
evidence_status = "adequate"
self._preview_evidence_status = "adequate"
self._write_preview_payload(force=True)
logging.info(
"Top-level claim-evidence final Gate passed: validated_references=%s sections=%s",
final_reference_count, len(citation_top_level_minimums),
)
evidence_coverage_summary = self._evidence_coverage_summary(
status=evidence_status,
early_risk=bool(state.get("evidence_risk_detected")),
final_reference_count=final_reference_count,
minimums=citation_top_level_minimums,
validated_by_top_level=validated_by_top_level,
claims_by_top_level=claims_by_top_level,
events=state.get("evidence_coverage_events") or self._evidence_coverage_events,
method_authority_count=len(method_authority_tags),
method_external_claims=method_external_claims,
)
state["evidence_status"] = evidence_status
state["evidence_coverage_summary"] = evidence_coverage_summary
full_draft_en = full_draft_en.rstrip() + "\n\n# References\n" + "\n\n".join(references)
actual_words = self._english_count_for_target(full_draft_en)
self._upsert_quality_diagnostic(
quality_diagnostics, self._validate_target_length(full_draft_en, target_words)
)
body_only, _reference_tail = agent.validate_reference_tail(
full_draft_en, minimum_references=final_reference_count
)
agent.validate_manuscript_structure(body_only, adaptive_structure, translated=False)
agent.enforce_resolved_claim_specific_citations(body_only)
agent.validate_final_citation_bijection(body_only, _reference_tail, final_citation_ledger)
# Body is now frozen. Generate both Final Abstract and Final Keywords
# from the completed manuscript/evidence, then replace the provisional
# front matter deterministically. No body rebalance may occur afterward.
self._progress(92, "final_front_matter")
final_front = agent.generate_final_front_matter(
topic, keywords, body_only, data_evidence=user_data_md, mode=mode,
hypothesis_contract=hypothesis_contract,
authoritative_keywords=self._user_authoritative_front_matter_keywords(data_metadata),
keyword_policy=self._resolved_front_matter_keyword_policy(data_metadata),
fallback_keywords=self._front_matter_keyword_fallback_terms(data_metadata),
)
final_abstract = str(final_front.get("abstract") or "").strip()
if hypothesis_ledger:
final_abstract = self._normalize_hypothesis_ordinal_aliases(
final_abstract, hypothesis_ledger
)
final_keywords = self._canonical_quantitative_front_matter_keywords(
mode, list(final_front.get("keywords") or []), data_metadata
)
if not final_abstract or not final_keywords:
raise RuntimeError("final_front_matter_missing")
final_front_fact_rows = self._study_fact_violations(
mode, agent, quant_engine, final_abstract, data_metadata,
section_name="Final Abstract",
)
if hypothesis_ledger:
final_front_fact_rows.extend(self._hypothesis_contract_violations(
final_abstract, hypothesis_ledger, section_name="Final Abstract"
))
if final_front_fact_rows:
feedback = self._section_retry_feedback(final_front_fact_rows)
logging.warning(
"Final Abstract pure-validation retry | mode=%s count=%s types=%s",
mode, len(final_front_fact_rows),
sorted({str(row.get("type") or "") for row in final_front_fact_rows if isinstance(row, dict)}),
)
final_front = agent.generate_final_front_matter(
topic, keywords, body_only, data_evidence=user_data_md, mode=mode,
validation_feedback=feedback, hypothesis_contract=hypothesis_contract,
authoritative_keywords=self._user_authoritative_front_matter_keywords(data_metadata),
keyword_policy=self._resolved_front_matter_keyword_policy(data_metadata),
fallback_keywords=self._front_matter_keyword_fallback_terms(data_metadata),
)
final_abstract = str(final_front.get("abstract") or "").strip()
if hypothesis_ledger:
final_abstract = self._normalize_hypothesis_ordinal_aliases(
final_abstract, hypothesis_ledger
)
final_keywords = self._canonical_quantitative_front_matter_keywords(
mode, list(final_front.get("keywords") or []), data_metadata
)
if not final_abstract or not final_keywords:
raise RuntimeError("final_front_matter_missing_after_retry")
remaining_final_fact_rows = self._study_fact_violations(
mode, agent, quant_engine, final_abstract, data_metadata,
section_name="Final Abstract",
)
if hypothesis_ledger:
remaining_final_fact_rows.extend(self._hypothesis_contract_violations(
final_abstract, hypothesis_ledger, section_name="Final Abstract"
))
if remaining_final_fact_rows:
final_abstract, residual_final_rows = self._adaptive_repair_front_matter(
agent=agent, quant_engine=quant_engine, abstract_text=final_abstract,
violations=remaining_final_fact_rows, mode=mode, data_metadata=data_metadata,
study_evidence=user_data_md, hypothesis_ledger=hypothesis_ledger,
quality_notes=quality_notes, label="Final Abstract",
)
if residual_final_rows:
logging.warning("Adaptive Final Abstract accepted with residual diagnostics | count=%s", len(residual_final_rows))
quality_diagnostics.append({
"scope":"final_abstract_study_fact_integrity", "name":"Final Abstract",
"status":"front_matter_retry_resolved", "count":len(final_front_fact_rows),
"examples":final_front_fact_rows[:3],
})
state.pop("final_abstract_retry_exhausted", None)
final_title = str(source_topic or topic or "").strip()
if mode == "amj" and str(data_metadata.get("source_type") or "") == "system_generated":
semantic_contract = dict(data_metadata.get("front_matter_contract") or {})
reconciled_front = agent.reconcile_quantitative_reader_front_matter(
topic=source_topic, keywords=source_keywords, contract=semantic_contract,
manuscript=body_only,
)
candidate_title = re.sub(r"\s+", " ", str(reconciled_front.get("title") or "")).strip()
candidate_keywords = self._canonical_quantitative_front_matter_keywords(
mode, list(reconciled_front.get("keywords") or []), data_metadata
)
if candidate_title:
final_title = candidate_title
if candidate_keywords:
# R7-R2C-R6B: the resolved policy owns keyword identity/count
# semantics. No local numeric cardinality branch is permitted.
final_keywords = candidate_keywords
else:
logging.warning(
"Adaptive final keyword reconciliation returned no usable keywords; preserving validated Final Front Matter keywords."
)
full_draft_en = replace_abstract(full_draft_en, final_abstract)
full_draft_en = self._replace_title(full_draft_en, final_title)
full_draft_en = self._replace_keywords(full_draft_en, final_keywords)
if mode == "amj" and hypothesis_ledger:
post_front_hypothesis_rows = self._hypothesis_contract_violations(
full_draft_en, hypothesis_ledger, section_name="Full Manuscript", require_all=True
)
if post_front_hypothesis_rows:
self._record_adaptive_completion_residual(
quality_notes, post_front_hypothesis_rows, section_name="Final Abstract",
stage="post_front_matter_hypothesis_review"
)
body_only, _reference_tail = agent.validate_reference_tail(
full_draft_en, minimum_references=final_reference_count
)
agent.validate_manuscript_structure(body_only, adaptive_structure, translated=False)
agent.enforce_resolved_claim_specific_citations(body_only)
post_front_ledger = agent.finalize_resolved_citation_ledger(body_only)
if set(post_front_ledger) != set(final_citation_ledger):
logging.warning(
"Adaptive finalization resealed citation ledger after Final Front Matter | before=%s after=%s",
len(final_citation_ledger), len(post_front_ledger),
)
# Final Front Matter is allowed to add/remove an already-authoritative
# citation. Rebuild References deterministically from the post-front
# resolved body instead of discarding an otherwise complete manuscript.
# Orphan/ambiguous citations still fail closed inside ledger finalization.
final_citation_ledger = list(post_front_ledger)
references = agent.build_final_reference_lines(final_citation_ledger)
final_reference_count = len(references)
full_draft_en = body_only.rstrip() + "\n\n# References\n" + "\n\n".join(references)
coverage_summary = dict(state.get("evidence_coverage_summary") or {})
if coverage_summary:
coverage_summary["final_reference_count"] = final_reference_count
state["evidence_coverage_summary"] = coverage_summary
body_only, _reference_tail = agent.validate_reference_tail(
full_draft_en, minimum_references=final_reference_count
)
agent.validate_final_citation_bijection(body_only, _reference_tail, final_citation_ledger)
self._upsert_quality_diagnostic(
quality_diagnostics, self._validate_target_length(full_draft_en, target_words)
)
# Final front matter has a different source hash from the Working
# front matter. Retranslate only __front_matter__; unchanged body
# translations are retained and reused.
section_order_for_front = self._unique_top_level_sections(adaptive_structure)
if section_order_for_front:
final_front_source = self._extract_front_matter(body_only, section_order_for_front[0][1])
self._enqueue_translation_unit(
translation_agent,
unit_key='__front_matter__', unit_name='Final Front Matter',
source_text=final_front_source,
translation_sections=translation_sections,
english_commit_text=full_draft_en,
)
translation_sections, translation_glossary = self._collect_translation_results(
translation_sections, translation_glossary, block=False
)
partial_target = self._assemble_partial_translation(
translation_sections, adaptive_structure, include_front=True
)
self._publish_preview(full_draft_en, "english_complete", force=True, complete=True)
state.update(
{
"full_draft_en": full_draft_en,
"full_draft_translated": partial_target,
"translation_sections": translation_sections,
"translation_glossary": translation_glossary,
"final_abstract": final_abstract,
"final_title": final_title,
"final_keywords": final_keywords,
"front_matter_phase": "final",
"final_reference_count": final_reference_count,
"final_citation_ledger": list(final_citation_ledger),
"final_citation_ledger_version": FINAL_CITATION_LEDGER_VERSION,
"evidence_status": str(state.get("evidence_status") or "adequate"),
"evidence_risk_detected": bool(state.get("evidence_risk_detected")),
"evidence_coverage_events": list(state.get("evidence_coverage_events") or []),
"evidence_coverage_summary": dict(state.get("evidence_coverage_summary") or {}),
"citation_plan": citation_plan,
"citation_minimums": citation_minimums,
"citation_reserve_targets": citation_reserve_targets,
"citation_top_level_minimums": citation_top_level_minimums,
"citation_top_level_reserve_targets": citation_top_level_reserve_targets,
"citation_unit_section_map": citation_unit_section_map,
"citation_required_claim_counts_by_unit": citation_required_claim_counts_by_unit,
"citation_validated_tags_by_unit": citation_validated_tags_by_unit,
"section_integrity_seals": section_integrity_seals,
"english_section_seal_version": ENGLISH_SECTION_SEAL_VERSION,
"quality_diagnostics": quality_diagnostics,
"english_finalized": True,
"translation_complete": False,
"agent_state": agent.export_state(),
}
)
self._checkpoint("english_complete", state, 93)
else:
full_draft_en = str(state.get("full_draft_en") or "")
full_draft_translated = str(state.get("full_draft_translated") or "")
translation_sections = dict(state.get('translation_sections') or translation_sections or {})
translation_glossary = dict(state.get('translation_glossary') or translation_glossary or {})
if not state.get("translation_complete") or not full_draft_translated:
self._progress(94, "translation")
# English is fully frozen. Wait for all previously queued section
# translations before the strict final gate; checkpoint ownership
# remains on this main execution thread.
translation_sections, translation_glossary = self._collect_translation_results(
translation_sections, translation_glossary, block=True
)
self._shutdown_translation_worker(wait=True)
body_only, reference_tail = agent.validate_reference_tail(
full_draft_en, minimum_references=max(0, int(state.get("final_reference_count") or 0))
)
section_order = self._unique_top_level_sections(adaptive_structure)
if not section_order:
raise RuntimeError('translation_section_plan_missing')
front_matter = self._extract_front_matter(body_only, section_order[0][1])
final_units = [('__front_matter__', 'Front Matter', front_matter)]
for key, title in section_order:
final_units.append((key, title, self._extract_top_level_section(body_only, title)))
for index, (unit_key, unit_name, source_text) in enumerate(final_units, start=1):
translation_source = source_text
if (
mode == "amj"
and str(unit_key).strip().casefold() == "results"
and data_metadata.get("quantitative_table_manifest")
):
translation_source = quant_engine._strip_target_quantitative_markdown_tables(
source_text
)
if mode == "asq" and str(unit_key).strip().casefold() == "appendix":
translation_sections, translation_glossary = self._translate_qualitative_appendix_unit(
translation_agent, qual_engine, unit_key=unit_key, source_text=source_text,
target_lang=target_lang, translation_sections=translation_sections,
translation_glossary=translation_glossary, data_metadata=data_metadata,
)
else:
translation_sections, translation_glossary = self._translate_frozen_unit(
translation_agent, unit_key=unit_key, unit_name=unit_name, source_text=translation_source,
target_lang=target_lang, translation_sections=translation_sections,
translation_glossary=translation_glossary, adaptive_structure=adaptive_structure,
include_front=True, defer_failures=False,
english_commit_text=full_draft_en, final_gate=True,
)
# R2: quantitative tables are manifest-owned artifacts, not translation-model output.
# Rebind them only after the frozen Results translation has passed its language/semantic QA.
# Re-running this block is idempotent because the engine first removes any prior/model table block.
if mode == "amj" and str(unit_key).strip().casefold() == "results":
table_manifest = dict(data_metadata.get("quantitative_table_manifest") or {})
current_result_translation = dict(translation_sections.get(unit_key) or {})
if (
table_manifest
and current_result_translation.get("qa_status") in {"passed", "passed_with_warning"}
and current_result_translation.get("translated_text")
):
current_result_translation["translated_text"] = quant_engine.bind_target_language_quantitative_tables(
current_result_translation.get("translated_text"),
table_manifest, target_lang, translation_glossary,
)
current_result_translation["quantitative_table_binding_version"] = (
quant_engine.TARGET_TABLE_BINDING_VERSION
)
translation_sections[unit_key] = current_result_translation
partial = self._assemble_partial_translation(
translation_sections, adaptive_structure, include_front=True
)
state.update({
'full_draft_en': full_draft_en,
'full_draft_translated': partial,
'translation_sections': translation_sections,
'translation_glossary': translation_glossary,
'agent_state': agent.export_state(),
})
pct = min(96, 94 + int(index * 2 / max(1, len(final_units))))
current_translation = dict(translation_sections.get(unit_key) or {})
blocking = (
current_translation.get('qa_status') not in {'passed', 'passed_with_warning'}
or not current_translation.get('translated_text')
)
if blocking:
state['translation_blocking_unit'] = str(unit_key)
state['translation_blocking_unit_name'] = str(unit_name)
state['translation_blocking_error_code'] = str(current_translation.get('error_code') or 'TRANSLATION_PENDING')
self._checkpoint(
f'translation_{unit_key}_repair_pending', state, pct,
display_node=f'translation_repair_pending:{index}:{len(final_units)}',
)
# A2: strict claim QA is diagnostic/recovery authority, not a
# manuscript kill switch. Escalate the same frozen source to
# a whole-unit provider-neutral fallback and keep going.
fallback = translation_agent.adaptive_translate_fallback(
translation_source, target_lang, section_name=unit_name, glossary=translation_glossary
)
current_translation.update({
'translated_text': str(fallback.get('text') or '').strip(),
'qa_status': str(fallback.get('qa_status') or 'passed_with_warning'),
'diagnostics': list(current_translation.get('diagnostics') or []) + list(fallback.get('diagnostics') or []),
'error_code': '',
'adaptive_completion_fallback': True,
})
translation_sections[unit_key] = current_translation
if mode == "amj" and str(unit_key).strip().casefold() == "results":
table_manifest = dict(data_metadata.get("quantitative_table_manifest") or {})
if table_manifest and current_translation.get('translated_text'):
current_translation['translated_text'] = quant_engine.bind_target_language_quantitative_tables(
current_translation.get('translated_text'), table_manifest, target_lang, translation_glossary
)
translation_sections[unit_key] = current_translation
self._record_adaptive_completion_residual(
quality_notes, [{'type':'translation_strict_gate_fallback','reason':str(state.get('translation_blocking_error_code') or '')}],
section_name=unit_name, stage='translation_fallback'
)
state.pop('translation_blocking_unit', None)
state.pop('translation_blocking_unit_name', None)
state.pop('translation_blocking_error_code', None)
self._checkpoint(
f'translation_{unit_key}', state, pct,
display_node=f'translation_final_gate:{index}:{len(final_units)}',
)
unresolved_units = [
key for key, _name, _source in final_units
if (translation_sections.get(key) or {}).get('qa_status') not in {'passed', 'passed_with_warning'}
or not (translation_sections.get(key) or {}).get('translated_text')
]
if unresolved_units:
# A2 should normally resolve these inside the per-unit fallback.
# Keep the manuscript moving if a stale checkpoint status remains
# but usable translated text exists; empty target text is still an
# infrastructure/completion failure and cannot produce a document.
empty_units = [key for key in unresolved_units if not (translation_sections.get(key) or {}).get('translated_text')]
if empty_units:
raise RuntimeError('translation_output_missing:' + ','.join(empty_units[:8]))
for key in unresolved_units:
translation_sections[key]['qa_status'] = 'passed_with_warning'
full_draft_translated = self._assemble_partial_translation(
translation_sections, adaptive_structure, include_front=True
).rstrip()
full_draft_translated += (
f"\n\n# {agent._localized_reference_heading(target_lang)}\n" + reference_tail.strip()
)
full_draft_translated = _normalize_target_manuscript_typography(
full_draft_translated, target_lang
)
full_draft_translated, intent_front_diagnostics = self._rebind_target_user_intent_front_matter(
full_draft_translated, target_lang, data_metadata, translation_glossary
)
for diagnostic in intent_front_diagnostics:
self._record_adaptive_completion_residual(
quality_notes, [diagnostic], section_name='Target-language front matter',
stage='user_research_intent_translation_recovery'
)
if mode == "asq":
appendix_manifest = qual_engine.build_appendix_manifest(data_metadata)
try:
qual_engine.validate_bound_target_appendix(
full_draft_translated, appendix_manifest, target_lang
)
except Exception as exc:
self._record_adaptive_completion_residual(
quality_notes, [{'type':'target_qualitative_appendix_review','reason':str(exc)[:500]}],
section_name='Appendix', stage='translation_final_review'
)
try:
agent.validate_translation_alignment(full_draft_en, full_draft_translated)
except Exception as exc:
self._record_adaptive_completion_residual(
quality_notes, [{'type':'translation_final_alignment_review','reason':str(exc)[:500]}],
section_name='Target-language manuscript', stage='translation_final_review'
)
try:
agent.validate_reference_preservation(full_draft_en, full_draft_translated, target_lang)
except Exception as exc:
logging.warning(
"Adaptive target References rebind | language=%s reason=%s",
target_lang, str(exc)[:300],
)
full_draft_translated = agent.rebind_target_reference_tail(
full_draft_translated, reference_tail, target_lang
)
# References are identity-bearing academic data. Recovery is
# deterministic and provider-free; if this still fails, package
# generation must stop rather than ship a corrupted bibliography.
agent.validate_reference_preservation(
full_draft_en, full_draft_translated, target_lang
)
self._record_adaptive_completion_residual(
quality_notes, [{'type':'target_reference_tail_rebound','reason':str(exc)[:500]}],
section_name='Target-language manuscript', stage='translation_final_recovery'
)
self._publish_target_preview(
full_draft_translated, 'translation_complete', force=True, complete=True
)
state.update(
{
"full_draft_translated": full_draft_translated,
"translation_sections": translation_sections,
"translation_glossary": translation_glossary,
"translation_complete": True,
"agent_state": agent.export_state(),
}
)
self._checkpoint("translation_complete", state, 96)
normalized_target = _normalize_target_manuscript_typography(
full_draft_translated, target_lang
)
if normalized_target != full_draft_translated:
full_draft_translated = normalized_target
state["full_draft_translated"] = full_draft_translated
existing_output = str(state.get("output_filename") or "").strip()
if state.get("package_validated") and existing_output:
existing_path = self.output_dir / os.path.basename(existing_output)
if existing_path.is_file() and existing_path.stat().st_size > 0:
self._progress(99, "package_validated")
result = {"output_filename": existing_path.name, "output_path": str(existing_path)}
if str(state.get("evidence_status") or "") == "limited":
result.update({
"quality_status": "evidence_limited",
"submission_readiness": "review_required",
"safe_error_code": "W_EVIDENCE_COVERAGE_LIMITED",
"safe_error_detail": self._evidence_warning_detail(),
})
return result
self._progress(97, "packaging")
bj_time = datetime.now(timezone(timedelta(hours=8))).strftime("%Y%m%d_%H%M")
target_file_language = (
"Chinese" if "chinese" in target_lang.casefold() or "中文" in target_lang
else "Russian" if "russian" in target_lang.casefold() or "рус" in target_lang.casefold()
else "Kazakh" if "kazakh" in target_lang.casefold() or "қазақ" in target_lang.casefold()
else re.sub(r"[^A-Za-z]+", "", output_suffix) or "Target"
)
package = build_validated_package(
english_markdown=full_draft_en,
translated_markdown=full_draft_translated,
english_filename="Paper_English.docx",
translated_filename=f"Paper_{target_file_language}.docx",
data_metadata=data_metadata,
mode=mode,
citation_audit=agent.citation_audit_log,
literature_metrics=agent.get_source_metrics(),
quality_diagnostics=quality_diagnostics,
quality_notes=quality_notes,
organize_appendix=True,
evidence_coverage=state.get("evidence_coverage_summary") or {
"status": str(state.get("evidence_status") or "adequate"),
"early_risk_detected": bool(state.get("evidence_risk_detected")),
"final_reference_count": int(state.get("final_reference_count") or 0),
"sections": [],
"events": list(state.get("evidence_coverage_events") or []),
},
)
self._progress(98, "package_validation")
output_filename = f"SmartPaper_{bj_time}.zip"
pending = self.output_dir / (".pending_" + output_filename)
pending.write_bytes(package.zip_bytes)
final = self.output_dir / output_filename
os.replace(pending, final)
state.update(
{
"output_filename": output_filename,
"package_validated": True,
"evidence_status": str(state.get("evidence_status") or "adequate"),
"evidence_coverage_summary": dict(state.get("evidence_coverage_summary") or {}),
"quality_notes": quality_notes,
"agent_state": agent.export_state(),
}
)
self._checkpoint("package_validated", state, 99, display_node="package_validated")
result = {"output_filename": output_filename, "output_path": str(final)}
if str(state.get("evidence_status") or "") == "limited":
result.update({
"quality_status": "evidence_limited",
"submission_readiness": "review_required",
"safe_error_code": "W_EVIDENCE_COVERAGE_LIMITED",
"safe_error_detail": self._evidence_warning_detail(),
})
return result
def run(self) -> dict[str, Any]:
try:
return self._run_impl()
finally:
self._shutdown_translation_worker(wait=False)
def _progress_activity(self) -> None:
# Activity remains Executor-owned. We also snapshot actual request
# counters so the next Executor heartbeat can persist live UI metrics.
ds_calls, s2_calls, oa_calls = self._compute_request_counts()
try:
self.progress_callback(
activity_only=True,
ds_calls=ds_calls,
s2_calls=s2_calls,
oa_calls=oa_calls,
)
except TypeError:
pass