Spaces:
Running
Running
S0-R1-C5.5-R6-R2 Typed Citation Provenance + Evidence Ownership
#90
by GuXiaopeng - opened
- core/generation_runner.py +12 -6
- core/pipeline_agent.py +229 -34
core/generation_runner.py
CHANGED
|
@@ -973,6 +973,11 @@ class GenerationRunner:
|
|
| 973 |
"phrases such as 'restricted to associations, not causal effects' and 'cannot establish "
|
| 974 |
"causality' are valid and should not be weakened further."
|
| 975 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 976 |
actor = str(item.get('actor') or '')
|
| 977 |
expected = item.get('expected')
|
| 978 |
observed = item.get('observed')
|
|
@@ -1023,7 +1028,7 @@ class GenerationRunner:
|
|
| 1023 |
if kind in {"author_facing_internal_workflow_language", "top_journal_critic"}:
|
| 1024 |
return "ADVISORY"
|
| 1025 |
|
| 1026 |
-
if kind in {"citation_claim_split_required", "citation_unsupported_claim", "citation_support_unverified"}:
|
| 1027 |
core_citation_section = bool(re.search(
|
| 1028 |
r"(?i)(?:Theoretical Mechanisms|Hypothesis Development|Methodology|Results?|Findings?)",
|
| 1029 |
section,
|
|
@@ -1047,8 +1052,10 @@ class GenerationRunner:
|
|
| 1047 |
item = dict(row or {}) if isinstance(row, dict) else {"type": "unknown_validation_issue"}
|
| 1048 |
kind = str(item.get("type") or "unknown_validation_issue")
|
| 1049 |
groups = {
|
| 1050 |
-
"
|
| 1051 |
-
|
|
|
|
|
|
|
| 1052 |
"causal_scope": {"causal_language_violation"},
|
| 1053 |
"quant_method": {"unsupported_quantitative_method", "quantitative_metric_mismatch", "unfrozen_p_value"},
|
| 1054 |
"study_fact": {"sample_size_mismatch", "sample_count_mismatch", "scale_points_mismatch",
|
|
@@ -3729,6 +3736,7 @@ Planning length: approximately {int(adjusted_target * min_ratio)}-{int(adjusted_
|
|
| 3729 |
{citation_rule}
|
| 3730 |
{table_rule}
|
| 3731 |
AUTHOR-FACING PURITY: Write only reader-facing scholarly prose. Do not mention internal orchestration, validation, state-management, evidence-management, or execution terminology. Describe the study, method, evidence, and limitations directly.
|
|
|
|
| 3732 |
{("CAUSAL-STRENGTH CONTRACT: This cross-sectional observational design supports association/covariation language only. Do not state that one construct causes, leads to, drives, determines, produces, or causally mediates another. Explicit limitation language is valid and preferred when relevant, including 'restricted to associations, not causal effects', 'cannot establish causality', and 'the indirect association is not causal mediation identification'." if mode == "amj" else "")}
|
| 3733 |
{self._hypothesis_contract_directive(hypothesis_ledger)}
|
| 3734 |
{str(sec.get('required_content') or '')}
|
|
@@ -3883,9 +3891,7 @@ The previous attempt for this same manuscript unit was rejected by validation. R
|
|
| 3883 |
set(attempt_assigned_tag_list) - set(retry_excluded_tags)
|
| 3884 |
if not is_quarantined else set()
|
| 3885 |
)
|
| 3886 |
-
attempt_section_text =
|
| 3887 |
-
prefix + en_raw, display_name
|
| 3888 |
-
)
|
| 3889 |
en_resolved = agent.audit_and_resolve_citations(
|
| 3890 |
attempt_section_text, section_name=display_name, section_id=section_id,
|
| 3891 |
quarantine=is_quarantined, allowed_tags=allowed_section_citation_tags
|
|
|
|
| 973 |
"phrases such as 'restricted to associations, not causal effects' and 'cannot establish "
|
| 974 |
"causality' are valid and should not be weakened further."
|
| 975 |
)
|
| 976 |
+
if kind == 'citation_evidence_ownership_mismatch':
|
| 977 |
+
detail = (
|
| 978 |
+
"Split the sentence by evidence ownership. Keep the study-owned fact as a separate statement grounded only in the study evidence and without an external citation. "
|
| 979 |
+
"State any methodological principle separately and attach only one directly supporting METHOD_AUTHORITY citation. Keep author inference separate, cautious, and uncited unless a source directly supports that exact inference."
|
| 980 |
+
)
|
| 981 |
actor = str(item.get('actor') or '')
|
| 982 |
expected = item.get('expected')
|
| 983 |
observed = item.get('observed')
|
|
|
|
| 1028 |
if kind in {"author_facing_internal_workflow_language", "top_journal_critic"}:
|
| 1029 |
return "ADVISORY"
|
| 1030 |
|
| 1031 |
+
if kind in {"citation_claim_split_required", "citation_unsupported_claim", "citation_support_unverified", "citation_evidence_ownership_mismatch"}:
|
| 1032 |
core_citation_section = bool(re.search(
|
| 1033 |
r"(?i)(?:Theoretical Mechanisms|Hypothesis Development|Methodology|Results?|Findings?)",
|
| 1034 |
section,
|
|
|
|
| 1052 |
item = dict(row or {}) if isinstance(row, dict) else {"type": "unknown_validation_issue"}
|
| 1053 |
kind = str(item.get("type") or "unknown_validation_issue")
|
| 1054 |
groups = {
|
| 1055 |
+
"citation_scope": {"citation_outside_frozen_section_whitelist"},
|
| 1056 |
+
"citation_ownership": {"citation_evidence_ownership_mismatch"},
|
| 1057 |
+
"citation_support": {"citation_unsupported_claim", "citation_support_unverified"},
|
| 1058 |
+
"citation_compound": {"citation_claim_split_required"},
|
| 1059 |
"causal_scope": {"causal_language_violation"},
|
| 1060 |
"quant_method": {"unsupported_quantitative_method", "quantitative_metric_mismatch", "unfrozen_p_value"},
|
| 1061 |
"study_fact": {"sample_size_mismatch", "sample_count_mismatch", "scale_points_mismatch",
|
|
|
|
| 3736 |
{citation_rule}
|
| 3737 |
{table_rule}
|
| 3738 |
AUTHOR-FACING PURITY: Write only reader-facing scholarly prose. Do not mention internal orchestration, validation, state-management, evidence-management, or execution terminology. Describe the study, method, evidence, and limitations directly.
|
| 3739 |
+
EVIDENCE-OWNERSHIP CONTRACT: Statements about this study's sample, measures, available or missing documentation, procedures, statistics, quotations, or findings are owned by the study evidence and must not borrow an external citation. General methodological principles may use only a directly supporting METHOD_AUTHORITY source. Author interpretation or limitation reasoning must not be presented as if an external source established this study-specific conclusion. If one sentence would combine a study-owned fact with an external methodological principle or author inference, split it into separate claim-specific sentences and attach the citation only to the directly supported external principle.
|
| 3740 |
{("CAUSAL-STRENGTH CONTRACT: This cross-sectional observational design supports association/covariation language only. Do not state that one construct causes, leads to, drives, determines, produces, or causally mediates another. Explicit limitation language is valid and preferred when relevant, including 'restricted to associations, not causal effects', 'cannot establish causality', and 'the indirect association is not causal mediation identification'." if mode == "amj" else "")}
|
| 3741 |
{self._hypothesis_contract_directive(hypothesis_ledger)}
|
| 3742 |
{str(sec.get('required_content') or '')}
|
|
|
|
| 3891 |
set(attempt_assigned_tag_list) - set(retry_excluded_tags)
|
| 3892 |
if not is_quarantined else set()
|
| 3893 |
)
|
| 3894 |
+
attempt_section_text = prefix + en_raw
|
|
|
|
|
|
|
| 3895 |
en_resolved = agent.audit_and_resolve_citations(
|
| 3896 |
attempt_section_text, section_name=display_name, section_id=section_id,
|
| 3897 |
quarantine=is_quarantined, allowed_tags=allowed_section_citation_tags
|
core/pipeline_agent.py
CHANGED
|
@@ -76,6 +76,7 @@ class SSciPipelineAgent:
|
|
| 76 |
PROTECTED_ENTITY_LEDGER_VERSION = 'protected-entity-ledger-v2'
|
| 77 |
CANONICAL_GLOSSARY_QUALITY_VERSION = 'canonical-glossary-quality-v1'
|
| 78 |
SEMANTIC_CLAIM_FRAME_VERSION = 'smartpaper-core-semantic-claim-frame-v1'
|
|
|
|
| 79 |
|
| 80 |
@staticmethod
|
| 81 |
def _translation_defect_family(codes):
|
|
@@ -2800,7 +2801,7 @@ INPUT:
|
|
| 2800 |
):
|
| 2801 |
continue
|
| 2802 |
provenance = self._citation_claim_provenance(claim, section)
|
| 2803 |
-
if provenance
|
| 2804 |
logging.info(
|
| 2805 |
'Citation planner rejected non-literature claim provenance: section=%s tag=%s provenance=%s',
|
| 2806 |
section, tag, provenance,
|
|
@@ -2929,7 +2930,10 @@ INPUT:
|
|
| 2929 |
method_concept = re.search(
|
| 2930 |
r'\b(?:cross[- ]sectional|observational|mediat(?:ion|or)|bootstrap(?:ping)?|confidence interval|'
|
| 2931 |
r'temporal ordering|experimental manipulation|instrumental variable|causal (?:effect|mediation|identification|pathway)|'
|
| 2932 |
-
r'measurement validity|construct validity|discriminant validity|convergent validity|
|
|
|
|
|
|
|
|
|
|
| 2933 |
value,
|
| 2934 |
)
|
| 2935 |
if not method_concept:
|
|
@@ -2937,7 +2941,7 @@ INPUT:
|
|
| 2937 |
rationale = re.search(
|
| 2938 |
r'\b(?:consistent with|methodolog(?:y|ical|ically)|cannot|can not|does not|do not|is not|are not|'
|
| 2939 |
r'without|insufficient|inability|limitation|limits?|threat|bias|validity|appropriate|requires?|should|must|'
|
| 2940 |
-
r'establish|identify|infer|interpret|justify|warrant|support|transmit|assess(?:es|ed|ing)?)\b',
|
| 2941 |
value,
|
| 2942 |
)
|
| 2943 |
return bool(rationale)
|
|
@@ -3274,15 +3278,8 @@ INPUT:
|
|
| 3274 |
))
|
| 3275 |
|
| 3276 |
@staticmethod
|
| 3277 |
-
def
|
| 3278 |
-
"""
|
| 3279 |
-
|
| 3280 |
-
This is deliberately conservative and deterministic. The recovery path may
|
| 3281 |
-
only add evidence to externally-oriented claims; it must not attach a source
|
| 3282 |
-
merely to satisfy a citation target when the sentence is owned by the current
|
| 3283 |
-
study, reports the current study's results, states the authors' positioning,
|
| 3284 |
-
or serves only as structure/transition.
|
| 3285 |
-
"""
|
| 3286 |
text = re.sub(r'\s+', ' ', str(sentence or '')).strip()
|
| 3287 |
lower = text.lower()
|
| 3288 |
section = str(section_name or '').lower()
|
|
@@ -3316,6 +3313,11 @@ INPUT:
|
|
| 3316 |
lower,
|
| 3317 |
):
|
| 3318 |
return 'STUDY_OWNED_FACT'
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3319 |
if any(token in section for token in ('result', 'finding')) and re.search(
|
| 3320 |
r'\b(?:we found|we find|our findings|our results|the results|the findings|coefficient|p\s*[<=>]|r\s*[²2]|β|beta)\b',
|
| 3321 |
lower,
|
|
@@ -3325,9 +3327,182 @@ INPUT:
|
|
| 3325 |
return 'STRUCTURAL'
|
| 3326 |
return 'EXTERNAL_LITERATURE_CLAIM'
|
| 3327 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3328 |
@classmethod
|
| 3329 |
def _citation_recovery_eligible_claim(cls, sentence, section_name=''):
|
| 3330 |
-
return cls._citation_claim_provenance(sentence, section_name)
|
| 3331 |
|
| 3332 |
def strip_ineligible_current_study_citations(self, text, section_name=''):
|
| 3333 |
"""Remove external citation tags from current-study/provenance-owned prose.
|
|
@@ -3345,7 +3520,7 @@ INPUT:
|
|
| 3345 |
original = str(sentence or '')
|
| 3346 |
if re.search(r'CITE-\d+', original, re.I):
|
| 3347 |
provenance = self._citation_claim_provenance(original, section_name)
|
| 3348 |
-
if provenance
|
| 3349 |
revised = self._citation_tag_only_rewrite(original, None)
|
| 3350 |
self._append_citation_audit(
|
| 3351 |
section_name,
|
|
@@ -4557,6 +4732,7 @@ INPUT:
|
|
| 4557 |
"tag": tag,
|
| 4558 |
"reference": metadata.get("ref", ""),
|
| 4559 |
"evidence": metadata.get("evidence", ""),
|
|
|
|
| 4560 |
}
|
| 4561 |
)
|
| 4562 |
records.append(
|
|
@@ -4567,6 +4743,9 @@ INPUT:
|
|
| 4567 |
"original": sentence,
|
| 4568 |
"allowed_tags": candidates,
|
| 4569 |
"evidence": evidence,
|
|
|
|
|
|
|
|
|
|
| 4570 |
}
|
| 4571 |
)
|
| 4572 |
|
|
@@ -4578,6 +4757,7 @@ INPUT:
|
|
| 4578 |
{
|
| 4579 |
"id": record["id"],
|
| 4580 |
"sentence": record["original"],
|
|
|
|
| 4581 |
"candidate_sources": record["evidence"],
|
| 4582 |
}
|
| 4583 |
for record in batch
|
|
@@ -4589,8 +4769,9 @@ Rules:
|
|
| 4589 |
2. Every output sentence may contain at most one [CITE-XX] tag.
|
| 4590 |
3. When several sources support the same claim, retain only the single source whose supplied evidence most directly supports that claim.
|
| 4591 |
4. Use status=split only when no single candidate source directly supports the complete original sentence AND at least two distinct candidate sources are needed for genuinely different claims. If one candidate source directly supports the complete sentence, status must be ok and that single tag must be selected; do not split merely because the sentence has multiple clauses.
|
| 4592 |
-
5.
|
| 4593 |
-
6.
|
|
|
|
| 4594 |
7. If no candidate source directly supports a claim that requires external evidence, set status to unsupported and leave revised_text equal to the original sentence.
|
| 4595 |
8. If the statement is ordinary connective prose that does not require external support, status may be uncited_common_knowledge and revised_text may omit the tag without changing the prose.
|
| 4596 |
|
|
@@ -4950,6 +5131,9 @@ Return exactly:
|
|
| 4950 |
self, text, section_name='', section_id='', quarantine=False, allowed_tags=None
|
| 4951 |
):
|
| 4952 |
self._last_citation_semantic_violations = []
|
|
|
|
|
|
|
|
|
|
| 4953 |
value = self.strip_ineligible_current_study_citations(str(text or ''), section_name)
|
| 4954 |
if quarantine:
|
| 4955 |
return self.resolve_citations(value, quarantine=True)
|
|
@@ -5007,10 +5191,12 @@ Return exactly:
|
|
| 5007 |
value = self._local_citation_recovery(value, section_name, retain_lexical=False)
|
| 5008 |
elif violations:
|
| 5009 |
value = self._local_citation_recovery(value, section_name)
|
| 5010 |
-
if outside_rows:
|
| 5011 |
-
self._last_citation_semantic_violations =
|
| 5012 |
-
|
| 5013 |
-
|
|
|
|
|
|
|
| 5014 |
if self.find_multi_citation_violations(value):
|
| 5015 |
value = self._local_citation_recovery(value, section_name)
|
| 5016 |
if self.find_multi_citation_violations(value):
|
|
@@ -6225,7 +6411,7 @@ TEXT:
|
|
| 6225 |
)
|
| 6226 |
|
| 6227 |
@classmethod
|
| 6228 |
-
def _source_semantic_claim_frame(cls, source_claim):
|
| 6229 |
"""Derive a conservative language-neutral frame once from frozen English.
|
| 6230 |
|
| 6231 |
This frame is not a substitute for bilingual QA. It is an immutable
|
|
@@ -6271,11 +6457,18 @@ TEXT:
|
|
| 6271 |
modality = "LIMITATION"
|
| 6272 |
else:
|
| 6273 |
modality = "ASSERTED"
|
|
|
|
| 6274 |
frame = {
|
| 6275 |
"version": cls.SEMANTIC_CLAIM_FRAME_VERSION,
|
| 6276 |
"polarity": polarity,
|
| 6277 |
"causal_strength": causal_strength,
|
| 6278 |
"modality": modality,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6279 |
}
|
| 6280 |
canonical = json.dumps(frame, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
| 6281 |
frame["frame_hash"] = hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
|
@@ -7232,7 +7425,7 @@ SECTION: {section_name}
|
|
| 7232 |
'claim_id': claim_id,
|
| 7233 |
'english': item['text'],
|
| 7234 |
'target': target_claims[claim_id],
|
| 7235 |
-
'source_semantic_frame': self._source_semantic_claim_frame(item['text']),
|
| 7236 |
})
|
| 7237 |
|
| 7238 |
issues = {}
|
|
@@ -7255,7 +7448,7 @@ SECTION: {section_name}
|
|
| 7255 |
profile = self._translation_language_profile(target_lang)
|
| 7256 |
lexical_contract = self._translation_lexical_retention_contract(target_lang)
|
| 7257 |
system_prompt = f"""You are a bilingual academic translation QA classifier for {target_lang}.
|
| 7258 |
-
Review EACH claim independently. Do not rewrite and do not provide free-form explanations. The supplied source_semantic_frame is server-frozen source authority: the target must preserve its polarity, causal_strength, and
|
| 7259 |
LANGUAGE ADAPTER: {profile['adapter_id']}; morphology={profile['morphology']}. {profile['term_policy']} {lexical_contract} {profile['contamination_policy']}
|
| 7260 |
For Russian, do not flag a terminology error merely because a translated term is inflected for grammatical case/number/gender; sustained non-Russian prose remains an additional contamination signal. For Kazakh, do not flag a terminology error merely because productive suffixes attach to the correct term stem; clearly Russian lexical/syntactic intrusion remains an additional language-specific signal when the pair supports that conclusion. Cyrillic script or absence of Kazakh-specific letters alone is never sufficient evidence.
|
| 7261 |
Return valid JSON only in this exact form: {{"items":[{{"id":"SP-P000001-C000001","ok":true,"codes":[],"evidence_codes":[]}}]}}.
|
|
@@ -7650,6 +7843,7 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
|
|
| 7650 |
|
| 7651 |
def _translate_marked_batch_with_recovery(self, batch, system_prompt, target_lang, _strict=False):
|
| 7652 |
"""Translate SP-P blocks through claim-bound, value-blind protected skeletons."""
|
|
|
|
| 7653 |
expected_ids = [block_id for block_id, _ in batch]
|
| 7654 |
protected_batch = []
|
| 7655 |
protected_maps = {}
|
|
@@ -7669,7 +7863,7 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
|
|
| 7669 |
}
|
| 7670 |
claim_semantic_manifest = {
|
| 7671 |
block_id: {
|
| 7672 |
-
claim_id: self._source_semantic_claim_frame(source_claim)
|
| 7673 |
for claim_id, source_claim in dict(claim_sources_by_block.get(block_id) or {}).items()
|
| 7674 |
}
|
| 7675 |
for block_id, _source, _protected in protected_batch
|
|
@@ -7681,7 +7875,7 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
|
|
| 7681 |
+ '<SOURCE_SEMANTIC_CLAIM_FRAMES>\n'
|
| 7682 |
+ json.dumps(claim_semantic_manifest, ensure_ascii=False)
|
| 7683 |
+ '\n</SOURCE_SEMANTIC_CLAIM_FRAMES>\n'
|
| 7684 |
-
'The source semantic claim frames are server-frozen. Preserve polarity, causal strength, and modality for each SP-C claim exactly;
|
| 7685 |
'This manifest intentionally contains no factual values. Preserve every SP-P, SP-C, and SP-T marker exactly. '
|
| 7686 |
'Each SP-C marker identifies one source claim. A target claim may contain one or more grammatical sentences, but every protected fact must remain inside its original SP-C claim. '
|
| 7687 |
'Never infer, spell out, calculate, or recreate a hidden number, date, citation, acronym, name, formula, or identifier. Do not print the manifest.'
|
|
@@ -7987,7 +8181,7 @@ Each code must be one of: {sorted(allowed)}. Return one or more specific dimensi
|
|
| 7987 |
|
| 7988 |
@classmethod
|
| 7989 |
def _snapshot_translation_claim_states(
|
| 7990 |
-
cls, source_blocks, translations, pending_issues=None, repair_evidence=None, repair_meta=None
|
| 7991 |
):
|
| 7992 |
"""Freeze claim text plus bounded repair history so Final Gate cannot restart a failed strategy."""
|
| 7993 |
pending = dict(pending_issues or {})
|
|
@@ -8014,8 +8208,8 @@ Each code must be one of: {sorted(allowed)}. Return one or more specific dimensi
|
|
| 8014 |
'block_id': str(block_id),
|
| 8015 |
'claim_id': str(claim_id),
|
| 8016 |
'source_hash': hashlib.sha256(str(item['text']).encode('utf-8')).hexdigest(),
|
| 8017 |
-
'semantic_frame': cls._source_semantic_claim_frame(item['text']),
|
| 8018 |
-
'semantic_frame_hash': cls._source_semantic_claim_frame(item['text']).get('frame_hash', ''),
|
| 8019 |
'target_text': target_text,
|
| 8020 |
'qa_status': 'repair_pending' if codes else 'passed',
|
| 8021 |
'issue_codes': [str(code or '').strip().upper() for code in codes if str(code or '').strip()][:5],
|
|
@@ -8029,7 +8223,7 @@ Each code must be one of: {sorted(allowed)}. Return one or more specific dimensi
|
|
| 8029 |
return states
|
| 8030 |
|
| 8031 |
@classmethod
|
| 8032 |
-
def _restore_translation_claim_states(cls, source_blocks, claim_states):
|
| 8033 |
"""Reassemble marked blocks only when every frozen source claim has a hash-matched durable target."""
|
| 8034 |
states = dict(claim_states or {})
|
| 8035 |
if not states:
|
|
@@ -8045,7 +8239,7 @@ Each code must be one of: {sorted(allowed)}. Return one or more specific dimensi
|
|
| 8045 |
key = f'SP-P{block_id}-{claim_id}'
|
| 8046 |
row = states.get(key)
|
| 8047 |
expected_hash = hashlib.sha256(str(item['text']).encode('utf-8')).hexdigest()
|
| 8048 |
-
expected_semantic_hash = cls._source_semantic_claim_frame(item['text']).get('frame_hash', '')
|
| 8049 |
if not isinstance(row, dict) or row.get('source_hash') != expected_hash or not str(row.get('target_text') or '').strip():
|
| 8050 |
return {}, {}, False
|
| 8051 |
# R6-R1: new states are semantic-frame bound. Pre-R6 states do
|
|
@@ -8421,6 +8615,7 @@ Decision must be CONFIRMED_BLOCKING or NOT_CONFIRMED. evidence_code must be one
|
|
| 8421 |
+ json.dumps(frozen_glossary, ensure_ascii=False)
|
| 8422 |
)
|
| 8423 |
source_blocks = {block_id: source for block_id, source in pending_blocks}
|
|
|
|
| 8424 |
attempts = max(1, int(max_attempts or 1))
|
| 8425 |
last_error = None
|
| 8426 |
self._last_translation_claim_states = {}
|
|
@@ -8472,7 +8667,7 @@ Decision must be CONFIRMED_BLOCKING or NOT_CONFIRMED. evidence_code must be one
|
|
| 8472 |
translations = {}
|
| 8473 |
try:
|
| 8474 |
translations, quality_issues, resumed_claims = self._restore_translation_claim_states(
|
| 8475 |
-
source_blocks, claim_states
|
| 8476 |
)
|
| 8477 |
if resumed_claims:
|
| 8478 |
for row in dict(claim_states or {}).values():
|
|
@@ -8505,7 +8700,7 @@ Decision must be CONFIRMED_BLOCKING or NOT_CONFIRMED. evidence_code must be one
|
|
| 8505 |
)
|
| 8506 |
self._last_translation_claim_states = self._snapshot_translation_claim_states(
|
| 8507 |
source_blocks, translations, quality_issues, self._translation_claim_qa_evidence,
|
| 8508 |
-
self._translation_claim_repair_meta,
|
| 8509 |
)
|
| 8510 |
repair_cycle = 0
|
| 8511 |
total_claims = sum(len(self._translation_claim_units(source)) for source in source_blocks.values())
|
|
@@ -8748,7 +8943,7 @@ Decision must be CONFIRMED_BLOCKING or NOT_CONFIRMED. evidence_code must be one
|
|
| 8748 |
quality_issues = refreshed
|
| 8749 |
self._last_translation_claim_states = self._snapshot_translation_claim_states(
|
| 8750 |
source_blocks, translations, quality_issues, self._translation_claim_qa_evidence,
|
| 8751 |
-
self._translation_claim_repair_meta,
|
| 8752 |
)
|
| 8753 |
if not touched_blocks and carried_exhausted:
|
| 8754 |
break
|
|
@@ -8772,7 +8967,7 @@ Decision must be CONFIRMED_BLOCKING or NOT_CONFIRMED. evidence_code must be one
|
|
| 8772 |
)
|
| 8773 |
self._last_translation_claim_states = self._snapshot_translation_claim_states(
|
| 8774 |
source_blocks, translations, quality_issues, self._translation_claim_qa_evidence,
|
| 8775 |
-
self._translation_claim_repair_meta,
|
| 8776 |
)
|
| 8777 |
if quality_issues:
|
| 8778 |
detail = '; '.join(
|
|
|
|
| 76 |
PROTECTED_ENTITY_LEDGER_VERSION = 'protected-entity-ledger-v2'
|
| 77 |
CANONICAL_GLOSSARY_QUALITY_VERSION = 'canonical-glossary-quality-v1'
|
| 78 |
SEMANTIC_CLAIM_FRAME_VERSION = 'smartpaper-core-semantic-claim-frame-v1'
|
| 79 |
+
CITATION_CLAIM_FRAME_VERSION = 'smartpaper-citation-claim-frame-v1'
|
| 80 |
|
| 81 |
@staticmethod
|
| 82 |
def _translation_defect_family(codes):
|
|
|
|
| 2801 |
):
|
| 2802 |
continue
|
| 2803 |
provenance = self._citation_claim_provenance(claim, section)
|
| 2804 |
+
if provenance not in {'EXTERNAL_LITERATURE_CLAIM', 'METHOD_PRINCIPLE'}:
|
| 2805 |
logging.info(
|
| 2806 |
'Citation planner rejected non-literature claim provenance: section=%s tag=%s provenance=%s',
|
| 2807 |
section, tag, provenance,
|
|
|
|
| 2930 |
method_concept = re.search(
|
| 2931 |
r'\b(?:cross[- ]sectional|observational|mediat(?:ion|or)|bootstrap(?:ping)?|confidence interval|'
|
| 2932 |
r'temporal ordering|experimental manipulation|instrumental variable|causal (?:effect|mediation|identification|pathway)|'
|
| 2933 |
+
r'measurement validity|content validity|construct validity|discriminant validity|convergent validity|internal consistency|'
|
| 2934 |
+
r'test[- ]retest|measurement invariance|confirmatory factor analysis|factor analysis|reliability|common method|endogeneity|'
|
| 2935 |
+
r'triangulation|member checking|reflexivity|qualitative content analysis|case study|process analysis|thematic analysis|'
|
| 2936 |
+
r'reflective measurement|formative measurement|measurement specification|item covariation|common underlying cause|latent construct)\b',
|
| 2937 |
value,
|
| 2938 |
)
|
| 2939 |
if not method_concept:
|
|
|
|
| 2941 |
rationale = re.search(
|
| 2942 |
r'\b(?:consistent with|methodolog(?:y|ical|ically)|cannot|can not|does not|do not|is not|are not|'
|
| 2943 |
r'without|insufficient|inability|limitation|limits?|threat|bias|validity|appropriate|requires?|should|must|'
|
| 2944 |
+
r'establish|identify|infer|interpret|justify|warrant|support|transmit|assess(?:es|ed|ing)?|assessment|precludes?|prevents?|trustworthiness|credibility|dependability|confirmability)\b',
|
| 2945 |
value,
|
| 2946 |
)
|
| 2947 |
return bool(rationale)
|
|
|
|
| 3278 |
))
|
| 3279 |
|
| 3280 |
@staticmethod
|
| 3281 |
+
def _citation_base_provenance(sentence, section_name=''):
|
| 3282 |
+
"""Legacy-compatible high-confidence provenance before R6-R2 ownership composition."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3283 |
text = re.sub(r'\s+', ' ', str(sentence or '')).strip()
|
| 3284 |
lower = text.lower()
|
| 3285 |
section = str(section_name or '').lower()
|
|
|
|
| 3313 |
lower,
|
| 3314 |
):
|
| 3315 |
return 'STUDY_OWNED_FACT'
|
| 3316 |
+
if re.search(
|
| 3317 |
+
r'\bthe study (?:employed|used|included|collected|recruited|interviewed|observed|conducted|analy[sz]ed|assessed|evaluated|estimated|reported|draws on|relies on)\b',
|
| 3318 |
+
lower,
|
| 3319 |
+
):
|
| 3320 |
+
return 'STUDY_OWNED_FACT'
|
| 3321 |
if any(token in section for token in ('result', 'finding')) and re.search(
|
| 3322 |
r'\b(?:we found|we find|our findings|our results|the results|the findings|coefficient|p\s*[<=>]|r\s*[²2]|β|beta)\b',
|
| 3323 |
lower,
|
|
|
|
| 3327 |
return 'STRUCTURAL'
|
| 3328 |
return 'EXTERNAL_LITERATURE_CLAIM'
|
| 3329 |
|
| 3330 |
+
@classmethod
|
| 3331 |
+
def _citation_study_evidence_signal(cls, sentence, section_name=''):
|
| 3332 |
+
text = re.sub(r'\s+', ' ', str(sentence or '')).casefold().strip()
|
| 3333 |
+
section = str(section_name or '').casefold()
|
| 3334 |
+
if not text:
|
| 3335 |
+
return False
|
| 3336 |
+
if cls._citation_base_provenance(text, section_name) in {'STUDY_OWNED_FACT', 'RESULT_OWNED_FACT'}:
|
| 3337 |
+
return True
|
| 3338 |
+
if re.search(
|
| 3339 |
+
r'\b(?:study evidence|available evidence|measurement evidence|empirical evidence in (?:this|the) study|'
|
| 3340 |
+
r'evidence (?:reported|provided|available|contained) in (?:this|the) study|reported in (?:this|the) study)\b',
|
| 3341 |
+
text,
|
| 3342 |
+
):
|
| 3343 |
+
return True
|
| 3344 |
+
if cls._citation_method_section(section_name=section_name):
|
| 3345 |
+
if re.search(
|
| 3346 |
+
r'\bthe absence of (?:such )?(?:documentation|item wording|source scale information|scale documentation|measurement documentation|direct content validation)\b',
|
| 3347 |
+
text,
|
| 3348 |
+
):
|
| 3349 |
+
return True
|
| 3350 |
+
if re.search(
|
| 3351 |
+
r'\bno (?:confirmatory factor analysis|cfa|fit indices?|modification indices?|alternative measurement structures?|'
|
| 3352 |
+
r'marker variables?|instrumental variables?|experimental controls?|endogeneity tests?|test[- ]retest evidence)\b.{0,90}\b(?:available|reported|provided|documented|claimed)\b',
|
| 3353 |
+
text,
|
| 3354 |
+
):
|
| 3355 |
+
return True
|
| 3356 |
+
if re.search(r'\bbootstrap(?:ping| resampling)?\b.{0,80}\b\d{2,6}\s+resamples?\b', text):
|
| 3357 |
+
return True
|
| 3358 |
+
if re.search(
|
| 3359 |
+
r'\b(?:was|were) (?:used|employed|estimated|assessed|evaluated|calculated|computed|reported)\b', text
|
| 3360 |
+
) and re.search(
|
| 3361 |
+
r'\b(?:sample|participants?|respondents?|questionnaire|survey|constructs?|measures?|items?|bootstrap|coefficient|validity|reliability|correlations?)\b', text
|
| 3362 |
+
):
|
| 3363 |
+
return True
|
| 3364 |
+
return False
|
| 3365 |
+
|
| 3366 |
+
@classmethod
|
| 3367 |
+
def _citation_author_inference_signal(cls, sentence, section_name=''):
|
| 3368 |
+
if not cls._citation_method_section(section_name=section_name):
|
| 3369 |
+
return False
|
| 3370 |
+
text = re.sub(r'\s+', ' ', str(sentence or '')).casefold().strip()
|
| 3371 |
+
if not text:
|
| 3372 |
+
return False
|
| 3373 |
+
external_orientation = bool(re.search(
|
| 3374 |
+
r'\b(?:prior|previous|existing) (?:studies|research|literature)|the literature|researchers? (?:argue|show|report|suggest)|'
|
| 3375 |
+
r'evidence from (?:prior|previous|existing) studies\b', text
|
| 3376 |
+
))
|
| 3377 |
+
if external_orientation:
|
| 3378 |
+
return False
|
| 3379 |
+
return bool(re.search(
|
| 3380 |
+
r'\b(?:therefore|thus|consequently|accordingly|means that|implies?|must be inferred|can be inferred|'
|
| 3381 |
+
r'should be interpreted|must be interpreted|internally coherent|supports? the (?:interpretation|conclusion)|'
|
| 3382 |
+
r'provides? partial mitigation|is appropriate when|are appropriate when|precludes? (?:an? )?assessment|'
|
| 3383 |
+
r'prevents? (?:an? )?assessment|does not permit|does not allow|which (?:captures|represents|indicates|suggests)|'
|
| 3384 |
+
r'captures? .{0,80} rather than|represents? .{0,80} rather than|enabl(?:e|es|ed|ing) .{0,80}(?:compar(?:e|ison)|identify|assess)|'
|
| 3385 |
+
r'allows? .{0,80}(?:compar(?:e|ison)|identify|assess))\b',
|
| 3386 |
+
text,
|
| 3387 |
+
))
|
| 3388 |
+
|
| 3389 |
+
@classmethod
|
| 3390 |
+
def build_citation_claim_frame(cls, sentence, section_name='', section_id=''):
|
| 3391 |
+
"""Freeze claim provenance, evidence owner, and citation scope before citation QA.
|
| 3392 |
+
|
| 3393 |
+
R6-R2 makes evidence ownership explicit so current-study facts, external
|
| 3394 |
+
methodological principles, literature findings, and author inferences do
|
| 3395 |
+
not inherit one another's citations merely because the writer placed them
|
| 3396 |
+
in one fluent sentence.
|
| 3397 |
+
"""
|
| 3398 |
+
raw = re.sub(r'\s+', ' ', str(sentence or '')).strip()
|
| 3399 |
+
citation_tags = [f'[{tag.upper()}]' for tag in re.findall(r'CITE-\d+', raw, flags=re.I)]
|
| 3400 |
+
clean = re.sub(r'\s*\[?CITE-\d+\]?\s*', ' ', raw, flags=re.I)
|
| 3401 |
+
clean = re.sub(r'\s+', ' ', clean).strip()
|
| 3402 |
+
base = cls._citation_base_provenance(clean, section_name)
|
| 3403 |
+
|
| 3404 |
+
# Structural/hypothesis/positioning/result identities stay authoritative.
|
| 3405 |
+
if base in {'STRUCTURAL', 'HYPOTHESIS_STATEMENT', 'STUDY_POSITIONING', 'RESULT_OWNED_FACT'}:
|
| 3406 |
+
provenance = base
|
| 3407 |
+
components = [base]
|
| 3408 |
+
else:
|
| 3409 |
+
study_fact = cls._citation_study_evidence_signal(clean, section_name)
|
| 3410 |
+
method_principle = cls._citation_claim_requires_method_authority(
|
| 3411 |
+
clean, section_id=section_id, section_name=section_name
|
| 3412 |
+
)
|
| 3413 |
+
author_inference = cls._citation_author_inference_signal(clean, section_name)
|
| 3414 |
+
# Reporting a study-owned reliability/validity value is not itself a
|
| 3415 |
+
# methodological principle merely because those nouns occur. A second
|
| 3416 |
+
# METHOD_AUTHORITY owner is introduced only when the study fact is tied
|
| 3417 |
+
# to an inferential/normative bridge (e.g. means that / must be inferred).
|
| 3418 |
+
if study_fact and method_principle and not author_inference:
|
| 3419 |
+
method_principle = False
|
| 3420 |
+
components = []
|
| 3421 |
+
if study_fact:
|
| 3422 |
+
components.append('STUDY_EVIDENCE_FACT')
|
| 3423 |
+
if method_principle:
|
| 3424 |
+
components.append('METHOD_PRINCIPLE')
|
| 3425 |
+
if author_inference and (study_fact or not method_principle):
|
| 3426 |
+
components.append('AUTHOR_INFERENCE')
|
| 3427 |
+
if len(components) > 1:
|
| 3428 |
+
provenance = 'MIXED_EVIDENCE_OWNERSHIP'
|
| 3429 |
+
elif components:
|
| 3430 |
+
provenance = components[0]
|
| 3431 |
+
else:
|
| 3432 |
+
provenance = 'EXTERNAL_LITERATURE_CLAIM'
|
| 3433 |
+
components = [provenance]
|
| 3434 |
+
|
| 3435 |
+
owner_map = {
|
| 3436 |
+
'STUDY_EVIDENCE_FACT': 'STUDY_EVIDENCE_LEDGER',
|
| 3437 |
+
'STUDY_OWNED_FACT': 'STUDY_EVIDENCE_LEDGER',
|
| 3438 |
+
'RESULT_OWNED_FACT': 'STUDY_EVIDENCE_LEDGER',
|
| 3439 |
+
'METHOD_PRINCIPLE': 'METHOD_AUTHORITY',
|
| 3440 |
+
'EXTERNAL_LITERATURE_CLAIM': 'LITERATURE_EVIDENCE',
|
| 3441 |
+
'AUTHOR_INFERENCE': 'AUTHOR_REASONING',
|
| 3442 |
+
'STUDY_POSITIONING': 'AUTHOR_REASONING',
|
| 3443 |
+
'HYPOTHESIS_STATEMENT': 'HYPOTHESIS_CONTRACT',
|
| 3444 |
+
'STRUCTURAL': 'NONE',
|
| 3445 |
+
'MIXED_EVIDENCE_OWNERSHIP': 'MIXED',
|
| 3446 |
+
}
|
| 3447 |
+
if provenance == 'MIXED_EVIDENCE_OWNERSHIP':
|
| 3448 |
+
citation_requirement = 'SPLIT_REQUIRED'
|
| 3449 |
+
elif provenance in {'METHOD_PRINCIPLE', 'EXTERNAL_LITERATURE_CLAIM'}:
|
| 3450 |
+
citation_requirement = 'REQUIRED'
|
| 3451 |
+
else:
|
| 3452 |
+
citation_requirement = 'NOT_REQUIRED'
|
| 3453 |
+
frame = {
|
| 3454 |
+
'version': cls.CITATION_CLAIM_FRAME_VERSION,
|
| 3455 |
+
'claim_provenance': provenance,
|
| 3456 |
+
'evidence_owner': owner_map.get(provenance, 'LITERATURE_EVIDENCE'),
|
| 3457 |
+
'citation_requirement': citation_requirement,
|
| 3458 |
+
'citation_scope': 'CLAIM_LOCAL' if citation_tags else 'NONE',
|
| 3459 |
+
'citation_count': len(citation_tags),
|
| 3460 |
+
'ownership_components': list(dict.fromkeys(components)),
|
| 3461 |
+
}
|
| 3462 |
+
canonical = json.dumps(frame, ensure_ascii=False, sort_keys=True, separators=(',', ':'))
|
| 3463 |
+
frame['frame_hash'] = hashlib.sha256(canonical.encode('utf-8')).hexdigest()
|
| 3464 |
+
return frame
|
| 3465 |
+
|
| 3466 |
+
@classmethod
|
| 3467 |
+
def _citation_claim_provenance(cls, sentence, section_name=''):
|
| 3468 |
+
provenance = cls.build_citation_claim_frame(sentence, section_name=section_name).get(
|
| 3469 |
+
'claim_provenance', 'EXTERNAL_LITERATURE_CLAIM'
|
| 3470 |
+
)
|
| 3471 |
+
# Legacy public classifier name remains stable for existing gates/callers;
|
| 3472 |
+
# the richer R6-R2 frame exposes STUDY_EVIDENCE_FACT explicitly.
|
| 3473 |
+
return 'STUDY_OWNED_FACT' if provenance == 'STUDY_EVIDENCE_FACT' else provenance
|
| 3474 |
+
|
| 3475 |
+
@classmethod
|
| 3476 |
+
def inspect_citation_evidence_ownership(cls, text, section_name='', section_id=''):
|
| 3477 |
+
"""Return deterministic mixed-ownership citation violations before source QA."""
|
| 3478 |
+
rows = []
|
| 3479 |
+
for _start, _end, sentence in cls._sentence_spans(str(text or '')):
|
| 3480 |
+
raw = str(sentence or '').strip()
|
| 3481 |
+
if not raw or not re.search(r'CITE-\d+', raw, flags=re.I):
|
| 3482 |
+
continue
|
| 3483 |
+
frame = cls.build_citation_claim_frame(raw, section_name=section_name, section_id=section_id)
|
| 3484 |
+
if frame.get('claim_provenance') != 'MIXED_EVIDENCE_OWNERSHIP':
|
| 3485 |
+
continue
|
| 3486 |
+
tags = [f'[{tag.upper()}]' for tag in re.findall(r'CITE-\d+', raw, flags=re.I)]
|
| 3487 |
+
rows.append({
|
| 3488 |
+
'type': 'citation_evidence_ownership_mismatch',
|
| 3489 |
+
'tag': tags[0] if tags else '',
|
| 3490 |
+
'sentence': raw[:500],
|
| 3491 |
+
'reason': (
|
| 3492 |
+
'One citation-bearing sentence mixes claims owned by different evidence authorities: '
|
| 3493 |
+
+ ', '.join(frame.get('ownership_components') or []) + '.'
|
| 3494 |
+
),
|
| 3495 |
+
'repair_instruction': (
|
| 3496 |
+
'Split the sentence by evidence ownership. State the current-study fact from the study evidence without an external citation; '
|
| 3497 |
+
'state a methodological principle separately and cite only a directly supporting METHOD_AUTHORITY source; keep author inference '
|
| 3498 |
+
'separate, cautious, and uncited unless a source directly supports that exact inference.'
|
| 3499 |
+
),
|
| 3500 |
+
})
|
| 3501 |
+
return rows
|
| 3502 |
+
|
| 3503 |
@classmethod
|
| 3504 |
def _citation_recovery_eligible_claim(cls, sentence, section_name=''):
|
| 3505 |
+
return cls._citation_claim_provenance(sentence, section_name) in {'EXTERNAL_LITERATURE_CLAIM', 'METHOD_PRINCIPLE'}
|
| 3506 |
|
| 3507 |
def strip_ineligible_current_study_citations(self, text, section_name=''):
|
| 3508 |
"""Remove external citation tags from current-study/provenance-owned prose.
|
|
|
|
| 3520 |
original = str(sentence or '')
|
| 3521 |
if re.search(r'CITE-\d+', original, re.I):
|
| 3522 |
provenance = self._citation_claim_provenance(original, section_name)
|
| 3523 |
+
if provenance not in {'EXTERNAL_LITERATURE_CLAIM', 'METHOD_PRINCIPLE'}:
|
| 3524 |
revised = self._citation_tag_only_rewrite(original, None)
|
| 3525 |
self._append_citation_audit(
|
| 3526 |
section_name,
|
|
|
|
| 4732 |
"tag": tag,
|
| 4733 |
"reference": metadata.get("ref", ""),
|
| 4734 |
"evidence": metadata.get("evidence", ""),
|
| 4735 |
+
"evidence_lanes": list(metadata.get("evidence_lanes") or []),
|
| 4736 |
}
|
| 4737 |
)
|
| 4738 |
records.append(
|
|
|
|
| 4743 |
"original": sentence,
|
| 4744 |
"allowed_tags": candidates,
|
| 4745 |
"evidence": evidence,
|
| 4746 |
+
"citation_claim_frame": self.build_citation_claim_frame(
|
| 4747 |
+
sentence, section_name=section_name, section_id=section_id
|
| 4748 |
+
),
|
| 4749 |
}
|
| 4750 |
)
|
| 4751 |
|
|
|
|
| 4757 |
{
|
| 4758 |
"id": record["id"],
|
| 4759 |
"sentence": record["original"],
|
| 4760 |
+
"citation_claim_frame": record.get("citation_claim_frame") or {},
|
| 4761 |
"candidate_sources": record["evidence"],
|
| 4762 |
}
|
| 4763 |
for record in batch
|
|
|
|
| 4769 |
2. Every output sentence may contain at most one [CITE-XX] tag.
|
| 4770 |
3. When several sources support the same claim, retain only the single source whose supplied evidence most directly supports that claim.
|
| 4771 |
4. Use status=split only when no single candidate source directly supports the complete original sentence AND at least two distinct candidate sources are needed for genuinely different claims. If one candidate source directly supports the complete sentence, status must be ok and that single tag must be selected; do not split merely because the sentence has multiple clauses.
|
| 4772 |
+
5. The supplied citation_claim_frame is server-frozen evidence-ownership authority. METHOD_PRINCIPLE claims may use only candidate sources marked method_authority. Current-study facts and AUTHOR_INFERENCE are not external-literature claims. A mixed-ownership sentence must not let one citation cover multiple evidence owners.
|
| 4773 |
+
6. A source may be used only when its supplied title/abstract evidence directly supports the adjacent claim. Do not infer support from the reference title alone when the evidence is empty or unrelated.
|
| 4774 |
+
7. Use only candidate tags supplied for that item. Never invent a source, fact, number, quotation, or citation tag.
|
| 4775 |
7. If no candidate source directly supports a claim that requires external evidence, set status to unsupported and leave revised_text equal to the original sentence.
|
| 4776 |
8. If the statement is ordinary connective prose that does not require external support, status may be uncited_common_knowledge and revised_text may omit the tag without changing the prose.
|
| 4777 |
|
|
|
|
| 5131 |
self, text, section_name='', section_id='', quarantine=False, allowed_tags=None
|
| 5132 |
):
|
| 5133 |
self._last_citation_semantic_violations = []
|
| 5134 |
+
ownership_rows = self.inspect_citation_evidence_ownership(
|
| 5135 |
+
str(text or ''), section_name=section_name, section_id=section_id
|
| 5136 |
+
)
|
| 5137 |
value = self.strip_ineligible_current_study_citations(str(text or ''), section_name)
|
| 5138 |
if quarantine:
|
| 5139 |
return self.resolve_citations(value, quarantine=True)
|
|
|
|
| 5191 |
value = self._local_citation_recovery(value, section_name, retain_lexical=False)
|
| 5192 |
elif violations:
|
| 5193 |
value = self._local_citation_recovery(value, section_name)
|
| 5194 |
+
if ownership_rows or outside_rows:
|
| 5195 |
+
self._last_citation_semantic_violations = (
|
| 5196 |
+
list(ownership_rows)
|
| 5197 |
+
+ list(self._last_citation_semantic_violations or [])
|
| 5198 |
+
+ list(outside_rows)
|
| 5199 |
+
)
|
| 5200 |
if self.find_multi_citation_violations(value):
|
| 5201 |
value = self._local_citation_recovery(value, section_name)
|
| 5202 |
if self.find_multi_citation_violations(value):
|
|
|
|
| 6411 |
)
|
| 6412 |
|
| 6413 |
@classmethod
|
| 6414 |
+
def _source_semantic_claim_frame(cls, source_claim, section_name=''):
|
| 6415 |
"""Derive a conservative language-neutral frame once from frozen English.
|
| 6416 |
|
| 6417 |
This frame is not a substitute for bilingual QA. It is an immutable
|
|
|
|
| 6457 |
modality = "LIMITATION"
|
| 6458 |
else:
|
| 6459 |
modality = "ASSERTED"
|
| 6460 |
+
citation_frame = cls.build_citation_claim_frame(text, section_name=section_name)
|
| 6461 |
frame = {
|
| 6462 |
"version": cls.SEMANTIC_CLAIM_FRAME_VERSION,
|
| 6463 |
"polarity": polarity,
|
| 6464 |
"causal_strength": causal_strength,
|
| 6465 |
"modality": modality,
|
| 6466 |
+
"citation_contract_version": cls.CITATION_CLAIM_FRAME_VERSION,
|
| 6467 |
+
"claim_provenance": citation_frame.get("claim_provenance", "EXTERNAL_LITERATURE_CLAIM"),
|
| 6468 |
+
"evidence_owner": citation_frame.get("evidence_owner", "LITERATURE_EVIDENCE"),
|
| 6469 |
+
"citation_requirement": citation_frame.get("citation_requirement", "NOT_REQUIRED"),
|
| 6470 |
+
"citation_scope": citation_frame.get("citation_scope", "NONE"),
|
| 6471 |
+
"citation_count": int(citation_frame.get("citation_count") or 0),
|
| 6472 |
}
|
| 6473 |
canonical = json.dumps(frame, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
| 6474 |
frame["frame_hash"] = hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
|
|
|
| 7425 |
'claim_id': claim_id,
|
| 7426 |
'english': item['text'],
|
| 7427 |
'target': target_claims[claim_id],
|
| 7428 |
+
'source_semantic_frame': self._source_semantic_claim_frame(item['text'], section_name),
|
| 7429 |
})
|
| 7430 |
|
| 7431 |
issues = {}
|
|
|
|
| 7448 |
profile = self._translation_language_profile(target_lang)
|
| 7449 |
lexical_contract = self._translation_lexical_retention_contract(target_lang)
|
| 7450 |
system_prompt = f"""You are a bilingual academic translation QA classifier for {target_lang}.
|
| 7451 |
+
Review EACH claim independently. Do not rewrite and do not provide free-form explanations. The supplied source_semantic_frame is server-frozen source authority: the target must preserve its polarity, causal_strength, modality, claim_provenance, evidence_owner, citation_scope, and citation_count. Check semantic equivalence and language quality: omissions/additions, negation, uncertainty, association-versus-causation strength, terminology, grammar, language contamination, citation attachment, and acronym attachment.
|
| 7452 |
LANGUAGE ADAPTER: {profile['adapter_id']}; morphology={profile['morphology']}. {profile['term_policy']} {lexical_contract} {profile['contamination_policy']}
|
| 7453 |
For Russian, do not flag a terminology error merely because a translated term is inflected for grammatical case/number/gender; sustained non-Russian prose remains an additional contamination signal. For Kazakh, do not flag a terminology error merely because productive suffixes attach to the correct term stem; clearly Russian lexical/syntactic intrusion remains an additional language-specific signal when the pair supports that conclusion. Cyrillic script or absence of Kazakh-specific letters alone is never sufficient evidence.
|
| 7454 |
Return valid JSON only in this exact form: {{"items":[{{"id":"SP-P000001-C000001","ok":true,"codes":[],"evidence_codes":[]}}]}}.
|
|
|
|
| 7843 |
|
| 7844 |
def _translate_marked_batch_with_recovery(self, batch, system_prompt, target_lang, _strict=False):
|
| 7845 |
"""Translate SP-P blocks through claim-bound, value-blind protected skeletons."""
|
| 7846 |
+
section_name = str(getattr(self, '_translation_section_name', '') or '')
|
| 7847 |
expected_ids = [block_id for block_id, _ in batch]
|
| 7848 |
protected_batch = []
|
| 7849 |
protected_maps = {}
|
|
|
|
| 7863 |
}
|
| 7864 |
claim_semantic_manifest = {
|
| 7865 |
block_id: {
|
| 7866 |
+
claim_id: self._source_semantic_claim_frame(source_claim, section_name)
|
| 7867 |
for claim_id, source_claim in dict(claim_sources_by_block.get(block_id) or {}).items()
|
| 7868 |
}
|
| 7869 |
for block_id, _source, _protected in protected_batch
|
|
|
|
| 7875 |
+ '<SOURCE_SEMANTIC_CLAIM_FRAMES>\n'
|
| 7876 |
+ json.dumps(claim_semantic_manifest, ensure_ascii=False)
|
| 7877 |
+ '\n</SOURCE_SEMANTIC_CLAIM_FRAMES>\n'
|
| 7878 |
+
'The source semantic claim frames are server-frozen. Preserve polarity, causal strength, and modality for each SP-C claim exactly; also preserve claim provenance, evidence owner, and citation scope/count. Do not strengthen an association into causation, drop a negation/hedge, merge citation scope across claims, or move citation attachment to a neighboring claim. '
|
| 7879 |
'This manifest intentionally contains no factual values. Preserve every SP-P, SP-C, and SP-T marker exactly. '
|
| 7880 |
'Each SP-C marker identifies one source claim. A target claim may contain one or more grammatical sentences, but every protected fact must remain inside its original SP-C claim. '
|
| 7881 |
'Never infer, spell out, calculate, or recreate a hidden number, date, citation, acronym, name, formula, or identifier. Do not print the manifest.'
|
|
|
|
| 8181 |
|
| 8182 |
@classmethod
|
| 8183 |
def _snapshot_translation_claim_states(
|
| 8184 |
+
cls, source_blocks, translations, pending_issues=None, repair_evidence=None, repair_meta=None, section_name=''
|
| 8185 |
):
|
| 8186 |
"""Freeze claim text plus bounded repair history so Final Gate cannot restart a failed strategy."""
|
| 8187 |
pending = dict(pending_issues or {})
|
|
|
|
| 8208 |
'block_id': str(block_id),
|
| 8209 |
'claim_id': str(claim_id),
|
| 8210 |
'source_hash': hashlib.sha256(str(item['text']).encode('utf-8')).hexdigest(),
|
| 8211 |
+
'semantic_frame': cls._source_semantic_claim_frame(item['text'], section_name),
|
| 8212 |
+
'semantic_frame_hash': cls._source_semantic_claim_frame(item['text'], section_name).get('frame_hash', ''),
|
| 8213 |
'target_text': target_text,
|
| 8214 |
'qa_status': 'repair_pending' if codes else 'passed',
|
| 8215 |
'issue_codes': [str(code or '').strip().upper() for code in codes if str(code or '').strip()][:5],
|
|
|
|
| 8223 |
return states
|
| 8224 |
|
| 8225 |
@classmethod
|
| 8226 |
+
def _restore_translation_claim_states(cls, source_blocks, claim_states, section_name=''):
|
| 8227 |
"""Reassemble marked blocks only when every frozen source claim has a hash-matched durable target."""
|
| 8228 |
states = dict(claim_states or {})
|
| 8229 |
if not states:
|
|
|
|
| 8239 |
key = f'SP-P{block_id}-{claim_id}'
|
| 8240 |
row = states.get(key)
|
| 8241 |
expected_hash = hashlib.sha256(str(item['text']).encode('utf-8')).hexdigest()
|
| 8242 |
+
expected_semantic_hash = cls._source_semantic_claim_frame(item['text'], section_name).get('frame_hash', '')
|
| 8243 |
if not isinstance(row, dict) or row.get('source_hash') != expected_hash or not str(row.get('target_text') or '').strip():
|
| 8244 |
return {}, {}, False
|
| 8245 |
# R6-R1: new states are semantic-frame bound. Pre-R6 states do
|
|
|
|
| 8615 |
+ json.dumps(frozen_glossary, ensure_ascii=False)
|
| 8616 |
)
|
| 8617 |
source_blocks = {block_id: source for block_id, source in pending_blocks}
|
| 8618 |
+
self._translation_section_name = str(section_name or 'frozen section')
|
| 8619 |
attempts = max(1, int(max_attempts or 1))
|
| 8620 |
last_error = None
|
| 8621 |
self._last_translation_claim_states = {}
|
|
|
|
| 8667 |
translations = {}
|
| 8668 |
try:
|
| 8669 |
translations, quality_issues, resumed_claims = self._restore_translation_claim_states(
|
| 8670 |
+
source_blocks, claim_states, section_name=section_name or 'frozen section'
|
| 8671 |
)
|
| 8672 |
if resumed_claims:
|
| 8673 |
for row in dict(claim_states or {}).values():
|
|
|
|
| 8700 |
)
|
| 8701 |
self._last_translation_claim_states = self._snapshot_translation_claim_states(
|
| 8702 |
source_blocks, translations, quality_issues, self._translation_claim_qa_evidence,
|
| 8703 |
+
self._translation_claim_repair_meta, section_name=section_name or 'frozen section',
|
| 8704 |
)
|
| 8705 |
repair_cycle = 0
|
| 8706 |
total_claims = sum(len(self._translation_claim_units(source)) for source in source_blocks.values())
|
|
|
|
| 8943 |
quality_issues = refreshed
|
| 8944 |
self._last_translation_claim_states = self._snapshot_translation_claim_states(
|
| 8945 |
source_blocks, translations, quality_issues, self._translation_claim_qa_evidence,
|
| 8946 |
+
self._translation_claim_repair_meta, section_name=section_name or 'frozen section',
|
| 8947 |
)
|
| 8948 |
if not touched_blocks and carried_exhausted:
|
| 8949 |
break
|
|
|
|
| 8967 |
)
|
| 8968 |
self._last_translation_claim_states = self._snapshot_translation_claim_states(
|
| 8969 |
source_blocks, translations, quality_issues, self._translation_claim_qa_evidence,
|
| 8970 |
+
self._translation_claim_repair_meta, section_name=section_name or 'frozen section',
|
| 8971 |
)
|
| 8972 |
if quality_issues:
|
| 8973 |
detail = '; '.join(
|