Spaces:
Running
Running
P0-R1.1-R1 Four-Language Dual-Engine Evidence Integrity
#70
by GuXiaopeng - opened
- app.py +21 -17
- core/generation_runner.py +174 -3
- core/pipeline_agent.py +27 -1
- core/quantitative_engine.py +304 -0
app.py
CHANGED
|
@@ -54,27 +54,23 @@ components.html(WECHAT_BLOCKER_JS, height=0, width=0)
|
|
| 54 |
|
| 55 |
|
| 56 |
def inject_file_uploader_i18n(language: str) -> None:
|
| 57 |
-
"""Localize
|
| 58 |
payload = {
|
| 59 |
"browse": tr(language, "browse_files"),
|
| 60 |
-
"uploaders":
|
| 61 |
-
{
|
| 62 |
"title": tr(language, "drop_docs"),
|
| 63 |
"limit": tr(language, "limit_docs"),
|
| 64 |
},
|
| 65 |
-
{
|
| 66 |
-
"title": tr(language, "drop_data"),
|
| 67 |
-
"limit": tr(language, "limit_data"),
|
| 68 |
-
},
|
| 69 |
-
{
|
| 70 |
-
"title": tr(language, "drop_docs"),
|
| 71 |
-
"limit": tr(language, "limit_docs"),
|
| 72 |
-
},
|
| 73 |
-
{
|
| 74 |
"title": tr(language, "drop_interviews"),
|
| 75 |
"limit": tr(language, "limit_interviews"),
|
| 76 |
},
|
| 77 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 78 |
}
|
| 79 |
|
| 80 |
script = f"""
|
|
@@ -98,14 +94,22 @@ def inject_file_uploader_i18n(language: str) -> None:
|
|
| 98 |
}}
|
| 99 |
}}
|
| 100 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 101 |
function applyUploaderTranslations() {{
|
| 102 |
-
const
|
| 103 |
-
'[data-testid="
|
| 104 |
);
|
| 105 |
|
| 106 |
-
|
| 107 |
-
const config =
|
| 108 |
if (!config) return;
|
|
|
|
|
|
|
| 109 |
|
| 110 |
const instructions = zone.querySelector(
|
| 111 |
'[data-testid="stFileUploaderDropzoneInstructions"]'
|
|
|
|
| 54 |
|
| 55 |
|
| 56 |
def inject_file_uploader_i18n(language: str) -> None:
|
| 57 |
+
"""Localize uploaders by semantic label, never by visual/tab position."""
|
| 58 |
payload = {
|
| 59 |
"browse": tr(language, "browse_files"),
|
| 60 |
+
"uploaders": {
|
| 61 |
+
tr(language, "upload_reference"): {
|
| 62 |
"title": tr(language, "drop_docs"),
|
| 63 |
"limit": tr(language, "limit_docs"),
|
| 64 |
},
|
| 65 |
+
tr(language, "upload_interviews"): {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 66 |
"title": tr(language, "drop_interviews"),
|
| 67 |
"limit": tr(language, "limit_interviews"),
|
| 68 |
},
|
| 69 |
+
tr(language, "upload_stats"): {
|
| 70 |
+
"title": tr(language, "drop_data"),
|
| 71 |
+
"limit": tr(language, "limit_data"),
|
| 72 |
+
},
|
| 73 |
+
},
|
| 74 |
}
|
| 75 |
|
| 76 |
script = f"""
|
|
|
|
| 94 |
}}
|
| 95 |
}}
|
| 96 |
|
| 97 |
+
function semanticUploaderConfig(uploader) {{
|
| 98 |
+
const label = uploader.querySelector('label');
|
| 99 |
+
const labelText = (label ? label.textContent : '').trim();
|
| 100 |
+
return payload.uploaders[labelText] || null;
|
| 101 |
+
}}
|
| 102 |
+
|
| 103 |
function applyUploaderTranslations() {{
|
| 104 |
+
const uploaders = parentDocument.querySelectorAll(
|
| 105 |
+
'[data-testid="stFileUploader"]'
|
| 106 |
);
|
| 107 |
|
| 108 |
+
uploaders.forEach((uploader) => {{
|
| 109 |
+
const config = semanticUploaderConfig(uploader);
|
| 110 |
if (!config) return;
|
| 111 |
+
const zone = uploader.querySelector('[data-testid="stFileUploaderDropzone"]');
|
| 112 |
+
if (!zone) return;
|
| 113 |
|
| 114 |
const instructions = zone.querySelector(
|
| 115 |
'[data-testid="stFileUploaderDropzoneInstructions"]'
|
core/generation_runner.py
CHANGED
|
@@ -901,6 +901,34 @@ class GenerationRunner:
|
|
| 901 |
match = re.search(rf'(?m)^#\s+{re.escape(str(first_section_title))}\s*$', value)
|
| 902 |
return value[:match.start()].strip() if match else ''
|
| 903 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 904 |
@staticmethod
|
| 905 |
def _replace_keywords(manuscript: str, keywords) -> str:
|
| 906 |
value = str(manuscript or '')
|
|
@@ -1133,6 +1161,15 @@ class GenerationRunner:
|
|
| 1133 |
source_hash = identity['source_hash']
|
| 1134 |
compiled_source_hash = identity['compiled_source_hash']
|
| 1135 |
fact_ledger_hash = identity['fact_ledger_hash']
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1136 |
existing = dict((translation_sections or {}).get(unit_key) or {})
|
| 1137 |
if not isinstance(getattr(self, '_translation_progress_queued', None), set):
|
| 1138 |
self._translation_progress_queued = set()
|
|
@@ -1149,6 +1186,8 @@ class GenerationRunner:
|
|
| 1149 |
and existing.get('target_lang') == str(self._translation_worker_context.get('target_lang') or '')
|
| 1150 |
and existing.get('translation_contract_version') == TRANSLATION_CONTRACT_VERSION
|
| 1151 |
and existing.get('fact_ledger_hash') == fact_ledger_hash
|
|
|
|
|
|
|
| 1152 |
and existing.get('translated_text')
|
| 1153 |
):
|
| 1154 |
return False
|
|
@@ -1157,11 +1196,13 @@ class GenerationRunner:
|
|
| 1157 |
if existing.get('source_hash') != source_hash: reasons.append('english_source_hash')
|
| 1158 |
if existing.get('compiled_source_hash') != compiled_source_hash: reasons.append('compiled_source_hash')
|
| 1159 |
if existing.get('fact_ledger_hash') != fact_ledger_hash: reasons.append('fact_ledger_hash')
|
|
|
|
|
|
|
| 1160 |
if existing.get('translation_contract_version') != TRANSLATION_CONTRACT_VERSION: reasons.append('contract_version')
|
| 1161 |
if existing.get('target_lang') != str(self._translation_worker_context.get('target_lang') or ''): reasons.append('target_lang')
|
| 1162 |
if existing.get('qa_status') not in {'passed', 'passed_with_warning'}: reasons.append('qa_status')
|
| 1163 |
logging.info('Translation reuse miss | unit=%s phase=enqueue reasons=%s', unit_name, ','.join(reasons) or 'unknown')
|
| 1164 |
-
pending_key = (str(unit_key), source_hash, compiled_source_hash, fact_ledger_hash)
|
| 1165 |
if pending_key in self._translation_worker_pending:
|
| 1166 |
return False
|
| 1167 |
self._translation_progress_inflight.add(str(unit_key))
|
|
@@ -1241,6 +1282,8 @@ class GenerationRunner:
|
|
| 1241 |
and existing.get('target_lang') == target_lang
|
| 1242 |
and existing.get('translation_contract_version') == contract_version
|
| 1243 |
and existing.get('fact_ledger_hash') == fact_ledger_hash
|
|
|
|
|
|
|
| 1244 |
and existing.get('translated_text')
|
| 1245 |
):
|
| 1246 |
self._translation_progress_inflight.discard(str(unit_key))
|
|
@@ -1250,6 +1293,8 @@ class GenerationRunner:
|
|
| 1250 |
if existing.get('source_hash') != source_hash: reasons.append('english_source_hash')
|
| 1251 |
if existing.get('compiled_source_hash') != compiled_source_hash: reasons.append('compiled_source_hash')
|
| 1252 |
if existing.get('fact_ledger_hash') != fact_ledger_hash: reasons.append('fact_ledger_hash')
|
|
|
|
|
|
|
| 1253 |
if existing.get('translation_contract_version') != contract_version: reasons.append('contract_version')
|
| 1254 |
if existing.get('target_lang') != target_lang: reasons.append('target_lang')
|
| 1255 |
if existing.get('qa_status') not in {'passed', 'passed_with_warning'}: reasons.append('qa_status')
|
|
@@ -2009,6 +2054,23 @@ class GenerationRunner:
|
|
| 2009 |
working_keywords = list(working_front.get("keywords") or [])
|
| 2010 |
if not working_abstract or not working_keywords:
|
| 2011 |
raise RuntimeError("working_front_matter_missing")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2012 |
agent.global_memory["working_abstract"] = working_abstract
|
| 2013 |
|
| 2014 |
self._progress(14, "outline_planning")
|
|
@@ -2230,6 +2292,26 @@ class GenerationRunner:
|
|
| 2230 |
state['canonical_concept_ledger'] = canonical_concept_ledger
|
| 2231 |
working_abstract = str(state.get("working_abstract") or "").strip()
|
| 2232 |
working_keywords = list(state.get("working_keywords") or [])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2233 |
if working_abstract:
|
| 2234 |
agent.global_memory["working_abstract"] = working_abstract
|
| 2235 |
if self._preview_target_text == '':
|
|
@@ -2249,6 +2331,29 @@ class GenerationRunner:
|
|
| 2249 |
state['translation_glossary'] = translation_glossary
|
| 2250 |
state['canonical_concept_ledger'] = canonical_concept_ledger
|
| 2251 |
self._checkpoint('canonical_concept_ledger_ready', state, 15)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2252 |
self._start_translation_worker(
|
| 2253 |
translation_agent,
|
| 2254 |
translation_sections=translation_sections,
|
|
@@ -2277,6 +2382,23 @@ class GenerationRunner:
|
|
| 2277 |
working_keywords = list(working_front.get("keywords") or [])
|
| 2278 |
if not working_abstract or not working_keywords:
|
| 2279 |
raise RuntimeError("working_front_matter_missing")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2280 |
agent.global_memory["working_abstract"] = working_abstract
|
| 2281 |
full_draft_en = replace_abstract(full_draft_en, working_abstract)
|
| 2282 |
full_draft_en = self._replace_keywords(full_draft_en, working_keywords)
|
|
@@ -2685,6 +2807,24 @@ Output only final academic prose. Do not report the word count."""
|
|
| 2685 |
agent.validate_qualitative_study_fact_contract(
|
| 2686 |
en_resolved, data_metadata, section_name=display_name
|
| 2687 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2688 |
en_resolved, enumeration_rows = agent.repair_and_validate_enumeration_contract(
|
| 2689 |
en_resolved, section_name=display_name
|
| 2690 |
)
|
|
@@ -2777,14 +2917,28 @@ Output only final academic prose. Do not report the word count."""
|
|
| 2777 |
if not state.get("english_finalized"):
|
| 2778 |
self._progress(91, "english_validation")
|
| 2779 |
full_draft_en = _clean_trailing_references(full_draft_en, ["Reference", "References", "Reference List"])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2780 |
if mode == "asq":
|
| 2781 |
fact_checked, fact_rows = agent.repair_qualitative_study_fact_contract(
|
| 2782 |
-
|
| 2783 |
)
|
| 2784 |
-
if fact_checked.strip() !=
|
| 2785 |
raise RuntimeError(
|
| 2786 |
"final_qualitative_study_fact_contract_requires_section_regeneration"
|
| 2787 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2788 |
enum_checked, enum_rows = agent.repair_and_validate_enumeration_contract(
|
| 2789 |
full_draft_en, section_name="Final English manuscript"
|
| 2790 |
)
|
|
@@ -2897,6 +3051,23 @@ Output only final academic prose. Do not report the word count."""
|
|
| 2897 |
final_keywords = list(final_front.get("keywords") or [])
|
| 2898 |
if not final_abstract or not final_keywords:
|
| 2899 |
raise RuntimeError("final_front_matter_missing")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2900 |
full_draft_en = replace_abstract(full_draft_en, final_abstract)
|
| 2901 |
full_draft_en = self._replace_keywords(full_draft_en, final_keywords)
|
| 2902 |
body_only, _reference_tail = agent.validate_reference_tail(
|
|
|
|
| 901 |
match = re.search(rf'(?m)^#\s+{re.escape(str(first_section_title))}\s*$', value)
|
| 902 |
return value[:match.start()].strip() if match else ''
|
| 903 |
|
| 904 |
+
@staticmethod
|
| 905 |
+
def _extract_body_from_first_section(manuscript: str, first_section_title: str) -> str:
|
| 906 |
+
value = str(manuscript or '')
|
| 907 |
+
match = re.search(rf'(?m)^#\s+{re.escape(str(first_section_title))}\s*$', value)
|
| 908 |
+
return value[match.start():].strip() if match else value.strip()
|
| 909 |
+
|
| 910 |
+
@staticmethod
|
| 911 |
+
def _repair_validate_study_fact_contract(
|
| 912 |
+
mode: str, agent, quant_engine, text: str, data_metadata: dict, *, section_name: str
|
| 913 |
+
):
|
| 914 |
+
value = str(text or '').strip()
|
| 915 |
+
diagnostics = []
|
| 916 |
+
if str(mode or '') == 'asq':
|
| 917 |
+
value, diagnostics = agent.repair_qualitative_study_fact_contract(
|
| 918 |
+
value, data_metadata, section_name=section_name
|
| 919 |
+
)
|
| 920 |
+
agent.validate_qualitative_study_fact_contract(
|
| 921 |
+
value, data_metadata, section_name=section_name
|
| 922 |
+
)
|
| 923 |
+
elif str(mode or '') == 'amj':
|
| 924 |
+
value, diagnostics = quant_engine.repair_quantitative_study_fact_contract(
|
| 925 |
+
value, data_metadata, section_name=section_name
|
| 926 |
+
)
|
| 927 |
+
quant_engine.validate_quantitative_study_fact_contract(
|
| 928 |
+
value, data_metadata, section_name=section_name
|
| 929 |
+
)
|
| 930 |
+
return value, list(diagnostics or [])
|
| 931 |
+
|
| 932 |
@staticmethod
|
| 933 |
def _replace_keywords(manuscript: str, keywords) -> str:
|
| 934 |
value = str(manuscript or '')
|
|
|
|
| 1161 |
source_hash = identity['source_hash']
|
| 1162 |
compiled_source_hash = identity['compiled_source_hash']
|
| 1163 |
fact_ledger_hash = identity['fact_ledger_hash']
|
| 1164 |
+
entity_ledger_method = getattr(agent, 'translation_protected_entity_ledger_hash', None)
|
| 1165 |
+
protected_entity_ledger_hash = str(
|
| 1166 |
+
entity_ledger_method(identity['compiled_source'], self._translation_worker_glossary)
|
| 1167 |
+
if callable(entity_ledger_method) else ''
|
| 1168 |
+
)
|
| 1169 |
+
protected_entity_ledger_version = str(
|
| 1170 |
+
getattr(agent, 'PROTECTED_ENTITY_LEDGER_VERSION', 'protected-entity-ledger-v1')
|
| 1171 |
+
or 'protected-entity-ledger-v1'
|
| 1172 |
+
)
|
| 1173 |
existing = dict((translation_sections or {}).get(unit_key) or {})
|
| 1174 |
if not isinstance(getattr(self, '_translation_progress_queued', None), set):
|
| 1175 |
self._translation_progress_queued = set()
|
|
|
|
| 1186 |
and existing.get('target_lang') == str(self._translation_worker_context.get('target_lang') or '')
|
| 1187 |
and existing.get('translation_contract_version') == TRANSLATION_CONTRACT_VERSION
|
| 1188 |
and existing.get('fact_ledger_hash') == fact_ledger_hash
|
| 1189 |
+
and existing.get('protected_entity_ledger_version') == protected_entity_ledger_version
|
| 1190 |
+
and existing.get('protected_entity_ledger_hash') == protected_entity_ledger_hash
|
| 1191 |
and existing.get('translated_text')
|
| 1192 |
):
|
| 1193 |
return False
|
|
|
|
| 1196 |
if existing.get('source_hash') != source_hash: reasons.append('english_source_hash')
|
| 1197 |
if existing.get('compiled_source_hash') != compiled_source_hash: reasons.append('compiled_source_hash')
|
| 1198 |
if existing.get('fact_ledger_hash') != fact_ledger_hash: reasons.append('fact_ledger_hash')
|
| 1199 |
+
if existing.get('protected_entity_ledger_version') != protected_entity_ledger_version: reasons.append('protected_entity_ledger_version')
|
| 1200 |
+
if existing.get('protected_entity_ledger_hash') != protected_entity_ledger_hash: reasons.append('protected_entity_ledger_hash')
|
| 1201 |
if existing.get('translation_contract_version') != TRANSLATION_CONTRACT_VERSION: reasons.append('contract_version')
|
| 1202 |
if existing.get('target_lang') != str(self._translation_worker_context.get('target_lang') or ''): reasons.append('target_lang')
|
| 1203 |
if existing.get('qa_status') not in {'passed', 'passed_with_warning'}: reasons.append('qa_status')
|
| 1204 |
logging.info('Translation reuse miss | unit=%s phase=enqueue reasons=%s', unit_name, ','.join(reasons) or 'unknown')
|
| 1205 |
+
pending_key = (str(unit_key), source_hash, compiled_source_hash, fact_ledger_hash, protected_entity_ledger_version, protected_entity_ledger_hash)
|
| 1206 |
if pending_key in self._translation_worker_pending:
|
| 1207 |
return False
|
| 1208 |
self._translation_progress_inflight.add(str(unit_key))
|
|
|
|
| 1282 |
and existing.get('target_lang') == target_lang
|
| 1283 |
and existing.get('translation_contract_version') == contract_version
|
| 1284 |
and existing.get('fact_ledger_hash') == fact_ledger_hash
|
| 1285 |
+
and existing.get('protected_entity_ledger_version') == protected_entity_ledger_version
|
| 1286 |
+
and existing.get('protected_entity_ledger_hash') == protected_entity_ledger_hash
|
| 1287 |
and existing.get('translated_text')
|
| 1288 |
):
|
| 1289 |
self._translation_progress_inflight.discard(str(unit_key))
|
|
|
|
| 1293 |
if existing.get('source_hash') != source_hash: reasons.append('english_source_hash')
|
| 1294 |
if existing.get('compiled_source_hash') != compiled_source_hash: reasons.append('compiled_source_hash')
|
| 1295 |
if existing.get('fact_ledger_hash') != fact_ledger_hash: reasons.append('fact_ledger_hash')
|
| 1296 |
+
if existing.get('protected_entity_ledger_version') != protected_entity_ledger_version: reasons.append('protected_entity_ledger_version')
|
| 1297 |
+
if existing.get('protected_entity_ledger_hash') != protected_entity_ledger_hash: reasons.append('protected_entity_ledger_hash')
|
| 1298 |
if existing.get('translation_contract_version') != contract_version: reasons.append('contract_version')
|
| 1299 |
if existing.get('target_lang') != target_lang: reasons.append('target_lang')
|
| 1300 |
if existing.get('qa_status') not in {'passed', 'passed_with_warning'}: reasons.append('qa_status')
|
|
|
|
| 2054 |
working_keywords = list(working_front.get("keywords") or [])
|
| 2055 |
if not working_abstract or not working_keywords:
|
| 2056 |
raise RuntimeError("working_front_matter_missing")
|
| 2057 |
+
working_abstract, front_fact_rows = self._repair_validate_study_fact_contract(
|
| 2058 |
+
mode, agent, quant_engine, working_abstract, data_metadata,
|
| 2059 |
+
section_name="Working Abstract",
|
| 2060 |
+
)
|
| 2061 |
+
if not working_abstract:
|
| 2062 |
+
raise RuntimeError("working_front_matter_empty_after_study_fact_contract")
|
| 2063 |
+
if front_fact_rows:
|
| 2064 |
+
logging.warning(
|
| 2065 |
+
"Working Abstract study-fact repair | mode=%s count=%s types=%s",
|
| 2066 |
+
mode, len(front_fact_rows),
|
| 2067 |
+
sorted({str(row.get("type") or "") for row in front_fact_rows}),
|
| 2068 |
+
)
|
| 2069 |
+
quality_diagnostics.append({
|
| 2070 |
+
"scope":"working_abstract_study_fact_integrity", "name":"Working Abstract",
|
| 2071 |
+
"status":"deterministic_repair", "count":len(front_fact_rows),
|
| 2072 |
+
"examples":front_fact_rows[:3],
|
| 2073 |
+
})
|
| 2074 |
agent.global_memory["working_abstract"] = working_abstract
|
| 2075 |
|
| 2076 |
self._progress(14, "outline_planning")
|
|
|
|
| 2292 |
state['canonical_concept_ledger'] = canonical_concept_ledger
|
| 2293 |
working_abstract = str(state.get("working_abstract") or "").strip()
|
| 2294 |
working_keywords = list(state.get("working_keywords") or [])
|
| 2295 |
+
if working_abstract and not state.get("english_finalized"):
|
| 2296 |
+
checked_working_abstract, resumed_front_rows = self._repair_validate_study_fact_contract(
|
| 2297 |
+
mode, agent, quant_engine, working_abstract, data_metadata,
|
| 2298 |
+
section_name="Working Abstract",
|
| 2299 |
+
)
|
| 2300 |
+
if not checked_working_abstract:
|
| 2301 |
+
raise RuntimeError("working_front_matter_empty_after_study_fact_contract")
|
| 2302 |
+
if checked_working_abstract != working_abstract or resumed_front_rows:
|
| 2303 |
+
working_abstract = checked_working_abstract
|
| 2304 |
+
state["working_abstract"] = working_abstract
|
| 2305 |
+
if str(state.get("full_draft_en") or "").strip():
|
| 2306 |
+
state["full_draft_en"] = replace_abstract(str(state.get("full_draft_en") or ""), working_abstract)
|
| 2307 |
+
full_draft_en = str(state["full_draft_en"] or "")
|
| 2308 |
+
quality_diagnostics.append({
|
| 2309 |
+
"scope":"working_abstract_study_fact_integrity", "name":"Working Abstract",
|
| 2310 |
+
"status":"resume_deterministic_repair", "count":len(resumed_front_rows),
|
| 2311 |
+
"examples":resumed_front_rows[:3],
|
| 2312 |
+
})
|
| 2313 |
+
state["quality_diagnostics"] = quality_diagnostics
|
| 2314 |
+
self._checkpoint("working_front_matter_fact_reconciled", state, 15)
|
| 2315 |
if working_abstract:
|
| 2316 |
agent.global_memory["working_abstract"] = working_abstract
|
| 2317 |
if self._preview_target_text == '':
|
|
|
|
| 2331 |
state['translation_glossary'] = translation_glossary
|
| 2332 |
state['canonical_concept_ledger'] = canonical_concept_ledger
|
| 2333 |
self._checkpoint('canonical_concept_ledger_ready', state, 15)
|
| 2334 |
+
|
| 2335 |
+
current_entity_ledger_version = str(
|
| 2336 |
+
getattr(translation_agent, 'PROTECTED_ENTITY_LEDGER_VERSION', 'protected-entity-ledger-v1')
|
| 2337 |
+
or 'protected-entity-ledger-v1'
|
| 2338 |
+
)
|
| 2339 |
+
stale_entity_units = [
|
| 2340 |
+
key for key, record in translation_sections.items()
|
| 2341 |
+
if isinstance(record, dict)
|
| 2342 |
+
and record.get('translated_text')
|
| 2343 |
+
and record.get('qa_status') in {'passed', 'passed_with_warning'}
|
| 2344 |
+
and record.get('protected_entity_ledger_version') != current_entity_ledger_version
|
| 2345 |
+
]
|
| 2346 |
+
if stale_entity_units:
|
| 2347 |
+
logging.warning(
|
| 2348 |
+
'Protected Entity Ledger contract advanced; invalidating target-language cache only | units=%s version=%s',
|
| 2349 |
+
stale_entity_units[:8], current_entity_ledger_version,
|
| 2350 |
+
)
|
| 2351 |
+
state['translation_complete'] = False
|
| 2352 |
+
state['package_validated'] = False
|
| 2353 |
+
state['full_draft_translated'] = ''
|
| 2354 |
+
self._preview_target_text = ''
|
| 2355 |
+
self._checkpoint('protected_entity_ledger_advanced', state, 15)
|
| 2356 |
+
|
| 2357 |
self._start_translation_worker(
|
| 2358 |
translation_agent,
|
| 2359 |
translation_sections=translation_sections,
|
|
|
|
| 2382 |
working_keywords = list(working_front.get("keywords") or [])
|
| 2383 |
if not working_abstract or not working_keywords:
|
| 2384 |
raise RuntimeError("working_front_matter_missing")
|
| 2385 |
+
working_abstract, front_fact_rows = self._repair_validate_study_fact_contract(
|
| 2386 |
+
mode, agent, quant_engine, working_abstract, data_metadata,
|
| 2387 |
+
section_name="Working Abstract",
|
| 2388 |
+
)
|
| 2389 |
+
if not working_abstract:
|
| 2390 |
+
raise RuntimeError("working_front_matter_empty_after_study_fact_contract")
|
| 2391 |
+
if front_fact_rows:
|
| 2392 |
+
logging.warning(
|
| 2393 |
+
"Working Abstract resume study-fact repair | mode=%s count=%s types=%s",
|
| 2394 |
+
mode, len(front_fact_rows),
|
| 2395 |
+
sorted({str(row.get("type") or "") for row in front_fact_rows}),
|
| 2396 |
+
)
|
| 2397 |
+
quality_diagnostics.append({
|
| 2398 |
+
"scope":"working_abstract_study_fact_integrity", "name":"Working Abstract",
|
| 2399 |
+
"status":"deterministic_repair", "count":len(front_fact_rows),
|
| 2400 |
+
"examples":front_fact_rows[:3],
|
| 2401 |
+
})
|
| 2402 |
agent.global_memory["working_abstract"] = working_abstract
|
| 2403 |
full_draft_en = replace_abstract(full_draft_en, working_abstract)
|
| 2404 |
full_draft_en = self._replace_keywords(full_draft_en, working_keywords)
|
|
|
|
| 2807 |
agent.validate_qualitative_study_fact_contract(
|
| 2808 |
en_resolved, data_metadata, section_name=display_name
|
| 2809 |
)
|
| 2810 |
+
elif mode == "amj":
|
| 2811 |
+
en_resolved, study_fact_rows = quant_engine.repair_quantitative_study_fact_contract(
|
| 2812 |
+
en_resolved, data_metadata, section_name=display_name
|
| 2813 |
+
)
|
| 2814 |
+
if study_fact_rows:
|
| 2815 |
+
logging.warning(
|
| 2816 |
+
"Quantitative Evidence Ledger repair | unit=%s count=%s types=%s",
|
| 2817 |
+
display_name, len(study_fact_rows),
|
| 2818 |
+
sorted({str(row.get("type") or "") for row in study_fact_rows}),
|
| 2819 |
+
)
|
| 2820 |
+
quality_diagnostics.append({
|
| 2821 |
+
"scope":"quantitative_study_fact_integrity", "name":display_name,
|
| 2822 |
+
"status":"deterministic_repair", "count":len(study_fact_rows),
|
| 2823 |
+
"examples":study_fact_rows[:3],
|
| 2824 |
+
})
|
| 2825 |
+
quant_engine.validate_quantitative_study_fact_contract(
|
| 2826 |
+
en_resolved, data_metadata, section_name=display_name
|
| 2827 |
+
)
|
| 2828 |
en_resolved, enumeration_rows = agent.repair_and_validate_enumeration_contract(
|
| 2829 |
en_resolved, section_name=display_name
|
| 2830 |
)
|
|
|
|
| 2917 |
if not state.get("english_finalized"):
|
| 2918 |
self._progress(91, "english_validation")
|
| 2919 |
full_draft_en = _clean_trailing_references(full_draft_en, ["Reference", "References", "Reference List"])
|
| 2920 |
+
section_order_for_fact = self._unique_top_level_sections(adaptive_structure)
|
| 2921 |
+
if not section_order_for_fact:
|
| 2922 |
+
raise RuntimeError("final_study_fact_section_plan_missing")
|
| 2923 |
+
final_body_for_fact = self._extract_body_from_first_section(
|
| 2924 |
+
full_draft_en, section_order_for_fact[0][1]
|
| 2925 |
+
)
|
| 2926 |
if mode == "asq":
|
| 2927 |
fact_checked, fact_rows = agent.repair_qualitative_study_fact_contract(
|
| 2928 |
+
final_body_for_fact, data_metadata, section_name="Final English body"
|
| 2929 |
)
|
| 2930 |
+
if fact_checked.strip() != final_body_for_fact.strip() or fact_rows:
|
| 2931 |
raise RuntimeError(
|
| 2932 |
"final_qualitative_study_fact_contract_requires_section_regeneration"
|
| 2933 |
)
|
| 2934 |
+
elif mode == "amj":
|
| 2935 |
+
fact_checked, fact_rows = quant_engine.repair_quantitative_study_fact_contract(
|
| 2936 |
+
final_body_for_fact, data_metadata, section_name="Final English body"
|
| 2937 |
+
)
|
| 2938 |
+
if fact_checked.strip() != final_body_for_fact.strip() or fact_rows:
|
| 2939 |
+
raise RuntimeError(
|
| 2940 |
+
"final_quantitative_study_fact_contract_requires_section_regeneration"
|
| 2941 |
+
)
|
| 2942 |
enum_checked, enum_rows = agent.repair_and_validate_enumeration_contract(
|
| 2943 |
full_draft_en, section_name="Final English manuscript"
|
| 2944 |
)
|
|
|
|
| 3051 |
final_keywords = list(final_front.get("keywords") or [])
|
| 3052 |
if not final_abstract or not final_keywords:
|
| 3053 |
raise RuntimeError("final_front_matter_missing")
|
| 3054 |
+
final_abstract, final_front_fact_rows = self._repair_validate_study_fact_contract(
|
| 3055 |
+
mode, agent, quant_engine, final_abstract, data_metadata,
|
| 3056 |
+
section_name="Final Abstract",
|
| 3057 |
+
)
|
| 3058 |
+
if not final_abstract:
|
| 3059 |
+
raise RuntimeError("final_abstract_empty_after_study_fact_contract")
|
| 3060 |
+
if final_front_fact_rows:
|
| 3061 |
+
logging.warning(
|
| 3062 |
+
"Final Abstract study-fact repair | mode=%s count=%s types=%s",
|
| 3063 |
+
mode, len(final_front_fact_rows),
|
| 3064 |
+
sorted({str(row.get("type") or "") for row in final_front_fact_rows}),
|
| 3065 |
+
)
|
| 3066 |
+
quality_diagnostics.append({
|
| 3067 |
+
"scope":"final_abstract_study_fact_integrity", "name":"Final Abstract",
|
| 3068 |
+
"status":"deterministic_repair", "count":len(final_front_fact_rows),
|
| 3069 |
+
"examples":final_front_fact_rows[:3],
|
| 3070 |
+
})
|
| 3071 |
full_draft_en = replace_abstract(full_draft_en, final_abstract)
|
| 3072 |
full_draft_en = self._replace_keywords(full_draft_en, final_keywords)
|
| 3073 |
body_only, _reference_tail = agent.validate_reference_tail(
|
core/pipeline_agent.py
CHANGED
|
@@ -72,7 +72,7 @@ class CitationAuditError(DeepSeekRequestError):
|
|
| 72 |
class SSciPipelineAgent:
|
| 73 |
TRANSLATION_CONTRACT_VERSION = 'hierarchical-multilingual-v8'
|
| 74 |
MAX_TRANSLATION_CLAIM_REPAIR_ATTEMPTS = 2
|
| 75 |
-
PROTECTED_ENTITY_LEDGER_VERSION = 'protected-entity-ledger-
|
| 76 |
|
| 77 |
def __init__(self, ds_key, s2_key=""):
|
| 78 |
self.ds_key = str(ds_key or "").strip()
|
|
@@ -4676,6 +4676,32 @@ Output only the repaired section."""
|
|
| 4676 |
if cls._translation_technical_entity_shape(token):
|
| 4677 |
add(token, 'MODEL_TOOL', claim_id, token)
|
| 4678 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4679 |
# Conservative standalone proper-name support. Hyphen-adjacent words
|
| 4680 |
# are excluded here and handled only by the bounded compound rule above,
|
| 4681 |
# preventing ``Open-ended`` from authorizing a bare ``Open``.
|
|
|
|
| 72 |
class SSciPipelineAgent:
|
| 73 |
TRANSLATION_CONTRACT_VERSION = 'hierarchical-multilingual-v8'
|
| 74 |
MAX_TRANSLATION_CLAIM_REPAIR_ATTEMPTS = 2
|
| 75 |
+
PROTECTED_ENTITY_LEDGER_VERSION = 'protected-entity-ledger-v2'
|
| 76 |
|
| 77 |
def __init__(self, ds_key, s2_key=""):
|
| 78 |
self.ds_key = str(ds_key or "").strip()
|
|
|
|
| 4676 |
if cls._translation_technical_entity_shape(token):
|
| 4677 |
add(token, 'MODEL_TOOL', claim_id, token)
|
| 4678 |
|
| 4679 |
+
# Source-owned multi-word proper names remain legal even at the
|
| 4680 |
+
# beginning of a claim (for example ``Krishi Saathi`` or
|
| 4681 |
+
# ``Stack Overflow``). This closes the v1 sentence-start blind spot
|
| 4682 |
+
# without authorizing ordinary sentence-initial words.
|
| 4683 |
+
multiword_pattern = re.compile(
|
| 4684 |
+
r'(?<![A-Za-z0-9_-])(' r'[A-Z][A-Za-z0-9+.#-]{2,}' r'(?:\s+(?:(?:of|the|and|for|in|at|on|by|to|with)\s+)?[A-Z][A-Za-z0-9+.#-]{2,}){1,5}' r')(?![A-Za-z0-9_-])'
|
| 4685 |
+
)
|
| 4686 |
+
for match in multiword_pattern.finditer(claim_source):
|
| 4687 |
+
phrase = re.sub(r'\s+', ' ', match.group(1)).strip()
|
| 4688 |
+
first = phrase.split()[0].casefold() if phrase else ''
|
| 4689 |
+
if first in ordinary_sentence_starters and len(phrase.split()) < 2:
|
| 4690 |
+
continue
|
| 4691 |
+
add(phrase, 'PROPER_NAME', claim_id, phrase)
|
| 4692 |
+
|
| 4693 |
+
# A sentence-initial Title-Case technical term is protected only
|
| 4694 |
+
# when its immediate source context names a bounded technical role.
|
| 4695 |
+
# Ordinary words such as ``Sensitizing`` do not qualify.
|
| 4696 |
+
technical_context = re.compile(
|
| 4697 |
+
r'(?<![A-Za-z0-9_-])([A-Z][A-Za-z0-9+.#-]{3,})\s+'
|
| 4698 |
+
r'(model|architecture|framework|network|encoder|decoder|algorithm|platform|system)\b'
|
| 4699 |
+
)
|
| 4700 |
+
for match in technical_context.finditer(claim_source):
|
| 4701 |
+
token = match.group(1)
|
| 4702 |
+
if token.casefold() not in ordinary_sentence_starters:
|
| 4703 |
+
add(token, 'TECHNICAL_TERM', claim_id, token)
|
| 4704 |
+
|
| 4705 |
# Conservative standalone proper-name support. Hyphen-adjacent words
|
| 4706 |
# are excluded here and handled only by the bounded compound rule above,
|
| 4707 |
# preventing ``Open-ended`` from authorizing a bare ``Open``.
|
core/quantitative_engine.py
CHANGED
|
@@ -17,6 +17,8 @@ from utils.file_parser import extract_data_from_files
|
|
| 17 |
class QuantitativeEngine:
|
| 18 |
"""Prepare one frozen quantitative evidence packet before manuscript writing."""
|
| 19 |
|
|
|
|
|
|
|
| 20 |
@staticmethod
|
| 21 |
def generate_psd_corr_matrix(n_constructs, max_corr=0.5):
|
| 22 |
if n_constructs == 1:
|
|
@@ -302,6 +304,306 @@ Keywords: {keywords}"""
|
|
| 302 |
}
|
| 303 |
return summary, path_table, spearman
|
| 304 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 305 |
def resolve_empirical_data(
|
| 306 |
self,
|
| 307 |
uploaded_files=None,
|
|
@@ -333,6 +635,7 @@ Keywords: {keywords}"""
|
|
| 333 |
"Do not infer absent setting, sample, measures, software, or results."
|
| 334 |
),
|
| 335 |
}
|
|
|
|
| 336 |
return None, user_md, tr(language, "actual_stats_parsed"), metadata
|
| 337 |
|
| 338 |
design = self._derive_system_design(agent, topic, keywords)
|
|
@@ -433,6 +736,7 @@ Keywords: {keywords}"""
|
|
| 433 |
"Frozen bootstrap and collinearity checks",
|
| 434 |
],
|
| 435 |
}
|
|
|
|
| 436 |
return (
|
| 437 |
df,
|
| 438 |
generated_prompt,
|
|
|
|
| 17 |
class QuantitativeEngine:
|
| 18 |
"""Prepare one frozen quantitative evidence packet before manuscript writing."""
|
| 19 |
|
| 20 |
+
QUANTITATIVE_EVIDENCE_LEDGER_VERSION = "quantitative-evidence-ledger-v1"
|
| 21 |
+
|
| 22 |
@staticmethod
|
| 23 |
def generate_psd_corr_matrix(n_constructs, max_corr=0.5):
|
| 24 |
if n_constructs == 1:
|
|
|
|
| 304 |
}
|
| 305 |
return summary, path_table, spearman
|
| 306 |
|
| 307 |
+
|
| 308 |
+
@staticmethod
|
| 309 |
+
def _sentence_spans(text):
|
| 310 |
+
"""Return sentence spans without splitting decimal statistics or citations."""
|
| 311 |
+
value = str(text or "")
|
| 312 |
+
spans = []
|
| 313 |
+
start = 0
|
| 314 |
+
round_depth = 0
|
| 315 |
+
square_depth = 0
|
| 316 |
+
index = 0
|
| 317 |
+
abbreviations = ("e.g.", "i.e.", "et al.", "fig.", "dr.", "mr.", "ms.", "prof.", "vs.", "no.", "vol.")
|
| 318 |
+
while index < len(value):
|
| 319 |
+
char = value[index]
|
| 320 |
+
if char == "(": round_depth += 1
|
| 321 |
+
elif char == ")" and round_depth: round_depth -= 1
|
| 322 |
+
elif char == "[": square_depth += 1
|
| 323 |
+
elif char == "]" and square_depth: square_depth -= 1
|
| 324 |
+
boundary = False
|
| 325 |
+
if char == "\n":
|
| 326 |
+
boundary = True
|
| 327 |
+
elif char in ".!?。!?" and round_depth == 0 and square_depth == 0:
|
| 328 |
+
if char == ".":
|
| 329 |
+
prev = value[index - 1] if index > 0 else ""
|
| 330 |
+
nxt = value[index + 1] if index + 1 < len(value) else ""
|
| 331 |
+
decimal = prev.isdigit() and nxt.isdigit()
|
| 332 |
+
prefix = value[max(0, index - 12): index + 1].lower()
|
| 333 |
+
abbreviation = any(prefix.endswith(item) for item in abbreviations)
|
| 334 |
+
boundary = not decimal and not abbreviation
|
| 335 |
+
else:
|
| 336 |
+
boundary = True
|
| 337 |
+
if boundary:
|
| 338 |
+
end = index + (0 if char == "\n" else 1)
|
| 339 |
+
if end > start and value[start:end].strip():
|
| 340 |
+
spans.append((start, end, value[start:end]))
|
| 341 |
+
start = index + 1
|
| 342 |
+
index += 1
|
| 343 |
+
if start < len(value) and value[start:].strip():
|
| 344 |
+
spans.append((start, len(value), value[start:]))
|
| 345 |
+
return spans
|
| 346 |
+
|
| 347 |
+
@classmethod
|
| 348 |
+
def build_quantitative_evidence_ledger(cls, metadata, evidence_text=""):
|
| 349 |
+
"""Freeze only high-confidence study-owned quantitative facts.
|
| 350 |
+
|
| 351 |
+
System-generated evidence has typed metadata and therefore supports a
|
| 352 |
+
strict contract. User-provided evidence is parsed conservatively: only
|
| 353 |
+
an unambiguous sample size / Likert scale is promoted to deterministic
|
| 354 |
+
authority; everything else remains owned by the uploaded evidence text.
|
| 355 |
+
"""
|
| 356 |
+
meta = dict(metadata or {})
|
| 357 |
+
source_type = str(meta.get("source_type") or "")
|
| 358 |
+
sample_size = meta.get("sample_size")
|
| 359 |
+
scale_points = meta.get("scale_points")
|
| 360 |
+
strict = source_type == "system_generated"
|
| 361 |
+
evidence = str(evidence_text or "")
|
| 362 |
+
if not strict and evidence:
|
| 363 |
+
sample_candidates = set()
|
| 364 |
+
for pattern in (
|
| 365 |
+
r"(?i)\b(?:sample\s+size|sample|N)\s*(?:=|:|of)?\s*(\d{2,6})\b",
|
| 366 |
+
r"(?i)\b(\d{2,6})\s+(?:respondents|participants|observations|cases)\b",
|
| 367 |
+
):
|
| 368 |
+
sample_candidates.update(int(x) for x in re.findall(pattern, evidence))
|
| 369 |
+
if len(sample_candidates) == 1:
|
| 370 |
+
sample_size = next(iter(sample_candidates))
|
| 371 |
+
scale_candidates = {int(x) for x in re.findall(r"(?i)\b([2-9]|1[0-1])[- ]point\s+Likert", evidence)}
|
| 372 |
+
if len(scale_candidates) == 1:
|
| 373 |
+
scale_points = next(iter(scale_candidates))
|
| 374 |
+
constructs = dict(meta.get("constructs") or {}) if isinstance(meta.get("constructs"), dict) else {}
|
| 375 |
+
results = dict(meta.get("association_results") or {}) if isinstance(meta.get("association_results"), dict) else {}
|
| 376 |
+
return {
|
| 377 |
+
"version": cls.QUANTITATIVE_EVIDENCE_LEDGER_VERSION,
|
| 378 |
+
"source_type": source_type,
|
| 379 |
+
"strict": bool(strict),
|
| 380 |
+
"sample_size": int(sample_size) if isinstance(sample_size, (int, np.integer)) or str(sample_size or "").isdigit() else None,
|
| 381 |
+
"scale_points": int(scale_points) if isinstance(scale_points, (int, np.integer)) or str(scale_points or "").isdigit() else None,
|
| 382 |
+
"study_design": str(meta.get("study_design") or "").strip(),
|
| 383 |
+
"constructs": {k: str(v or "").strip() for k, v in constructs.items() if str(v or "").strip()},
|
| 384 |
+
"association_results": results,
|
| 385 |
+
"causal_identification": "none" if strict else "evidence_owned",
|
| 386 |
+
"allowed_methods": [
|
| 387 |
+
"reliability", "composite reliability", "average variance extracted",
|
| 388 |
+
"Fornell-Larcker", "standardized association", "bootstrap",
|
| 389 |
+
"Spearman", "VIF", "descriptive statistics",
|
| 390 |
+
] if strict else [],
|
| 391 |
+
"forbidden_unowned_methods": [
|
| 392 |
+
"confirmatory factor analysis", "CFA", "structural equation modeling", "SEM",
|
| 393 |
+
"PLS-SEM", "RMSEA", "CFI", "TLI", "SRMR", "Harman",
|
| 394 |
+
"common method bias", "endogeneity", "instrumental variable",
|
| 395 |
+
"fixed effects", "random effects", "difference-in-differences",
|
| 396 |
+
] if strict else [],
|
| 397 |
+
}
|
| 398 |
+
|
| 399 |
+
@classmethod
|
| 400 |
+
def repair_quantitative_study_fact_contract(cls, text, metadata, section_name=""):
|
| 401 |
+
"""Repair only deterministic AMJ study-fact drift owned by frozen evidence."""
|
| 402 |
+
value = str(text or "")
|
| 403 |
+
meta = dict(metadata or {})
|
| 404 |
+
ledger = dict(meta.get("quantitative_evidence_ledger") or {})
|
| 405 |
+
if not ledger:
|
| 406 |
+
ledger = cls.build_quantitative_evidence_ledger(meta)
|
| 407 |
+
diagnostics = []
|
| 408 |
+
sample_size = ledger.get("sample_size")
|
| 409 |
+
scale_points = ledger.get("scale_points")
|
| 410 |
+
strict = bool(ledger.get("strict"))
|
| 411 |
+
constructs = dict(ledger.get("constructs") or {})
|
| 412 |
+
results = dict(ledger.get("association_results") or {})
|
| 413 |
+
|
| 414 |
+
# Sample/scale claims are repaired only when the sentence is clearly
|
| 415 |
+
# study-owned and not an external cited finding.
|
| 416 |
+
spans = cls._sentence_spans(value)
|
| 417 |
+
sample_scale_replacements = []
|
| 418 |
+
for start, end, sentence in spans:
|
| 419 |
+
has_citation = bool(re.search(r"\([^()]*?(?:19|20)\d{2}[a-z]?[^()]*?\)", sentence))
|
| 420 |
+
study_owned = bool(re.search(
|
| 421 |
+
r"(?i)\b(?:this\s+study|our\s+study|we|our\s+sample|the\s+sample|sample\s+(?:comprised|consisted|included)|respondents?|participants?)\b",
|
| 422 |
+
sentence,
|
| 423 |
+
))
|
| 424 |
+
if has_citation or not study_owned:
|
| 425 |
+
continue
|
| 426 |
+
revised = sentence
|
| 427 |
+
if sample_size:
|
| 428 |
+
sample_pattern = re.compile(r"(?i)(\b(?:sample(?:\s+size)?(?:\s+of)?|N)\s*(?:=|:)?\s*)(\d{2,6})\b")
|
| 429 |
+
def sample_repl(match):
|
| 430 |
+
observed = int(match.group(2))
|
| 431 |
+
if observed == int(sample_size):
|
| 432 |
+
return match.group(0)
|
| 433 |
+
diagnostics.append({"type":"sample_size_repaired","observed":observed,"expected":int(sample_size)})
|
| 434 |
+
return match.group(1) + str(int(sample_size))
|
| 435 |
+
revised = sample_pattern.sub(sample_repl, revised)
|
| 436 |
+
count_pattern = re.compile(r"(?i)\b(\d{2,6})\s+(respondents|participants|observations|cases)\b")
|
| 437 |
+
def count_repl(match):
|
| 438 |
+
observed = int(match.group(1))
|
| 439 |
+
if observed == int(sample_size):
|
| 440 |
+
return match.group(0)
|
| 441 |
+
diagnostics.append({"type":"sample_count_repaired","observed":observed,"expected":int(sample_size)})
|
| 442 |
+
return f"{int(sample_size)} {match.group(2)}"
|
| 443 |
+
revised = count_pattern.sub(count_repl, revised)
|
| 444 |
+
if scale_points:
|
| 445 |
+
scale_pattern = re.compile(r"(?i)\b([2-9]|1[0-1])[- ]point\s+(Likert(?:-type)?\s+scale)")
|
| 446 |
+
def scale_repl(match):
|
| 447 |
+
observed = int(match.group(1))
|
| 448 |
+
if observed == int(scale_points):
|
| 449 |
+
return match.group(0)
|
| 450 |
+
diagnostics.append({"type":"scale_points_repaired","observed":observed,"expected":int(scale_points)})
|
| 451 |
+
return f"{int(scale_points)}-point {match.group(2)}"
|
| 452 |
+
revised = scale_pattern.sub(scale_repl, revised)
|
| 453 |
+
if revised != sentence:
|
| 454 |
+
sample_scale_replacements.append((start, end, revised))
|
| 455 |
+
for start, end, replacement in sorted(sample_scale_replacements, reverse=True):
|
| 456 |
+
value = value[:start] + replacement + value[end:]
|
| 457 |
+
|
| 458 |
+
# Explicit construct-role labels are deterministic and safe to correct.
|
| 459 |
+
role_patterns = {
|
| 460 |
+
"predictor": r"(?:predictor|independent\s+variable|predictor\s+construct)",
|
| 461 |
+
"mediator": r"(?:mediator|mediating\s+variable|mediating\s+construct)",
|
| 462 |
+
"outcome": r"(?:outcome|dependent\s+variable|outcome\s+construct)",
|
| 463 |
+
}
|
| 464 |
+
known_constructs = [str(v) for v in constructs.values() if str(v)]
|
| 465 |
+
for role, role_pattern in role_patterns.items():
|
| 466 |
+
expected = str(constructs.get(role) or "")
|
| 467 |
+
if not expected:
|
| 468 |
+
continue
|
| 469 |
+
for candidate in known_constructs:
|
| 470 |
+
if candidate.casefold() == expected.casefold():
|
| 471 |
+
continue
|
| 472 |
+
pattern = re.compile(rf"(?i)({role_pattern}\s*(?:was|is|:|=)?\s*){re.escape(candidate)}")
|
| 473 |
+
if pattern.search(value):
|
| 474 |
+
value, count = pattern.subn(lambda m: m.group(1) + expected, value)
|
| 475 |
+
if count:
|
| 476 |
+
diagnostics.append({"type":"construct_role_repaired","role":role,"observed":candidate,"expected":expected,"count":count})
|
| 477 |
+
|
| 478 |
+
if strict:
|
| 479 |
+
# Frozen system-generated evidence has no causal identification.
|
| 480 |
+
replacements = (
|
| 481 |
+
(r"(?i)\bhad\s+a\s+causal\s+effect\s+on\b", "was associated with"),
|
| 482 |
+
(r"(?i)\bexerted\s+a\s+causal\s+effect\s+on\b", "was associated with"),
|
| 483 |
+
(r"(?i)\bcausal\s+effect\s+on\b", "association with"),
|
| 484 |
+
(r"(?i)\ban\s+causal\s+effect\b", "an association"),
|
| 485 |
+
(r"(?i)\ba\s+causal\s+effect\b", "an association"),
|
| 486 |
+
(r"(?i)\bthe\s+causal\s+effect\b", "the association"),
|
| 487 |
+
(r"(?i)\bcausal\s+effects\b", "associations"),
|
| 488 |
+
(r"(?i)\bcausal\s+effect\b", "association"),
|
| 489 |
+
(r"(?i)\bcausal\s+mediation\b", "indirect association"),
|
| 490 |
+
)
|
| 491 |
+
for pattern, replacement in replacements:
|
| 492 |
+
value, count = re.subn(pattern, replacement, value)
|
| 493 |
+
if count:
|
| 494 |
+
diagnostics.append({"type":"causal_language_repaired","replacement":replacement,"count":count})
|
| 495 |
+
|
| 496 |
+
# Remove p-values attached to this study's frozen constructs because
|
| 497 |
+
# no p-values are generated/frozen by this engine. External cited
|
| 498 |
+
# literature claims are left untouched.
|
| 499 |
+
construct_folded = [c.casefold() for c in known_constructs]
|
| 500 |
+
spans = cls._sentence_spans(value)
|
| 501 |
+
replacements_by_span = []
|
| 502 |
+
forbidden = [str(x) for x in ledger.get("forbidden_unowned_methods") or []]
|
| 503 |
+
section_sensitive = any(token in str(section_name or "").casefold() for token in ("method", "result", "abstract", "final", "analysis"))
|
| 504 |
+
for start, end, sentence in spans:
|
| 505 |
+
lower = sentence.casefold()
|
| 506 |
+
has_construct = any(c in lower for c in construct_folded) if construct_folded else False
|
| 507 |
+
has_citation = bool(re.search(r"\([^()]*?(?:19|20)\d{2}[a-z]?[^()]*?\)", sentence))
|
| 508 |
+
revised = sentence
|
| 509 |
+
if has_construct and not has_citation:
|
| 510 |
+
revised, p_count = re.subn(r"(?i)\s*[,;]?\s*p\s*(?:=|<|>|≤|≥)\s*(?:0?\.\d+|\d+(?:\.\d+)?)", "", revised)
|
| 511 |
+
if p_count:
|
| 512 |
+
diagnostics.append({"type":"unfrozen_p_value_removed","count":p_count})
|
| 513 |
+
if section_sensitive and not has_citation and re.search(r"(?i)\b(?:we|our\s+study|this\s+study|the\s+study|our\s+analysis|the\s+analysis)\b", sentence):
|
| 514 |
+
bad = next((term for term in forbidden if re.search(rf"(?i)(?<![A-Za-z]){re.escape(term)}(?![A-Za-z])", sentence)), "")
|
| 515 |
+
if bad:
|
| 516 |
+
revised = ""
|
| 517 |
+
diagnostics.append({"type":"unsupported_quantitative_method_removed","method":bad,"preview":sentence.strip()[:180]})
|
| 518 |
+
if revised != sentence:
|
| 519 |
+
replacements_by_span.append((start, end, revised))
|
| 520 |
+
for start, end, replacement in sorted(replacements_by_span, reverse=True):
|
| 521 |
+
value = value[:start] + replacement + value[end:]
|
| 522 |
+
|
| 523 |
+
# High-confidence frozen numeric anchors.
|
| 524 |
+
def replace_metric(pattern, expected, dtype="float", label="metric"):
|
| 525 |
+
nonlocal value
|
| 526 |
+
if expected is None:
|
| 527 |
+
return
|
| 528 |
+
expected_text = str(int(expected)) if dtype == "int" else f"{float(expected):.3f}".rstrip("0").rstrip(".")
|
| 529 |
+
regex = re.compile(pattern, re.I)
|
| 530 |
+
def repl(match):
|
| 531 |
+
observed = match.group(2)
|
| 532 |
+
try:
|
| 533 |
+
same = int(float(observed)) == int(expected) if dtype == "int" else abs(float(observed) - float(expected)) < 0.0006
|
| 534 |
+
except Exception:
|
| 535 |
+
same = False
|
| 536 |
+
if same:
|
| 537 |
+
return match.group(0)
|
| 538 |
+
diagnostics.append({"type":"quantitative_metric_repaired","metric":label,"observed":observed,"expected":expected_text})
|
| 539 |
+
return match.group(1) + expected_text
|
| 540 |
+
value = regex.sub(repl, value)
|
| 541 |
+
|
| 542 |
+
if results:
|
| 543 |
+
replace_metric(r"(\bVIF\s*(?:=|:)?\s*)([-+]?\d+(?:\.\d+)?)", results.get("predictor_mediator_vif"), label="VIF")
|
| 544 |
+
replace_metric(r"(\bbootstrap\s+(?:resamples?|iterations?)\s*(?:=|:)?\s*)(\d{2,6})", results.get("bootstrap_iterations"), dtype="int", label="bootstrap_iterations")
|
| 545 |
+
for role, key in (("mediator", "r2_mediator"), ("outcome", "r2_outcome")):
|
| 546 |
+
construct = str(constructs.get(role) or "")
|
| 547 |
+
if construct and results.get(key) is not None:
|
| 548 |
+
metric_pattern = rf"(R(?:²|\^2)\s*(?:for|of)?\s*{re.escape(construct)}\s*(?:=|:)?\s*)([-+]?\d+(?:\.\d+)?)"
|
| 549 |
+
replace_metric(metric_pattern, results.get(key), label=key)
|
| 550 |
+
|
| 551 |
+
association_specs = [
|
| 552 |
+
("indirect", results.get("indirect"), [constructs.get("predictor"), constructs.get("mediator"), constructs.get("outcome")]),
|
| 553 |
+
("direct", results.get("direct"), [constructs.get("predictor"), constructs.get("outcome")]),
|
| 554 |
+
("a", results.get("a"), [constructs.get("predictor"), constructs.get("mediator")]),
|
| 555 |
+
("b", results.get("b"), [constructs.get("mediator"), constructs.get("outcome")]),
|
| 556 |
+
]
|
| 557 |
+
sentences = cls._sentence_spans(value)
|
| 558 |
+
metric_spans = []
|
| 559 |
+
for start, end, sentence in sentences:
|
| 560 |
+
lower = sentence.casefold()
|
| 561 |
+
selected = None
|
| 562 |
+
if "indirect" in lower and all(str(x or "").casefold() in lower for x in association_specs[0][2] if x):
|
| 563 |
+
selected = association_specs[0]
|
| 564 |
+
elif (
|
| 565 |
+
f"controlling {str(constructs.get('mediator') or '').casefold()}" in lower
|
| 566 |
+
and str(constructs.get("predictor") or "").casefold() in lower
|
| 567 |
+
and str(constructs.get("outcome") or "").casefold() in lower
|
| 568 |
+
):
|
| 569 |
+
selected = association_specs[1]
|
| 570 |
+
elif (
|
| 571 |
+
f"controlling {str(constructs.get('predictor') or '').casefold()}" in lower
|
| 572 |
+
and str(constructs.get("mediator") or "").casefold() in lower
|
| 573 |
+
and str(constructs.get("outcome") or "").casefold() in lower
|
| 574 |
+
):
|
| 575 |
+
selected = association_specs[3]
|
| 576 |
+
elif str(constructs.get("predictor") or "").casefold() in lower and str(constructs.get("mediator") or "").casefold() in lower and str(constructs.get("outcome") or "").casefold() not in lower:
|
| 577 |
+
selected = association_specs[2]
|
| 578 |
+
elif str(constructs.get("mediator") or "").casefold() in lower and str(constructs.get("outcome") or "").casefold() in lower and str(constructs.get("predictor") or "").casefold() not in lower:
|
| 579 |
+
selected = association_specs[3]
|
| 580 |
+
if selected and selected[1] is not None:
|
| 581 |
+
expected = float(selected[1])
|
| 582 |
+
def beta_repl(match):
|
| 583 |
+
observed = float(match.group(2))
|
| 584 |
+
if abs(observed - expected) < 0.0006:
|
| 585 |
+
return match.group(0)
|
| 586 |
+
expected_text = f"{expected:.3f}".rstrip("0").rstrip(".")
|
| 587 |
+
diagnostics.append({"type":"quantitative_metric_repaired","metric":selected[0],"observed":match.group(2),"expected":expected_text})
|
| 588 |
+
return match.group(1) + expected_text
|
| 589 |
+
revised = re.sub(r"(?i)((?:β|standardized\s+estimate)\s*(?:=|:)?\s*)([-+]?\d+(?:\.\d+)?)", beta_repl, sentence)
|
| 590 |
+
if revised != sentence:
|
| 591 |
+
metric_spans.append((start, end, revised))
|
| 592 |
+
for start, end, replacement in sorted(metric_spans, reverse=True):
|
| 593 |
+
value = value[:start] + replacement + value[end:]
|
| 594 |
+
|
| 595 |
+
value = re.sub(r"[ \t]+\n", "\n", value)
|
| 596 |
+
value = re.sub(r"\n{3,}", "\n\n", value)
|
| 597 |
+
value = re.sub(r" {2,}", " ", value)
|
| 598 |
+
return value.strip(), diagnostics
|
| 599 |
+
|
| 600 |
+
@classmethod
|
| 601 |
+
def validate_quantitative_study_fact_contract(cls, text, metadata, section_name=""):
|
| 602 |
+
repaired, diagnostics = cls.repair_quantitative_study_fact_contract(text, metadata, section_name=section_name)
|
| 603 |
+
if repaired.strip() != str(text or "").strip():
|
| 604 |
+
raise RuntimeError(f"quantitative_study_fact_contract_unrepaired:{section_name}:{diagnostics[:3]}")
|
| 605 |
+
return True
|
| 606 |
+
|
| 607 |
def resolve_empirical_data(
|
| 608 |
self,
|
| 609 |
uploaded_files=None,
|
|
|
|
| 635 |
"Do not infer absent setting, sample, measures, software, or results."
|
| 636 |
),
|
| 637 |
}
|
| 638 |
+
metadata["quantitative_evidence_ledger"] = self.build_quantitative_evidence_ledger(metadata, user_md)
|
| 639 |
return None, user_md, tr(language, "actual_stats_parsed"), metadata
|
| 640 |
|
| 641 |
design = self._derive_system_design(agent, topic, keywords)
|
|
|
|
| 736 |
"Frozen bootstrap and collinearity checks",
|
| 737 |
],
|
| 738 |
}
|
| 739 |
+
metadata["quantitative_evidence_ledger"] = self.build_quantitative_evidence_ledger(metadata, generated_prompt)
|
| 740 |
return (
|
| 741 |
df,
|
| 742 |
generated_prompt,
|