# -*- coding: utf-8 -*- """Quantitative evidence preparation for SmartPaper's AMJ-oriented track.""" from __future__ import annotations import json import logging import re import numpy as np import pandas as pd from i18n import tr from utils.file_parser import extract_data_from_files class QuantitativeEngine: """Prepare one frozen quantitative evidence packet before manuscript writing.""" QUANTITATIVE_EVIDENCE_LEDGER_VERSION = "quantitative-evidence-ledger-v1" @staticmethod def generate_psd_corr_matrix(n_constructs, max_corr=0.5): if n_constructs == 1: return np.array([[1.0]]) base_corr = np.random.uniform(0.2, max_corr - 0.1) matrix = np.random.uniform( base_corr - 0.1, base_corr + 0.1, (n_constructs, n_constructs), ) matrix = (matrix + matrix.T) / 2 np.fill_diagonal(matrix, 1.0) min_eig = np.min(np.linalg.eigvals(matrix)) if min_eig < 0: matrix -= (min_eig - 0.01) * np.eye(n_constructs) diagonal = np.sqrt(np.diag(matrix)) return matrix / np.outer(diagonal, diagonal) @staticmethod def calculate_validity_metrics(df, constructs_info, true_loadings): metrics = [] for construct in constructs_info: construct_name = construct["name"] items = construct["items"] sub_df = df[items] var_total = sub_df.sum(axis=1).var(ddof=1) sum_var = sub_df.var(ddof=1).sum() n_items = len(items) alpha = ( 0.0 if var_total <= 0 else (n_items / (n_items - 1)) * (1 - sum_var / var_total) ) loadings = np.array(true_loadings[construct_name]) sum_loadings_sq = np.sum(loadings) ** 2 sum_sq_loadings = np.sum(loadings**2) sum_error = np.sum(1 - loadings**2) cr = sum_loadings_sq / (sum_loadings_sq + sum_error) ave = sum_sq_loadings / len(loadings) metrics.append( { "Construct": construct_name, "Items": n_items, "Cronbach's α": round(alpha, 3), "CR": round(cr, 3), "AVE": round(ave, 3), } ) return pd.DataFrame(metrics) @staticmethod def _safe_constructs(values): cleaned = [] for value in values or []: name = re.sub(r"\s+", " ", str(value or "")).strip(" .,:;-_\n\t") if not name or len(name) > 80 or name.lower() in {item.lower() for item in cleaned}: continue cleaned.append(name) return cleaned[:3] def _derive_system_design(self, agent, topic="", keywords=""): """Freeze a parsimonious topic-aligned design without inventing named sites.""" fallback = { "constructs": [ "Focal Predictor", "Focal Mediator", "Focal Outcome", ], "research_context": "respondents with experience relevant to the focal management phenomenon", } if agent is None: return fallback prompt = f"""Design the minimum coherent quantitative management-study skeleton for the topic below. Return ONLY valid JSON: {{"constructs":["predictor construct","mediating construct","outcome construct"],"research_context":"one generic respondent context"}} Rules: 1. Exactly three distinct constructs in predictor -> mediator -> outcome order. 2. Use concise construct labels that can appear throughout a journal article. 3. The context must be generic and directly relevant to the topic. Do NOT name or invent a country, university, firm, industry sample frame, software package, time period, or institution. 4. This is a cross-sectional single-wave survey design, so do not imply experimental or causal identification. 5. Do not create any number; sample size and scale are frozen elsewhere. Topic: {topic} Keywords: {keywords}""" try: raw = agent.call_deepseek( "You are a rigorous management research-design editor.", prompt, max_tokens=500, temp=0.0, model_role="fast", thinking=False, allow_fallback=True, ) parsed = agent._extract_json_from_text(raw) if not isinstance(parsed, dict): return fallback constructs = self._safe_constructs(parsed.get("constructs")) context = re.sub(r"\s+", " ", str(parsed.get("research_context") or "")).strip() if len(constructs) != 3: return fallback if not context or len(context) > 240: context = fallback["research_context"] # Named-location/site details are deliberately not accepted from this # design step. The manuscript may remain generic when no user evidence # supplies a real setting. if re.search(r"\b(university|company|corporation|inc\.?|ltd\.?|country|china|usa|united states|europe|asia)\b", context, re.I): context = fallback["research_context"] return {"constructs": constructs, "research_context": context} except Exception as exc: logging.warning("Quantitative study-design freeze fell back safely: %s", exc) return fallback def generate_empirical_data_robust( self, construct_names=None, n=300, scale_points=7, ): if not construct_names or len(construct_names) < 2: construct_names = [ "Focal Predictor", "Focal Mediator", "Focal Outcome", ] n_constructs = len(construct_names) corr = self.generate_psd_corr_matrix(n_constructs, max_corr=0.55) latent_scores = np.random.multivariate_normal( mean=[0] * n_constructs, cov=corr, size=n, ) data = {} true_loadings = {} constructs_info = [] target_mean = scale_points / 2.0 + 0.5 target_std = scale_points / 5.0 for index, construct_name in enumerate(construct_names): true_score = latent_scores[:, index] n_items = np.random.randint(3, 6) loadings = np.random.uniform(0.72, 0.88, n_items) true_loadings[construct_name] = loadings item_names = [] safe_prefix = re.sub(r"[^A-Za-z0-9]+", "_", construct_name).strip("_") or f"Construct_{index + 1}" for item_index, loading in enumerate(loadings): error = np.random.normal( 0, np.sqrt(1 - loading**2), n, ) item_score = loading * true_score + error raw_scaled = item_score * target_std + target_mean item_int = np.clip( np.round(raw_scaled), 1, scale_points, ).astype(int) column_name = f"{safe_prefix}_{item_index + 1}" data[column_name] = item_int item_names.append(column_name) constructs_info.append( {"name": construct_name, "items": item_names} ) df = pd.DataFrame(data) validity_df = self.calculate_validity_metrics( df, constructs_info, true_loadings, ) mean_scores = pd.DataFrame( { construct["name"]: df[construct["items"]].mean(axis=1) for construct in constructs_info } ) corr_matrix = mean_scores.corr() fl_matrix = corr_matrix.astype(object) for _, row in validity_df.iterrows(): construct_name = row["Construct"] sqrt_ave = np.sqrt(row["AVE"]) fl_matrix.loc[construct_name, construct_name] = ( f"({sqrt_ave:.3f})" ) for row_index in range(len(fl_matrix.columns)): for column_index in range(row_index + 1, len(fl_matrix.columns)): fl_matrix.iloc[row_index, column_index] = "" for column_index in range(row_index): value = fl_matrix.iloc[row_index, column_index] if isinstance(value, (int, float, np.integer, np.floating)): fl_matrix.iloc[row_index, column_index] = f"{float(value):.3f}" desc_stats = mean_scores.mean().to_frame("Mean").join( mean_scores.std().to_frame("SD") ) return df, validity_df, desc_stats.join(fl_matrix), mean_scores, corr_matrix @staticmethod def _standardized_ols(y, predictors): y = np.asarray(y, dtype=float) x = np.column_stack([np.asarray(value, dtype=float) for value in predictors]) y_sd = y.std(ddof=1) x_sd = x.std(axis=0, ddof=1) if y_sd <= 0 or np.any(x_sd <= 0): raise RuntimeError("quantitative_zero_variance") yz = (y - y.mean()) / y_sd xz = (x - x.mean(axis=0)) / x_sd design = np.column_stack([np.ones(len(yz)), xz]) coef = np.linalg.lstsq(design, yz, rcond=None)[0] fitted = design @ coef ss_res = float(np.sum((yz - fitted) ** 2)) ss_tot = float(np.sum((yz - yz.mean()) ** 2)) r2 = 0.0 if ss_tot <= 0 else 1.0 - ss_res / ss_tot return coef[1:], max(0.0, min(1.0, r2)) def _path_evidence(self, mean_scores, construct_names, bootstrap_iterations=1500): if len(construct_names) < 3: return {}, pd.DataFrame(), pd.DataFrame() x_name, m_name, y_name = construct_names[0], construct_names[1], construct_names[2] x = mean_scores[x_name].to_numpy(dtype=float) m = mean_scores[m_name].to_numpy(dtype=float) y = mean_scores[y_name].to_numpy(dtype=float) a_coef, r2_m = self._standardized_ols(m, [x]) y_coef, r2_y = self._standardized_ols(y, [x, m]) a = float(a_coef[0]) direct = float(y_coef[0]) b = float(y_coef[1]) indirect = a * b rng = np.random.default_rng(20260809) indirect_boot = [] direct_boot = [] n = len(mean_scores) for _ in range(int(bootstrap_iterations)): idx = rng.integers(0, n, n) try: a_b, _ = self._standardized_ols(m[idx], [x[idx]]) y_b, _ = self._standardized_ols(y[idx], [x[idx], m[idx]]) indirect_boot.append(float(a_b[0] * y_b[1])) direct_boot.append(float(y_b[0])) except Exception: continue if len(indirect_boot) < 100: raise RuntimeError("quantitative_bootstrap_insufficient") indirect_ci = np.percentile(indirect_boot, [2.5, 97.5]) direct_ci = np.percentile(direct_boot, [2.5, 97.5]) path_table = pd.DataFrame( [ {"Association": f"{x_name} → {m_name}", "Standardized estimate": round(a, 3), "R² of outcome": round(r2_m, 3)}, {"Association": f"{m_name} → {y_name} (controlling {x_name})", "Standardized estimate": round(b, 3), "R² of outcome": round(r2_y, 3)}, {"Association": f"{x_name} → {y_name} (controlling {m_name})", "Standardized estimate": round(direct, 3), "R² of outcome": round(r2_y, 3)}, {"Association": f"Indirect association: {x_name} → {m_name} → {y_name}", "Standardized estimate": round(indirect, 3), "R² of outcome": "—"}, ] ) bootstrap_table = pd.DataFrame( [ {"Estimate": "Indirect association", "Point estimate": round(indirect, 3), "95% bootstrap CI lower": round(float(indirect_ci[0]), 3), "95% bootstrap CI upper": round(float(indirect_ci[1]), 3)}, {"Estimate": "Residual direct association", "Point estimate": round(direct, 3), "95% bootstrap CI lower": round(float(direct_ci[0]), 3), "95% bootstrap CI upper": round(float(direct_ci[1]), 3)}, ] ) x_m_corr = float(np.corrcoef(x, m)[0, 1]) vif = 1.0 / max(1e-9, 1.0 - x_m_corr**2) spearman = mean_scores.corr(method="spearman") summary = { "predictor": x_name, "mediator": m_name, "outcome": y_name, "a": round(a, 3), "b": round(b, 3), "direct": round(direct, 3), "indirect": round(indirect, 3), "r2_mediator": round(r2_m, 3), "r2_outcome": round(r2_y, 3), "indirect_ci": [round(float(indirect_ci[0]), 3), round(float(indirect_ci[1]), 3)], "direct_ci": [round(float(direct_ci[0]), 3), round(float(direct_ci[1]), 3)], "predictor_mediator_vif": round(float(vif), 3), "bootstrap_iterations": int(bootstrap_iterations), } return summary, path_table, spearman @staticmethod def _sentence_spans(text): """Return sentence spans without splitting decimal statistics or citations.""" value = str(text or "") spans = [] start = 0 round_depth = 0 square_depth = 0 index = 0 abbreviations = ("e.g.", "i.e.", "et al.", "fig.", "dr.", "mr.", "ms.", "prof.", "vs.", "no.", "vol.") while index < len(value): char = value[index] if char == "(": round_depth += 1 elif char == ")" and round_depth: round_depth -= 1 elif char == "[": square_depth += 1 elif char == "]" and square_depth: square_depth -= 1 boundary = False if char == "\n": boundary = True elif char in ".!?。!?" and round_depth == 0 and square_depth == 0: if char == ".": prev = value[index - 1] if index > 0 else "" nxt = value[index + 1] if index + 1 < len(value) else "" decimal = prev.isdigit() and nxt.isdigit() prefix = value[max(0, index - 12): index + 1].lower() abbreviation = any(prefix.endswith(item) for item in abbreviations) boundary = not decimal and not abbreviation else: boundary = True if boundary: end = index + (0 if char == "\n" else 1) if end > start and value[start:end].strip(): spans.append((start, end, value[start:end])) start = index + 1 index += 1 if start < len(value) and value[start:].strip(): spans.append((start, len(value), value[start:])) return spans @classmethod def build_quantitative_evidence_ledger(cls, metadata, evidence_text=""): """Freeze only high-confidence study-owned quantitative facts. System-generated evidence has typed metadata and therefore supports a strict contract. User-provided evidence is parsed conservatively: only an unambiguous sample size / Likert scale is promoted to deterministic authority; everything else remains owned by the uploaded evidence text. """ meta = dict(metadata or {}) source_type = str(meta.get("source_type") or "") sample_size = meta.get("sample_size") scale_points = meta.get("scale_points") strict = source_type == "system_generated" evidence = str(evidence_text or "") if not strict and evidence: sample_candidates = set() for pattern in ( r"(?i)\b(?:sample\s+size|sample|N)\s*(?:=|:|of)?\s*(\d{2,6})\b", r"(?i)\b(\d{2,6})\s+(?:respondents|participants|observations|cases)\b", ): sample_candidates.update(int(x) for x in re.findall(pattern, evidence)) if len(sample_candidates) == 1: sample_size = next(iter(sample_candidates)) scale_candidates = {int(x) for x in re.findall(r"(?i)\b([2-9]|1[0-1])[- ]point\s+Likert", evidence)} if len(scale_candidates) == 1: scale_points = next(iter(scale_candidates)) constructs = dict(meta.get("constructs") or {}) if isinstance(meta.get("constructs"), dict) else {} results = dict(meta.get("association_results") or {}) if isinstance(meta.get("association_results"), dict) else {} return { "version": cls.QUANTITATIVE_EVIDENCE_LEDGER_VERSION, "source_type": source_type, "strict": bool(strict), "sample_size": int(sample_size) if isinstance(sample_size, (int, np.integer)) or str(sample_size or "").isdigit() else None, "scale_points": int(scale_points) if isinstance(scale_points, (int, np.integer)) or str(scale_points or "").isdigit() else None, "study_design": str(meta.get("study_design") or "").strip(), "constructs": {k: str(v or "").strip() for k, v in constructs.items() if str(v or "").strip()}, "association_results": results, "causal_identification": "none" if strict else "evidence_owned", "allowed_methods": [ "reliability", "composite reliability", "average variance extracted", "Fornell-Larcker", "standardized association", "bootstrap", "Spearman", "VIF", "descriptive statistics", ] if strict else [], "forbidden_unowned_methods": [ "confirmatory factor analysis", "CFA", "structural equation modeling", "SEM", "PLS-SEM", "RMSEA", "CFI", "TLI", "SRMR", "Harman", "common method bias", "endogeneity", "instrumental variable", "fixed effects", "random effects", "difference-in-differences", ] if strict else [], } @classmethod def repair_quantitative_study_fact_contract(cls, text, metadata, section_name=""): """Repair only deterministic AMJ study-fact drift owned by frozen evidence.""" value = str(text or "") meta = dict(metadata or {}) ledger = dict(meta.get("quantitative_evidence_ledger") or {}) if not ledger: ledger = cls.build_quantitative_evidence_ledger(meta) diagnostics = [] sample_size = ledger.get("sample_size") scale_points = ledger.get("scale_points") strict = bool(ledger.get("strict")) constructs = dict(ledger.get("constructs") or {}) results = dict(ledger.get("association_results") or {}) # Sample/scale claims are repaired only when the sentence is clearly # study-owned and not an external cited finding. spans = cls._sentence_spans(value) sample_scale_replacements = [] for start, end, sentence in spans: has_citation = bool(re.search(r"\([^()]*?(?:19|20)\d{2}[a-z]?[^()]*?\)", sentence)) study_owned = bool(re.search( r"(?i)\b(?:this\s+study|our\s+study|we|our\s+sample|the\s+sample|sample\s+(?:comprised|consisted|included)|respondents?|participants?)\b", sentence, )) if has_citation or not study_owned: continue revised = sentence if sample_size: sample_pattern = re.compile(r"(?i)(\b(?:sample(?:\s+size)?(?:\s+of)?|N)\s*(?:=|:)?\s*)(\d{2,6})\b") def sample_repl(match): observed = int(match.group(2)) if observed == int(sample_size): return match.group(0) diagnostics.append({"type":"sample_size_repaired","observed":observed,"expected":int(sample_size)}) return match.group(1) + str(int(sample_size)) revised = sample_pattern.sub(sample_repl, revised) count_pattern = re.compile(r"(?i)\b(\d{2,6})\s+(respondents|participants|observations|cases)\b") def count_repl(match): observed = int(match.group(1)) if observed == int(sample_size): return match.group(0) diagnostics.append({"type":"sample_count_repaired","observed":observed,"expected":int(sample_size)}) return f"{int(sample_size)} {match.group(2)}" revised = count_pattern.sub(count_repl, revised) if scale_points: scale_pattern = re.compile(r"(?i)\b([2-9]|1[0-1])[- ]point\s+(Likert(?:-type)?\s+scale)") def scale_repl(match): observed = int(match.group(1)) if observed == int(scale_points): return match.group(0) diagnostics.append({"type":"scale_points_repaired","observed":observed,"expected":int(scale_points)}) return f"{int(scale_points)}-point {match.group(2)}" revised = scale_pattern.sub(scale_repl, revised) if revised != sentence: sample_scale_replacements.append((start, end, revised)) for start, end, replacement in sorted(sample_scale_replacements, reverse=True): value = value[:start] + replacement + value[end:] # Explicit construct-role labels are deterministic and safe to correct. role_patterns = { "predictor": r"(?:predictor|independent\s+variable|predictor\s+construct)", "mediator": r"(?:mediator|mediating\s+variable|mediating\s+construct)", "outcome": r"(?:outcome|dependent\s+variable|outcome\s+construct)", } known_constructs = [str(v) for v in constructs.values() if str(v)] for role, role_pattern in role_patterns.items(): expected = str(constructs.get(role) or "") if not expected: continue for candidate in known_constructs: if candidate.casefold() == expected.casefold(): continue pattern = re.compile(rf"(?i)({role_pattern}\s*(?:was|is|:|=)?\s*){re.escape(candidate)}") if pattern.search(value): value, count = pattern.subn(lambda m: m.group(1) + expected, value) if count: diagnostics.append({"type":"construct_role_repaired","role":role,"observed":candidate,"expected":expected,"count":count}) if strict: # Frozen system-generated evidence has no causal identification. replacements = ( (r"(?i)\bhad\s+a\s+causal\s+effect\s+on\b", "was associated with"), (r"(?i)\bexerted\s+a\s+causal\s+effect\s+on\b", "was associated with"), (r"(?i)\bcausal\s+effect\s+on\b", "association with"), (r"(?i)\ban\s+causal\s+effect\b", "an association"), (r"(?i)\ba\s+causal\s+effect\b", "an association"), (r"(?i)\bthe\s+causal\s+effect\b", "the association"), (r"(?i)\bcausal\s+effects\b", "associations"), (r"(?i)\bcausal\s+effect\b", "association"), (r"(?i)\bcausal\s+mediation\b", "indirect association"), ) for pattern, replacement in replacements: value, count = re.subn(pattern, replacement, value) if count: diagnostics.append({"type":"causal_language_repaired","replacement":replacement,"count":count}) # Remove p-values attached to this study's frozen constructs because # no p-values are generated/frozen by this engine. External cited # literature claims are left untouched. construct_folded = [c.casefold() for c in known_constructs] spans = cls._sentence_spans(value) replacements_by_span = [] forbidden = [str(x) for x in ledger.get("forbidden_unowned_methods") or []] section_sensitive = any(token in str(section_name or "").casefold() for token in ("method", "result", "abstract", "final", "analysis")) for start, end, sentence in spans: lower = sentence.casefold() has_construct = any(c in lower for c in construct_folded) if construct_folded else False has_citation = bool(re.search(r"\([^()]*?(?:19|20)\d{2}[a-z]?[^()]*?\)", sentence)) revised = sentence if has_construct and not has_citation: revised, p_count = re.subn(r"(?i)\s*[,;]?\s*p\s*(?:=|<|>|≤|≥)\s*(?:0?\.\d+|\d+(?:\.\d+)?)", "", revised) if p_count: diagnostics.append({"type":"unfrozen_p_value_removed","count":p_count}) if section_sensitive and not has_citation and re.search(r"(?i)\b(?:we|our\s+study|this\s+study|the\s+study|our\s+analysis|the\s+analysis)\b", sentence): bad = next((term for term in forbidden if re.search(rf"(?i)(?