Upload 2 files

app.py

@@ -3,6 +3,7 @@
 from core.meta_extractor import extract_golden_metadata
 from core.rag_engine import initialize_rag
 from core.qualitative_engine import get_qualitative_engine
+from core.quantitative_engine import get_quantitative_engine
 import streamlit as st
 
 st.set_page_config(
@@ -24,14 +25,12 @@ import zipfile
 import math
 import json
 from datetime import datetime, timezone, timedelta
-import pandas as pd
-import numpy as np
 import threading
 from concurrent.futures import ThreadPoolExecutor
 from streamlit.runtime.scriptrunner import add_script_run_ctx, get_script_run_ctx
 
 from config import APP_CONFIG, AMJ_PROMPTS, ASQ_PROMPTS, GENERIC_PROMPTS
-from utils.file_parser import extract_text_from_files, extract_data_from_files
+from utils.file_parser import extract_text_from_files
 from utils.docx_builder import text_to_docx
 from core.pipeline_agent import SSciPipelineAgent
 from managed_auth import require_alipay_login, render_logged_in_account
@@ -124,90 +123,62 @@ def get_global_lock():
 
 global_lock = get_global_lock()
 
-def generate_psd_corr_matrix(n_constructs, max_corr=0.5):
-    if n_constructs == 1: return np.array([[1.0]])
-    base_corr = np.random.uniform(0.2, max_corr - 0.1) 
-    A = np.random.uniform(base_corr - 0.1, base_corr + 0.1, (n_constructs, n_constructs))
-    A = (A + A.T) / 2
-    np.fill_diagonal(A, 1.0)
-    min_eig = np.min(np.linalg.eigvals(A))
-    if min_eig < 0: A -= (min_eig - 0.01) * np.eye(n_constructs)
-    d = np.sqrt(np.diag(A))
-    return A / np.outer(d, d)
-
-def calculate_validity_metrics(df, constructs_info, true_loadings):
-    metrics = []
-    for const in constructs_info:
-        c_name = const['name']
-        items = const['items']
-        sub_df = df[items]
-        var_total = sub_df.sum(axis=1).var(ddof=1)
-        sum_var = sub_df.var(ddof=1).sum()
-        n_items = len(items)
-        alpha = 0.0 if var_total <= 0 else (n_items / (n_items - 1)) * (1 - sum_var / var_total)
-        c_loadings = np.array(true_loadings[c_name])
-        sum_loadings_sq = np.sum(c_loadings)**2
-        sum_sq_loadings = np.sum(c_loadings**2)
-        sum_error = np.sum(1 - c_loadings**2)
-        cr = sum_loadings_sq / (sum_loadings_sq + sum_error)
-        ave = sum_sq_loadings / len(c_loadings)
-        metrics.append({"Construct": c_name, "Items": n_items, "Cronbach's α": round(alpha, 3), "CR": round(cr, 3), "AVE": round(ave, 3)})
-    return pd.DataFrame(metrics)
-
-def generate_empirical_data_robust(construct_names=None, n=300, scale_points=7):
-    if not construct_names or len(construct_names) < 2: 
-        construct_names = ["Independent_Var", "Mediator_Var", "Dependent_Var"]
-        
-    n_constructs = len(construct_names)
-    corr = generate_psd_corr_matrix(n_constructs, max_corr=0.55)
-    latent_scores = np.random.multivariate_normal(mean=[0]*n_constructs, cov=corr, size=n)
-    data, true_loadings, constructs_info = {}, {}, []
-    target_mean = scale_points / 2.0 + 0.5
-    target_std = scale_points / 5.0
-    
-    for idx, c_name in enumerate(construct_names):
-        true_score = latent_scores[:, idx]
-        n_items = np.random.randint(3, 6) 
-        loadings = np.random.uniform(0.72, 0.88, n_items)
-        true_loadings[c_name] = loadings
-        item_names = []
-        for i, load in enumerate(loadings):
-            error = np.random.normal(0, np.sqrt(1 - load**2), n)
-            item_score = load * true_score + error 
-            raw_scaled = item_score * target_std + target_mean
-            item_int = np.clip(np.round(raw_scaled), 1, scale_points).astype(int)
-            col_name = f"{c_name}_{i+1}"
-            data[col_name] = item_int
-            item_names.append(col_name)
-        constructs_info.append({"name": c_name, "items": item_names})
-        
-    df = pd.DataFrame(data)
-    validity_df = calculate_validity_metrics(df, constructs_info, true_loadings)
-    mean_scores = pd.DataFrame({c['name']: df[c['items']].mean(axis=1) for c in constructs_info})
-    fl_matrix = mean_scores.corr().astype(object) 
-    
-    for i, row in validity_df.iterrows():
-        c_name = row['Construct']
-        sqrt_ave = np.sqrt(row['AVE'])
-        fl_matrix.loc[c_name, c_name] = f"({sqrt_ave:.3f})"
-        
-    for i in range(len(fl_matrix.columns)):
-        for j in range(i + 1, len(fl_matrix.columns)): 
-            fl_matrix.iloc[i, j] = ""
-            
-    desc_stats = mean_scores.mean().to_frame('Mean').join(mean_scores.std().to_frame('SD'))
-    return df, validity_df, desc_stats.join(fl_matrix)
-
-def resolve_empirical_data(uploaded_files=None, language="zh"):
-    if uploaded_files:
-        user_md = extract_data_from_files(uploaded_files)
-        if user_md:
-            return None, user_md, tr(language, "actual_stats_parsed")
-    
-    df, validity_df, fl_matrix_df = generate_empirical_data_robust()
-    simulated_md_prompt = f"""### Table 1: Reliability and Convergent Validity\n{validity_df.to_markdown(index=False)}\n### Table 2: Descriptive Statistics and Discriminant Validity (Fornell-Larcker Criterion)\n*Note: Diagonal elements in parentheses are the square roots of AVE.*\n{fl_matrix_df.to_markdown()}\n[STRICT EMPIRICAL INSTRUCTION]\n1. ZERO HALLUCINATION: You MUST report the exact metrics (Cronbach's α, CR, AVE, correlations) provided in the tables above.\n2. VALIDITY DEFENSE: Explicitly state that Convergent Validity is established because all AVEs > 0.5 and CRs > 0.7.\n3. DISCRIMINANT DEFENSE: Explicitly state that Discriminant Validity is satisfied per the Fornell-Larcker criterion, as the square root of AVE (diagonal values) exceed inter-construct correlations."""
-    status_msg = tr(language, "synthetic_stats_generated")
-    return df, simulated_md_prompt, status_msg
+def build_generation_disclosure(run_mode, data_metadata):
+    metadata = data_metadata or {}
+    source_type = metadata.get("source_type", "unknown")
+    generated = source_type == "system_generated"
+    mode_label = "quantitative" if run_mode == "amj" else "qualitative"
+
+    lines = [
+        "SmartPaper Generation Disclosure / SmartPaper 生成说明",
+        "",
+        f"Research mode / 研究模式: {mode_label}",
+        f"Evidence source / 证据来源: {source_type}",
+        "",
+    ]
+    if generated:
+        lines.extend(
+            [
+                "The statistical results or qualitative materials used by "
+                "this draft were generated by the system because no "
+                "complete user-supplied evidence was available.",
+                "本初稿使用的统计结果或定性材料由系统生成，原因是用户未提供完整的相应证据材料。",
+                "",
+                "These materials support drafting and methodological "
+                "demonstration only. They must not be represented as "
+                "observed, collected, or independently verified real data.",
+                "这些材料仅用于论文起草和方法展示，不得被表述为已经观察、收集或独立核验的真实数据。",
+                "",
+                "The generated evidence is bounded by the configured "
+                "parameters and prompts. Users must replace or validate it "
+                "before research submission, publication, policy use, or "
+                "other consequential use.",
+                "生成证据受系统参数和提示词约束。用于投稿、发表、政策或其他重要用途前，用户必须替换或核验。",
+            ]
+        )
+    else:
+        lines.extend(
+            [
+                "The draft used evidence material supplied by the user. "
+                "SmartPaper did not independently verify its accuracy, "
+                "completeness, consent status, or provenance.",
+                "本初稿使用了用户提供的证据材料。SmartPaper未独立核验其准确性、完整性、授权状态或来源。",
+            ]
+        )
+
+    method = metadata.get("method")
+    if method:
+        lines.extend(["", f"Generation method / 生成方法: {method}"])
+    facts = metadata.get("study_facts") or {}
+    if facts:
+        lines.extend(
+            [
+                "",
+                "Frozen qualitative study facts / 已冻结的定性研究事实:",
+                json.dumps(facts, ensure_ascii=False, indent=2),
+            ]
+        )
+    return "\n".join(lines).strip() + "\n"
 
 def _translation_system_prompt(target_lang):
     common = """Translate the supplied academic manuscript faithfully. Preserve every citation tag, number, percentage, coefficient, p-value, confidence interval, DOI, table structure, heading level, hypothesis label, variable abbreviation, and conclusion direction. Do not add, remove, strengthen, or weaken empirical claims. Use concise, accurate, and rigorous academic language. Do not use metaphors, analogies, rhetorical questions, promotional language, formulaic transitions, or mechanical repetition. Output only the translated manuscript."""
@@ -221,7 +192,14 @@ def _translation_system_prompt(target_lang):
 def async_translate_task(agent, en_text, placeholder, target_lang, quarantine_flag, interface_lang):
     try:
         sys_p = _translation_system_prompt(target_lang)
-        trans_text = agent.call_deepseek(sys_p, f"TEXT:\n{en_text}", max_tokens=6000, model_name="deepseek-chat")
+        trans_text = agent.call_deepseek(
+            sys_p,
+            f"TEXT:\n{en_text}",
+            max_tokens=6000,
+            model_role="translation",
+            thinking=False,
+            allow_fallback=False,
+        )
         trans_resolved = agent.resolve_citations(trans_text, quarantine=quarantine_flag)
         placeholder.markdown(trans_resolved)
         return trans_resolved
@@ -604,6 +582,7 @@ if st.session_state.start_pipeline or st.session_state.resume_pipeline:
     SAFE_CHUNK_WORDS = agent.chunk_words_cyrillic if target_lang in ["Russian (Русский)", "Kazakh (Қазақша)"] else agent.chunk_words_default
     
     qual_engine = get_qualitative_engine(agent)
+    quant_engine = get_quantitative_engine()
     
     try:
         update_heartbeat()
@@ -624,6 +603,7 @@ if st.session_state.start_pipeline or st.session_state.resume_pipeline:
             words = state["words"]
             user_data_md = state["user_data_md"]
             user_lit_context = state["user_lit_context"]
+            data_metadata = state.get("data_metadata", {})
             
             agent.load_state(state["agent_state"])
             adaptive_structure = state["adaptive_structure"]
@@ -647,6 +627,7 @@ if st.session_state.start_pipeline or st.session_state.resume_pipeline:
             pb = progress_bar.progress(1, text=tr(lang_code, "pipeline_start"))
             user_lit_context = ""
             user_data_md = ""
+            data_metadata = {}
             
             if st.session_state.run_mode == "amj":
                 if uploaded_papers_amj:
@@ -669,7 +650,12 @@ if st.session_state.start_pipeline or st.session_state.resume_pipeline:
                 
                 with st.spinner(tr(lang_code, "extract_quant")):
                     update_heartbeat()
-                    _, user_data_md, msg = resolve_empirical_data(uploaded_data_amj, lang_code)
+                    _, user_data_md, msg, data_metadata = (
+                        quant_engine.resolve_empirical_data(
+                            uploaded_data_amj,
+                            lang_code,
+                        )
+                    )
                     st.info(msg)
                     
             elif st.session_state.run_mode == "asq":
@@ -693,14 +679,21 @@ if st.session_state.start_pipeline or st.session_state.resume_pipeline:
                 
                 update_heartbeat()
                 raw_interviews = extract_text_from_files([uploaded_interviews]) if uploaded_interviews else ""
-                user_data_md, msg = qual_engine.resolve_qualitative_data(topic, keywords, raw_interviews)
+                user_data_md, msg, data_metadata = (
+                    qual_engine.resolve_qualitative_data(
+                        topic,
+                        keywords,
+                        raw_interviews,
+                    )
+                )
                 st.info(tr(lang_code, "qualitative_ready"))
 
             state = {
                 "topic": topic, "keywords": keywords, "target_lang": target_lang, "words": words,
                 "user_data_md": user_data_md, "user_lit_context": user_lit_context,
                 "completed_sections": [], "full_draft_en": "", "full_draft_translated": "",
-                "context_tail": "", "run_mode": st.session_state.run_mode, "interface_language": lang_code
+                "context_tail": "", "run_mode": st.session_state.run_mode, "interface_language": lang_code,
+                "data_metadata": data_metadata
             }
 
             pb.progress(5, text=tr(lang_code, "config_route"))
@@ -755,7 +748,18 @@ if st.session_state.start_pipeline or st.session_state.resume_pipeline:
             pb.progress(14, text=tr(lang_code, "anchor_progress"))
             with st.status(tr(lang_code, "anchor_status"), expanded=True) as status:
                 ph = st.empty()
-                anchor_response = ph.write_stream(stream_with_heartbeat(agent.stream_generate_anchors(topic, keywords, synthesis_result, refs_text)))
+                anchor_response = ph.write_stream(
+                    stream_with_heartbeat(
+                        agent.stream_generate_anchors(
+                            topic,
+                            keywords,
+                            synthesis_result,
+                            refs_text,
+                            data_evidence=user_data_md,
+                            mode=st.session_state.run_mode,
+                        )
+                    )
+                )
                 status.update(label=tr(lang_code, "anchor_done"), state="complete", expanded=False)
             
             # ==========================================================
@@ -767,6 +771,8 @@ if st.session_state.start_pipeline or st.session_state.resume_pipeline:
             conclusion_match = re.search(r'-+\s*CONCLUSION\s*-+\s*(.*?)$', anchor_response_clean, re.DOTALL | re.IGNORECASE)
             abstract_anchor = abstract_match.group(1).strip() if abstract_match else anchor_response_clean.strip()
             conclusion_anchor = conclusion_match.group(1).strip() if conclusion_match else ""
+            agent.global_memory["abstract_anchor"] = abstract_anchor
+            agent.global_memory["conclusion_anchor"] = conclusion_anchor
             # ==========================================================
             
             pb.progress(15, text=tr(lang_code, "outline_progress"))
@@ -776,7 +782,14 @@ if st.session_state.start_pipeline or st.session_state.resume_pipeline:
             structure = plan.get("structure", [])
 
             en_abstract_clean = clean_markdown_headings(re.sub(r'(?i)^abstract[:\s]*', '', str(abstract_anchor)).strip())
-            trans_abstract_clean = clean_markdown_headings(re.sub(r'(?i)^(摘要|abstract|аннотация|түйіндеме)[:\s]*', '', agent.call_deepseek(_translation_system_prompt(target_lang), abstract_anchor, max_tokens=3000)).strip())
+            trans_abstract_clean = clean_markdown_headings(re.sub(r'(?i)^(摘要|abstract|аннотация|түйіндеме)[:\s]*', '', agent.call_deepseek(
+                _translation_system_prompt(target_lang),
+                abstract_anchor,
+                max_tokens=3000,
+                model_role="translation",
+                thinking=False,
+                allow_fallback=False,
+            )).strip())
 
             loc_abs = {"Simplified Chinese (简体中文)": "摘要", "Russian (Русский)": "Аннотация", "Kazakh (Қазақша)": "Түйіндеме"}.get(target_lang, "Abstract")
             loc_kw = {"Simplified Chinese (简体中文)": "关键词", "Russian (Русский)": "Ключевые слова", "Kazakh (Қазақша)": "Кілт сөздер"}.get(target_lang, "Keywords")
@@ -992,11 +1005,18 @@ if st.session_state.start_pipeline or st.session_state.resume_pipeline:
                     update_heartbeat()
                     st.write(tr(lang_code, "api_requested"))
                     
-                    target_model = "deepseek-v4-flash"
+                    target_model = agent.model_for("writing")
                     st.write(tr(lang_code, "model_writing", model=target_model))
                     
                     en_draft_clean = re.sub(r'<think>.*?</think>', '', en_ph.write_stream(
-                        stream_with_heartbeat(agent.stream_deepseek(sys_prompt_m3, base_user, max_tokens=6000, model_name=target_model, thinking=False))
+                        stream_with_heartbeat(agent.stream_deepseek(
+                            sys_prompt_m3,
+                            base_user,
+                            max_tokens=6000,
+                            model_role="writing",
+                            thinking=False,
+                            allow_fallback=False,
+                        ))
                     ), flags=re.DOTALL).strip()
                     
                     status.update(label=tr(lang_code, "draft_done", name=display_name), state="complete", expanded=False)
@@ -1101,7 +1121,14 @@ if st.session_state.start_pipeline or st.session_state.resume_pipeline:
             if total_words_en > upper_limit:
                 conclusion_match = re.search(r'(#+\s*Conclusion.*?)(?=\n#+ |\Z)', full_draft_en, re.DOTALL | re.IGNORECASE)
                 if conclusion_match:
-                    compressed_en = agent.call_deepseek("Condense the following conclusion by 15-20% preserving citations.", conclusion_match.group(1), max_tokens=1500, model_name="deepseek-chat")
+                    compressed_en = agent.call_deepseek(
+                        "Condense the following conclusion by 15-20% preserving citations.",
+                        conclusion_match.group(1),
+                        max_tokens=1500,
+                        model_role="writing",
+                        thinking=False,
+                        allow_fallback=False,
+                    )
                     if compressed_en:
                         compressed_en = agent.resolve_citations(compressed_en, quarantine=True)
                         full_draft_en = full_draft_en.replace(conclusion_match.group(1), compressed_en.strip())
@@ -1114,7 +1141,14 @@ if st.session_state.start_pipeline or st.session_state.resume_pipeline:
                             else: 
                                 full_draft_translated += "\n\n" + compressed_trans.strip()
             elif total_words_en < lower_limit:
-                expanded_en = agent.call_deepseek("Continue conclusion by adding 150 words (future research).", full_draft_en[-2000:], max_tokens=800, model_name="deepseek-chat")
+                expanded_en = agent.call_deepseek(
+                    "Continue conclusion by adding 150 words (future research).",
+                    full_draft_en[-2000:],
+                    max_tokens=800,
+                    model_role="writing",
+                    thinking=False,
+                    allow_fallback=False,
+                )
                 if expanded_en:
                     expanded_en = agent.resolve_citations(expanded_en, quarantine=True)
                     full_draft_en += "\n\n" + expanded_en.strip()
@@ -1136,6 +1170,13 @@ if st.session_state.start_pipeline or st.session_state.resume_pipeline:
             with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zf:
                 zf.writestr(f"Research_EN_{bj_time}.docx", text_to_docx(full_draft_en).getvalue())
                 zf.writestr(f"Research_{output_suffix}_{bj_time}.docx", text_to_docx(full_draft_translated).getvalue())
+                zf.writestr(
+                    "GENERATION_DISCLOSURE.txt",
+                    build_generation_disclosure(
+                        st.session_state.run_mode,
+                        data_metadata,
+                    ),
+                )
             zip_buffer.seek(0)
             st.session_state.zip_buffer = zip_buffer.getvalue()
             st.session_state.zip_filename = f"Draft_{bj_time}.zip"

config.py

@@ -23,32 +23,65 @@ def load_config():
         
         # --- API 节点配置 ---
         "deepseek_api_url": os.environ.get(
-        "DEEPSEEK_API_URL",
-        "https://api.deepseek.com/chat/completions",
-    ),
-    "deepseek_default_model": os.environ.get(
-        "DEEPSEEK_DEFAULT_MODEL",
-        "deepseek-v4-flash",
-    ),
-    "deepseek_fallback_model": os.environ.get(
-        "DEEPSEEK_FALLBACK_MODEL",
-        "deepseek-v4-flash",
-    ),
-    "deepseek_default_thinking": os.environ.get(
-        "DEEPSEEK_DEFAULT_THINKING",
-        "true",
-    ).strip().lower() in {"1", "true", "yes", "on"},
-    "deepseek_reasoning_effort": os.environ.get(
-        "DEEPSEEK_REASONING_EFFORT",
-        "high",
-    ),
-    "deepseek_max_retries": int(
-        os.environ.get("DEEPSEEK_MAX_RETRIES", "3")
-    ),
-    "deepseek_show_reasoning": os.environ.get(
-        "DEEPSEEK_SHOW_REASONING",
-        "false",
-    ).strip().lower() in {"1", "true", "yes", "on"},
+            "DEEPSEEK_API_URL",
+            "https://api.deepseek.com/chat/completions",
+        ),
+        # Model names are centralized here and can be upgraded in HF Variables.
+        # Application and engine code select a task role, never a concrete model.
+        "deepseek_models": {
+            "reasoning": os.environ.get(
+                "DEEPSEEK_MODEL_REASONING",
+                "deepseek-v4-pro",
+            ),
+            "writing": os.environ.get(
+                "DEEPSEEK_MODEL_WRITING",
+                "deepseek-v4-pro",
+            ),
+            "translation": os.environ.get(
+                "DEEPSEEK_MODEL_TRANSLATION",
+                "deepseek-v4-pro",
+            ),
+            "fast": os.environ.get(
+                "DEEPSEEK_MODEL_FAST",
+                "deepseek-v4-flash",
+            ),
+            "fallback": os.environ.get(
+                "DEEPSEEK_MODEL_FALLBACK",
+                "deepseek-v4-flash",
+            ),
+        },
+        "deepseek_task_settings": {
+            "reasoning": {
+                "thinking": True,
+                "reasoning_effort": os.environ.get(
+                    "DEEPSEEK_REASONING_EFFORT",
+                    "high",
+                ),
+                "allow_fallback": False,
+            },
+            "writing": {
+                "thinking": False,
+                "reasoning_effort": "high",
+                "allow_fallback": False,
+            },
+            "translation": {
+                "thinking": False,
+                "reasoning_effort": "high",
+                "allow_fallback": False,
+            },
+            "fast": {
+                "thinking": False,
+                "reasoning_effort": "low",
+                "allow_fallback": True,
+            },
+        },
+        "deepseek_max_retries": int(
+            os.environ.get("DEEPSEEK_MAX_RETRIES", "3")
+        ),
+        "deepseek_show_reasoning": os.environ.get(
+            "DEEPSEEK_SHOW_REASONING",
+            "false",
+        ).strip().lower() in {"1", "true", "yes", "on"},
         "openalex_api_url": os.environ.get("OPENALEX_API_URL", "https://api.openalex.org/works"),
         "s2_api_url": os.environ.get("S2_API_URL", "https://api.semanticscholar.org/graph/v1"),