diff --git "a/BioMod/interpretability/module_text_alignment/module_text_alignment_publication.py" "b/BioMod/interpretability/module_text_alignment/module_text_alignment_publication.py" --- "a/BioMod/interpretability/module_text_alignment/module_text_alignment_publication.py" +++ "b/BioMod/interpretability/module_text_alignment/module_text_alignment_publication.py" @@ -1,1277 +1,1277 @@ -"""Publication-quality analysis for Experiment 2 — Module-Text Alignment. - -Produces 6 main figures + comprehensive metrics with bootstrap CIs for paper. -Supports multi-model comparison and cross-dataset validation. - -Usage: - python module_text_alignment_publication.py \ - --module_embedding_path ../results/geometry/pretrain_subset/module_embeddings.npy \ - --module_embedding_path2 ../results/geometry/pancreas_scib/module_embeddings.npy \ - --module_json ../../module_data_all/merged_modules.json \ - --sentence_model ./S-PubMedBert-MS-MARCO \ - --sentence_models_compare "sentence-transformers/all-mpnet-base-v2" \ - --descriptions_file ./reactome_descriptions.json \ - --n_permutations 1000 \ - --n_bootstrap 2000 \ - --output_dir ../results/text_alignment/publication -""" -from __future__ import annotations - -import argparse -import json -import logging -import sys -from collections import Counter, defaultdict -from pathlib import Path -from typing import Dict, List, Optional, Tuple - -import numpy as np -import pandas as pd -from scipy.cluster.hierarchy import dendrogram, fcluster, linkage -from scipy.spatial.distance import pdist, squareform -from scipy.stats import kruskal, spearmanr -from sklearn.metrics.pairwise import cosine_similarity - -import matplotlib -matplotlib.use("Agg") -import matplotlib.pyplot as plt -from matplotlib.patches import Patch as LegendHandle -from matplotlib.lines import Line2D - -HERE = Path(__file__).resolve().parent -INTERP_DIR = HERE.parent -PERFORMER_MODULE = INTERP_DIR.parent -REPO_ROOT = PERFORMER_MODULE.parent -for p in (str(HERE), str(INTERP_DIR), str(PERFORMER_MODULE), str(REPO_ROOT)): - if p not in sys.path: - sys.path.insert(0, p) - -from interpretability.utils import ( - HALLMARK_PROCESS, - load_module_names, - parse_hallmark_process, - parse_module_family, -) -from module_text_alignment.module_text_alignment import ( - build_text_descriptions, - compute_cka, - compute_rsa, - encode_texts, - mantel_test, - permutation_test_cka, -) -from module_text_alignment.module_text_alignment_extended import ( - REACTOME_CATEGORIES, - categorize_reactome, -) - -logger = logging.getLogger(__name__) - -# ============================================================ -# Color palette for MSigDB hallmark process categories -# ============================================================ - -PROCESS_COLORS = { - "immune": "#2196F3", - "metabolic": "#4CAF50", - "signaling": "#FF9800", - "proliferation": "#E91E63", - "development": "#9C27B0", - "dna_damage": "#F44336", - "pathway": "#607D8B", - "cellular_component": "#795548", -} - -PROCESS_LABELS = { - "immune": "Immune", - "metabolic": "Metabolic", - "signaling": "Signaling", - "proliferation": "Proliferation", - "development": "Development", - "dna_damage": "DNA Damage", - "pathway": "Pathway", - "cellular_component": "Cellular Component", -} - -# Reactome super-categories (merge related categories for granularity sweep) -REACTOME_SUPER_CATEGORIES = { - "Information Processing": ["Gene Expression", "DNA Replication", "DNA Repair", - "Chromatin & Epigenetics"], - "Signal & Communication": ["Signal Transduction", "Cell-Cell Communication"], - "Cell Fate": ["Cell Cycle", "Apoptosis", "Cellular Responses"], - "Metabolism & Transport": ["Metabolism", "Transport"], - "Immune & Defense": ["Immune System", "Hemostasis", "Disease"], - "Development": ["Developmental Biology", "Protein Homeostasis"], -} - - -# ============================================================ -# Utility functions -# ============================================================ - -def savefig(fig, path_stem: str, dpi: int = 300): - """Save figure as both PDF and PNG.""" - fig.savefig(f"{path_stem}.pdf", dpi=dpi, bbox_inches="tight") - fig.savefig(f"{path_stem}.png", dpi=dpi, bbox_inches="tight") - plt.close(fig) - logger.info("Saved: %s.{pdf,png}", path_stem) - - -def adaptive_vlim(matrix: np.ndarray, percentile: int = 98, symmetric: bool = True): - """Compute adaptive vmin/vmax from data percentiles.""" - vals = matrix[np.triu_indices_from(matrix, k=1)] - if symmetric: - vlim = np.percentile(np.abs(vals), percentile) - return -vlim, vlim - else: - return np.percentile(vals, [100 - percentile, percentile]) - - -def bootstrap_ci( - values: np.ndarray, - stat_fn=np.mean, - n_boot: int = 2000, - ci: float = 0.95, - seed: int = 42, -) -> Tuple[float, float, float]: - """Bootstrap confidence interval. Returns (estimate, ci_low, ci_high).""" - rng = np.random.default_rng(seed) - n = len(values) - boot_stats = np.zeros(n_boot) - for i in range(n_boot): - idx = rng.integers(0, n, size=n) - boot_stats[i] = stat_fn(values[idx]) - alpha = (1 - ci) / 2 - return float(stat_fn(values)), float(np.percentile(boot_stats, alpha * 100)), \ - float(np.percentile(boot_stats, (1 - alpha) * 100)) - - -def bootstrap_rsa( - module_emb: np.ndarray, - text_emb: np.ndarray, - n_boot: int = 2000, - seed: int = 42, -) -> Tuple[float, float, float]: - """Bootstrap CI for RSA Spearman r by resampling modules.""" - rng = np.random.default_rng(seed) - n = module_emb.shape[0] - boot_rs = np.zeros(n_boot) - for i in range(n_boot): - idx = rng.choice(n, size=n, replace=True) - dist_p = pdist(module_emb[idx], metric="cosine") - dist_t = pdist(text_emb[idx], metric="cosine") - boot_rs[i], _ = spearmanr(dist_p, dist_t) - observed, _ = spearmanr(pdist(module_emb, "cosine"), pdist(text_emb, "cosine")) - return float(observed), float(np.percentile(boot_rs, 2.5)), \ - float(np.percentile(boot_rs, 97.5)) - - -def get_module_sizes(module_json: str) -> Dict[str, int]: - """Get number of genes per module from merged_modules.json.""" - with open(module_json, "r", encoding="utf-8") as f: - modules = json.load(f) - return {name: len(genes) for name, genes in modules.items()} - - -# ============================================================ -# CLI -# ============================================================ - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - parser.add_argument("--module_embedding_path", type=str, required=True, - help="Primary module_embeddings.npy (e.g. pretrain_subset)") - parser.add_argument("--module_embedding_path2", type=str, default=None, - help="Second dataset module_embeddings.npy for cross-dataset comparison") - parser.add_argument("--module_json", type=str, - default=str(PERFORMER_MODULE / "module_data_all" / "merged_modules.json")) - parser.add_argument("--descriptions_file", type=str, default=None) - parser.add_argument("--sentence_model", type=str, - default="pritamdeka/S-PubMedBert-MS-MARCO") - parser.add_argument("--sentence_models_compare", type=str, default=None, - help="Comma-separated additional models for comparison") - parser.add_argument("--device", type=str, default="cpu") - parser.add_argument("--n_permutations", type=int, default=1000) - parser.add_argument("--n_bootstrap", type=int, default=2000) - parser.add_argument("--seed", type=int, default=42) - parser.add_argument("--output_dir", type=str, required=True) - return parser.parse_args() - - -# ============================================================ -# Figure 1: RSA Overview (hexbin + null distribution) -# ============================================================ - -def fig1_rsa_overview( - module_emb: np.ndarray, - text_emb: np.ndarray, - mantel_results: Dict, - output_dir: Path, - metric: str = "cosine", - n_bootstrap: int = 2000, - seed: int = 42, -) -> Dict: - """RSA overview: hexbin density + Mantel null distribution.""" - dist_module = pdist(module_emb, metric=metric) - dist_text = pdist(text_emb, metric=metric) - - observed_r = mantel_results["rsa_spearman_observed"] - null_mean = mantel_results["rsa_null_mean"] - null_std = mantel_results["rsa_null_std"] - p_value = mantel_results["rsa_p_value"] - z_score = mantel_results["rsa_effect_size_z"] - - # Bootstrap CI for RSA - rsa_obs, rsa_ci_lo, rsa_ci_hi = bootstrap_rsa( - module_emb, text_emb, n_boot=n_bootstrap, seed=seed - ) - - fig, axes = plt.subplots(1, 2, figsize=(12, 5)) - - # Panel A: Hexbin density - ax = axes[0] - hb = ax.hexbin(dist_text, dist_module, gridsize=40, cmap="YlOrRd", - mincnt=1, linewidths=0.2) - cb = plt.colorbar(hb, ax=ax, fraction=0.046, pad=0.02) - cb.set_label("Count", fontsize=9) - - # Trend line - z = np.polyfit(dist_text, dist_module, 1) - p_line = np.poly1d(z) - x_range = np.linspace(dist_text.min(), dist_text.max(), 100) - ax.plot(x_range, p_line(x_range), "k-", linewidth=2, alpha=0.8) - - ax.set_xlabel("Text pairwise distance (cosine)", fontsize=10) - ax.set_ylabel("Module embedding pairwise distance (cosine)", fontsize=10) - ax.set_title(f"A. RSA: r = {observed_r:.3f} [{rsa_ci_lo:.3f}, {rsa_ci_hi:.3f}]", - fontsize=11, fontweight="bold") - ax.text(0.05, 0.95, f"n = {module_emb.shape[0]} modules\n" - f"pairs = {len(dist_module):,}", - transform=ax.transAxes, fontsize=8, va="top", - bbox=dict(boxstyle="round", facecolor="white", alpha=0.8)) - - # Panel B: Null distribution - ax = axes[1] - from scipy.stats import norm as norm_dist - x = np.linspace(null_mean - 4.5 * null_std, - max(observed_r + 2 * null_std, null_mean + 4.5 * null_std), 300) - y = norm_dist.pdf(x, null_mean, null_std) - ax.fill_between(x, y, alpha=0.25, color="gray") - ax.plot(x, y, color="gray", linewidth=1.5, label="Null distribution") - - # Shade p-value region - x_tail = x[x >= observed_r] - y_tail = norm_dist.pdf(x_tail, null_mean, null_std) - ax.fill_between(x_tail, y_tail, alpha=0.4, color="red", label=f"p = {p_value:.4f}") - - ax.axvline(observed_r, color="red", linewidth=2.5, linestyle="-", - label=f"Observed r = {observed_r:.4f}") - ax.axvline(null_mean, color="gray", linewidth=1, linestyle="--", alpha=0.7) - - ax.set_xlabel("Spearman r (distance matrix correlation)", fontsize=10) - ax.set_ylabel("Density", fontsize=10) - ax.set_title(f"B. Mantel Test: z = {z_score:.1f}σ, p = {p_value:.4f}", - fontsize=11, fontweight="bold") - ax.legend(fontsize=9, loc="upper left") - - plt.tight_layout() - savefig(fig, str(output_dir / "fig1_rsa_overview")) - - return { - "rsa_r": rsa_obs, - "rsa_ci_95": [rsa_ci_lo, rsa_ci_hi], - "mantel_p": p_value, - "mantel_z": z_score, - } - - -# ============================================================ -# Figure 2: Hallmarks Similarity Matrices with Dendrograms -# ============================================================ - -def _rank_transform_matrix(sim_matrix: np.ndarray) -> np.ndarray: - """Rank-transform off-diagonal elements to [0,1] for visualization.""" - from scipy.stats import rankdata - n = sim_matrix.shape[0] - result = np.zeros_like(sim_matrix) - triu_idx = np.triu_indices(n, k=1) - vals = sim_matrix[triu_idx] - ranks = rankdata(vals) / len(vals) - result[triu_idx] = ranks - result[triu_idx[1], triu_idx[0]] = ranks - np.fill_diagonal(result, 1.0) - return result - - -def fig2_hallmarks_heatmaps( - module_emb: np.ndarray, - text_emb: np.ndarray, - names: List[str], - texts: List[str], - output_dir: Path, -) -> Dict: - """Hallmarks 50x50 similarity matrices with rank-transform and clustering.""" - families = [parse_module_family(n) for n in names] - hallmark_mask = np.array([f == "Hallmarks" for f in families]) - hallmark_idx = np.where(hallmark_mask)[0] - - h_module = module_emb[hallmark_idx] - h_text = text_emb[hallmark_idx] - h_names = [names[i] for i in hallmark_idx] - h_texts = [texts[i] for i in hallmark_idx] - n_h = len(hallmark_idx) - - h_processes = [] - for n in h_names: - proc = parse_hallmark_process(n) - h_processes.append(proc if proc else "pathway") - - module_sim = cosine_similarity(h_module) - text_sim = cosine_similarity(h_text) - - linkage_mat = linkage(pdist(h_module, "cosine"), method="ward") - dendro = dendrogram(linkage_mat, no_plot=True) - order = dendro["leaves"] - - module_sim_ord = module_sim[order][:, order] - text_sim_ord = text_sim[order][:, order] - texts_ord = [h_texts[i] for i in order] - procs_ord = [h_processes[i] for i in order] - - # Rank-transform to spread narrow cosine similarity ranges - module_rank_ord = _rank_transform_matrix(module_sim_ord) - text_rank_ord = _rank_transform_matrix(text_sim_ord) - diff_rank = module_rank_ord - text_rank_ord - - # Adaptive limit for difference panel - d_vals = diff_rank[np.triu_indices(n_h, k=1)] - vlim_d = float(np.percentile(np.abs(d_vals), 95)) - - # Figure - fig = plt.figure(figsize=(20, 8.5)) - gs = fig.add_gridspec(2, 3, height_ratios=[0.08, 1], hspace=0.02, wspace=0.3) - - # Dendrogram - ax_dend = fig.add_subplot(gs[0, :]) - dendrogram(linkage_mat, ax=ax_dend, no_labels=True, - color_threshold=0, above_threshold_color="gray") - ax_dend.set_xticks([]) - ax_dend.set_yticks([]) - ax_dend.spines[:].set_visible(False) - - for i, proc in enumerate(procs_ord): - color = PROCESS_COLORS.get(proc, "#999999") - ax_dend.axvspan(i * 10 + 5 - 4, i * 10 + 5 + 4, ymin=-0.2, ymax=-0.02, - color=color, clip_on=False) - - # Panel A: Module similarity (rank-transformed) - ax0 = fig.add_subplot(gs[1, 0]) - im0 = ax0.imshow(module_rank_ord, cmap="YlOrRd", vmin=0, vmax=1, - aspect="equal", interpolation="nearest") - ax0.set_title("A. Module Embedding Similarity\n(rank-transformed)", - fontsize=10, fontweight="bold") - ax0.set_xticks(range(n_h)) - ax0.set_xticklabels(texts_ord, rotation=90, fontsize=5.5, ha="center") - ax0.set_yticks(range(n_h)) - ax0.set_yticklabels(texts_ord, fontsize=5.5) - cb0 = plt.colorbar(im0, ax=ax0, fraction=0.046, pad=0.02) - cb0.set_label("Rank percentile", fontsize=8) - - # Panel B: Text similarity (rank-transformed) - ax1 = fig.add_subplot(gs[1, 1]) - im1 = ax1.imshow(text_rank_ord, cmap="YlOrRd", vmin=0, vmax=1, - aspect="equal", interpolation="nearest") - ax1.set_title("B. Text Embedding Similarity\n(rank-transformed)", - fontsize=10, fontweight="bold") - ax1.set_xticks(range(n_h)) - ax1.set_xticklabels(texts_ord, rotation=90, fontsize=5.5, ha="center") - ax1.set_yticks([]) - cb1 = plt.colorbar(im1, ax=ax1, fraction=0.046, pad=0.02) - cb1.set_label("Rank percentile", fontsize=8) - - # Panel C: Difference - ax2 = fig.add_subplot(gs[1, 2]) - im2 = ax2.imshow(diff_rank, cmap="PiYG", vmin=-vlim_d, vmax=vlim_d, - aspect="equal", interpolation="nearest") - ax2.set_title("C. Difference (Module − Text)\n(rank space)", - fontsize=10, fontweight="bold") - ax2.set_xticks(range(n_h)) - ax2.set_xticklabels(texts_ord, rotation=90, fontsize=5.5, ha="center") - ax2.set_yticks([]) - cb2 = plt.colorbar(im2, ax=ax2, fraction=0.046, pad=0.02) - cb2.set_label("Rank difference", fontsize=8) - - legend_elements = [LegendHandle(facecolor=PROCESS_COLORS[k], label=PROCESS_LABELS[k]) - for k in PROCESS_COLORS] - fig.legend(handles=legend_elements, loc="lower center", ncol=4, - fontsize=9, frameon=True, bbox_to_anchor=(0.5, -0.01)) - - plt.tight_layout(rect=[0, 0.03, 1, 1]) - savefig(fig, str(output_dir / "fig2_hallmarks_heatmaps")) - - return { - "module_sim_range": [float(module_sim.min()), float(module_sim.max())], - "text_sim_range": [float(text_sim.min()), float(text_sim.max())], - "clustering_order": [int(i) for i in order], - } - - -# ============================================================ -# Figure 3: Per-Hallmark Alignment (lollipop chart) -# ============================================================ - -def fig3_per_hallmark_alignment( - module_emb: np.ndarray, - text_emb: np.ndarray, - names: List[str], - texts: List[str], - output_dir: Path, - n_bootstrap: int = 2000, - seed: int = 42, -) -> Dict: - """Lollipop chart of per-hallmark alignment scores, colored by process.""" - families = [parse_module_family(n) for n in names] - hallmark_mask = np.array([f == "Hallmarks" for f in families]) - hallmark_idx = np.where(hallmark_mask)[0] - - h_module = module_emb[hallmark_idx] - h_text = text_emb[hallmark_idx] - h_names = [names[i] for i in hallmark_idx] - h_texts = [texts[i] for i in hallmark_idx] - n_h = len(hallmark_idx) - - h_processes = [] - for n in h_names: - proc = parse_hallmark_process(n) - h_processes.append(proc if proc else "pathway") - - # Per-module alignment: Spearman r of similarity profiles - module_sim = cosine_similarity(h_module) - text_sim = cosine_similarity(h_text) - per_module_corr = np.zeros(n_h) - for i in range(n_h): - mask = np.ones(n_h, dtype=bool) - mask[i] = False - r, _ = spearmanr(module_sim[i, mask], text_sim[i, mask]) - per_module_corr[i] = r - - # Sort by alignment - sort_idx = np.argsort(per_module_corr) # ascending for bottom-to-top lollipop - - # Bootstrap CI for mean alignment - mean_val, ci_lo, ci_hi = bootstrap_ci(per_module_corr, np.mean, n_bootstrap, seed=seed) - n_positive = int((per_module_corr > 0).sum()) - - # Figure - fig, ax = plt.subplots(figsize=(8, 12)) - - y_pos = np.arange(n_h) - for i, idx in enumerate(sort_idx): - color = PROCESS_COLORS.get(h_processes[idx], "#999999") - ax.hlines(y=i, xmin=0, xmax=per_module_corr[idx], color=color, - linewidth=1.5, alpha=0.7) - ax.scatter(per_module_corr[idx], i, color=color, s=40, zorder=3) - - ax.axvline(0, color="black", linewidth=0.8, linestyle="-") - ax.axvline(mean_val, color="black", linewidth=1.5, linestyle="--", alpha=0.6) - - ax.set_yticks(y_pos) - ax.set_yticklabels([h_texts[i] for i in sort_idx], fontsize=7) - ax.set_xlabel("Alignment Score (Spearman r of similarity profiles)", fontsize=10) - ax.set_title("Per-Hallmark Alignment Score", fontsize=12, fontweight="bold") - - # Summary text - summary = (f"Mean r = {mean_val:.3f} [{ci_lo:.3f}, {ci_hi:.3f}]\n" - f"Positive: {n_positive}/{n_h} ({100*n_positive/n_h:.0f}%)") - ax.text(0.98, 0.02, summary, transform=ax.transAxes, fontsize=9, - ha="right", va="bottom", - bbox=dict(boxstyle="round", facecolor="lightyellow", alpha=0.9)) - - # Legend - legend_elements = [Line2D([0], [0], marker="o", color="w", markerfacecolor=c, - markersize=8, label=PROCESS_LABELS[k]) - for k, c in PROCESS_COLORS.items()] - ax.legend(handles=legend_elements, loc="upper left", fontsize=7, - frameon=True, ncol=2) - - ax.set_xlim(min(per_module_corr) - 0.05, max(per_module_corr) + 0.05) - plt.tight_layout() - savefig(fig, str(output_dir / "fig3_per_hallmark_alignment")) - - # Save CSV - df = pd.DataFrame({ - "hallmark": h_texts, - "module_name": h_names, - "process_category": h_processes, - "alignment_r": per_module_corr, - }).sort_values("alignment_r", ascending=False) - df.to_csv(output_dir / "hallmarks_alignment_table.csv", index=False) - - return { - "mean_alignment": mean_val, - "mean_alignment_ci_95": [ci_lo, ci_hi], - "n_positive": n_positive, - "n_total": n_h, - "per_module_scores": {h_texts[i]: float(per_module_corr[i]) for i in range(n_h)}, - } - - -# ============================================================ -# Figure 4: Alignment by Biological Process Category -# ============================================================ - -def fig4_category_alignment( - module_emb: np.ndarray, - text_emb: np.ndarray, - names: List[str], - texts: List[str], - output_dir: Path, -) -> Dict: - """Violin/box plot of alignment scores grouped by MSigDB process category.""" - families = [parse_module_family(n) for n in names] - hallmark_mask = np.array([f == "Hallmarks" for f in families]) - hallmark_idx = np.where(hallmark_mask)[0] - - h_module = module_emb[hallmark_idx] - h_text = text_emb[hallmark_idx] - h_names = [names[i] for i in hallmark_idx] - n_h = len(hallmark_idx) - - h_processes = [] - for n in h_names: - proc = parse_hallmark_process(n) - h_processes.append(proc if proc else "pathway") - - # Per-module alignment - module_sim = cosine_similarity(h_module) - text_sim = cosine_similarity(h_text) - per_module_corr = np.zeros(n_h) - for i in range(n_h): - mask = np.ones(n_h, dtype=bool) - mask[i] = False - r, _ = spearmanr(module_sim[i, mask], text_sim[i, mask]) - per_module_corr[i] = r - - # Group by category - cat_scores = defaultdict(list) - for i in range(n_h): - cat_scores[h_processes[i]].append(per_module_corr[i]) - - # Sort categories by median alignment - cats_sorted = sorted(cat_scores.keys(), - key=lambda c: np.median(cat_scores[c]), reverse=True) - - # Kruskal-Wallis test - groups = [np.array(cat_scores[c]) for c in cats_sorted if len(cat_scores[c]) >= 2] - if len(groups) >= 2: - kw_stat, kw_p = kruskal(*groups) - else: - kw_stat, kw_p = 0.0, 1.0 - - # Figure - fig, ax = plt.subplots(figsize=(10, 6)) - - positions = range(len(cats_sorted)) - data_list = [np.array(cat_scores[c]) for c in cats_sorted] - colors_list = [PROCESS_COLORS.get(c, "#999999") for c in cats_sorted] - - # Box + strip plot - bp = ax.boxplot(data_list, positions=positions, widths=0.5, - patch_artist=True, showfliers=False) - for module, color in zip(bp["boxes"], colors_list): - module.set_facecolor(color) - module.set_alpha(0.4) - for median in bp["medians"]: - median.set_color("black") - median.set_linewidth(1.5) - - # Overlay individual points - for i, (data, color) in enumerate(zip(data_list, colors_list)): - jitter = np.random.default_rng(42).uniform(-0.15, 0.15, size=len(data)) - ax.scatter(np.full(len(data), i) + jitter, data, color=color, - s=30, alpha=0.7, edgecolors="white", linewidths=0.5, zorder=3) - - ax.axhline(0, color="gray", linewidth=0.8, linestyle="--") - ax.set_xticks(positions) - ax.set_xticklabels([PROCESS_LABELS.get(c, c) for c in cats_sorted], - rotation=30, ha="right", fontsize=9) - ax.set_ylabel("Alignment Score (Spearman r)", fontsize=10) - ax.set_title(f"Alignment by Biological Process Category\n" - f"Kruskal-Wallis H = {kw_stat:.2f}, p = {kw_p:.4f}", - fontsize=11, fontweight="bold") - - # Annotate counts - for i, c in enumerate(cats_sorted): - n = len(cat_scores[c]) - ax.text(i, ax.get_ylim()[1] * 0.95, f"n={n}", ha="center", fontsize=8) - - plt.tight_layout() - savefig(fig, str(output_dir / "fig4_category_alignment")) - - return { - "kruskal_wallis_H": float(kw_stat), - "kruskal_wallis_p": float(kw_p), - "category_medians": {c: float(np.median(cat_scores[c])) for c in cats_sorted}, - "category_counts": {c: len(cat_scores[c]) for c in cats_sorted}, - } - - -# ============================================================ -# Figure 5: Granularity Effect (Reactome hierarchy) -# ============================================================ - -def fig5_granularity_effect( - module_emb: np.ndarray, - text_emb: np.ndarray, - names: List[str], - texts: List[str], - n_permutations: int = 1000, - n_bootstrap: int = 2000, - seed: int = 42, - output_dir: Path = None, -) -> Dict: - """RSA at multiple granularity levels: fine → category → super-category.""" - families = [parse_module_family(n) for n in names] - reactome_mask = np.array([f == "Reactome" for f in families]) - hallmark_mask = np.array([f == "Hallmarks" for f in families]) - reactome_idx = np.where(reactome_mask)[0] - hallmark_idx = np.where(hallmark_mask)[0] - - results = {} - - # Level 0: Hallmarks (50 modules, already shown to align) - h_module = module_emb[hallmark_idx] - h_text = text_emb[hallmark_idx] - r_hall, ci_lo_h, ci_hi_h = bootstrap_rsa(h_module, h_text, n_bootstrap, seed) - results["hallmarks_50"] = {"r": r_hall, "ci": [ci_lo_h, ci_hi_h], "n": len(hallmark_idx)} - - # Level 1: Reactome fine-grained (491 pathways) - r_module = module_emb[reactome_idx] - r_text = text_emb[reactome_idx] - r_fine, ci_lo_f, ci_hi_f = bootstrap_rsa(r_module, r_text, n_bootstrap, seed) - results["reactome_fine_491"] = {"r": r_fine, "ci": [ci_lo_f, ci_hi_f], "n": len(reactome_idx)} - - # Level 2: Reactome categories (~16) - r_names = [names[i] for i in reactome_idx] - r_texts = [texts[i] for i in reactome_idx] - cat_map = {} - for i, n in enumerate(r_names): - cat = categorize_reactome(n) - if cat not in cat_map: - cat_map[cat] = {"module_embs": [], "text_embs": []} - cat_map[cat]["module_embs"].append(r_module[i]) - cat_map[cat]["text_embs"].append(r_text[i]) - - cats_with_data = {k: v for k, v in cat_map.items() if len(v["module_embs"]) >= 3} - if len(cats_with_data) >= 4: - cat_module_means = np.array([np.mean(v["module_embs"], axis=0) - for v in cats_with_data.values()]) - cat_text_means = np.array([np.mean(v["text_embs"], axis=0) - for v in cats_with_data.values()]) - r_cat, ci_lo_c, ci_hi_c = bootstrap_rsa(cat_module_means, cat_text_means, - n_bootstrap, seed) - results["reactome_category"] = { - "r": r_cat, "ci": [ci_lo_c, ci_hi_c], "n": len(cats_with_data) - } - else: - results["reactome_category"] = {"r": 0.0, "ci": [0.0, 0.0], "n": 0} - - # Level 3: Super-categories (5-6) - super_map = {} - for super_name, sub_cats in REACTOME_SUPER_CATEGORIES.items(): - embs_p, embs_t = [], [] - for sub in sub_cats: - if sub in cats_with_data: - embs_p.extend(cats_with_data[sub]["module_embs"]) - embs_t.extend(cats_with_data[sub]["text_embs"]) - if len(embs_p) >= 3: - super_map[super_name] = { - "module_mean": np.mean(embs_p, axis=0), - "text_mean": np.mean(embs_t, axis=0), - } - - if len(super_map) >= 4: - sup_module = np.array([v["module_mean"] for v in super_map.values()]) - sup_text = np.array([v["text_mean"] for v in super_map.values()]) - r_sup, ci_lo_s, ci_hi_s = bootstrap_rsa(sup_module, sup_text, n_bootstrap, seed) - results["reactome_super_category"] = { - "r": r_sup, "ci": [ci_lo_s, ci_hi_s], "n": len(super_map) - } - else: - results["reactome_super_category"] = {"r": 0.0, "ci": [0.0, 0.0], "n": 0} - - # Figure - fig, ax = plt.subplots(figsize=(8, 5)) - - levels = ["Reactome\nFine (491)", "Reactome\nCategory (~16)", - "Reactome\nSuper-Cat (~6)", "Hallmarks\n(50)"] - keys = ["reactome_fine_491", "reactome_category", - "reactome_super_category", "hallmarks_50"] - rs = [results[k]["r"] for k in keys] - cis_lo = [results[k]["ci"][0] for k in keys] - cis_hi = [results[k]["ci"][1] for k in keys] - ns = [results[k]["n"] for k in keys] - - colors = ["#90CAF9", "#42A5F5", "#1565C0", "#E91E63"] - x = np.arange(len(levels)) - bars = ax.bar(x, rs, color=colors, edgecolor="black", linewidth=0.8, width=0.6) - - # Error bars (CI) - for i in range(len(levels)): - ax.plot([i, i], [cis_lo[i], cis_hi[i]], color="black", linewidth=2) - ax.plot([i - 0.1, i + 0.1], [cis_lo[i], cis_lo[i]], color="black", linewidth=1.5) - ax.plot([i - 0.1, i + 0.1], [cis_hi[i], cis_hi[i]], color="black", linewidth=1.5) - - ax.axhline(0, color="gray", linewidth=0.8, linestyle="--") - ax.set_xticks(x) - ax.set_xticklabels(levels, fontsize=9) - ax.set_ylabel("RSA Spearman r", fontsize=10) - ax.set_title("Alignment Improves at Coarser Granularity", - fontsize=12, fontweight="bold") - - # Annotate n - for i, n in enumerate(ns): - ax.text(i, max(rs[i], cis_hi[i]) + 0.01, f"n={n}", ha="center", fontsize=8) - - plt.tight_layout() - savefig(fig, str(output_dir / "fig5_granularity_effect")) - - return results - - -# ============================================================ -# Figure 6: Multi-Model / Cross-Dataset Comparison -# ============================================================ - -def fig6_model_comparison( - module_emb_primary: np.ndarray, - module_emb_secondary: Optional[np.ndarray], - text_emb_primary: np.ndarray, - names: List[str], - texts: List[str], - primary_model_name: str, - compare_model_names: List[str], - compare_text_embs: List[np.ndarray], - n_permutations: int = 1000, - n_bootstrap: int = 2000, - seed: int = 42, - output_dir: Path = None, -) -> Dict: - """Grouped bar chart comparing RSA across models and datasets.""" - families = [parse_module_family(n) for n in names] - hallmark_mask = np.array([f == "Hallmarks" for f in families]) - hallmark_idx = np.where(hallmark_mask)[0] - - h_module = module_emb_primary[hallmark_idx] - - # Compute RSA for ALL models including primary - all_models = [primary_model_name] + compare_model_names - all_text_embs = [text_emb_primary] + compare_text_embs - - model_results = {} - for model_name, text_emb_full in zip(all_models, all_text_embs): - h_text = text_emb_full[hallmark_idx] - r, ci_lo, ci_hi = bootstrap_rsa(h_module, h_text, n_bootstrap, seed) - mantel_res = mantel_test(h_module, h_text, n_perm=n_permutations) - p = mantel_res["rsa_p_value"] - model_results[model_name] = { - "dataset": "primary", - "r": r, "ci": [ci_lo, ci_hi], "p": p - } - - # Cross-dataset (secondary) - dataset_results = {} - if module_emb_secondary is not None: - h_module2 = module_emb_secondary[hallmark_idx] - for model_name, text_emb_full in zip(all_models, all_text_embs): - h_text = text_emb_full[hallmark_idx] - r, ci_lo, ci_hi = bootstrap_rsa(h_module2, h_text, n_bootstrap, seed) - mantel_res = mantel_test(h_module2, h_text, n_perm=n_permutations) - p = mantel_res["rsa_p_value"] - dataset_results[model_name] = { - "dataset": "secondary", - "r": r, "ci": [ci_lo, ci_hi], "p": p - } - - # Figure - has_secondary = len(dataset_results) > 0 - n_models = len(model_results) - - if n_models == 0: - logger.warning("No models to compare, skipping fig6") - return {} - - fig, ax = plt.subplots(figsize=(max(8, n_models * 2.5), 5)) - - model_names_short = [m.split("/")[-1][:25] for m in model_results.keys()] - x = np.arange(len(model_names_short)) - width = 0.35 if has_secondary else 0.5 - - # Primary dataset bars - rs_primary = [model_results[m]["r"] for m in model_results] - cis_primary = [model_results[m]["ci"] for m in model_results] - yerr_lo = [r - ci[0] for r, ci in zip(rs_primary, cis_primary)] - yerr_hi = [ci[1] - r for r, ci in zip(rs_primary, cis_primary)] - - # Color: highlight primary model differently - colors_primary = ["#1E88E5"] + ["#90CAF9"] * (n_models - 1) - - offset = -width / 2 if has_secondary else 0 - bars1 = ax.bar(x + offset, rs_primary, width, label="Dataset 1 (pretrain)", - color=colors_primary, edgecolor="black", linewidth=0.8) - ax.errorbar(x + offset, rs_primary, yerr=[yerr_lo, yerr_hi], - fmt="none", color="black", capsize=4) - - if has_secondary: - rs_sec = [dataset_results.get(m, {}).get("r", 0) for m in model_results] - cis_sec = [dataset_results.get(m, {}).get("ci", [0, 0]) for m in model_results] - yerr_lo2 = [r - ci[0] for r, ci in zip(rs_sec, cis_sec)] - yerr_hi2 = [ci[1] - r for r, ci in zip(rs_sec, cis_sec)] - colors_sec = ["#E53935"] + ["#EF9A9A"] * (n_models - 1) - bars2 = ax.bar(x + width / 2, rs_sec, width, label="Dataset 2 (pancreas)", - color=colors_sec, edgecolor="black", linewidth=0.8) - ax.errorbar(x + width / 2, rs_sec, yerr=[yerr_lo2, yerr_hi2], - fmt="none", color="black", capsize=4) - - ax.axhline(0, color="gray", linewidth=0.8, linestyle="--") - ax.set_xticks(x) - ax.set_xticklabels(model_names_short, rotation=15, ha="right", fontsize=9) - ax.set_ylabel("RSA Spearman r (Hallmarks)", fontsize=10) - ax.set_title("Alignment Across Sentence Encoders", - fontsize=12, fontweight="bold") - ax.legend(fontsize=9) - - # Annotate p-values - for i, m in enumerate(model_results): - p = model_results[m]["p"] - sig = "***" if p < 0.001 else "**" if p < 0.01 else "*" if p < 0.05 else "ns" - y_pos = max(rs_primary[i], cis_primary[i][1]) + 0.02 - ax.text(i + offset, y_pos, sig, ha="center", fontsize=9, fontweight="bold") - - plt.tight_layout() - savefig(fig, str(output_dir / "fig6_model_comparison")) - - return {"model_results": model_results, "dataset_results": dataset_results} - - -# ============================================================ -# Deep Analysis: Alignment vs Module Size -# ============================================================ - -def analysis_alignment_vs_module_size( - module_emb: np.ndarray, - text_emb: np.ndarray, - names: List[str], - texts: List[str], - module_json: str, - output_dir: Path, -) -> Dict: - """Scatter: n_genes vs alignment score. Tests if larger modules align better.""" - families = [parse_module_family(n) for n in names] - hallmark_mask = np.array([f == "Hallmarks" for f in families]) - hallmark_idx = np.where(hallmark_mask)[0] - - h_module = module_emb[hallmark_idx] - h_text = text_emb[hallmark_idx] - h_names = [names[i] for i in hallmark_idx] - h_texts = [texts[i] for i in hallmark_idx] - n_h = len(hallmark_idx) - - # Per-module alignment - module_sim = cosine_similarity(h_module) - text_sim = cosine_similarity(h_text) - per_module_corr = np.zeros(n_h) - for i in range(n_h): - mask = np.ones(n_h, dtype=bool) - mask[i] = False - r, _ = spearmanr(module_sim[i, mask], text_sim[i, mask]) - per_module_corr[i] = r - - # Module sizes - module_sizes = get_module_sizes(module_json) - h_sizes = np.array([module_sizes.get(n, 0) for n in h_names]) - - # Correlation - r_size, p_size = spearmanr(h_sizes, per_module_corr) - - # Figure - fig, ax = plt.subplots(figsize=(7, 5)) - h_processes = [parse_hallmark_process(n) or "pathway" for n in h_names] - colors = [PROCESS_COLORS.get(p, "#999999") for p in h_processes] - - ax.scatter(h_sizes, per_module_corr, c=colors, s=50, alpha=0.7, - edgecolors="white", linewidths=0.5) - - # Trend line - z = np.polyfit(h_sizes, per_module_corr, 1) - p_line = np.poly1d(z) - x_range = np.linspace(h_sizes.min(), h_sizes.max(), 100) - ax.plot(x_range, p_line(x_range), "k--", linewidth=1.5, alpha=0.6) - - ax.axhline(0, color="gray", linewidth=0.8, linestyle=":") - ax.set_xlabel("Number of Genes in Module", fontsize=10) - ax.set_ylabel("Alignment Score (Spearman r)", fontsize=10) - ax.set_title(f"Alignment vs Module Size\n" - f"Spearman ρ = {r_size:.3f}, p = {p_size:.4f}", - fontsize=11, fontweight="bold") - - # Label outliers - for i in range(n_h): - if abs(per_module_corr[i]) > 0.35 or h_sizes[i] > 250: - ax.annotate(h_texts[i][:15], (h_sizes[i], per_module_corr[i]), - fontsize=6, alpha=0.8, xytext=(5, 3), - textcoords="offset points") - - plt.tight_layout() - savefig(fig, str(output_dir / "fig_alignment_vs_size")) - - return {"spearman_r": float(r_size), "spearman_p": float(p_size)} - - -# ============================================================ -# Deep Analysis: Negative Result Deep-Dive -# ============================================================ - -def analysis_negative_results( - module_emb: np.ndarray, - text_emb: np.ndarray, - names: List[str], - texts: List[str], - output_dir: Path, - k: int = 5, - n_show: int = 5, -) -> Dict: - """For poorly-aligned modules, show neighbor divergence.""" - families = [parse_module_family(n) for n in names] - hallmark_mask = np.array([f == "Hallmarks" for f in families]) - hallmark_idx = np.where(hallmark_mask)[0] - - h_module = module_emb[hallmark_idx] - h_text = text_emb[hallmark_idx] - h_names = [names[i] for i in hallmark_idx] - h_texts = [texts[i] for i in hallmark_idx] - n_h = len(hallmark_idx) - - # Per-module alignment - module_sim = cosine_similarity(h_module) - text_sim = cosine_similarity(h_text) - per_module_corr = np.zeros(n_h) - for i in range(n_h): - mask = np.ones(n_h, dtype=bool) - mask[i] = False - r, _ = spearmanr(module_sim[i, mask], text_sim[i, mask]) - per_module_corr[i] = r - - # Bottom n_show and top n_show - bottom_idx = np.argsort(per_module_corr)[:n_show] - top_idx = np.argsort(per_module_corr)[-n_show:][::-1] - - rows = [] - for idx_set, label in [(bottom_idx, "bottom"), (top_idx, "top")]: - for i in idx_set: - # kNN in module space - module_nn = np.argsort(-module_sim[i]) - module_nn = [j for j in module_nn if j != i][:k] - # kNN in text space - text_nn = np.argsort(-text_sim[i]) - text_nn = [j for j in text_nn if j != i][:k] - overlap = set(module_nn) & set(text_nn) - - rows.append({ - "rank": label, - "hallmark": h_texts[i], - "alignment_r": float(per_module_corr[i]), - "module_neighbors": ", ".join([h_texts[j] for j in module_nn]), - "text_neighbors": ", ".join([h_texts[j] for j in text_nn]), - "knn_overlap": len(overlap), - }) - - df = pd.DataFrame(rows) - df.to_csv(output_dir / "negative_analysis.csv", index=False) - logger.info("Saved negative_analysis.csv (%d rows)", len(df)) - - return { - "bottom_5": [{ - "hallmark": r["hallmark"], - "r": r["alignment_r"], - "module_nn": r["module_neighbors"], - "text_nn": r["text_neighbors"], - } for _, r in df[df["rank"] == "bottom"].iterrows()], - "top_5": [{ - "hallmark": r["hallmark"], - "r": r["alignment_r"], - } for _, r in df[df["rank"] == "top"].iterrows()], - } - - -# ============================================================ -# Deep Analysis: kNN Overlap with Bootstrap CI -# ============================================================ - -def analysis_knn_overlap( - module_emb: np.ndarray, - text_emb: np.ndarray, - names: List[str], - n_permutations: int = 1000, - n_bootstrap: int = 2000, - k: int = 5, - seed: int = 42, - output_dir: Path = None, -) -> Dict: - """kNN overlap with bootstrap CI and per-category breakdown.""" - families = [parse_module_family(n) for n in names] - hallmark_mask = np.array([f == "Hallmarks" for f in families]) - hallmark_idx = np.where(hallmark_mask)[0] - - h_module = module_emb[hallmark_idx] - h_text = text_emb[hallmark_idx] - h_names = [names[i] for i in hallmark_idx] - n_h = len(hallmark_idx) - - # Compute kNN overlap - module_sim = cosine_similarity(h_module) - text_sim = cosine_similarity(h_text) - - overlaps = np.zeros(n_h) - for i in range(n_h): - module_nn = set(np.argsort(-module_sim[i])[1:k + 1]) - text_nn = set(np.argsort(-text_sim[i])[1:k + 1]) - overlaps[i] = len(module_nn & text_nn) / k - - # Bootstrap CI - mean_obs, ci_lo, ci_hi = bootstrap_ci(overlaps, np.mean, n_bootstrap, seed=seed) - - # Null distribution (permutation) - rng = np.random.default_rng(seed) - null_means = np.zeros(n_permutations) - for p in range(n_permutations): - perm = rng.permutation(n_h) - text_sim_perm = text_sim[perm][:, perm] - ov_perm = np.zeros(n_h) - for i in range(n_h): - module_nn = set(np.argsort(-module_sim[i])[1:k + 1]) - text_nn = set(np.argsort(-text_sim_perm[i])[1:k + 1]) - ov_perm[i] = len(module_nn & text_nn) / k - null_means[p] = ov_perm.mean() - - p_value = float((null_means >= mean_obs).sum() / n_permutations) - null_mean = float(null_means.mean()) - null_std = float(null_means.std()) - z_score = (mean_obs - null_mean) / null_std if null_std > 0 else 0.0 - - # Per-category breakdown - h_processes = [parse_hallmark_process(n) or "pathway" for n in h_names] - cat_overlaps = defaultdict(list) - for i in range(n_h): - cat_overlaps[h_processes[i]].append(overlaps[i]) - - return { - "knn_overlap_mean": mean_obs, - "knn_overlap_ci_95": [ci_lo, ci_hi], - "null_mean": null_mean, - "null_std": null_std, - "p_value": p_value, - "z_score": z_score, - "k": k, - "category_means": {c: float(np.mean(v)) for c, v in cat_overlaps.items()}, - } - - -# ============================================================ -# Main -# ============================================================ - -def main(): - args = parse_args() - output_dir = Path(args.output_dir) - output_dir.mkdir(parents=True, exist_ok=True) - - # Setup logging - log_file = output_dir / "run_publication.log" - logging.basicConfig( - level=logging.INFO, - format="%(asctime)s [%(levelname)s] %(message)s", - handlers=[ - logging.FileHandler(log_file, mode="w", encoding="utf-8"), - logging.StreamHandler(), - ], - ) - logger.info("Publication analysis starting") - logger.info("Args: %s", vars(args)) - - # ---- Load data ---- - module_names = load_module_names(args.module_json) - all_module_emb = np.load(args.module_embedding_path) - logger.info("Loaded module embeddings: %s from %s", - all_module_emb.shape, args.module_embedding_path) - - # Filter to semantic modules (Hallmarks + Reactome) - families = [parse_module_family(n) for n in module_names] - semantic_mask = np.array([f in ("Hallmarks", "Reactome") for f in families]) - semantic_idx = np.where(semantic_mask)[0] - module_emb = all_module_emb[semantic_idx] - names = [module_names[i] for i in semantic_idx] - logger.info("Semantic modules: %d (Hallmarks + Reactome)", len(names)) - - # ---- Build text descriptions and encode ---- - desc_file = getattr(args, "descriptions_file", None) - _, _, texts = build_text_descriptions(names, descriptions_file=desc_file) - logger.info("Encoding %d texts with primary model: %s", len(texts), args.sentence_model) - text_emb = encode_texts(texts, args.sentence_model, device=args.device) - logger.info("Text embeddings: %s", text_emb.shape) - - # ---- Core metrics (RSA + Mantel) ---- - logger.info("Computing RSA + Mantel test (n_perm=%d)...", args.n_permutations) - mantel_results = mantel_test(module_emb, text_emb, n_perm=args.n_permutations) - logger.info("RSA r=%.4f, p=%.4f, z=%.1f", - mantel_results["rsa_spearman_observed"], - mantel_results["rsa_p_value"], - mantel_results["rsa_effect_size_z"]) - - # ---- Figure 1: RSA Overview ---- - logger.info("Generating Figure 1: RSA Overview...") - metrics_fig1 = fig1_rsa_overview( - module_emb, text_emb, mantel_results, output_dir, - n_bootstrap=args.n_bootstrap, seed=args.seed, - ) - - # ---- Figure 2: Hallmarks Heatmaps ---- - logger.info("Generating Figure 2: Hallmarks Heatmaps...") - metrics_fig2 = fig2_hallmarks_heatmaps( - module_emb, text_emb, names, texts, output_dir, - ) - - # ---- Figure 3: Per-Hallmark Alignment ---- - logger.info("Generating Figure 3: Per-Hallmark Alignment...") - metrics_fig3 = fig3_per_hallmark_alignment( - module_emb, text_emb, names, texts, output_dir, - n_bootstrap=args.n_bootstrap, seed=args.seed, - ) - - # ---- Figure 4: Category Alignment ---- - logger.info("Generating Figure 4: Category Alignment...") - metrics_fig4 = fig4_category_alignment( - module_emb, text_emb, names, texts, output_dir, - ) - - # ---- Figure 5: Granularity Effect ---- - logger.info("Generating Figure 5: Granularity Effect...") - metrics_fig5 = fig5_granularity_effect( - module_emb, text_emb, names, texts, - n_permutations=args.n_permutations, - n_bootstrap=args.n_bootstrap, - seed=args.seed, - output_dir=output_dir, - ) - - # ---- Figure 6: Multi-Model Comparison (optional) ---- - metrics_fig6 = {} - compare_models = [] - compare_embs = [] - if args.sentence_models_compare: - compare_models = [m.strip() for m in args.sentence_models_compare.split(",") - if m.strip()] - logger.info("Loading %d comparison models...", len(compare_models)) - for model_name in compare_models: - logger.info(" Encoding with: %s", model_name) - try: - emb = encode_texts(texts, model_name, device=args.device) - compare_embs.append(emb) - except Exception as e: - logger.warning(" Failed to load %s: %s", model_name, e) - compare_models.remove(model_name) - - module_emb2 = None - if args.module_embedding_path2: - all_module_emb2 = np.load(args.module_embedding_path2) - module_emb2 = all_module_emb2[semantic_idx] - logger.info("Loaded secondary dataset: %s", all_module_emb2.shape) - - if compare_models: - logger.info("Generating Figure 6: Model Comparison...") - metrics_fig6 = fig6_model_comparison( - module_emb, module_emb2, text_emb, names, texts, - primary_model_name=args.sentence_model.split("/")[-1], - compare_model_names=compare_models, - compare_text_embs=compare_embs, - n_permutations=args.n_permutations, - n_bootstrap=args.n_bootstrap, - seed=args.seed, - output_dir=output_dir, - ) - - # ---- Deep Analysis ---- - logger.info("Running deep analysis: alignment vs module size...") - metrics_size = analysis_alignment_vs_module_size( - module_emb, text_emb, names, texts, args.module_json, output_dir, - ) - - logger.info("Running deep analysis: negative results...") - metrics_neg = analysis_negative_results( - module_emb, text_emb, names, texts, output_dir, - ) - - logger.info("Running deep analysis: kNN overlap with bootstrap...") - metrics_knn = analysis_knn_overlap( - module_emb, text_emb, names, - n_permutations=args.n_permutations, - n_bootstrap=args.n_bootstrap, - seed=args.seed, - output_dir=output_dir, - ) - logger.info("kNN overlap: %.3f [%.3f, %.3f], null=%.3f, p=%.4f", - metrics_knn["knn_overlap_mean"], - metrics_knn["knn_overlap_ci_95"][0], - metrics_knn["knn_overlap_ci_95"][1], - metrics_knn["null_mean"], - metrics_knn["p_value"]) - - # ---- Cross-dataset RSA (if secondary available) ---- - metrics_cross = {} - if module_emb2 is not None: - logger.info("Cross-dataset RSA (secondary dataset)...") - mantel2 = mantel_test(module_emb2, text_emb, n_perm=args.n_permutations) - r2, ci2_lo, ci2_hi = bootstrap_rsa( - module_emb2, text_emb, args.n_bootstrap, args.seed - ) - metrics_cross = { - "secondary_rsa_r": r2, - "secondary_rsa_ci_95": [ci2_lo, ci2_hi], - "secondary_mantel_p": mantel2["rsa_p_value"], - "secondary_mantel_z": mantel2["rsa_effect_size_z"], - } - logger.info("Secondary RSA: r=%.4f [%.4f, %.4f], p=%.4f", - r2, ci2_lo, ci2_hi, mantel2["rsa_p_value"]) - - # ---- Aggregate and save metrics ---- - all_metrics = { - "primary_dataset": args.module_embedding_path, - "secondary_dataset": args.module_embedding_path2, - "sentence_model": args.sentence_model, - "n_modules": len(names), - "n_permutations": args.n_permutations, - "n_bootstrap": args.n_bootstrap, - "fig1_rsa_overview": metrics_fig1, - "fig2_heatmaps": metrics_fig2, - "fig3_per_hallmark": metrics_fig3, - "fig4_category": metrics_fig4, - "fig5_granularity": metrics_fig5, - "fig6_model_comparison": metrics_fig6, - "alignment_vs_size": metrics_size, - "negative_analysis": metrics_neg, - "knn_overlap": metrics_knn, - "cross_dataset": metrics_cross, - } - - metrics_path = output_dir / "publication_metrics.json" - with open(metrics_path, "w", encoding="utf-8") as f: - json.dump(all_metrics, f, indent=2, ensure_ascii=False, default=str) - logger.info("Saved: %s", metrics_path) - logger.info("Publication analysis complete. Output: %s", output_dir) - - -if __name__ == "__main__": - main() +"""Publication-quality analysis for Experiment 2 — Module-Text Alignment. + +Produces 6 main figures + comprehensive metrics with bootstrap CIs for paper. +Supports multi-model comparison and cross-dataset validation. + +Usage (run from the BioMod repository root): + python BioMod/interpretability/module_text_alignment/module_text_alignment_publication.py \ + --module_embedding_path outputs/interpretability/geometry/pretrain_subset/module_embeddings.npy \ + --module_embedding_path2 outputs/interpretability/geometry/pancreas_scib/module_embeddings.npy \ + --module_json model_dependencies/module_data_all/merged_modules.json \ + --sentence_model pritamdeka/S-PubMedBert-MS-MARCO \ + --sentence_models_compare "sentence-transformers/all-mpnet-base-v2" \ + --descriptions_file BioMod/interpretability/module_text_alignment/reactome_descriptions.json \ + --n_permutations 1000 \ + --n_bootstrap 2000 \ + --output_dir outputs/interpretability/text_alignment/publication +""" +from __future__ import annotations + +import argparse +import json +import logging +import sys +from collections import Counter, defaultdict +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +import numpy as np +import pandas as pd +from scipy.cluster.hierarchy import dendrogram, fcluster, linkage +from scipy.spatial.distance import pdist, squareform +from scipy.stats import kruskal, spearmanr +from sklearn.metrics.pairwise import cosine_similarity + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +from matplotlib.patches import Patch as LegendHandle +from matplotlib.lines import Line2D + +HERE = Path(__file__).resolve().parent +INTERP_DIR = HERE.parent +PERFORMER_MODULE = INTERP_DIR.parent +REPO_ROOT = PERFORMER_MODULE.parent +for p in (str(HERE), str(INTERP_DIR), str(PERFORMER_MODULE), str(REPO_ROOT)): + if p not in sys.path: + sys.path.insert(0, p) + +from interpretability.utils import ( + HALLMARK_PROCESS, + load_module_names, + parse_hallmark_process, + parse_module_family, +) +from module_text_alignment.module_text_alignment import ( + build_text_descriptions, + compute_cka, + compute_rsa, + encode_texts, + mantel_test, + permutation_test_cka, +) +from module_text_alignment.module_text_alignment_extended import ( + REACTOME_CATEGORIES, + categorize_reactome, +) + +logger = logging.getLogger(__name__) + +# ============================================================ +# Color palette for MSigDB hallmark process categories +# ============================================================ + +PROCESS_COLORS = { + "immune": "#2196F3", + "metabolic": "#4CAF50", + "signaling": "#FF9800", + "proliferation": "#E91E63", + "development": "#9C27B0", + "dna_damage": "#F44336", + "pathway": "#607D8B", + "cellular_component": "#795548", +} + +PROCESS_LABELS = { + "immune": "Immune", + "metabolic": "Metabolic", + "signaling": "Signaling", + "proliferation": "Proliferation", + "development": "Development", + "dna_damage": "DNA Damage", + "pathway": "Pathway", + "cellular_component": "Cellular Component", +} + +# Reactome super-categories (merge related categories for granularity sweep) +REACTOME_SUPER_CATEGORIES = { + "Information Processing": ["Gene Expression", "DNA Replication", "DNA Repair", + "Chromatin & Epigenetics"], + "Signal & Communication": ["Signal Transduction", "Cell-Cell Communication"], + "Cell Fate": ["Cell Cycle", "Apoptosis", "Cellular Responses"], + "Metabolism & Transport": ["Metabolism", "Transport"], + "Immune & Defense": ["Immune System", "Hemostasis", "Disease"], + "Development": ["Developmental Biology", "Protein Homeostasis"], +} + + +# ============================================================ +# Utility functions +# ============================================================ + +def savefig(fig, path_stem: str, dpi: int = 300): + """Save figure as both PDF and PNG.""" + fig.savefig(f"{path_stem}.pdf", dpi=dpi, bbox_inches="tight") + fig.savefig(f"{path_stem}.png", dpi=dpi, bbox_inches="tight") + plt.close(fig) + logger.info("Saved: %s.{pdf,png}", path_stem) + + +def adaptive_vlim(matrix: np.ndarray, percentile: int = 98, symmetric: bool = True): + """Compute adaptive vmin/vmax from data percentiles.""" + vals = matrix[np.triu_indices_from(matrix, k=1)] + if symmetric: + vlim = np.percentile(np.abs(vals), percentile) + return -vlim, vlim + else: + return np.percentile(vals, [100 - percentile, percentile]) + + +def bootstrap_ci( + values: np.ndarray, + stat_fn=np.mean, + n_boot: int = 2000, + ci: float = 0.95, + seed: int = 42, +) -> Tuple[float, float, float]: + """Bootstrap confidence interval. Returns (estimate, ci_low, ci_high).""" + rng = np.random.default_rng(seed) + n = len(values) + boot_stats = np.zeros(n_boot) + for i in range(n_boot): + idx = rng.integers(0, n, size=n) + boot_stats[i] = stat_fn(values[idx]) + alpha = (1 - ci) / 2 + return float(stat_fn(values)), float(np.percentile(boot_stats, alpha * 100)), \ + float(np.percentile(boot_stats, (1 - alpha) * 100)) + + +def bootstrap_rsa( + module_emb: np.ndarray, + text_emb: np.ndarray, + n_boot: int = 2000, + seed: int = 42, +) -> Tuple[float, float, float]: + """Bootstrap CI for RSA Spearman r by resampling modules.""" + rng = np.random.default_rng(seed) + n = module_emb.shape[0] + boot_rs = np.zeros(n_boot) + for i in range(n_boot): + idx = rng.choice(n, size=n, replace=True) + dist_p = pdist(module_emb[idx], metric="cosine") + dist_t = pdist(text_emb[idx], metric="cosine") + boot_rs[i], _ = spearmanr(dist_p, dist_t) + observed, _ = spearmanr(pdist(module_emb, "cosine"), pdist(text_emb, "cosine")) + return float(observed), float(np.percentile(boot_rs, 2.5)), \ + float(np.percentile(boot_rs, 97.5)) + + +def get_module_sizes(module_json: str) -> Dict[str, int]: + """Get number of genes per module from merged_modules.json.""" + with open(module_json, "r", encoding="utf-8") as f: + modules = json.load(f) + return {name: len(genes) for name, genes in modules.items()} + + +# ============================================================ +# CLI +# ============================================================ + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("--module_embedding_path", type=str, required=True, + help="Primary module_embeddings.npy (e.g. pretrain_subset)") + parser.add_argument("--module_embedding_path2", type=str, default=None, + help="Second dataset module_embeddings.npy for cross-dataset comparison") + parser.add_argument("--module_json", type=str, + default=str(REPO_ROOT / "model_dependencies" / "module_data_all" / "merged_modules.json")) + parser.add_argument("--descriptions_file", type=str, default=None) + parser.add_argument("--sentence_model", type=str, + default="pritamdeka/S-PubMedBert-MS-MARCO") + parser.add_argument("--sentence_models_compare", type=str, default=None, + help="Comma-separated additional models for comparison") + parser.add_argument("--device", type=str, default="cpu") + parser.add_argument("--n_permutations", type=int, default=1000) + parser.add_argument("--n_bootstrap", type=int, default=2000) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--output_dir", type=str, required=True) + return parser.parse_args() + + +# ============================================================ +# Figure 1: RSA Overview (hexbin + null distribution) +# ============================================================ + +def fig1_rsa_overview( + module_emb: np.ndarray, + text_emb: np.ndarray, + mantel_results: Dict, + output_dir: Path, + metric: str = "cosine", + n_bootstrap: int = 2000, + seed: int = 42, +) -> Dict: + """RSA overview: hexbin density + Mantel null distribution.""" + dist_module = pdist(module_emb, metric=metric) + dist_text = pdist(text_emb, metric=metric) + + observed_r = mantel_results["rsa_spearman_observed"] + null_mean = mantel_results["rsa_null_mean"] + null_std = mantel_results["rsa_null_std"] + p_value = mantel_results["rsa_p_value"] + z_score = mantel_results["rsa_effect_size_z"] + + # Bootstrap CI for RSA + rsa_obs, rsa_ci_lo, rsa_ci_hi = bootstrap_rsa( + module_emb, text_emb, n_boot=n_bootstrap, seed=seed + ) + + fig, axes = plt.subplots(1, 2, figsize=(12, 5)) + + # Panel A: Hexbin density + ax = axes[0] + hb = ax.hexbin(dist_text, dist_module, gridsize=40, cmap="YlOrRd", + mincnt=1, linewidths=0.2) + cb = plt.colorbar(hb, ax=ax, fraction=0.046, pad=0.02) + cb.set_label("Count", fontsize=9) + + # Trend line + z = np.polyfit(dist_text, dist_module, 1) + p_line = np.poly1d(z) + x_range = np.linspace(dist_text.min(), dist_text.max(), 100) + ax.plot(x_range, p_line(x_range), "k-", linewidth=2, alpha=0.8) + + ax.set_xlabel("Text pairwise distance (cosine)", fontsize=10) + ax.set_ylabel("Module embedding pairwise distance (cosine)", fontsize=10) + ax.set_title(f"A. RSA: r = {observed_r:.3f} [{rsa_ci_lo:.3f}, {rsa_ci_hi:.3f}]", + fontsize=11, fontweight="bold") + ax.text(0.05, 0.95, f"n = {module_emb.shape[0]} modules\n" + f"pairs = {len(dist_module):,}", + transform=ax.transAxes, fontsize=8, va="top", + bbox=dict(boxstyle="round", facecolor="white", alpha=0.8)) + + # Panel B: Null distribution + ax = axes[1] + from scipy.stats import norm as norm_dist + x = np.linspace(null_mean - 4.5 * null_std, + max(observed_r + 2 * null_std, null_mean + 4.5 * null_std), 300) + y = norm_dist.pdf(x, null_mean, null_std) + ax.fill_between(x, y, alpha=0.25, color="gray") + ax.plot(x, y, color="gray", linewidth=1.5, label="Null distribution") + + # Shade p-value region + x_tail = x[x >= observed_r] + y_tail = norm_dist.pdf(x_tail, null_mean, null_std) + ax.fill_between(x_tail, y_tail, alpha=0.4, color="red", label=f"p = {p_value:.4f}") + + ax.axvline(observed_r, color="red", linewidth=2.5, linestyle="-", + label=f"Observed r = {observed_r:.4f}") + ax.axvline(null_mean, color="gray", linewidth=1, linestyle="--", alpha=0.7) + + ax.set_xlabel("Spearman r (distance matrix correlation)", fontsize=10) + ax.set_ylabel("Density", fontsize=10) + ax.set_title(f"B. Mantel Test: z = {z_score:.1f}σ, p = {p_value:.4f}", + fontsize=11, fontweight="bold") + ax.legend(fontsize=9, loc="upper left") + + plt.tight_layout() + savefig(fig, str(output_dir / "fig1_rsa_overview")) + + return { + "rsa_r": rsa_obs, + "rsa_ci_95": [rsa_ci_lo, rsa_ci_hi], + "mantel_p": p_value, + "mantel_z": z_score, + } + + +# ============================================================ +# Figure 2: Hallmarks Similarity Matrices with Dendrograms +# ============================================================ + +def _rank_transform_matrix(sim_matrix: np.ndarray) -> np.ndarray: + """Rank-transform off-diagonal elements to [0,1] for visualization.""" + from scipy.stats import rankdata + n = sim_matrix.shape[0] + result = np.zeros_like(sim_matrix) + triu_idx = np.triu_indices(n, k=1) + vals = sim_matrix[triu_idx] + ranks = rankdata(vals) / len(vals) + result[triu_idx] = ranks + result[triu_idx[1], triu_idx[0]] = ranks + np.fill_diagonal(result, 1.0) + return result + + +def fig2_hallmarks_heatmaps( + module_emb: np.ndarray, + text_emb: np.ndarray, + names: List[str], + texts: List[str], + output_dir: Path, +) -> Dict: + """Hallmarks 50x50 similarity matrices with rank-transform and clustering.""" + families = [parse_module_family(n) for n in names] + hallmark_mask = np.array([f == "Hallmarks" for f in families]) + hallmark_idx = np.where(hallmark_mask)[0] + + h_module = module_emb[hallmark_idx] + h_text = text_emb[hallmark_idx] + h_names = [names[i] for i in hallmark_idx] + h_texts = [texts[i] for i in hallmark_idx] + n_h = len(hallmark_idx) + + h_processes = [] + for n in h_names: + proc = parse_hallmark_process(n) + h_processes.append(proc if proc else "pathway") + + module_sim = cosine_similarity(h_module) + text_sim = cosine_similarity(h_text) + + linkage_mat = linkage(pdist(h_module, "cosine"), method="ward") + dendro = dendrogram(linkage_mat, no_plot=True) + order = dendro["leaves"] + + module_sim_ord = module_sim[order][:, order] + text_sim_ord = text_sim[order][:, order] + texts_ord = [h_texts[i] for i in order] + procs_ord = [h_processes[i] for i in order] + + # Rank-transform to spread narrow cosine similarity ranges + module_rank_ord = _rank_transform_matrix(module_sim_ord) + text_rank_ord = _rank_transform_matrix(text_sim_ord) + diff_rank = module_rank_ord - text_rank_ord + + # Adaptive limit for difference panel + d_vals = diff_rank[np.triu_indices(n_h, k=1)] + vlim_d = float(np.percentile(np.abs(d_vals), 95)) + + # Figure + fig = plt.figure(figsize=(20, 8.5)) + gs = fig.add_gridspec(2, 3, height_ratios=[0.08, 1], hspace=0.02, wspace=0.3) + + # Dendrogram + ax_dend = fig.add_subplot(gs[0, :]) + dendrogram(linkage_mat, ax=ax_dend, no_labels=True, + color_threshold=0, above_threshold_color="gray") + ax_dend.set_xticks([]) + ax_dend.set_yticks([]) + ax_dend.spines[:].set_visible(False) + + for i, proc in enumerate(procs_ord): + color = PROCESS_COLORS.get(proc, "#999999") + ax_dend.axvspan(i * 10 + 5 - 4, i * 10 + 5 + 4, ymin=-0.2, ymax=-0.02, + color=color, clip_on=False) + + # Panel A: Module similarity (rank-transformed) + ax0 = fig.add_subplot(gs[1, 0]) + im0 = ax0.imshow(module_rank_ord, cmap="YlOrRd", vmin=0, vmax=1, + aspect="equal", interpolation="nearest") + ax0.set_title("A. Module Embedding Similarity\n(rank-transformed)", + fontsize=10, fontweight="bold") + ax0.set_xticks(range(n_h)) + ax0.set_xticklabels(texts_ord, rotation=90, fontsize=5.5, ha="center") + ax0.set_yticks(range(n_h)) + ax0.set_yticklabels(texts_ord, fontsize=5.5) + cb0 = plt.colorbar(im0, ax=ax0, fraction=0.046, pad=0.02) + cb0.set_label("Rank percentile", fontsize=8) + + # Panel B: Text similarity (rank-transformed) + ax1 = fig.add_subplot(gs[1, 1]) + im1 = ax1.imshow(text_rank_ord, cmap="YlOrRd", vmin=0, vmax=1, + aspect="equal", interpolation="nearest") + ax1.set_title("B. Text Embedding Similarity\n(rank-transformed)", + fontsize=10, fontweight="bold") + ax1.set_xticks(range(n_h)) + ax1.set_xticklabels(texts_ord, rotation=90, fontsize=5.5, ha="center") + ax1.set_yticks([]) + cb1 = plt.colorbar(im1, ax=ax1, fraction=0.046, pad=0.02) + cb1.set_label("Rank percentile", fontsize=8) + + # Panel C: Difference + ax2 = fig.add_subplot(gs[1, 2]) + im2 = ax2.imshow(diff_rank, cmap="PiYG", vmin=-vlim_d, vmax=vlim_d, + aspect="equal", interpolation="nearest") + ax2.set_title("C. Difference (Module − Text)\n(rank space)", + fontsize=10, fontweight="bold") + ax2.set_xticks(range(n_h)) + ax2.set_xticklabels(texts_ord, rotation=90, fontsize=5.5, ha="center") + ax2.set_yticks([]) + cb2 = plt.colorbar(im2, ax=ax2, fraction=0.046, pad=0.02) + cb2.set_label("Rank difference", fontsize=8) + + legend_elements = [LegendHandle(facecolor=PROCESS_COLORS[k], label=PROCESS_LABELS[k]) + for k in PROCESS_COLORS] + fig.legend(handles=legend_elements, loc="lower center", ncol=4, + fontsize=9, frameon=True, bbox_to_anchor=(0.5, -0.01)) + + plt.tight_layout(rect=[0, 0.03, 1, 1]) + savefig(fig, str(output_dir / "fig2_hallmarks_heatmaps")) + + return { + "module_sim_range": [float(module_sim.min()), float(module_sim.max())], + "text_sim_range": [float(text_sim.min()), float(text_sim.max())], + "clustering_order": [int(i) for i in order], + } + + +# ============================================================ +# Figure 3: Per-Hallmark Alignment (lollipop chart) +# ============================================================ + +def fig3_per_hallmark_alignment( + module_emb: np.ndarray, + text_emb: np.ndarray, + names: List[str], + texts: List[str], + output_dir: Path, + n_bootstrap: int = 2000, + seed: int = 42, +) -> Dict: + """Lollipop chart of per-hallmark alignment scores, colored by process.""" + families = [parse_module_family(n) for n in names] + hallmark_mask = np.array([f == "Hallmarks" for f in families]) + hallmark_idx = np.where(hallmark_mask)[0] + + h_module = module_emb[hallmark_idx] + h_text = text_emb[hallmark_idx] + h_names = [names[i] for i in hallmark_idx] + h_texts = [texts[i] for i in hallmark_idx] + n_h = len(hallmark_idx) + + h_processes = [] + for n in h_names: + proc = parse_hallmark_process(n) + h_processes.append(proc if proc else "pathway") + + # Per-module alignment: Spearman r of similarity profiles + module_sim = cosine_similarity(h_module) + text_sim = cosine_similarity(h_text) + per_module_corr = np.zeros(n_h) + for i in range(n_h): + mask = np.ones(n_h, dtype=bool) + mask[i] = False + r, _ = spearmanr(module_sim[i, mask], text_sim[i, mask]) + per_module_corr[i] = r + + # Sort by alignment + sort_idx = np.argsort(per_module_corr) # ascending for bottom-to-top lollipop + + # Bootstrap CI for mean alignment + mean_val, ci_lo, ci_hi = bootstrap_ci(per_module_corr, np.mean, n_bootstrap, seed=seed) + n_positive = int((per_module_corr > 0).sum()) + + # Figure + fig, ax = plt.subplots(figsize=(8, 12)) + + y_pos = np.arange(n_h) + for i, idx in enumerate(sort_idx): + color = PROCESS_COLORS.get(h_processes[idx], "#999999") + ax.hlines(y=i, xmin=0, xmax=per_module_corr[idx], color=color, + linewidth=1.5, alpha=0.7) + ax.scatter(per_module_corr[idx], i, color=color, s=40, zorder=3) + + ax.axvline(0, color="black", linewidth=0.8, linestyle="-") + ax.axvline(mean_val, color="black", linewidth=1.5, linestyle="--", alpha=0.6) + + ax.set_yticks(y_pos) + ax.set_yticklabels([h_texts[i] for i in sort_idx], fontsize=7) + ax.set_xlabel("Alignment Score (Spearman r of similarity profiles)", fontsize=10) + ax.set_title("Per-Hallmark Alignment Score", fontsize=12, fontweight="bold") + + # Summary text + summary = (f"Mean r = {mean_val:.3f} [{ci_lo:.3f}, {ci_hi:.3f}]\n" + f"Positive: {n_positive}/{n_h} ({100*n_positive/n_h:.0f}%)") + ax.text(0.98, 0.02, summary, transform=ax.transAxes, fontsize=9, + ha="right", va="bottom", + bbox=dict(boxstyle="round", facecolor="lightyellow", alpha=0.9)) + + # Legend + legend_elements = [Line2D([0], [0], marker="o", color="w", markerfacecolor=c, + markersize=8, label=PROCESS_LABELS[k]) + for k, c in PROCESS_COLORS.items()] + ax.legend(handles=legend_elements, loc="upper left", fontsize=7, + frameon=True, ncol=2) + + ax.set_xlim(min(per_module_corr) - 0.05, max(per_module_corr) + 0.05) + plt.tight_layout() + savefig(fig, str(output_dir / "fig3_per_hallmark_alignment")) + + # Save CSV + df = pd.DataFrame({ + "hallmark": h_texts, + "module_name": h_names, + "process_category": h_processes, + "alignment_r": per_module_corr, + }).sort_values("alignment_r", ascending=False) + df.to_csv(output_dir / "hallmarks_alignment_table.csv", index=False) + + return { + "mean_alignment": mean_val, + "mean_alignment_ci_95": [ci_lo, ci_hi], + "n_positive": n_positive, + "n_total": n_h, + "per_module_scores": {h_texts[i]: float(per_module_corr[i]) for i in range(n_h)}, + } + + +# ============================================================ +# Figure 4: Alignment by Biological Process Category +# ============================================================ + +def fig4_category_alignment( + module_emb: np.ndarray, + text_emb: np.ndarray, + names: List[str], + texts: List[str], + output_dir: Path, +) -> Dict: + """Violin/box plot of alignment scores grouped by MSigDB process category.""" + families = [parse_module_family(n) for n in names] + hallmark_mask = np.array([f == "Hallmarks" for f in families]) + hallmark_idx = np.where(hallmark_mask)[0] + + h_module = module_emb[hallmark_idx] + h_text = text_emb[hallmark_idx] + h_names = [names[i] for i in hallmark_idx] + n_h = len(hallmark_idx) + + h_processes = [] + for n in h_names: + proc = parse_hallmark_process(n) + h_processes.append(proc if proc else "pathway") + + # Per-module alignment + module_sim = cosine_similarity(h_module) + text_sim = cosine_similarity(h_text) + per_module_corr = np.zeros(n_h) + for i in range(n_h): + mask = np.ones(n_h, dtype=bool) + mask[i] = False + r, _ = spearmanr(module_sim[i, mask], text_sim[i, mask]) + per_module_corr[i] = r + + # Group by category + cat_scores = defaultdict(list) + for i in range(n_h): + cat_scores[h_processes[i]].append(per_module_corr[i]) + + # Sort categories by median alignment + cats_sorted = sorted(cat_scores.keys(), + key=lambda c: np.median(cat_scores[c]), reverse=True) + + # Kruskal-Wallis test + groups = [np.array(cat_scores[c]) for c in cats_sorted if len(cat_scores[c]) >= 2] + if len(groups) >= 2: + kw_stat, kw_p = kruskal(*groups) + else: + kw_stat, kw_p = 0.0, 1.0 + + # Figure + fig, ax = plt.subplots(figsize=(10, 6)) + + positions = range(len(cats_sorted)) + data_list = [np.array(cat_scores[c]) for c in cats_sorted] + colors_list = [PROCESS_COLORS.get(c, "#999999") for c in cats_sorted] + + # Box + strip plot + bp = ax.boxplot(data_list, positions=positions, widths=0.5, + patch_artist=True, showfliers=False) + for module, color in zip(bp["boxes"], colors_list): + module.set_facecolor(color) + module.set_alpha(0.4) + for median in bp["medians"]: + median.set_color("black") + median.set_linewidth(1.5) + + # Overlay individual points + for i, (data, color) in enumerate(zip(data_list, colors_list)): + jitter = np.random.default_rng(42).uniform(-0.15, 0.15, size=len(data)) + ax.scatter(np.full(len(data), i) + jitter, data, color=color, + s=30, alpha=0.7, edgecolors="white", linewidths=0.5, zorder=3) + + ax.axhline(0, color="gray", linewidth=0.8, linestyle="--") + ax.set_xticks(positions) + ax.set_xticklabels([PROCESS_LABELS.get(c, c) for c in cats_sorted], + rotation=30, ha="right", fontsize=9) + ax.set_ylabel("Alignment Score (Spearman r)", fontsize=10) + ax.set_title(f"Alignment by Biological Process Category\n" + f"Kruskal-Wallis H = {kw_stat:.2f}, p = {kw_p:.4f}", + fontsize=11, fontweight="bold") + + # Annotate counts + for i, c in enumerate(cats_sorted): + n = len(cat_scores[c]) + ax.text(i, ax.get_ylim()[1] * 0.95, f"n={n}", ha="center", fontsize=8) + + plt.tight_layout() + savefig(fig, str(output_dir / "fig4_category_alignment")) + + return { + "kruskal_wallis_H": float(kw_stat), + "kruskal_wallis_p": float(kw_p), + "category_medians": {c: float(np.median(cat_scores[c])) for c in cats_sorted}, + "category_counts": {c: len(cat_scores[c]) for c in cats_sorted}, + } + + +# ============================================================ +# Figure 5: Granularity Effect (Reactome hierarchy) +# ============================================================ + +def fig5_granularity_effect( + module_emb: np.ndarray, + text_emb: np.ndarray, + names: List[str], + texts: List[str], + n_permutations: int = 1000, + n_bootstrap: int = 2000, + seed: int = 42, + output_dir: Path = None, +) -> Dict: + """RSA at multiple granularity levels: fine → category → super-category.""" + families = [parse_module_family(n) for n in names] + reactome_mask = np.array([f == "Reactome" for f in families]) + hallmark_mask = np.array([f == "Hallmarks" for f in families]) + reactome_idx = np.where(reactome_mask)[0] + hallmark_idx = np.where(hallmark_mask)[0] + + results = {} + + # Level 0: Hallmarks (50 modules, already shown to align) + h_module = module_emb[hallmark_idx] + h_text = text_emb[hallmark_idx] + r_hall, ci_lo_h, ci_hi_h = bootstrap_rsa(h_module, h_text, n_bootstrap, seed) + results["hallmarks_50"] = {"r": r_hall, "ci": [ci_lo_h, ci_hi_h], "n": len(hallmark_idx)} + + # Level 1: Reactome fine-grained (491 pathways) + r_module = module_emb[reactome_idx] + r_text = text_emb[reactome_idx] + r_fine, ci_lo_f, ci_hi_f = bootstrap_rsa(r_module, r_text, n_bootstrap, seed) + results["reactome_fine_491"] = {"r": r_fine, "ci": [ci_lo_f, ci_hi_f], "n": len(reactome_idx)} + + # Level 2: Reactome categories (~16) + r_names = [names[i] for i in reactome_idx] + r_texts = [texts[i] for i in reactome_idx] + cat_map = {} + for i, n in enumerate(r_names): + cat = categorize_reactome(n) + if cat not in cat_map: + cat_map[cat] = {"module_embs": [], "text_embs": []} + cat_map[cat]["module_embs"].append(r_module[i]) + cat_map[cat]["text_embs"].append(r_text[i]) + + cats_with_data = {k: v for k, v in cat_map.items() if len(v["module_embs"]) >= 3} + if len(cats_with_data) >= 4: + cat_module_means = np.array([np.mean(v["module_embs"], axis=0) + for v in cats_with_data.values()]) + cat_text_means = np.array([np.mean(v["text_embs"], axis=0) + for v in cats_with_data.values()]) + r_cat, ci_lo_c, ci_hi_c = bootstrap_rsa(cat_module_means, cat_text_means, + n_bootstrap, seed) + results["reactome_category"] = { + "r": r_cat, "ci": [ci_lo_c, ci_hi_c], "n": len(cats_with_data) + } + else: + results["reactome_category"] = {"r": 0.0, "ci": [0.0, 0.0], "n": 0} + + # Level 3: Super-categories (5-6) + super_map = {} + for super_name, sub_cats in REACTOME_SUPER_CATEGORIES.items(): + embs_p, embs_t = [], [] + for sub in sub_cats: + if sub in cats_with_data: + embs_p.extend(cats_with_data[sub]["module_embs"]) + embs_t.extend(cats_with_data[sub]["text_embs"]) + if len(embs_p) >= 3: + super_map[super_name] = { + "module_mean": np.mean(embs_p, axis=0), + "text_mean": np.mean(embs_t, axis=0), + } + + if len(super_map) >= 4: + sup_module = np.array([v["module_mean"] for v in super_map.values()]) + sup_text = np.array([v["text_mean"] for v in super_map.values()]) + r_sup, ci_lo_s, ci_hi_s = bootstrap_rsa(sup_module, sup_text, n_bootstrap, seed) + results["reactome_super_category"] = { + "r": r_sup, "ci": [ci_lo_s, ci_hi_s], "n": len(super_map) + } + else: + results["reactome_super_category"] = {"r": 0.0, "ci": [0.0, 0.0], "n": 0} + + # Figure + fig, ax = plt.subplots(figsize=(8, 5)) + + levels = ["Reactome\nFine (491)", "Reactome\nCategory (~16)", + "Reactome\nSuper-Cat (~6)", "Hallmarks\n(50)"] + keys = ["reactome_fine_491", "reactome_category", + "reactome_super_category", "hallmarks_50"] + rs = [results[k]["r"] for k in keys] + cis_lo = [results[k]["ci"][0] for k in keys] + cis_hi = [results[k]["ci"][1] for k in keys] + ns = [results[k]["n"] for k in keys] + + colors = ["#90CAF9", "#42A5F5", "#1565C0", "#E91E63"] + x = np.arange(len(levels)) + bars = ax.bar(x, rs, color=colors, edgecolor="black", linewidth=0.8, width=0.6) + + # Error bars (CI) + for i in range(len(levels)): + ax.plot([i, i], [cis_lo[i], cis_hi[i]], color="black", linewidth=2) + ax.plot([i - 0.1, i + 0.1], [cis_lo[i], cis_lo[i]], color="black", linewidth=1.5) + ax.plot([i - 0.1, i + 0.1], [cis_hi[i], cis_hi[i]], color="black", linewidth=1.5) + + ax.axhline(0, color="gray", linewidth=0.8, linestyle="--") + ax.set_xticks(x) + ax.set_xticklabels(levels, fontsize=9) + ax.set_ylabel("RSA Spearman r", fontsize=10) + ax.set_title("Alignment Improves at Coarser Granularity", + fontsize=12, fontweight="bold") + + # Annotate n + for i, n in enumerate(ns): + ax.text(i, max(rs[i], cis_hi[i]) + 0.01, f"n={n}", ha="center", fontsize=8) + + plt.tight_layout() + savefig(fig, str(output_dir / "fig5_granularity_effect")) + + return results + + +# ============================================================ +# Figure 6: Multi-Model / Cross-Dataset Comparison +# ============================================================ + +def fig6_model_comparison( + module_emb_primary: np.ndarray, + module_emb_secondary: Optional[np.ndarray], + text_emb_primary: np.ndarray, + names: List[str], + texts: List[str], + primary_model_name: str, + compare_model_names: List[str], + compare_text_embs: List[np.ndarray], + n_permutations: int = 1000, + n_bootstrap: int = 2000, + seed: int = 42, + output_dir: Path = None, +) -> Dict: + """Grouped bar chart comparing RSA across models and datasets.""" + families = [parse_module_family(n) for n in names] + hallmark_mask = np.array([f == "Hallmarks" for f in families]) + hallmark_idx = np.where(hallmark_mask)[0] + + h_module = module_emb_primary[hallmark_idx] + + # Compute RSA for ALL models including primary + all_models = [primary_model_name] + compare_model_names + all_text_embs = [text_emb_primary] + compare_text_embs + + model_results = {} + for model_name, text_emb_full in zip(all_models, all_text_embs): + h_text = text_emb_full[hallmark_idx] + r, ci_lo, ci_hi = bootstrap_rsa(h_module, h_text, n_bootstrap, seed) + mantel_res = mantel_test(h_module, h_text, n_perm=n_permutations) + p = mantel_res["rsa_p_value"] + model_results[model_name] = { + "dataset": "primary", + "r": r, "ci": [ci_lo, ci_hi], "p": p + } + + # Cross-dataset (secondary) + dataset_results = {} + if module_emb_secondary is not None: + h_module2 = module_emb_secondary[hallmark_idx] + for model_name, text_emb_full in zip(all_models, all_text_embs): + h_text = text_emb_full[hallmark_idx] + r, ci_lo, ci_hi = bootstrap_rsa(h_module2, h_text, n_bootstrap, seed) + mantel_res = mantel_test(h_module2, h_text, n_perm=n_permutations) + p = mantel_res["rsa_p_value"] + dataset_results[model_name] = { + "dataset": "secondary", + "r": r, "ci": [ci_lo, ci_hi], "p": p + } + + # Figure + has_secondary = len(dataset_results) > 0 + n_models = len(model_results) + + if n_models == 0: + logger.warning("No models to compare, skipping fig6") + return {} + + fig, ax = plt.subplots(figsize=(max(8, n_models * 2.5), 5)) + + model_names_short = [m.split("/")[-1][:25] for m in model_results.keys()] + x = np.arange(len(model_names_short)) + width = 0.35 if has_secondary else 0.5 + + # Primary dataset bars + rs_primary = [model_results[m]["r"] for m in model_results] + cis_primary = [model_results[m]["ci"] for m in model_results] + yerr_lo = [r - ci[0] for r, ci in zip(rs_primary, cis_primary)] + yerr_hi = [ci[1] - r for r, ci in zip(rs_primary, cis_primary)] + + # Color: highlight primary model differently + colors_primary = ["#1E88E5"] + ["#90CAF9"] * (n_models - 1) + + offset = -width / 2 if has_secondary else 0 + bars1 = ax.bar(x + offset, rs_primary, width, label="Dataset 1 (pretrain)", + color=colors_primary, edgecolor="black", linewidth=0.8) + ax.errorbar(x + offset, rs_primary, yerr=[yerr_lo, yerr_hi], + fmt="none", color="black", capsize=4) + + if has_secondary: + rs_sec = [dataset_results.get(m, {}).get("r", 0) for m in model_results] + cis_sec = [dataset_results.get(m, {}).get("ci", [0, 0]) for m in model_results] + yerr_lo2 = [r - ci[0] for r, ci in zip(rs_sec, cis_sec)] + yerr_hi2 = [ci[1] - r for r, ci in zip(rs_sec, cis_sec)] + colors_sec = ["#E53935"] + ["#EF9A9A"] * (n_models - 1) + bars2 = ax.bar(x + width / 2, rs_sec, width, label="Dataset 2 (pancreas)", + color=colors_sec, edgecolor="black", linewidth=0.8) + ax.errorbar(x + width / 2, rs_sec, yerr=[yerr_lo2, yerr_hi2], + fmt="none", color="black", capsize=4) + + ax.axhline(0, color="gray", linewidth=0.8, linestyle="--") + ax.set_xticks(x) + ax.set_xticklabels(model_names_short, rotation=15, ha="right", fontsize=9) + ax.set_ylabel("RSA Spearman r (Hallmarks)", fontsize=10) + ax.set_title("Alignment Across Sentence Encoders", + fontsize=12, fontweight="bold") + ax.legend(fontsize=9) + + # Annotate p-values + for i, m in enumerate(model_results): + p = model_results[m]["p"] + sig = "***" if p < 0.001 else "**" if p < 0.01 else "*" if p < 0.05 else "ns" + y_pos = max(rs_primary[i], cis_primary[i][1]) + 0.02 + ax.text(i + offset, y_pos, sig, ha="center", fontsize=9, fontweight="bold") + + plt.tight_layout() + savefig(fig, str(output_dir / "fig6_model_comparison")) + + return {"model_results": model_results, "dataset_results": dataset_results} + + +# ============================================================ +# Deep Analysis: Alignment vs Module Size +# ============================================================ + +def analysis_alignment_vs_module_size( + module_emb: np.ndarray, + text_emb: np.ndarray, + names: List[str], + texts: List[str], + module_json: str, + output_dir: Path, +) -> Dict: + """Scatter: n_genes vs alignment score. Tests if larger modules align better.""" + families = [parse_module_family(n) for n in names] + hallmark_mask = np.array([f == "Hallmarks" for f in families]) + hallmark_idx = np.where(hallmark_mask)[0] + + h_module = module_emb[hallmark_idx] + h_text = text_emb[hallmark_idx] + h_names = [names[i] for i in hallmark_idx] + h_texts = [texts[i] for i in hallmark_idx] + n_h = len(hallmark_idx) + + # Per-module alignment + module_sim = cosine_similarity(h_module) + text_sim = cosine_similarity(h_text) + per_module_corr = np.zeros(n_h) + for i in range(n_h): + mask = np.ones(n_h, dtype=bool) + mask[i] = False + r, _ = spearmanr(module_sim[i, mask], text_sim[i, mask]) + per_module_corr[i] = r + + # Module sizes + module_sizes = get_module_sizes(module_json) + h_sizes = np.array([module_sizes.get(n, 0) for n in h_names]) + + # Correlation + r_size, p_size = spearmanr(h_sizes, per_module_corr) + + # Figure + fig, ax = plt.subplots(figsize=(7, 5)) + h_processes = [parse_hallmark_process(n) or "pathway" for n in h_names] + colors = [PROCESS_COLORS.get(p, "#999999") for p in h_processes] + + ax.scatter(h_sizes, per_module_corr, c=colors, s=50, alpha=0.7, + edgecolors="white", linewidths=0.5) + + # Trend line + z = np.polyfit(h_sizes, per_module_corr, 1) + p_line = np.poly1d(z) + x_range = np.linspace(h_sizes.min(), h_sizes.max(), 100) + ax.plot(x_range, p_line(x_range), "k--", linewidth=1.5, alpha=0.6) + + ax.axhline(0, color="gray", linewidth=0.8, linestyle=":") + ax.set_xlabel("Number of Genes in Module", fontsize=10) + ax.set_ylabel("Alignment Score (Spearman r)", fontsize=10) + ax.set_title(f"Alignment vs Module Size\n" + f"Spearman ρ = {r_size:.3f}, p = {p_size:.4f}", + fontsize=11, fontweight="bold") + + # Label outliers + for i in range(n_h): + if abs(per_module_corr[i]) > 0.35 or h_sizes[i] > 250: + ax.annotate(h_texts[i][:15], (h_sizes[i], per_module_corr[i]), + fontsize=6, alpha=0.8, xytext=(5, 3), + textcoords="offset points") + + plt.tight_layout() + savefig(fig, str(output_dir / "fig_alignment_vs_size")) + + return {"spearman_r": float(r_size), "spearman_p": float(p_size)} + + +# ============================================================ +# Deep Analysis: Negative Result Deep-Dive +# ============================================================ + +def analysis_negative_results( + module_emb: np.ndarray, + text_emb: np.ndarray, + names: List[str], + texts: List[str], + output_dir: Path, + k: int = 5, + n_show: int = 5, +) -> Dict: + """For poorly-aligned modules, show neighbor divergence.""" + families = [parse_module_family(n) for n in names] + hallmark_mask = np.array([f == "Hallmarks" for f in families]) + hallmark_idx = np.where(hallmark_mask)[0] + + h_module = module_emb[hallmark_idx] + h_text = text_emb[hallmark_idx] + h_names = [names[i] for i in hallmark_idx] + h_texts = [texts[i] for i in hallmark_idx] + n_h = len(hallmark_idx) + + # Per-module alignment + module_sim = cosine_similarity(h_module) + text_sim = cosine_similarity(h_text) + per_module_corr = np.zeros(n_h) + for i in range(n_h): + mask = np.ones(n_h, dtype=bool) + mask[i] = False + r, _ = spearmanr(module_sim[i, mask], text_sim[i, mask]) + per_module_corr[i] = r + + # Bottom n_show and top n_show + bottom_idx = np.argsort(per_module_corr)[:n_show] + top_idx = np.argsort(per_module_corr)[-n_show:][::-1] + + rows = [] + for idx_set, label in [(bottom_idx, "bottom"), (top_idx, "top")]: + for i in idx_set: + # kNN in module space + module_nn = np.argsort(-module_sim[i]) + module_nn = [j for j in module_nn if j != i][:k] + # kNN in text space + text_nn = np.argsort(-text_sim[i]) + text_nn = [j for j in text_nn if j != i][:k] + overlap = set(module_nn) & set(text_nn) + + rows.append({ + "rank": label, + "hallmark": h_texts[i], + "alignment_r": float(per_module_corr[i]), + "module_neighbors": ", ".join([h_texts[j] for j in module_nn]), + "text_neighbors": ", ".join([h_texts[j] for j in text_nn]), + "knn_overlap": len(overlap), + }) + + df = pd.DataFrame(rows) + df.to_csv(output_dir / "negative_analysis.csv", index=False) + logger.info("Saved negative_analysis.csv (%d rows)", len(df)) + + return { + "bottom_5": [{ + "hallmark": r["hallmark"], + "r": r["alignment_r"], + "module_nn": r["module_neighbors"], + "text_nn": r["text_neighbors"], + } for _, r in df[df["rank"] == "bottom"].iterrows()], + "top_5": [{ + "hallmark": r["hallmark"], + "r": r["alignment_r"], + } for _, r in df[df["rank"] == "top"].iterrows()], + } + + +# ============================================================ +# Deep Analysis: kNN Overlap with Bootstrap CI +# ============================================================ + +def analysis_knn_overlap( + module_emb: np.ndarray, + text_emb: np.ndarray, + names: List[str], + n_permutations: int = 1000, + n_bootstrap: int = 2000, + k: int = 5, + seed: int = 42, + output_dir: Path = None, +) -> Dict: + """kNN overlap with bootstrap CI and per-category breakdown.""" + families = [parse_module_family(n) for n in names] + hallmark_mask = np.array([f == "Hallmarks" for f in families]) + hallmark_idx = np.where(hallmark_mask)[0] + + h_module = module_emb[hallmark_idx] + h_text = text_emb[hallmark_idx] + h_names = [names[i] for i in hallmark_idx] + n_h = len(hallmark_idx) + + # Compute kNN overlap + module_sim = cosine_similarity(h_module) + text_sim = cosine_similarity(h_text) + + overlaps = np.zeros(n_h) + for i in range(n_h): + module_nn = set(np.argsort(-module_sim[i])[1:k + 1]) + text_nn = set(np.argsort(-text_sim[i])[1:k + 1]) + overlaps[i] = len(module_nn & text_nn) / k + + # Bootstrap CI + mean_obs, ci_lo, ci_hi = bootstrap_ci(overlaps, np.mean, n_bootstrap, seed=seed) + + # Null distribution (permutation) + rng = np.random.default_rng(seed) + null_means = np.zeros(n_permutations) + for p in range(n_permutations): + perm = rng.permutation(n_h) + text_sim_perm = text_sim[perm][:, perm] + ov_perm = np.zeros(n_h) + for i in range(n_h): + module_nn = set(np.argsort(-module_sim[i])[1:k + 1]) + text_nn = set(np.argsort(-text_sim_perm[i])[1:k + 1]) + ov_perm[i] = len(module_nn & text_nn) / k + null_means[p] = ov_perm.mean() + + p_value = float((null_means >= mean_obs).sum() / n_permutations) + null_mean = float(null_means.mean()) + null_std = float(null_means.std()) + z_score = (mean_obs - null_mean) / null_std if null_std > 0 else 0.0 + + # Per-category breakdown + h_processes = [parse_hallmark_process(n) or "pathway" for n in h_names] + cat_overlaps = defaultdict(list) + for i in range(n_h): + cat_overlaps[h_processes[i]].append(overlaps[i]) + + return { + "knn_overlap_mean": mean_obs, + "knn_overlap_ci_95": [ci_lo, ci_hi], + "null_mean": null_mean, + "null_std": null_std, + "p_value": p_value, + "z_score": z_score, + "k": k, + "category_means": {c: float(np.mean(v)) for c, v in cat_overlaps.items()}, + } + + +# ============================================================ +# Main +# ============================================================ + +def main(): + args = parse_args() + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + # Setup logging + log_file = output_dir / "run_publication.log" + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + handlers=[ + logging.FileHandler(log_file, mode="w", encoding="utf-8"), + logging.StreamHandler(), + ], + ) + logger.info("Publication analysis starting") + logger.info("Args: %s", vars(args)) + + # ---- Load data ---- + module_names = load_module_names(args.module_json) + all_module_emb = np.load(args.module_embedding_path) + logger.info("Loaded module embeddings: %s from %s", + all_module_emb.shape, args.module_embedding_path) + + # Filter to semantic modules (Hallmarks + Reactome) + families = [parse_module_family(n) for n in module_names] + semantic_mask = np.array([f in ("Hallmarks", "Reactome") for f in families]) + semantic_idx = np.where(semantic_mask)[0] + module_emb = all_module_emb[semantic_idx] + names = [module_names[i] for i in semantic_idx] + logger.info("Semantic modules: %d (Hallmarks + Reactome)", len(names)) + + # ---- Build text descriptions and encode ---- + desc_file = getattr(args, "descriptions_file", None) + _, _, texts = build_text_descriptions(names, descriptions_file=desc_file) + logger.info("Encoding %d texts with primary model: %s", len(texts), args.sentence_model) + text_emb = encode_texts(texts, args.sentence_model, device=args.device) + logger.info("Text embeddings: %s", text_emb.shape) + + # ---- Core metrics (RSA + Mantel) ---- + logger.info("Computing RSA + Mantel test (n_perm=%d)...", args.n_permutations) + mantel_results = mantel_test(module_emb, text_emb, n_perm=args.n_permutations) + logger.info("RSA r=%.4f, p=%.4f, z=%.1f", + mantel_results["rsa_spearman_observed"], + mantel_results["rsa_p_value"], + mantel_results["rsa_effect_size_z"]) + + # ---- Figure 1: RSA Overview ---- + logger.info("Generating Figure 1: RSA Overview...") + metrics_fig1 = fig1_rsa_overview( + module_emb, text_emb, mantel_results, output_dir, + n_bootstrap=args.n_bootstrap, seed=args.seed, + ) + + # ---- Figure 2: Hallmarks Heatmaps ---- + logger.info("Generating Figure 2: Hallmarks Heatmaps...") + metrics_fig2 = fig2_hallmarks_heatmaps( + module_emb, text_emb, names, texts, output_dir, + ) + + # ---- Figure 3: Per-Hallmark Alignment ---- + logger.info("Generating Figure 3: Per-Hallmark Alignment...") + metrics_fig3 = fig3_per_hallmark_alignment( + module_emb, text_emb, names, texts, output_dir, + n_bootstrap=args.n_bootstrap, seed=args.seed, + ) + + # ---- Figure 4: Category Alignment ---- + logger.info("Generating Figure 4: Category Alignment...") + metrics_fig4 = fig4_category_alignment( + module_emb, text_emb, names, texts, output_dir, + ) + + # ---- Figure 5: Granularity Effect ---- + logger.info("Generating Figure 5: Granularity Effect...") + metrics_fig5 = fig5_granularity_effect( + module_emb, text_emb, names, texts, + n_permutations=args.n_permutations, + n_bootstrap=args.n_bootstrap, + seed=args.seed, + output_dir=output_dir, + ) + + # ---- Figure 6: Multi-Model Comparison (optional) ---- + metrics_fig6 = {} + compare_models = [] + compare_embs = [] + if args.sentence_models_compare: + compare_models = [m.strip() for m in args.sentence_models_compare.split(",") + if m.strip()] + logger.info("Loading %d comparison models...", len(compare_models)) + for model_name in compare_models: + logger.info(" Encoding with: %s", model_name) + try: + emb = encode_texts(texts, model_name, device=args.device) + compare_embs.append(emb) + except Exception as e: + logger.warning(" Failed to load %s: %s", model_name, e) + compare_models.remove(model_name) + + module_emb2 = None + if args.module_embedding_path2: + all_module_emb2 = np.load(args.module_embedding_path2) + module_emb2 = all_module_emb2[semantic_idx] + logger.info("Loaded secondary dataset: %s", all_module_emb2.shape) + + if compare_models: + logger.info("Generating Figure 6: Model Comparison...") + metrics_fig6 = fig6_model_comparison( + module_emb, module_emb2, text_emb, names, texts, + primary_model_name=args.sentence_model.split("/")[-1], + compare_model_names=compare_models, + compare_text_embs=compare_embs, + n_permutations=args.n_permutations, + n_bootstrap=args.n_bootstrap, + seed=args.seed, + output_dir=output_dir, + ) + + # ---- Deep Analysis ---- + logger.info("Running deep analysis: alignment vs module size...") + metrics_size = analysis_alignment_vs_module_size( + module_emb, text_emb, names, texts, args.module_json, output_dir, + ) + + logger.info("Running deep analysis: negative results...") + metrics_neg = analysis_negative_results( + module_emb, text_emb, names, texts, output_dir, + ) + + logger.info("Running deep analysis: kNN overlap with bootstrap...") + metrics_knn = analysis_knn_overlap( + module_emb, text_emb, names, + n_permutations=args.n_permutations, + n_bootstrap=args.n_bootstrap, + seed=args.seed, + output_dir=output_dir, + ) + logger.info("kNN overlap: %.3f [%.3f, %.3f], null=%.3f, p=%.4f", + metrics_knn["knn_overlap_mean"], + metrics_knn["knn_overlap_ci_95"][0], + metrics_knn["knn_overlap_ci_95"][1], + metrics_knn["null_mean"], + metrics_knn["p_value"]) + + # ---- Cross-dataset RSA (if secondary available) ---- + metrics_cross = {} + if module_emb2 is not None: + logger.info("Cross-dataset RSA (secondary dataset)...") + mantel2 = mantel_test(module_emb2, text_emb, n_perm=args.n_permutations) + r2, ci2_lo, ci2_hi = bootstrap_rsa( + module_emb2, text_emb, args.n_bootstrap, args.seed + ) + metrics_cross = { + "secondary_rsa_r": r2, + "secondary_rsa_ci_95": [ci2_lo, ci2_hi], + "secondary_mantel_p": mantel2["rsa_p_value"], + "secondary_mantel_z": mantel2["rsa_effect_size_z"], + } + logger.info("Secondary RSA: r=%.4f [%.4f, %.4f], p=%.4f", + r2, ci2_lo, ci2_hi, mantel2["rsa_p_value"]) + + # ---- Aggregate and save metrics ---- + all_metrics = { + "primary_dataset": args.module_embedding_path, + "secondary_dataset": args.module_embedding_path2, + "sentence_model": args.sentence_model, + "n_modules": len(names), + "n_permutations": args.n_permutations, + "n_bootstrap": args.n_bootstrap, + "fig1_rsa_overview": metrics_fig1, + "fig2_heatmaps": metrics_fig2, + "fig3_per_hallmark": metrics_fig3, + "fig4_category": metrics_fig4, + "fig5_granularity": metrics_fig5, + "fig6_model_comparison": metrics_fig6, + "alignment_vs_size": metrics_size, + "negative_analysis": metrics_neg, + "knn_overlap": metrics_knn, + "cross_dataset": metrics_cross, + } + + metrics_path = output_dir / "publication_metrics.json" + with open(metrics_path, "w", encoding="utf-8") as f: + json.dump(all_metrics, f, indent=2, ensure_ascii=False, default=str) + logger.info("Saved: %s", metrics_path) + logger.info("Publication analysis complete. Output: %s", output_dir) + + +if __name__ == "__main__": + main()