Upload significance_test.py with huggingface_hub

significance_test.py

@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+"""Significance testing for Hadith vs Dense NDCG@10 differences.
+Reads query_deltas.csv, runs paired t-test and randomization test."""
+import csv, math, random, sys
+import numpy as np
+from scipy import stats as scipy_stats
+
+CSV_PATH = "/workspace/rankings/query_deltas.csv" if len(sys.argv) < 2 else sys.argv[1]
+
+def paired_t_test(a, b):
+    """Paired two-sided t-test. Returns t-statistic, p-value."""
+    diffs = np.array(a) - np.array(b)
+    n = len(diffs)
+    mean_d = np.mean(diffs)
+    std_d = np.std(diffs, ddof=1)
+    t_stat = mean_d / (std_d / math.sqrt(n)) if std_d > 0 else 0.0
+    # approximate p-value from t-distribution
+    from scipy.stats import t
+    p = 2 * t.sf(abs(t_stat), df=n-1)
+    return t_stat, p
+
+def randomization_test(a, b, n_permutations=100_000):
+    """Paired randomization test. Returns p-value (fraction of permutations with |delta| >= observed)."""
+    diffs = np.array(a) - np.array(b)
+    observed = np.abs(np.mean(diffs))
+    n = len(diffs)
+    count_extreme = 0
+    for _ in range(n_permutations):
+        # Randomly flip each pair's sign
+        signs = np.random.choice([-1, 1], size=n)
+        perm_mean = np.abs(np.mean(diffs * signs))
+        if perm_mean >= observed:
+            count_extreme += 1
+    return (count_extreme + 1) / (n_permutations + 1)  # +1 for continuity
+
+def cohens_d(a, b):
+    """Cohen's d effect size."""
+    diffs = np.array(a) - np.array(b)
+    mean_d = np.mean(diffs)
+    std_d = np.std(diffs, ddof=1)
+    return mean_d / std_d if std_d > 0 else 0.0
+
+# ── Load ──
+print(f"Reading {CSV_PATH} ...")
+rows = []
+with open(CSV_PATH) as f:
+    reader = csv.DictReader(f)
+    for row in reader:
+        rows.append(row)
+
+print(f"Loaded {len(rows)} queries\n")
+
+# Extract named variants
+variants = ["dense+T", "dense+S", "dense+M", "dense+TS", "dense+MT", "dense+MS", "dense+MTS"]
+dense_col = "dense"
+
+dense_ndcg = np.array([float(r[dense_col]) for r in rows])
+
+print("=" * 80)
+print("SIGNIFICANCE TEST REPORT — Hadith vs Dense (paired test)")
+print("=" * 80)
+print(f"{'Variant':<12} {'Mean Δ':>8} {'Std Δ':>8} {'t':>8} {'p(t-test)':>10} {'p(random)':>10} {'Cohen d':>8} {'Signif':>8}")
+print("-" * 80)
+
+results = []
+for v in variants:
+    v_ndcg = np.array([float(r[v]) for r in rows])
+    deltas = v_ndcg - dense_ndcg
+    
+    mean_d = np.mean(deltas)
+    std_d = np.std(deltas, ddof=1)
+    
+    # Paired t-test
+    t_stat, p_ttest = paired_t_test(v_ndcg, dense_ndcg)
+    
+    # Randomization test (10K permutations for speed; increase for final)
+    p_rand = randomization_test(v_ndcg, dense_ndcg, n_permutations=10_000)
+    
+    # Cohen's d
+    d_cohen = cohens_d(v_ndcg, dense_ndcg)
+    
+    # Significance label
+    if p_ttest < 0.001:
+        sig = "***"
+    elif p_ttest < 0.01:
+        sig = "**"
+    elif p_ttest < 0.05:
+        sig = "*"
+    else:
+        sig = "n.s."
+    
+    results.append((v, mean_d, std_d, t_stat, p_ttest, p_rand, d_cohen, sig))
+    print(f"{v:<12} {mean_d:>+8.6f} {std_d:>8.6f} {t_stat:>+8.3f} {p_ttest:>10.6f} {p_rand:>10.6f} {d_cohen:>+8.4f} {sig:>8}")
+
+print("-" * 80)
+print("\nSignificance codes: *** p<0.001  ** p<0.01  * p<0.05  n.s. = not significant")
+print()
+
+# ── Summary statistics ──
+print("\n" + "=" * 80)
+print("SUMMARY STATISTICS — Per-Query Deltas")
+print("=" * 80)
+print(f"{'Variant':<12} {'Mean':>8} {'Median':>8} {'Std':>8} {'Min':>8} {'Max':>8} {'Q1':>8} {'Q3':>8} {'%Neg':>8} {'%Zero':>8}")
+print("-" * 80)
+
+for v, md, std_d, _, _, _, _, _ in results:
+    deltas = np.array([float(r[v]) for r in rows]) - dense_ndcg
+    median = np.median(deltas)
+    mn = np.min(deltas)
+    mx = np.max(deltas)
+    q1 = np.percentile(deltas, 25)
+    q3 = np.percentile(deltas, 75)
+    pct_neg = 100 * np.mean(deltas < 0)
+    pct_zero = 100 * np.mean(deltas == 0)
+    print(f"{v:<12} {md:>+8.6f} {median:>+8.6f} {std_d:>8.6f} {mn:>+8.4f} {mx:>+8.4f} {q1:>+8.6f} {q3:>+8.6f} {pct_neg:>7.1f}% {pct_zero:>7.1f}%")
+
+print("-" * 80)
+
+# ── Best sweep ──
+print("\n" + "=" * 80)
+print("BEST SWEEP — Tawatur threshold/tau sweep vs Dense")
+print("=" * 80)
+# Find best sweep variant
+sweep_keys = [k for k in rows[0].keys() if k.startswith("tawatur_ct=")]
+sweep_results = []
+for k in sweep_keys:
+    v = np.array([float(r[k]) for r in rows])
+    deltas = v - dense_ndcg
+    mean_d = np.mean(deltas)
+    sweep_results.append((k, mean_d, np.std(deltas, ddof=1)))
+
+sweep_results.sort(key=lambda x: -x[1])
+print(f"{'Sweep':<30} {'Mean Δ':>10} {'Std Δ':>10}")
+print("-" * 50)
+for k, md, sd in sweep_results[:5]:
+    print(f"{k:<30} {md:>+10.6f} {sd:>10.6f}")
+print(f"{'...':<30}")
+for k, md, sd in sweep_results[-3:]:
+    print(f"{k:<30} {md:>+10.6f} {sd:>10.6f}")
+
+# ── N queries helped/hurt ──
+print("\n" + "=" * 80)
+print("HELPED vs HURT — Query counts")
+print("=" * 80)
+for v in variants:
+    deltas = np.array([float(r[v]) for r in rows]) - dense_ndcg
+    helped = np.sum(deltas > 0.001)  # >0.001 to avoid floating-point noise
+    hurt = np.sum(deltas < -0.001)
+    neutral = len(deltas) - helped - hurt
+    print(f"{v:<12}: helped={helped:>4} ({100*helped/len(deltas):>5.1f}%)  hurt={hurt:>4} ({100*hurt/len(deltas):>5.1f}%)  neutral={neutral:>4} ({100*neutral/len(deltas):>5.1f}%)")