Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+analysis/dataset_level_analysis/figures/04_baseline_sensitivity.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,11 @@
+__pycache__/
+*.log
+
+# Inference and analysis outputs are regenerated from the scripts; only the two
+# source files that live alongside them are tracked. Git needs each parent
+# directory un-ignored before a file inside it can be negated.
+result/*
+!result/analysis/
+result/analysis/*
+!result/analysis/readme.md
+!result/analysis/baseline_intervention.ipynb

analysis/analyze_baseline_overlap.py

@@ -0,0 +1,284 @@
+"""Cross-baseline stability and failure-case overlap.
+
+Reads the per-baseline prediction files written by inference/run_inference.py
+and, for each (model, task), lines the samples up by sample_id to answer:
+
+  accuracy        per baseline, plus how much it moves across the sweep.
+  flip rate       fraction of samples whose ANSWER changes relative to the
+                  reference baseline (default 0.00). This is the direct test of
+                  the premise: a visual tower that modelled the scene perfectly
+                  would answer identically at every baseline, so the flip rate
+                  is the size of the effect the intervention has, independent of
+                  whether the answer was right to begin with.
+  stability       fraction of samples correct at every baseline / wrong at every
+                  baseline / unstable (correct at some, wrong at others).
+  failure overlap for every pair of baselines, |F_a & F_b| / |F_a | F_b| over
+                  the failure sets, plus the raw counts. High overlap means the
+                  baselines fail on the SAME samples (a shared difficulty);
+                  low overlap means each baseline breaks a different subset,
+                  i.e. the failures are driven by the intervention itself.
+
+Usage
+    python analysis/analyze_baseline_overlap.py --result_dir result
+    python analysis/analyze_baseline_overlap.py --result_dir result --dump_per_sample
+
+Writes analysis_summary.json and one <model>__<task>_overlap.csv per run next
+to the results, and prints a readable table.
+"""
+
+import argparse
+import csv
+import itertools
+import json
+import os
+import pathlib
+import sys
+from collections import defaultdict
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from dataset import BASELINES  # noqa: E402
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description=__doc__.split("\n")[0])
+    default_result = pathlib.Path(__file__).resolve().parent.parent / "result"
+    parser.add_argument("--result_dir", default=str(default_result))
+    parser.add_argument(
+        "--output_dir",
+        default=str(pathlib.Path(__file__).resolve().parent / "intervention_analysis"),
+        help="Defaults to analysis/intervention_analysis, beside the notebook that reads it.",
+    )
+    parser.add_argument(
+        "--reference_baseline",
+        default="0.00",
+        help="Baseline that flip rate and per-pair comparisons are measured against.",
+    )
+    parser.add_argument(
+        "--dump_per_sample",
+        action="store_true",
+        help="Also write a per-sample correctness/answer matrix for each run.",
+    )
+    return parser.parse_args()
+
+
+def sample_key(row):
+    """A key that identifies one question, unique within a task.
+
+    VSR reuses an image across several statements, so its `sample_id` is the
+    image id and repeats (2195 rows over 1874 ids). Keying the cross-baseline
+    alignment on `sample_id` alone would silently keep only the last question
+    per image and drop the rest, so the question text is part of the key. The
+    Youtube levels already have unique ids and are unaffected.
+    """
+    return (row["sample_id"], row.get("question", ""))
+
+
+def load_runs(result_dir):
+    """{(model, task): {baseline: {sample_key: row}}} from result/<model>/<task>/baseline_<b>/."""
+    runs = defaultdict(dict)
+    for metrics_path in sorted(pathlib.Path(result_dir).glob("*/*/baseline_*/metrics.json")):
+        predictions_path = metrics_path.parent / "predictions.jsonl"
+        if not predictions_path.is_file():
+            print(f"  skipping {metrics_path.parent} (no predictions.jsonl)", file=sys.stderr)
+            continue
+        with metrics_path.open(encoding="utf-8") as f:
+            metrics = json.load(f)
+        with predictions_path.open(encoding="utf-8") as f:
+            rows = {}
+            for line in f:
+                if not line.strip():
+                    continue
+                row = json.loads(line)
+                rows[sample_key(row)] = row
+        key = (metrics.get("model_id", metrics_path.parents[2].name), metrics.get("task", metrics_path.parents[1].name))
+        runs[key][metrics["baseline"]] = {"metrics": metrics, "rows": rows}
+    return runs
+
+
+def analyse_run(baselines_data, reference_baseline):
+    """Accuracy / flip rate / stability / pairwise failure overlap for one run."""
+    baselines = [b for b in BASELINES if b in baselines_data]
+    common = set.intersection(*(set(baselines_data[b]["rows"]) for b in baselines))
+    dropped = {b: len(baselines_data[b]["rows"]) - len(common) for b in baselines}
+
+    correct = {b: {sid for sid in common if baselines_data[b]["rows"][sid]["correct"]} for b in baselines}
+    answers = {b: {sid: baselines_data[b]["rows"][sid]["prediction_norm"] for sid in common} for b in baselines}
+    failures = {b: common - correct[b] for b in baselines}
+
+    per_baseline = []
+    reference = reference_baseline if reference_baseline in baselines else baselines[0]
+    for b in baselines:
+        flipped = {sid for sid in common if answers[b][sid] != answers[reference][sid]}
+        per_baseline.append(
+            {
+                "baseline": b,
+                "num_samples": len(common),
+                "accuracy": len(correct[b]) / len(common) if common else 0.0,
+                "num_failures": len(failures[b]),
+                "flip_rate_vs_reference": len(flipped) / len(common) if common else 0.0,
+                "unparsed_rate": baselines_data[b]["metrics"].get("unparsed_rate"),
+            }
+        )
+
+    always_correct = set.intersection(*(correct[b] for b in baselines)) if baselines else set()
+    always_wrong = set.intersection(*(failures[b] for b in baselines)) if baselines else set()
+    unstable = common - always_correct - always_wrong
+
+    pairwise = []
+    for a, b in itertools.combinations(baselines, 2):
+        intersection = failures[a] & failures[b]
+        union = failures[a] | failures[b]
+        agree = sum(1 for sid in common if answers[a][sid] == answers[b][sid])
+        pairwise.append(
+            {
+                "baseline_a": a,
+                "baseline_b": b,
+                "failures_a": len(failures[a]),
+                "failures_b": len(failures[b]),
+                "failures_shared": len(intersection),
+                "failure_jaccard": len(intersection) / len(union) if union else 1.0,
+                # Of the samples a fails, how many does b also fail.
+                "failure_recall_a_in_b": len(intersection) / len(failures[a]) if failures[a] else float("nan"),
+                # The same measure over the samples each baseline gets RIGHT. It is
+                # not redundant with the failure Jaccard: with 4-way answers the
+                # success sets are small and the failure sets large, so the two
+                # move independently and a task can share its successes while its
+                # failures scatter (or the reverse).
+                "successes_a": len(correct[a]),
+                "successes_b": len(correct[b]),
+                "successes_shared": len(correct[a] & correct[b]),
+                "success_jaccard": (
+                    len(correct[a] & correct[b]) / len(correct[a] | correct[b])
+                    if (correct[a] | correct[b]) else 1.0
+                ),
+                "answer_agreement": agree / len(common) if common else 0.0,
+            }
+        )
+
+    return {
+        "baselines": baselines,
+        "reference_baseline": reference,
+        "num_common_samples": len(common),
+        "dropped_per_baseline": dropped,
+        "per_baseline": per_baseline,
+        "accuracy_spread": (
+            max(r["accuracy"] for r in per_baseline) - min(r["accuracy"] for r in per_baseline)
+            if per_baseline
+            else 0.0
+        ),
+        "stability": {
+            "always_correct": len(always_correct),
+            "always_wrong": len(always_wrong),
+            "unstable": len(unstable),
+            "always_correct_rate": len(always_correct) / len(common) if common else 0.0,
+            "always_wrong_rate": len(always_wrong) / len(common) if common else 0.0,
+            "unstable_rate": len(unstable) / len(common) if common else 0.0,
+        },
+        "pairwise": pairwise,
+        "_common": common,
+        "_answers": answers,
+        "_correct": correct,
+    }
+
+
+def write_pairwise_csv(path, result):
+    with path.open("w", newline="", encoding="utf-8") as f:
+        writer = csv.DictWriter(
+            f,
+            fieldnames=[
+                "baseline_a", "baseline_b",
+                "failures_a", "failures_b", "failures_shared",
+                "failure_jaccard", "failure_recall_a_in_b",
+                "successes_a", "successes_b", "successes_shared", "success_jaccard",
+                "answer_agreement",
+            ],
+        )
+        writer.writeheader()
+        writer.writerows(result["pairwise"])
+
+
+def write_per_sample_csv(path, result):
+    baselines = result["baselines"]
+    with path.open("w", newline="", encoding="utf-8") as f:
+        writer = csv.writer(f)
+        writer.writerow(
+            ["sample_id", "question"]
+            + [f"answer_{b}" for b in baselines]
+            + [f"correct_{b}" for b in baselines]
+            + ["num_correct", "stability"]
+        )
+        for sid in sorted(result["_common"]):
+            flags = [sid in result["_correct"][b] for b in baselines]
+            total = sum(flags)
+            stability = "always_correct" if total == len(baselines) else (
+                "always_wrong" if total == 0 else "unstable"
+            )
+            writer.writerow(
+                list(sid)
+                + [result["_answers"][b][sid] for b in baselines]
+                + [int(flag) for flag in flags]
+                + [total, stability]
+            )
+
+
+def main():
+    args = parse_args()
+    runs = load_runs(args.result_dir)
+    if not runs:
+        raise SystemExit(
+            f"No results under {args.result_dir}; expected "
+            "<result_dir>/<model>/<task>/baseline_<b>/{metrics.json,predictions.jsonl}"
+        )
+
+    output_dir = pathlib.Path(args.output_dir or os.path.join(args.result_dir, "analysis"))
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    summary = []
+    for (model_id, task), baselines_data in sorted(runs.items()):
+        result = analyse_run(baselines_data, args.reference_baseline)
+        tag = f"{model_id.replace('/', '_')}__{task}"
+
+        write_pairwise_csv(output_dir / f"{tag}_failure_overlap.csv", result)
+        if args.dump_per_sample:
+            write_per_sample_csv(output_dir / f"{tag}_per_sample.csv", result)
+
+        print(f"\n=== {model_id}  |  {task}  |  n={result['num_common_samples']} ===")
+        print(f"{'baseline':>10} {'accuracy':>10} {'failures':>10} {'flip vs ' + result['reference_baseline']:>14}")
+        for row in result["per_baseline"]:
+            print(
+                f"{row['baseline']:>10} {row['accuracy']:>10.4f} {row['num_failures']:>10d} "
+                f"{row['flip_rate_vs_reference']:>14.4f}"
+            )
+        stability = result["stability"]
+        print(
+            f"  accuracy spread {result['accuracy_spread']:.4f} | "
+            f"always correct {stability['always_correct_rate']:.4f} | "
+            f"always wrong {stability['always_wrong_rate']:.4f} | "
+            f"unstable {stability['unstable_rate']:.4f}"
+        )
+        print("  failure-set overlap (Jaccard):")
+        for pair in result["pairwise"]:
+            print(
+                f"    {pair['baseline_a']} vs {pair['baseline_b']}: "
+                f"J={pair['failure_jaccard']:.4f} shared={pair['failures_shared']:>5d} "
+                f"(|F_a|={pair['failures_a']}, |F_b|={pair['failures_b']}) "
+                f"answer agreement={pair['answer_agreement']:.4f}"
+            )
+
+        summary.append(
+            {
+                "model_id": model_id,
+                "task": task,
+                **{k: v for k, v in result.items() if not k.startswith("_")},
+            }
+        )
+
+    summary_path = output_dir / "analysis_summary.json"
+    with summary_path.open("w", encoding="utf-8") as f:
+        json.dump({"result_dir": os.path.abspath(args.result_dir), "runs": summary}, f, indent=2)
+    print(f"\nSummary saved to {summary_path}")
+
+
+if __name__ == "__main__":
+    main()

analysis/dataset_level_analysis/accuracy_by_baseline.csv

@@ -0,0 +1,97 @@
+model,task,baseline,n,accuracy,num_correct,unparsed_rate,scoring,chance,floor,headroom_captured
+Qwen_Qwen2.5-VL-3B-Instruct,vsr,0.00,2195,0.7713,1693,0.0,generate,0.5,0.5,0.5426
+Qwen_Qwen2.5-VL-3B-Instruct,vsr,0.05,2195,0.7677,1685,0.0,generate,0.5,0.5,0.5353
+Qwen_Qwen2.5-VL-3B-Instruct,vsr,0.10,2195,0.7595,1667,0.0,generate,0.5,0.5,0.5189
+Qwen_Qwen2.5-VL-3B-Instruct,vsr,0.15,2195,0.7581,1664,0.0,generate,0.5,0.5,0.5162
+Qwen_Qwen2.5-VL-3B-Instruct,vsr,0.20,2195,0.7526,1652,0.0,generate,0.5,0.5,0.5052
+Qwen_Qwen2.5-VL-3B-Instruct,vsr,0.25,2195,0.7481,1642,0.0,generate,0.5,0.5,0.4961
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level1,0.00,821,0.3995,328,0.0,generate,0.25,0.25,0.1994
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level1,0.05,821,0.4056,333,0.0,generate,0.25,0.25,0.2075
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level1,0.10,821,0.4117,338,0.0,generate,0.25,0.25,0.2156
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level1,0.15,821,0.4105,337,0.0,generate,0.25,0.25,0.214
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level1,0.20,821,0.3995,328,0.0,generate,0.25,0.25,0.1994
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level1,0.25,821,0.4007,329,0.0,generate,0.25,0.25,0.201
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level2,0.00,764,0.3626,277,0.0,generate,0.25,0.25,0.1501
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level2,0.05,764,0.3743,286,0.0,generate,0.25,0.25,0.1658
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level2,0.10,764,0.3783,289,0.0,generate,0.25,0.25,0.171
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level2,0.15,764,0.3796,290,0.0,generate,0.25,0.25,0.1728
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level2,0.20,764,0.3809,291,0.0,generate,0.25,0.25,0.1745
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level2,0.25,764,0.3704,283,0.0,generate,0.25,0.25,0.1606
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level3,0.00,836,0.2667,223,0.0,generate,0.25,0.25,0.0223
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level3,0.05,836,0.2452,205,0.0,generate,0.25,0.25,-0.0064
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level3,0.10,836,0.2392,200,0.0,generate,0.25,0.25,-0.0144
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level3,0.15,836,0.2524,211,0.0,generate,0.25,0.25,0.0032
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level3,0.20,836,0.2596,217,0.0,generate,0.25,0.25,0.0128
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level3,0.25,836,0.2656,222,0.0,generate,0.25,0.25,0.0207
+Qwen_Qwen2.5-VL-7B-Instruct,vsr,0.00,2195,0.8223,1805,0.0,generate,0.5,0.5,0.6446
+Qwen_Qwen2.5-VL-7B-Instruct,vsr,0.05,2195,0.8077,1773,0.0,generate,0.5,0.5,0.6155
+Qwen_Qwen2.5-VL-7B-Instruct,vsr,0.10,2195,0.8027,1762,0.0,generate,0.5,0.5,0.6055
+Qwen_Qwen2.5-VL-7B-Instruct,vsr,0.15,2195,0.8068,1771,0.0,generate,0.5,0.5,0.6137
+Qwen_Qwen2.5-VL-7B-Instruct,vsr,0.20,2195,0.8023,1761,0.0,generate,0.5,0.5,0.6046
+Qwen_Qwen2.5-VL-7B-Instruct,vsr,0.25,2195,0.7882,1730,0.0,generate,0.5,0.5,0.5763
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level1,0.00,821,0.447,367,0.0,generate,0.25,0.25,0.2627
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level1,0.05,821,0.4482,368,0.0,generate,0.25,0.25,0.2643
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level1,0.10,821,0.4555,374,0.0,generate,0.25,0.25,0.2741
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level1,0.15,821,0.4665,383,0.0,generate,0.25,0.25,0.2887
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level1,0.20,821,0.4421,363,0.0,generate,0.25,0.25,0.2562
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level1,0.25,821,0.4519,371,0.0,generate,0.25,0.25,0.2692
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level2,0.00,764,0.4031,308,0.0,generate,0.25,0.25,0.2042
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level2,0.05,764,0.394,301,0.0,generate,0.25,0.25,0.192
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level2,0.10,764,0.3979,304,0.0,generate,0.25,0.25,0.1972
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level2,0.15,764,0.4018,307,0.0,generate,0.25,0.25,0.2024
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level2,0.20,764,0.377,288,0.0,generate,0.25,0.25,0.1693
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level2,0.25,764,0.377,288,0.0,generate,0.25,0.25,0.1693
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level3,0.00,836,0.2799,234,0.0,generate,0.25,0.25,0.0399
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level3,0.05,836,0.2871,240,0.0,generate,0.25,0.25,0.0494
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level3,0.10,836,0.2679,224,0.0,generate,0.25,0.25,0.0239
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level3,0.15,836,0.2823,236,0.0,generate,0.25,0.25,0.0431
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level3,0.20,836,0.2763,231,0.0,generate,0.25,0.25,0.0351
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level3,0.25,836,0.2739,229,0.0,generate,0.25,0.25,0.0319
+Qwen_Qwen3.5-4B,vsr,0.00,2195,0.8214,1803,0.0,generate,0.5,0.5,0.6428
+Qwen_Qwen3.5-4B,vsr,0.05,2195,0.8173,1794,0.0,generate,0.5,0.5,0.6346
+Qwen_Qwen3.5-4B,vsr,0.10,2195,0.8187,1797,0.0,generate,0.5,0.5,0.6374
+Qwen_Qwen3.5-4B,vsr,0.15,2195,0.8114,1781,0.0,generate,0.5,0.5,0.6228
+Qwen_Qwen3.5-4B,vsr,0.20,2195,0.8073,1772,0.0,generate,0.5,0.5,0.6146
+Qwen_Qwen3.5-4B,vsr,0.25,2195,0.8046,1766,0.0,generate,0.5,0.5,0.6091
+Qwen_Qwen3.5-4B,youtube_level1,0.00,821,0.5274,433,0.0037,generate,0.25,0.25,0.3699
+Qwen_Qwen3.5-4B,youtube_level1,0.05,821,0.5213,428,0.0073,generate,0.25,0.25,0.3618
+Qwen_Qwen3.5-4B,youtube_level1,0.10,821,0.5189,426,0.0012,generate,0.25,0.25,0.3585
+Qwen_Qwen3.5-4B,youtube_level1,0.15,821,0.5408,444,0.0049,generate,0.25,0.25,0.3877
+Qwen_Qwen3.5-4B,youtube_level1,0.20,821,0.5347,439,0.0012,generate,0.25,0.25,0.3796
+Qwen_Qwen3.5-4B,youtube_level1,0.25,821,0.5347,439,0.0,generate,0.25,0.25,0.3796
+Qwen_Qwen3.5-4B,youtube_level2,0.00,764,0.3914,299,0.0,generate,0.25,0.25,0.1885
+Qwen_Qwen3.5-4B,youtube_level2,0.05,764,0.3901,298,0.0,generate,0.25,0.25,0.1867
+Qwen_Qwen3.5-4B,youtube_level2,0.10,764,0.3822,292,0.0,generate,0.25,0.25,0.1763
+Qwen_Qwen3.5-4B,youtube_level2,0.15,764,0.3874,296,0.0,generate,0.25,0.25,0.1832
+Qwen_Qwen3.5-4B,youtube_level2,0.20,764,0.3796,290,0.0,generate,0.25,0.25,0.1728
+Qwen_Qwen3.5-4B,youtube_level2,0.25,764,0.3848,294,0.0,generate,0.25,0.25,0.1798
+Qwen_Qwen3.5-4B,youtube_level3,0.00,836,0.2931,245,0.0,generate,0.25,0.25,0.0574
+Qwen_Qwen3.5-4B,youtube_level3,0.05,836,0.299,250,0.0,generate,0.25,0.25,0.0654
+Qwen_Qwen3.5-4B,youtube_level3,0.10,836,0.2967,248,0.0,generate,0.25,0.25,0.0622
+Qwen_Qwen3.5-4B,youtube_level3,0.15,836,0.3038,254,0.0,generate,0.25,0.25,0.0718
+Qwen_Qwen3.5-4B,youtube_level3,0.20,836,0.2883,241,0.0,generate,0.25,0.25,0.051
+Qwen_Qwen3.5-4B,youtube_level3,0.25,836,0.2931,245,0.0,generate,0.25,0.25,0.0574
+Qwen_Qwen3.5-4B-Base,vsr,0.00,2195,0.8319,1826,0.0,likelihood,0.5,0.5,0.6638
+Qwen_Qwen3.5-4B-Base,vsr,0.05,2195,0.8314,1825,0.0,likelihood,0.5,0.5,0.6629
+Qwen_Qwen3.5-4B-Base,vsr,0.10,2195,0.8323,1827,0.0,likelihood,0.5,0.5,0.6647
+Qwen_Qwen3.5-4B-Base,vsr,0.15,2195,0.8278,1817,0.0,likelihood,0.5,0.5,0.6556
+Qwen_Qwen3.5-4B-Base,vsr,0.20,2195,0.8241,1809,0.0,likelihood,0.5,0.5,0.6483
+Qwen_Qwen3.5-4B-Base,vsr,0.25,2195,0.8164,1792,0.0,likelihood,0.5,0.5,0.6328
+Qwen_Qwen3.5-4B-Base,youtube_level1,0.00,821,0.542,445,0.0,likelihood,0.25,0.25,0.3894
+Qwen_Qwen3.5-4B-Base,youtube_level1,0.05,821,0.5457,448,0.0,likelihood,0.25,0.25,0.3942
+Qwen_Qwen3.5-4B-Base,youtube_level1,0.10,821,0.5603,460,0.0,likelihood,0.25,0.25,0.4137
+Qwen_Qwen3.5-4B-Base,youtube_level1,0.15,821,0.5725,470,0.0,likelihood,0.25,0.25,0.43
+Qwen_Qwen3.5-4B-Base,youtube_level1,0.20,821,0.5664,465,0.0,likelihood,0.25,0.25,0.4218
+Qwen_Qwen3.5-4B-Base,youtube_level1,0.25,821,0.5554,456,0.0,likelihood,0.25,0.25,0.4072
+Qwen_Qwen3.5-4B-Base,youtube_level2,0.00,764,0.4424,338,0.0,likelihood,0.25,0.25,0.2565
+Qwen_Qwen3.5-4B-Base,youtube_level2,0.05,764,0.4529,346,0.0,likelihood,0.25,0.25,0.2705
+Qwen_Qwen3.5-4B-Base,youtube_level2,0.10,764,0.4503,344,0.0,likelihood,0.25,0.25,0.267
+Qwen_Qwen3.5-4B-Base,youtube_level2,0.15,764,0.4542,347,0.0,likelihood,0.25,0.25,0.2723
+Qwen_Qwen3.5-4B-Base,youtube_level2,0.20,764,0.4503,344,0.0,likelihood,0.25,0.25,0.267
+Qwen_Qwen3.5-4B-Base,youtube_level2,0.25,764,0.4398,336,0.0,likelihood,0.25,0.25,0.2531
+Qwen_Qwen3.5-4B-Base,youtube_level3,0.00,836,0.3026,253,0.0,likelihood,0.25,0.25,0.0702
+Qwen_Qwen3.5-4B-Base,youtube_level3,0.05,836,0.2715,227,0.0,likelihood,0.25,0.25,0.0287
+Qwen_Qwen3.5-4B-Base,youtube_level3,0.10,836,0.2775,232,0.0,likelihood,0.25,0.25,0.0367
+Qwen_Qwen3.5-4B-Base,youtube_level3,0.15,836,0.2667,223,0.0,likelihood,0.25,0.25,0.0223
+Qwen_Qwen3.5-4B-Base,youtube_level3,0.20,836,0.2727,228,0.0,likelihood,0.25,0.25,0.0303
+Qwen_Qwen3.5-4B-Base,youtube_level3,0.25,836,0.2823,236,0.0,likelihood,0.25,0.25,0.0431

analysis/dataset_level_analysis/analyze_dataset_level.py

@@ -0,0 +1,291 @@
+#!/usr/bin/env python3
+"""Compare every model across the four evaluation splits.
+
+    python analysis/dataset_level_analysis/analyze_dataset_level.py
+
+Reads result/<model>/<task>/baseline_<b>/{metrics.json,predictions.jsonl} and
+writes the CSVs next to this script.
+
+Raw accuracy is not comparable across the four splits: VSR is a two-way
+true/false question while the three Youtube levels are four-way, and level 2
+carries a structural floor of its own (see FLOORS).  Everything is therefore
+also reported as *headroom captured*, (acc - floor) / (1 - floor): the share of
+the distance between guessing and perfect that the model actually covers.  It
+is negative when a model does worse than the floor, which happens here.
+"""
+from __future__ import annotations
+
+import argparse
+import collections
+import csv
+import json
+import math
+import os
+import statistics
+
+BASELINES = ["0.00", "0.05", "0.10", "0.15", "0.20", "0.25"]
+TASKS = ["vsr", "youtube_level1", "youtube_level2", "youtube_level3"]
+
+# What a model scores on each split without looking at the image: pure guessing
+# among the choices.  Level 2's three distractors are drawn uniformly from the
+# 23 wrong orderings, so the four options are exchangeable and every text-only
+# heuristic sits at chance -- measured on the split: prefix vote 0.252, medoid
+# 0.256, outlier 0.254, positional majority 0.245.  Earlier structured variants
+# did NOT have this property; see disparity_qa/level2_choices.py.
+FLOORS = {"vsr": 0.500, "youtube_level1": 0.250,
+          "youtube_level2": 0.250, "youtube_level3": 0.250}
+CHANCE = dict(FLOORS)
+
+
+def load_metrics(root: str, model: str, task: str, baseline: str) -> dict:
+    with open(os.path.join(root, "result", model, task,
+                           f"baseline_{baseline}", "metrics.json")) as fh:
+        return json.load(fh)
+
+
+def load_predictions(root: str, model: str, task: str, baseline: str):
+    path = os.path.join(root, "result", model, task,
+                        f"baseline_{baseline}", "predictions.jsonl")
+    with open(path) as fh:
+        for line in fh:
+            line = line.strip()
+            if line:
+                yield json.loads(line)
+
+
+def entropy(counter: collections.Counter) -> float:
+    n = sum(counter.values())
+    if not n:
+        return 0.0
+    return -sum((v / n) * math.log2(v / n) for v in counter.values() if v)
+
+
+def slope(xs, ys) -> float:
+    """Least-squares slope of accuracy against baseline (per unit of baseline)."""
+    mx, my = statistics.mean(xs), statistics.mean(ys)
+    den = sum((x - mx) ** 2 for x in xs)
+    return sum((x - mx) * (y - my) for x, y in zip(xs, ys)) / den if den else 0.0
+
+
+def main() -> int:
+    here = os.path.dirname(os.path.abspath(__file__))
+    p = argparse.ArgumentParser()
+    p.add_argument("--root", default=os.path.abspath(os.path.join(here, "..", "..")))
+    p.add_argument("--out_dir", default=here)
+    args = p.parse_args()
+
+    models = sorted(m for m in os.listdir(os.path.join(args.root, "result"))
+                    if not m.startswith("_")
+                    and os.path.isdir(os.path.join(args.root, "result", m)))
+
+    acc = {}          # (model, task, baseline) -> metrics dict
+    for m in models:
+        for t in TASKS:
+            for b in BASELINES:
+                acc[(m, t, b)] = load_metrics(args.root, m, t, b)
+
+    # ---------------------------------------------------------- per baseline
+    rows = []
+    for m in models:
+        for t in TASKS:
+            for b in BASELINES:
+                d = acc[(m, t, b)]
+                rows.append({
+                    "model": m, "task": t, "baseline": b,
+                    "n": d["num_samples"], "accuracy": round(d["accuracy"], 4),
+                    "num_correct": d["num_correct"],
+                    "unparsed_rate": round(d["unparsed_rate"], 4),
+                    "scoring": d["scoring"],
+                    "chance": CHANCE[t], "floor": FLOORS[t],
+                    "headroom_captured": round(
+                        (d["accuracy"] - FLOORS[t]) / (1 - FLOORS[t]), 4),
+                })
+    write_csv(os.path.join(args.out_dir, "accuracy_by_baseline.csv"), rows)
+
+    # ------------------------------------------------------- per model/task
+    summary = []
+    for m in models:
+        for t in TASKS:
+            a = [acc[(m, t, b)]["accuracy"] for b in BASELINES]
+            xs = [float(b) for b in BASELINES]
+            summary.append({
+                "model": m, "task": t, "n": acc[(m, t, "0.00")]["num_samples"],
+                "scoring": acc[(m, t, "0.00")]["scoring"],
+                "chance": CHANCE[t], "floor": FLOORS[t],
+                "acc_b0": round(a[0], 4), "acc_mean": round(statistics.mean(a), 4),
+                "acc_min": round(min(a), 4), "acc_max": round(max(a), 4),
+                "acc_range": round(max(a) - min(a), 4),
+                "acc_std": round(statistics.pstdev(a), 4),
+                "slope_per_baseline": round(slope(xs, a), 4),
+                "delta_b025_minus_b0": round(a[-1] - a[0], 4),
+                "headroom_captured_mean": round(
+                    (statistics.mean(a) - FLOORS[t]) / (1 - FLOORS[t]), 4),
+                "above_floor": statistics.mean(a) > FLOORS[t],
+            })
+    write_csv(os.path.join(args.out_dir, "summary_by_model_task.csv"), summary)
+
+    # --------------------------------------------------------------- biases
+    bias = []
+    for m in models:
+        for t in TASKS:
+            for b in BASELINES:
+                pred = collections.Counter()
+                ref = collections.Counter()
+                n = 0
+                for r in load_predictions(args.root, m, t, b):
+                    pred[r["prediction_norm"]] += 1
+                    ref[r["reference_norm"]] += 1
+                    n += 1
+                top_label, top_n = pred.most_common(1)[0]
+                # what a model would score by always naming its own modal answer
+                majority_acc = ref[top_label] / n if n else 0.0
+                bias.append({
+                    "model": m, "task": t, "baseline": b, "n": n,
+                    "n_distinct_predictions": len(pred),
+                    "modal_prediction": top_label,
+                    "modal_share": round(top_n / n, 4),
+                    "prediction_entropy_bits": round(entropy(pred), 4),
+                    "reference_entropy_bits": round(entropy(ref), 4),
+                    "always_modal_accuracy": round(majority_acc, 4),
+                    "accuracy": round(acc[(m, t, b)]["accuracy"], 4),
+                    "prediction_counts": json.dumps(dict(sorted(pred.items()))),
+                })
+    write_csv(os.path.join(args.out_dir, "prediction_bias.csv"), bias)
+
+    ladder = level2_ladder(args.root, models)
+    if ladder:
+        write_csv(os.path.join(args.out_dir, "level2_structure_use.csv"), ladder)
+
+    report(models, acc, summary, bias, ladder)
+    print(f"\nwrote {3 + bool(ladder)} CSVs to {args.out_dir}")
+    return 0
+
+
+# the split lives in the repo, one level above this experiment directory
+LEVEL2_TEST = os.path.join("..", "02_data", "Youtube_self_depth_QA",
+                           "level2", "test.jsonl")
+
+
+def _prefix_survivors(options):
+    """The options a majority vote on successive positions cannot rule out.
+
+    A regression guard.  With random distractors this should leave most items
+    undecided and score at chance; if it ever climbs toward 0.5 again, the
+    option construction has reacquired a structural leak.
+    """
+    alive = list(options)
+    seqs = {lab: txt.split(" > ") for lab, txt in options.items()}
+    for k in range(4):
+        if len(alive) <= 1:
+            break
+        counts = collections.Counter(seqs[a][k] for a in alive)
+        top = counts.most_common(1)[0][1]
+        winners = [l for l, v in counts.items() if v == top]
+        if len(winners) == 1:
+            alive = [a for a in alive if seqs[a][k] == winners[0]]
+    return alive
+
+
+def level2_ladder(root: str, models):
+    """How often each model's answer lands inside the structure-only survivor set."""
+    path = os.path.abspath(os.path.join(root, LEVEL2_TEST))
+    if not os.path.exists(path):
+        print(f"[warn] level-2 split not found at {path}; skipping ladder analysis")
+        return []
+    test = {}
+    with open(path) as fh:
+        for line in fh:
+            rec = json.loads(line)
+            if rec.get("options"):
+                test[rec["sample_id"]] = rec["options"]
+    if not test:
+        return []
+    out = []
+    for m in models:
+        for b in BASELINES:
+            inside = hit = n = 0
+            for r in load_predictions(root, m, "youtube_level2", b):
+                opts = test.get(r["sample_id"])
+                if not opts:
+                    continue
+                alive = _prefix_survivors(opts)
+                n += 1
+                inside += int(r["prediction_norm"] in alive)
+                hit += int(r["correct"])
+            out.append({
+                "model": m, "baseline": b, "n": n,
+                "answer_in_survivor_set": round(inside / n, 4) if n else 0.0,
+                "accuracy": round(hit / n, 4) if n else 0.0,
+                # of the answers that did land in the survivor set, how many were
+                # right: this is the coin flip the structure cannot decide
+                "accuracy_within_survivors": round(hit / inside, 4) if inside else 0.0,
+            })
+    return out
+
+
+def write_csv(path: str, rows) -> None:
+    with open(path, "w", newline="") as fh:
+        w = csv.DictWriter(fh, fieldnames=list(rows[0]))
+        w.writeheader()
+        w.writerows(rows)
+
+
+def report(models, acc, summary, bias, ladder=()) -> None:
+    short = {m: m.replace("Qwen_Qwen", "").replace("-Instruct", "") for m in models}
+    idx = {(r["model"], r["task"]): r for r in summary}
+
+    print("=" * 96)
+    print("ACCURACY  (mean over the six baselines; floor = what the split gives "
+          "away for free)")
+    print("=" * 96)
+    print(f'{"model":22s}' + "".join(f"{t.replace('youtube_','yt_'):>16s}" for t in TASKS))
+    print(f'{"floor":22s}' + "".join(f"{FLOORS[t]:>16.3f}" for t in TASKS))
+    print("-" * 96)
+    for m in models:
+        print(f"{short[m]:22s}" + "".join(
+            f'{idx[(m,t)]["acc_mean"]:>16.3f}' for t in TASKS))
+    print()
+    print("HEADROOM CAPTURED  (acc - floor) / (1 - floor);  negative = below floor")
+    print("-" * 96)
+    for m in models:
+        cells = []
+        for t in TASKS:
+            v = idx[(m, t)]["headroom_captured_mean"]
+            cells.append(f"{v:>+15.3f} " if v < 0 else f"{v:>16.3f}")
+        print(f"{short[m]:22s}" + "".join(cells))
+    print()
+    print("BASELINE SENSITIVITY  range over 0.00-0.25  /  slope per unit baseline")
+    print("-" * 96)
+    for m in models:
+        print(f"{short[m]:22s}" + "".join(
+            f'{idx[(m,t)]["acc_range"]:>7.3f}/{idx[(m,t)]["slope_per_baseline"]:>+8.3f}'
+            for t in TASKS))
+    print()
+    print("ANSWER COLLAPSE  modal predicted label, its share, and what always "
+          "answering it would score (baseline 0.00)")
+    print("-" * 96)
+    bidx = {(r["model"], r["task"], r["baseline"]): r for r in bias}
+    for m in models:
+        cells = []
+        for t in TASKS:
+            r = bidx[(m, t, "0.00")]
+            cells.append(f'{r["modal_prediction"]:>7s}{r["modal_share"]:>6.2f}'
+                         f'{r["always_modal_accuracy"]:>7.3f}')
+        print(f"{short[m]:22s}" + "".join(cells))
+    print("  (reference labels are near-uniform, so 'always modal' sits at chance)")
+
+    if ladder:
+        print()
+        print("LEVEL 2 REGRESSION GUARD -- can a text-only majority vote still help?")
+        print("-" * 96)
+        lidx = {(r["model"], r["baseline"]): r for r in ladder}
+        for m in models:
+            r = lidx[(m, "0.00")]
+            print(f'{short[m]:22s} answer inside the vote survivors '
+                  f'{r["answer_in_survivor_set"]:.3f}   accuracy {r["accuracy"]:.3f}')
+        print("  distractors are random, so the vote rarely decides anything; a")
+        print("  survivor rate near 1.0 means it eliminated nothing.")
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

analysis/dataset_level_analysis/build_notebook.py

@@ -0,0 +1,463 @@
+#!/usr/bin/env python3
+"""Build dataset_difficulty.ipynb from the cell sources below."""
+import nbformat as nbf, os
+
+OUT = ("/HDD2/disheng/Spatial_VLM_Stereo/June/Spatial_VLM_Stereo/"
+       "01_3_prelimilary_intervention_different_baseline/analysis/"
+       "dataset_level_analysis/dataset_difficulty.ipynb")
+
+C = []
+md = lambda s: C.append(nbf.v4.new_markdown_cell(s.strip("\n")))
+co = lambda s: C.append(nbf.v4.new_code_cell(s.strip("\n")))
+
+md(r"""
+# How hard is each split, and do the models agree?
+
+Four models answer four splits at six stereo baselines. This notebook asks two
+things the raw accuracy table cannot: **how hard is each split once its own
+guessing floor is taken out**, and **do the models find the same items hard**.
+
+Raw accuracy is not comparable across splits. VSR is a two-way true/false
+question, so guessing already scores 0.500; the three Youtube levels are
+four-way, so guessing scores 0.250. Everything below is therefore reported as
+
+$$\text{headroom captured} = \frac{\text{acc} - \text{floor}}{1 - \text{floor}}$$
+
+the share of the distance between guessing and perfect that a model actually
+covers. It goes negative when a model does worse than free.
+""")
+
+md("## Setup")
+
+co(r"""
+import json, pathlib, itertools, warnings
+import numpy as np
+import pandas as pd
+import matplotlib as mpl
+import matplotlib.pyplot as plt
+
+warnings.filterwarnings("ignore")
+pd.set_option("display.width", 200)
+
+HERE = pathlib.Path.cwd()
+ROOT = HERE if (HERE / "result").is_dir() else HERE.parents[1]
+RESULT = ROOT / "result"
+REPO = ROOT.parent
+FIGDIR = ROOT / "analysis" / "dataset_level_analysis" / "figures"
+FIGDIR.mkdir(parents=True, exist_ok=True)
+
+BASELINES = ["0.00", "0.05", "0.10", "0.15", "0.20", "0.25"]
+TASKS = ["vsr", "youtube_level1", "youtube_level2", "youtube_level3"]
+NICE = {"vsr": "VSR", "youtube_level1": "YT L1", "youtube_level2": "YT L2",
+        "youtube_level3": "YT L3"}
+
+# guessing floor per split -- all four are plain chance among the options
+FLOOR = {"vsr": 0.500, "youtube_level1": 0.250,
+         "youtube_level2": 0.250, "youtube_level3": 0.250}
+CHANCE = dict(FLOOR)
+
+MODELS = sorted(p.name for p in RESULT.iterdir()
+                if p.is_dir() and not p.name.startswith("_"))
+SHORT = {m: m.replace("Qwen_Qwen", "").replace("-Instruct", "") for m in MODELS}
+COLOR = dict(zip(MODELS, ["#4C72B0", "#DD8452", "#55A868", "#C44E52"]))
+
+mpl.rcParams.update({
+    "figure.dpi": 120, "savefig.dpi": 150, "font.size": 10,
+    "axes.spines.top": False, "axes.spines.right": False,
+    "axes.grid": True, "grid.alpha": .25, "grid.linewidth": .6,
+    "axes.axisbelow": True, "legend.frameon": False,
+})
+print(f"root   : {ROOT}")
+print(f"models : {[SHORT[m] for m in MODELS]}")
+""")
+
+md("### Load every run")
+
+co(r"""
+rows = []
+for m in MODELS:
+    for t in TASKS:
+        for b in BASELINES:
+            d = json.loads((RESULT / m / t / f"baseline_{b}" / "metrics.json").read_text())
+            rows.append(dict(model=m, short=SHORT[m], task=t, baseline=float(b),
+                             n=d["num_samples"], accuracy=d["accuracy"],
+                             scoring=d["scoring"], unparsed=d["unparsed_rate"]))
+runs = pd.DataFrame(rows)
+runs["floor"] = runs["task"].map(FLOOR)
+runs["headroom"] = (runs["accuracy"] - runs["floor"]) / (1 - runs["floor"])
+assert len(runs) == len(MODELS) * len(TASKS) * len(BASELINES), "missing runs"
+
+# An unparsed answer is scored wrong, so a model that narrates instead of naming
+# a letter is penalised twice. Report the rate rather than assume it is zero.
+bad = runs[runs.unparsed > 0]
+if len(bad):
+    print("runs with unparsable answers (counted as wrong):")
+    print(bad.groupby(["short", "task"]).unparsed.agg(["max", "mean"]).round(4)
+          .to_string())
+else:
+    print("every answer parsed in every run")
+
+acc = runs.pivot_table(index="short", columns="task", values="accuracy", aggfunc="mean")[TASKS]
+head = runs.pivot_table(index="short", columns="task", values="headroom", aggfunc="mean")[TASKS]
+acc.columns = head.columns = [NICE[t] for t in TASKS]
+display(acc.round(3).rename_axis("accuracy (mean over baselines)"))
+display(head.round(3).rename_axis("headroom captured"))
+""")
+
+md(r"""
+## 1. Where each model sits relative to its free floor
+
+The bar is the accuracy; the dashed rule is what the split hands over for
+nothing. Anything below the rule is a model doing worse than a rule that ignores
+the picture.
+""")
+
+co(r"""
+fig, axes = plt.subplots(1, 4, figsize=(13, 3.4), sharey=True)
+for ax, t in zip(axes, TASKS):
+    sub = runs[runs.task == t].groupby("short", sort=False)
+    means = [runs[(runs.task == t) & (runs.model == m)].accuracy.mean() for m in MODELS]
+    errs = [runs[(runs.task == t) & (runs.model == m)].accuracy.std() for m in MODELS]
+    x = np.arange(len(MODELS))
+    ax.bar(x, means, yerr=errs, capsize=3, color=[COLOR[m] for m in MODELS],
+           edgecolor="white", linewidth=.8)
+    ax.axhline(FLOOR[t], ls="--", lw=1.4, color="#333")
+    ax.text(len(MODELS) - .45, FLOOR[t] + .012, f"floor {FLOOR[t]:.2f}",
+            ha="right", fontsize=8.5, color="#333")
+    if CHANCE[t] != FLOOR[t]:
+        ax.axhline(CHANCE[t], ls=":", lw=1.1, color="#999")
+        ax.text(-.45, CHANCE[t] + .012, f"guessing {CHANCE[t]:.2f}",
+                fontsize=8, color="#888")
+    ax.set_xticks(x); ax.set_xticklabels([SHORT[m] for m in MODELS], rotation=30, ha="right")
+    ax.set_title(f"{NICE[t]}  (n={runs[runs.task==t].n.iloc[0]})", fontsize=11)
+axes[0].set_ylabel("accuracy"); axes[0].set_ylim(0, 1)
+fig.suptitle("Accuracy against the free floor, averaged over the six baselines",
+             y=1.04, fontsize=12)
+fig.tight_layout(); fig.savefig(FIGDIR / "01_accuracy_vs_floor.png", bbox_inches="tight")
+""")
+
+md(r"""
+## 2. Difficulty, on one scale
+
+Headroom captured puts the four splits on a common axis. Red is a model losing
+to its own floor.
+""")
+
+co(r"""
+fig, ax = plt.subplots(figsize=(7.2, 3.2))
+data = head.values
+vmax = np.abs(data).max()
+im = ax.imshow(data, cmap="RdYlGn", vmin=-vmax, vmax=vmax, aspect="auto")
+ax.set_xticks(range(len(TASKS))); ax.set_xticklabels(head.columns)
+ax.set_yticks(range(len(MODELS))); ax.set_yticklabels(head.index)
+for i in range(data.shape[0]):
+    for j in range(data.shape[1]):
+        ax.text(j, i, f"{data[i,j]:+.2f}", ha="center", va="center", fontsize=10,
+                color="black")
+ax.grid(False)
+ax.set_title("Headroom captured   (acc - floor) / (1 - floor)")
+fig.colorbar(im, ax=ax, shrink=.85, label="share of headroom")
+fig.tight_layout(); fig.savefig(FIGDIR / "02_headroom_heatmap.png", bbox_inches="tight")
+""")
+
+md(r"""
+## 3. Do the models rank the splits the same way?
+
+One line per model across the four splits. Parallel lines mean the models agree
+on which split is hard and differ only in overall strength; crossings mean a
+model finds a particular split unusually easy or hard.
+""")
+
+co(r"""
+order = list(head.mean().sort_values(ascending=False).index)
+fig, ax = plt.subplots(figsize=(7.6, 4.0))
+for m in MODELS:
+    y = [head.loc[SHORT[m], c] for c in order]
+    ax.plot(range(len(order)), y, "-o", color=COLOR[m], lw=2, ms=6, label=SHORT[m])
+ax.axhline(0, color="#333", lw=1.2, ls="--")
+ax.text(len(order) - 1, .015, "floor", ha="right", fontsize=8.5, color="#333")
+ax.set_xticks(range(len(order))); ax.set_xticklabels(order)
+ax.set_ylabel("headroom captured"); ax.set_title("Difficulty profile per model")
+ax.legend(loc="upper right", ncol=2)
+fig.tight_layout(); fig.savefig(FIGDIR / "03_difficulty_profile.png", bbox_inches="tight")
+
+ranks = head.rank(axis=1, ascending=False).astype(int)
+print("difficulty rank each model assigns (1 = most headroom captured):")
+display(ranks)
+print("Spearman between models (only four splits, so 1.00 means identical order):")
+display(head.T.corr(method="spearman").round(2))
+""")
+
+md(r"""
+## 4. Does the stereo baseline move anything?
+
+Each panel is one split; each line is one model across the six baselines. A
+visual tower that models the scene properly should be flat here — the question
+and the answer never change, only the viewpoint.
+""")
+
+co(r"""
+fig, axes = plt.subplots(1, 4, figsize=(13.5, 3.2))
+for ax, t in zip(axes, TASKS):
+    for m in MODELS:
+        s = runs[(runs.task == t) & (runs.model == m)].sort_values("baseline")
+        ax.plot(s.baseline, s.accuracy, "-o", ms=4, color=COLOR[m], label=SHORT[m])
+    ax.axhline(FLOOR[t], ls="--", lw=1.1, color="#666")
+    ax.set_title(NICE[t]); ax.set_xlabel("baseline")
+    lo = min(runs[runs.task == t].accuracy.min(), FLOOR[t]) - .03
+    hi = max(runs[runs.task == t].accuracy.max(), FLOOR[t]) + .03
+    ax.set_ylim(lo, hi)
+axes[0].set_ylabel("accuracy")
+axes[-1].legend(loc="center left", bbox_to_anchor=(1.02, .5))
+fig.suptitle("Accuracy across the intervention, per split", y=1.05, fontsize=12)
+fig.tight_layout(); fig.savefig(FIGDIR / "04_baseline_sensitivity.png", bbox_inches="tight")
+
+spread = (runs.groupby(["short", "task"]).accuracy.agg(["min", "max"])
+          .assign(range=lambda d: d["max"] - d["min"])["range"]
+          .unstack()[TASKS].rename(columns=NICE))
+display(spread.round(3).rename_axis("accuracy range over the six baselines"))
+""")
+
+md(r"""
+## 5. Item-level difficulty: do the models fail on the same pictures?
+
+For every item, count how many of the four models get it right at baseline 0.00.
+If difficulty were a property of the item, the counts would pile up at 0 and 4.
+If each model failed idiosyncratically, the counts would follow the independent
+(Poisson-binomial) null drawn behind the bars.
+""")
+
+co(r"""
+def load_pred(m, t, b="0.00"):
+    path = RESULT / m / t / f"baseline_{b}" / "predictions.jsonl"
+    return {json.loads(l)["sample_id"]: json.loads(l)
+            for l in path.read_text().splitlines() if l.strip()}
+
+correct_by_task = {}
+for t in TASKS:
+    per = {m: load_pred(m, t) for m in MODELS}
+    ids = sorted(set.intersection(*(set(v) for v in per.values())))
+    correct_by_task[t] = pd.DataFrame(
+        {SHORT[m]: [per[m][i]["correct"] for i in ids] for m in MODELS}, index=ids)
+
+def poisson_binomial(ps):
+    dist = np.zeros(len(ps) + 1); dist[0] = 1.0
+    for p in ps:
+        dist[1:] = dist[1:] * (1 - p) + dist[:-1] * p
+        dist[0] *= (1 - p)
+    return dist
+
+fig, axes = plt.subplots(1, 4, figsize=(13.5, 3.2), sharey=True)
+conc = {}
+for ax, t in zip(axes, TASKS):
+    df = correct_by_task[t]
+    k = df.sum(axis=1).value_counts().reindex(range(5), fill_value=0).sort_index()
+    obs = k / k.sum()
+    null = poisson_binomial(df.mean().values)
+    ax.bar(range(5), null, color="#CCCCCC", width=.82, label="independent null")
+    ax.bar(range(5), obs, color="#4C72B0", width=.45, label="observed")
+    ax.set_title(f"{NICE[t]}  (n={len(df)})"); ax.set_xlabel("# models correct")
+    ax.set_xticks(range(5))
+    # how much mass sits at the two extremes, versus the null
+    conc[NICE[t]] = dict(observed_0_or_4=obs.iloc[[0, 4]].sum(),
+                         null_0_or_4=null[[0, 4]].sum())
+axes[0].set_ylabel("share of items"); axes[0].legend(fontsize=8.5)
+fig.suptitle("Agreement on which items are hard", y=1.05, fontsize=12)
+fig.tight_layout(); fig.savefig(FIGDIR / "05_item_difficulty.png", bbox_inches="tight")
+
+conc = pd.DataFrame(conc).T
+conc["excess"] = conc["observed_0_or_4"] - conc["null_0_or_4"]
+display(conc.round(3).rename_axis("mass at 0 or 4 models correct"))
+""")
+
+md(r"""
+## 6. Failure overlap between models
+
+Jaccard overlap of the failure sets, per split. High values mean the models
+stumble on the same items — shared difficulty rather than independent noise.
+""")
+
+co(r"""
+fig, axes = plt.subplots(1, 4, figsize=(14, 3.4))
+for ax, t in zip(axes, TASKS):
+    df = correct_by_task[t]
+    names = list(df.columns)
+    M = np.ones((len(names), len(names)))
+    for i, a in enumerate(names):
+        for j, b in enumerate(names):
+            if i == j:
+                continue
+            fa, fb = ~df[a].values, ~df[b].values
+            M[i, j] = (fa & fb).sum() / max((fa | fb).sum(), 1)
+    im = ax.imshow(M, cmap="Blues", vmin=0, vmax=1)
+    ax.set_xticks(range(len(names))); ax.set_xticklabels(names, rotation=40, ha="right", fontsize=8)
+    ax.set_yticks(range(len(names))); ax.set_yticklabels(names, fontsize=8)
+    for i in range(len(names)):
+        for j in range(len(names)):
+            ax.text(j, i, f"{M[i,j]:.2f}", ha="center", va="center", fontsize=8,
+                    color="white" if M[i, j] > .55 else "black")
+    ax.grid(False); ax.set_title(NICE[t], fontsize=11)
+fig.suptitle("Jaccard overlap of failure sets (baseline 0.00)", y=1.06, fontsize=12)
+fig.tight_layout(); fig.savefig(FIGDIR / "06_failure_overlap.png", bbox_inches="tight")
+""")
+
+md(r"""
+## 7. Is the model answering, or just picking a letter?
+
+The reference labels are near-uniform in every split, so a flat prediction
+histogram is what an unbiased model produces. A spike means the model has
+collapsed onto one option, and that spike caps how well it can possibly do.
+""")
+
+co(r"""
+fig, axes = plt.subplots(len(MODELS), 4, figsize=(12, 2.3 * len(MODELS)),
+                         sharey="row")
+for r, m in enumerate(MODELS):
+    for c, t in enumerate(TASKS):
+        ax = axes[r, c]
+        pred = load_pred(m, t)
+        pc = pd.Series([v["prediction_norm"] for v in pred.values()]).value_counts()
+        rc = pd.Series([v["reference_norm"] for v in pred.values()]).value_counts()
+        labels = sorted(set(pc.index) | set(rc.index))
+        x = np.arange(len(labels))
+        ax.bar(x - .2, [rc.get(l, 0) / len(pred) for l in labels], width=.4,
+               color="#BBBBBB", label="reference")
+        ax.bar(x + .2, [pc.get(l, 0) / len(pred) for l in labels], width=.4,
+               color=COLOR[m], label="prediction")
+        ax.set_xticks(x); ax.set_xticklabels(labels, fontsize=8)
+        ax.axhline(1 / len(labels), ls=":", lw=1, color="#666")
+        if r == 0:
+            ax.set_title(NICE[t])
+        if c == 0:
+            ax.set_ylabel(SHORT[m], fontsize=9)
+        if r == 0 and c == 0:
+            ax.legend(fontsize=7.5)
+fig.suptitle("Predicted vs reference label distribution (baseline 0.00)",
+             y=1.01, fontsize=12)
+fig.tight_layout(); fig.savefig(FIGDIR / "07_answer_bias.png", bbox_inches="tight")
+""")
+
+md(r"""
+## 8. Level 2 regression guard: is the option text still uninformative?
+
+Level 2's three distractors are drawn uniformly from the 23 wrong orderings, so
+the four options should be statistically exchangeable and no text-only rule
+should beat chance. Earlier, structured distractor sets did not have that
+property — the worst of them let a majority vote on successive positions narrow
+every item to two candidates and score 0.500 blind.
+
+This cell re-runs that vote. `in survivor set` near 1.0 means the vote eliminated
+nothing, which is what random distractors should produce. If it ever drops
+toward 0.5 while accuracy rises, the construction has reacquired a leak.
+""")
+
+co(r"""
+L2_TEST = REPO / "02_data" / "Youtube_self_depth_QA" / "level2" / "test.jsonl"
+rows = []
+if L2_TEST.exists():
+    opts = {}
+    for line in L2_TEST.read_text().splitlines():
+        rec = json.loads(line)
+        if rec.get("options"):
+            opts[rec["sample_id"]] = {k: v.split(" > ") for k, v in rec["options"].items()}
+
+    def survivors(o):
+        alive = list(o)
+        for k in range(4):
+            if len(alive) <= 1:
+                break
+            counts = pd.Series([o[a][k] for a in alive]).value_counts()
+            if len(counts) > 1 and counts.iloc[0] == counts.iloc[1]:
+                continue
+            alive = [a for a in alive if o[a][k] == counts.index[0]]
+        return alive
+
+    surv = {sid: survivors(o) for sid, o in opts.items()}
+    for m in MODELS:
+        pred = load_pred(m, "youtube_level2")
+        ins = [p["prediction_norm"] in surv[s] for s, p in pred.items() if s in surv]
+        hit = [p["correct"] for s, p in pred.items() if s in surv]
+        rows.append(dict(model=SHORT[m], accuracy=np.mean(hit),
+                         in_survivor_set=np.mean(ins)))
+    l2 = pd.DataFrame(rows).set_index("model")
+    surv = np.mean([len(v) for v in surv.values()])
+    print(f"mean survivor-set size after the vote: {surv:.2f} of 4 options")
+    display(l2.round(3))
+
+    fig, ax = plt.subplots(figsize=(7, 3.2))
+    x = np.arange(len(l2))
+    ax.bar(x - .2, l2.in_survivor_set, width=.4, color="#999",
+           label="answer survives the text-only vote")
+    ax.bar(x + .2, l2.accuracy, width=.4, color="#C44E52", label="accuracy")
+    ax.axhline(.25, ls="--", color="#333", lw=1.3)
+    ax.text(len(l2) - .4, .265, "chance", ha="right", fontsize=8.5)
+    ax.set_xticks(x); ax.set_xticklabels(l2.index, rotation=25, ha="right")
+    ax.set_ylim(0, 1); ax.set_ylabel("share")
+    ax.set_title("Level 2: random distractors leave the text-only vote powerless")
+    ax.legend()
+    fig.tight_layout(); fig.savefig(FIGDIR / "08_level2_guard.png", bbox_inches="tight")
+else:
+    print(f"level-2 split not found at {L2_TEST}; skipping")
+""")
+
+md(r"""
+## Findings
+
+**1. Every model puts VSR first and YT L1 second; the two families then
+disagree.** Qwen2.5-VL finds level 3 hardest, Qwen3.5 finds level 2 hardest.
+Spearman is 1.00 within a family and 0.80 across, so the ordering of the two
+Youtube reasoning levels is the one place where architecture, not just
+strength, changes what looks hard.
+
+**2. Only VSR is comfortably solved** (0.52-0.66 of headroom). It is also the
+easiest question type: a binary relation that is often inferable from object
+identity alone, without metric depth.
+
+**3. Level 3 is the sharpest discriminator.** Qwen2.5-VL captures 0.03 (3B) and
+0.07 (7B) — indistinguishable from guessing — against 0.29 and 0.33 for
+Qwen3.5. A 4-10x gap, far wider than anywhere else in the suite. Continuous
+trajectory reasoning is what separates the two families.
+
+**4. Level 2 sits at 0.17-0.26 of headroom for everyone**, a narrow band that
+does not track model size or family. Its distractors are drawn at random from
+the 23 wrong orderings, so nothing in the option text helps (figure 8: the
+text-only vote leaves ~2 of 4 options standing and scores 0.252 on the split,
+i.e. chance). The scores are what four-way depth ranking actually costs these
+models: clearly above guessing, nowhere near solved.
+
+**5. The base model beats the instruct model on every split.** Qwen3.5-4B-Base
+scored by likelihood outperforms Qwen3.5-4B scored by generation everywhere,
+and 4B-Instruct is the only model producing unparsable answers (up to 0.7% on
+L1), which are scored wrong. Part of the instruct model's deficit is format
+compliance, not perception.
+
+**6. The stereo intervention barely moves the aggregate.** Accuracy ranges over
+the six baselines are 0.012-0.044 with no consistent sign across models or
+splits. Whether the same *items* flip is a separate question, answered in
+`../intervention_analysis`.
+
+**7. Models agree most on which L1 items are hard, least on L3.** Excess mass at
+"0 or 4 models correct" over the independent null: +0.36 (L1), +0.27 (VSR),
++0.23 (L2), +0.14 (L3). On L3 the failures are close to independent noise, which
+is what near-chance accuracy looks like — there is little shared signal to agree
+about.
+
+### A note on the level-2 numbers
+
+Level 2 was rebuilt three times before this run. Two earlier option
+constructions leaked: distractors grown outward from the ground truth made it
+the option most similar to the others (0.388 blind), and a prefix ladder let a
+majority vote narrow every item to two (0.500 blind). Under the leaking
+constructions these same models scored 0.40-0.47 on level 2; with random
+distractors they score 0.37-0.45 against a real 0.25 floor. The drop of
+0.02-0.10 is the part of the old score that came from the option text rather
+than the picture.
+""")
+
+nb = nbf.v4.new_notebook(cells=C)
+nb.metadata.kernelspec = {"display_name": "Spatial_VLM", "language": "python",
+                          "name": "python3"}
+nb.metadata.language_info = {"name": "python"}
+os.makedirs(os.path.dirname(OUT), exist_ok=True)
+nbf.write(nb, OUT)
+print("wrote", OUT, f"({len(C)} cells)")

analysis/dataset_level_analysis/level2_structure_use.csv

@@ -0,0 +1,25 @@
+model,baseline,n,answer_in_survivor_set,accuracy,accuracy_within_survivors
+Qwen_Qwen2.5-VL-3B-Instruct,0.00,764,0.6898,0.3626,0.5256
+Qwen_Qwen2.5-VL-3B-Instruct,0.05,764,0.6963,0.3743,0.5376
+Qwen_Qwen2.5-VL-3B-Instruct,0.10,764,0.7042,0.3783,0.5372
+Qwen_Qwen2.5-VL-3B-Instruct,0.15,764,0.7134,0.3796,0.5321
+Qwen_Qwen2.5-VL-3B-Instruct,0.20,764,0.7186,0.3809,0.5301
+Qwen_Qwen2.5-VL-3B-Instruct,0.25,764,0.7094,0.3704,0.5221
+Qwen_Qwen2.5-VL-7B-Instruct,0.00,764,0.6898,0.4031,0.5844
+Qwen_Qwen2.5-VL-7B-Instruct,0.05,764,0.7003,0.394,0.5626
+Qwen_Qwen2.5-VL-7B-Instruct,0.10,764,0.699,0.3979,0.5693
+Qwen_Qwen2.5-VL-7B-Instruct,0.15,764,0.6937,0.4018,0.5792
+Qwen_Qwen2.5-VL-7B-Instruct,0.20,764,0.7016,0.377,0.5373
+Qwen_Qwen2.5-VL-7B-Instruct,0.25,764,0.7068,0.377,0.5333
+Qwen_Qwen3.5-4B,0.00,764,0.623,0.3914,0.6282
+Qwen_Qwen3.5-4B,0.05,764,0.6178,0.3901,0.6314
+Qwen_Qwen3.5-4B,0.10,764,0.6113,0.3822,0.6253
+Qwen_Qwen3.5-4B,0.15,764,0.6191,0.3874,0.6258
+Qwen_Qwen3.5-4B,0.20,764,0.6099,0.3796,0.6223
+Qwen_Qwen3.5-4B,0.25,764,0.6034,0.3848,0.6377
+Qwen_Qwen3.5-4B-Base,0.00,764,0.6178,0.4424,0.7161
+Qwen_Qwen3.5-4B-Base,0.05,764,0.606,0.4529,0.7473
+Qwen_Qwen3.5-4B-Base,0.10,764,0.6073,0.4503,0.7414
+Qwen_Qwen3.5-4B-Base,0.15,764,0.6126,0.4542,0.7415
+Qwen_Qwen3.5-4B-Base,0.20,764,0.6178,0.4503,0.7288
+Qwen_Qwen3.5-4B-Base,0.25,764,0.6204,0.4398,0.7089

analysis/dataset_level_analysis/prediction_bias.csv

@@ -0,0 +1,97 @@
+model,task,baseline,n,n_distinct_predictions,modal_prediction,modal_share,prediction_entropy_bits,reference_entropy_bits,always_modal_accuracy,accuracy,prediction_counts
+Qwen_Qwen2.5-VL-3B-Instruct,vsr,0.00,2195,2,false,0.513,0.9995,0.9958,0.462,0.7713,"{""false"": 1126, ""true"": 1069}"
+Qwen_Qwen2.5-VL-3B-Instruct,vsr,0.05,2195,2,false,0.523,0.9985,0.9958,0.462,0.7677,"{""false"": 1148, ""true"": 1047}"
+Qwen_Qwen2.5-VL-3B-Instruct,vsr,0.10,2195,2,false,0.5303,0.9974,0.9958,0.462,0.7595,"{""false"": 1164, ""true"": 1031}"
+Qwen_Qwen2.5-VL-3B-Instruct,vsr,0.15,2195,2,false,0.5235,0.9984,0.9958,0.462,0.7581,"{""false"": 1149, ""true"": 1046}"
+Qwen_Qwen2.5-VL-3B-Instruct,vsr,0.20,2195,2,false,0.528,0.9977,0.9958,0.462,0.7526,"{""false"": 1159, ""true"": 1036}"
+Qwen_Qwen2.5-VL-3B-Instruct,vsr,0.25,2195,2,false,0.5308,0.9973,0.9958,0.462,0.7481,"{""false"": 1165, ""true"": 1030}"
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level1,0.00,821,4,B,0.3642,1.9258,1.9983,0.2704,0.3995,"{""A"": 212, ""B"": 299, ""C"": 116, ""D"": 194}"
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level1,0.05,821,4,B,0.3752,1.9189,1.9983,0.2704,0.4056,"{""A"": 202, ""B"": 308, ""C"": 115, ""D"": 196}"
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level1,0.10,821,4,B,0.3715,1.9203,1.9983,0.2704,0.4117,"{""A"": 202, ""B"": 305, ""C"": 114, ""D"": 200}"
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level1,0.15,821,4,B,0.3788,1.922,1.9983,0.2704,0.4105,"{""A"": 196, ""B"": 311, ""C"": 121, ""D"": 193}"
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level1,0.20,821,4,B,0.3825,1.9177,1.9983,0.2704,0.3995,"{""A"": 190, ""B"": 314, ""C"": 119, ""D"": 198}"
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level1,0.25,821,4,B,0.3691,1.9332,1.9983,0.2704,0.4007,"{""A"": 194, ""B"": 303, ""C"": 127, ""D"": 197}"
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level2,0.00,764,4,A,0.3128,1.9633,1.9962,0.2343,0.3626,"{""A"": 239, ""B"": 227, ""C"": 139, ""D"": 159}"
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level2,0.05,764,4,B,0.3089,1.9612,1.9962,0.2814,0.3743,"{""A"": 234, ""B"": 236, ""C"": 144, ""D"": 150}"
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level2,0.10,764,4,B,0.3115,1.9653,1.9962,0.2814,0.3783,"{""A"": 227, ""B"": 238, ""C"": 146, ""D"": 153}"
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level2,0.15,764,4,B,0.3063,1.9627,1.9962,0.2814,0.3796,"{""A"": 234, ""B"": 234, ""C"": 142, ""D"": 154}"
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level2,0.20,764,4,B,0.3259,1.9659,1.9962,0.2814,0.3809,"{""A"": 212, ""B"": 249, ""C"": 147, ""D"": 156}"
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level2,0.25,764,4,B,0.3115,1.9677,1.9962,0.2814,0.3704,"{""A"": 224, ""B"": 238, ""C"": 149, ""D"": 153}"
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level3,0.00,836,4,A,0.5694,1.5259,1.9927,0.2847,0.2667,"{""A"": 476, ""B"": 233, ""C"": 40, ""D"": 87}"
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level3,0.05,836,4,A,0.5443,1.5853,1.9927,0.2847,0.2452,"{""A"": 455, ""B"": 239, ""C"": 48, ""D"": 94}"
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level3,0.10,836,4,A,0.5622,1.5541,1.9927,0.2847,0.2392,"{""A"": 470, ""B"": 231, ""C"": 46, ""D"": 89}"
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level3,0.15,836,4,A,0.5323,1.5953,1.9927,0.2847,0.2524,"{""A"": 445, ""B"": 250, ""C"": 49, ""D"": 92}"
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level3,0.20,836,4,A,0.5275,1.6189,1.9927,0.2847,0.2596,"{""A"": 441, ""B"": 246, ""C"": 55, ""D"": 94}"
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level3,0.25,836,4,A,0.5467,1.5792,1.9927,0.2847,0.2656,"{""A"": 457, ""B"": 240, ""C"": 49, ""D"": 90}"
+Qwen_Qwen2.5-VL-7B-Instruct,vsr,0.00,2195,2,false,0.5385,0.9957,0.9958,0.462,0.8223,"{""false"": 1182, ""true"": 1013}"
+Qwen_Qwen2.5-VL-7B-Instruct,vsr,0.05,2195,2,false,0.5421,0.9949,0.9958,0.462,0.8077,"{""false"": 1190, ""true"": 1005}"
+Qwen_Qwen2.5-VL-7B-Instruct,vsr,0.10,2195,2,false,0.5417,0.995,0.9958,0.462,0.8027,"{""false"": 1189, ""true"": 1006}"
+Qwen_Qwen2.5-VL-7B-Instruct,vsr,0.15,2195,2,false,0.5476,0.9935,0.9958,0.462,0.8068,"{""false"": 1202, ""true"": 993}"
+Qwen_Qwen2.5-VL-7B-Instruct,vsr,0.20,2195,2,false,0.5485,0.9932,0.9958,0.462,0.8023,"{""false"": 1204, ""true"": 991}"
+Qwen_Qwen2.5-VL-7B-Instruct,vsr,0.25,2195,2,false,0.5544,0.9914,0.9958,0.462,0.7882,"{""false"": 1217, ""true"": 978}"
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level1,0.00,821,4,D,0.4799,1.8231,1.9983,0.2436,0.447,"{""A"": 141, ""B"": 144, ""C"": 142, ""D"": 394}"
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level1,0.05,821,4,D,0.4641,1.8454,1.9983,0.2436,0.4482,"{""A"": 152, ""B"": 146, ""C"": 142, ""D"": 381}"
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level1,0.10,821,4,D,0.4555,1.8564,1.9983,0.2436,0.4555,"{""A"": 140, ""B"": 157, ""C"": 150, ""D"": 374}"
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level1,0.15,821,4,D,0.4641,1.8455,1.9983,0.2436,0.4665,"{""A"": 142, ""B"": 150, ""C"": 148, ""D"": 381}"
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level1,0.20,821,4,D,0.4507,1.8634,1.9983,0.2436,0.4421,"{""A"": 146, ""B"": 154, ""C"": 151, ""D"": 370}"
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level1,0.25,821,4,D,0.4677,1.8404,1.9983,0.2436,0.4519,"{""A"": 141, ""B"": 148, ""C"": 148, ""D"": 384}"
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level2,0.00,764,4,D,0.3455,1.9146,1.9962,0.2408,0.4031,"{""A"": 107, ""B"": 242, ""C"": 151, ""D"": 264}"
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level2,0.05,764,4,B,0.3325,1.9084,1.9962,0.2814,0.394,"{""A"": 100, ""B"": 254, ""C"": 156, ""D"": 254}"
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level2,0.10,764,4,D,0.3521,1.9016,1.9962,0.2408,0.3979,"{""A"": 97, ""B"": 242, ""C"": 156, ""D"": 269}"
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level2,0.15,764,4,D,0.3547,1.9012,1.9962,0.2408,0.4018,"{""A"": 97, ""B"": 240, ""C"": 156, ""D"": 271}"
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level2,0.20,764,4,D,0.3573,1.921,1.9962,0.2408,0.377,"{""A"": 108, ""B"": 222, ""C"": 161, ""D"": 273}"
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level2,0.25,764,4,D,0.3442,1.9182,1.9962,0.2408,0.377,"{""A"": 104, ""B"": 236, ""C"": 161, ""D"": 263}"
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level3,0.00,836,4,B,0.5598,1.6491,1.9927,0.2608,0.2799,"{""A"": 72, ""B"": 468, ""C"": 175, ""D"": 121}"
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level3,0.05,836,4,B,0.561,1.6452,1.9927,0.2608,0.2871,"{""A"": 71, ""B"": 469, ""C"": 176, ""D"": 120}"
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level3,0.10,836,4,B,0.5646,1.6367,1.9927,0.2608,0.2679,"{""A"": 67, ""B"": 472, ""C"": 172, ""D"": 125}"
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level3,0.15,836,4,B,0.5586,1.6493,1.9927,0.2608,0.2823,"{""A"": 67, ""B"": 467, ""C"": 169, ""D"": 133}"
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level3,0.20,836,4,B,0.5658,1.6353,1.9927,0.2608,0.2763,"{""A"": 66, ""B"": 473, ""C"": 168, ""D"": 129}"
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level3,0.25,836,4,B,0.5658,1.6529,1.9927,0.2608,0.2739,"{""A"": 81, ""B"": 473, ""C"": 160, ""D"": 122}"
+Qwen_Qwen3.5-4B,vsr,0.00,2195,2,true,0.5882,0.9775,0.9958,0.538,0.8214,"{""false"": 904, ""true"": 1291}"
+Qwen_Qwen3.5-4B,vsr,0.05,2195,2,true,0.5941,0.9743,0.9958,0.538,0.8173,"{""false"": 891, ""true"": 1304}"
+Qwen_Qwen3.5-4B,vsr,0.10,2195,2,true,0.5854,0.9788,0.9958,0.538,0.8187,"{""false"": 910, ""true"": 1285}"
+Qwen_Qwen3.5-4B,vsr,0.15,2195,2,true,0.5872,0.9779,0.9958,0.538,0.8114,"{""false"": 906, ""true"": 1289}"
+Qwen_Qwen3.5-4B,vsr,0.20,2195,2,true,0.585,0.9791,0.9958,0.538,0.8073,"{""false"": 911, ""true"": 1284}"
+Qwen_Qwen3.5-4B,vsr,0.25,2195,2,true,0.5831,0.98,0.9958,0.538,0.8046,"{""false"": 915, ""true"": 1280}"
+Qwen_Qwen3.5-4B,youtube_level1,0.00,821,7,A,0.2984,2.02,1.9983,0.2473,0.5274,"{""A"": 245, ""B"": 181, ""C"": 176, ""D"": 216, ""G"": 1, ""O"": 1, ""R"": 1}"
+Qwen_Qwen3.5-4B,youtube_level1,0.05,821,6,A,0.2899,2.0402,1.9983,0.2473,0.5213,"{""A"": 238, ""B"": 183, ""C"": 173, ""D"": 221, ""G"": 1, ""O"": 5}"
+Qwen_Qwen3.5-4B,youtube_level1,0.10,821,5,A,0.2911,2.0004,1.9983,0.2473,0.5189,"{""A"": 239, ""B"": 180, ""C"": 182, ""D"": 219, ""O"": 1}"
+Qwen_Qwen3.5-4B,youtube_level1,0.15,821,6,A,0.3021,2.0238,1.9983,0.2473,0.5408,"{""A"": 248, ""B"": 177, ""C"": 177, ""D"": 215, ""G"": 1, ""O"": 3}"
+Qwen_Qwen3.5-4B,youtube_level1,0.20,821,5,A,0.2801,2.0008,1.9983,0.2473,0.5347,"{""A"": 230, ""B"": 171, ""C"": 193, ""D"": 226, ""O"": 1}"
+Qwen_Qwen3.5-4B,youtube_level1,0.25,821,4,A,0.2887,1.9889,1.9983,0.2473,0.5347,"{""A"": 237, ""B"": 173, ""C"": 189, ""D"": 222}"
+Qwen_Qwen3.5-4B,youtube_level2,0.00,764,4,A,0.5537,1.6082,1.9962,0.2343,0.3914,"{""A"": 423, ""B"": 60, ""C"": 201, ""D"": 80}"
+Qwen_Qwen3.5-4B,youtube_level2,0.05,764,4,A,0.5785,1.56,1.9962,0.2343,0.3901,"{""A"": 442, ""B"": 68, ""C"": 193, ""D"": 61}"
+Qwen_Qwen3.5-4B,youtube_level2,0.10,764,4,A,0.5877,1.5489,1.9962,0.2343,0.3822,"{""A"": 449, ""B"": 62, ""C"": 186, ""D"": 67}"
+Qwen_Qwen3.5-4B,youtube_level2,0.15,764,4,A,0.5759,1.5768,1.9962,0.2343,0.3874,"{""A"": 440, ""B"": 63, ""C"": 188, ""D"": 73}"
+Qwen_Qwen3.5-4B,youtube_level2,0.20,764,4,A,0.5746,1.5784,1.9962,0.2343,0.3796,"{""A"": 439, ""B"": 63, ""C"": 189, ""D"": 73}"
+Qwen_Qwen3.5-4B,youtube_level2,0.25,764,4,A,0.5785,1.5697,1.9962,0.2343,0.3848,"{""A"": 442, ""B"": 62, ""C"": 188, ""D"": 72}"
+Qwen_Qwen3.5-4B,youtube_level3,0.00,836,4,A,0.4677,1.778,1.9927,0.2847,0.2931,"{""A"": 391, ""B"": 132, ""C"": 229, ""D"": 84}"
+Qwen_Qwen3.5-4B,youtube_level3,0.05,836,4,A,0.4426,1.8225,1.9927,0.2847,0.299,"{""A"": 370, ""B"": 147, ""C"": 227, ""D"": 92}"
+Qwen_Qwen3.5-4B,youtube_level3,0.10,836,4,A,0.4474,1.8167,1.9927,0.2847,0.2967,"{""A"": 374, ""B"": 146, ""C"": 225, ""D"": 91}"
+Qwen_Qwen3.5-4B,youtube_level3,0.15,836,4,A,0.4438,1.8283,1.9927,0.2847,0.3038,"{""A"": 371, ""B"": 144, ""C"": 223, ""D"": 98}"
+Qwen_Qwen3.5-4B,youtube_level3,0.20,836,4,A,0.4438,1.8289,1.9927,0.2847,0.2883,"{""A"": 371, ""B"": 143, ""C"": 223, ""D"": 99}"
+Qwen_Qwen3.5-4B,youtube_level3,0.25,836,4,A,0.4486,1.8317,1.9927,0.2847,0.2931,"{""A"": 375, ""B"": 141, ""C"": 215, ""D"": 105}"
+Qwen_Qwen3.5-4B-Base,vsr,0.00,2195,2,true,0.5194,0.9989,0.9958,0.538,0.8319,"{""false"": 1055, ""true"": 1140}"
+Qwen_Qwen3.5-4B-Base,vsr,0.05,2195,2,true,0.5162,0.9992,0.9958,0.538,0.8314,"{""false"": 1062, ""true"": 1133}"
+Qwen_Qwen3.5-4B-Base,vsr,0.10,2195,2,true,0.5125,0.9995,0.9958,0.538,0.8323,"{""false"": 1070, ""true"": 1125}"
+Qwen_Qwen3.5-4B-Base,vsr,0.15,2195,2,true,0.5116,0.9996,0.9958,0.538,0.8278,"{""false"": 1072, ""true"": 1123}"
+Qwen_Qwen3.5-4B-Base,vsr,0.20,2195,2,true,0.5153,0.9993,0.9958,0.538,0.8241,"{""false"": 1064, ""true"": 1131}"
+Qwen_Qwen3.5-4B-Base,vsr,0.25,2195,2,true,0.5075,0.9998,0.9958,0.538,0.8164,"{""false"": 1081, ""true"": 1114}"
+Qwen_Qwen3.5-4B-Base,youtube_level1,0.00,821,4,A,0.3179,1.9803,1.9983,0.2473,0.542,"{""A"": 261, ""B"": 193, ""C"": 167, ""D"": 200}"
+Qwen_Qwen3.5-4B-Base,youtube_level1,0.05,821,4,A,0.3228,1.9769,1.9983,0.2473,0.5457,"{""A"": 265, ""B"": 192, ""C"": 163, ""D"": 201}"
+Qwen_Qwen3.5-4B-Base,youtube_level1,0.10,821,4,A,0.3337,1.971,1.9983,0.2473,0.5603,"{""A"": 274, ""B"": 187, ""C"": 161, ""D"": 199}"
+Qwen_Qwen3.5-4B-Base,youtube_level1,0.15,821,4,A,0.3191,1.9772,1.9983,0.2473,0.5725,"{""A"": 262, ""B"": 190, ""C"": 161, ""D"": 208}"
+Qwen_Qwen3.5-4B-Base,youtube_level1,0.20,821,4,A,0.3228,1.9773,1.9983,0.2473,0.5664,"{""A"": 265, ""B"": 184, ""C"": 167, ""D"": 205}"
+Qwen_Qwen3.5-4B-Base,youtube_level1,0.25,821,4,A,0.3167,1.9769,1.9983,0.2473,0.5554,"{""A"": 260, ""B"": 192, ""C"": 158, ""D"": 211}"
+Qwen_Qwen3.5-4B-Base,youtube_level2,0.00,764,4,A,0.4921,1.8009,1.9962,0.2343,0.4424,"{""A"": 376, ""B"": 112, ""C"": 144, ""D"": 132}"
+Qwen_Qwen3.5-4B-Base,youtube_level2,0.05,764,4,A,0.5065,1.7772,1.9962,0.2343,0.4529,"{""A"": 387, ""B"": 112, ""C"": 146, ""D"": 119}"
+Qwen_Qwen3.5-4B-Base,youtube_level2,0.10,764,4,A,0.5052,1.7811,1.9962,0.2343,0.4503,"{""A"": 386, ""B"": 113, ""C"": 141, ""D"": 124}"
+Qwen_Qwen3.5-4B-Base,youtube_level2,0.15,764,4,A,0.4908,1.803,1.9962,0.2343,0.4542,"{""A"": 375, ""B"": 117, ""C"": 148, ""D"": 124}"
+Qwen_Qwen3.5-4B-Base,youtube_level2,0.20,764,4,A,0.5026,1.7802,1.9962,0.2343,0.4503,"{""A"": 384, ""B"": 113, ""C"": 154, ""D"": 113}"
+Qwen_Qwen3.5-4B-Base,youtube_level2,0.25,764,4,A,0.5262,1.7432,1.9962,0.2343,0.4398,"{""A"": 402, ""B"": 107, ""C"": 143, ""D"": 112}"
+Qwen_Qwen3.5-4B-Base,youtube_level3,0.00,836,4,B,0.3457,1.9035,1.9927,0.2608,0.3026,"{""A"": 212, ""B"": 289, ""C"": 241, ""D"": 94}"
+Qwen_Qwen3.5-4B-Base,youtube_level3,0.05,836,4,B,0.3481,1.8942,1.9927,0.2608,0.2715,"{""A"": 205, ""B"": 291, ""C"": 250, ""D"": 90}"
+Qwen_Qwen3.5-4B-Base,youtube_level3,0.10,836,4,B,0.3301,1.9053,1.9927,0.2608,0.2775,"{""A"": 221, ""B"": 276, ""C"": 247, ""D"": 92}"
+Qwen_Qwen3.5-4B-Base,youtube_level3,0.15,836,4,B,0.3577,1.8933,1.9927,0.2608,0.2667,"{""A"": 212, ""B"": 299, ""C"": 235, ""D"": 90}"
+Qwen_Qwen3.5-4B-Base,youtube_level3,0.20,836,4,B,0.323,1.9153,1.9927,0.2608,0.2727,"{""A"": 225, ""B"": 270, ""C"": 244, ""D"": 97}"
+Qwen_Qwen3.5-4B-Base,youtube_level3,0.25,836,4,B,0.3289,1.9209,1.9927,0.2608,0.2823,"{""A"": 211, ""B"": 275, ""C"": 247, ""D"": 103}"

analysis/dataset_level_analysis/readme.md

@@ -0,0 +1,133 @@
+# Dataset-level difficulty
+
+How hard each split is for each model, once the split's own guessing floor is
+taken out, and whether the models find the same items hard.
+
+```
+dataset_difficulty.ipynb    the analysis, 8 figures, executed with outputs
+build_notebook.py           regenerates the notebook from source (nbformat)
+analyze_dataset_level.py    the same numbers as CSVs, no notebook needed
+figures/*.png               every figure, also written by the notebook
+```
+
+Run either entry point from the experiment root:
+
+```bash
+python analysis/dataset_level_analysis/analyze_dataset_level.py     # CSVs + terminal tables
+/home/disheng/miniconda3/envs/Spatial_VLM/bin/python \
+    analysis/dataset_level_analysis/build_notebook.py               # rebuild the .ipynb
+```
+
+## Why not just compare accuracy
+
+The four splits do not share a scale. VSR is a two-way true/false question, so
+guessing scores 0.500; the three Youtube levels are four-way, so guessing scores
+0.250. Level 2's three distractors are drawn uniformly from the 23 wrong
+orderings, so nothing in the option text helps and its floor is plain chance —
+measured on the split, the strongest text-only heuristic scores 0.252.
+
+Everything is therefore reported as **headroom captured**,
+`(acc - floor) / (1 - floor)`: the share of the distance from guessing to
+perfect that the model actually covers. It is negative when a model does worse
+than free.
+
+| split | n | choices | guessing | floor used |
+|---|---|---|---|---|
+| VSR | 2195 | 2 | 0.500 | 0.500 |
+| YT level 1 | 821 | 4 | 0.250 | 0.250 |
+| YT level 2 | 764 | 4 | 0.250 | 0.250 |
+| YT level 3 | 836 | 4 | 0.250 | 0.250 |
+
+## Results
+
+Accuracy, mean over the six baselines:
+
+| model | VSR | YT L1 | YT L2 | YT L3 |
+|---|---|---|---|---|
+| Qwen2.5-VL-3B | 0.759 | 0.405 | 0.374 | 0.255 |
+| Qwen2.5-VL-7B | 0.805 | 0.452 | 0.392 | 0.278 |
+| Qwen3.5-4B | 0.813 | 0.530 | 0.386 | 0.296 |
+| Qwen3.5-4B-Base | 0.827 | 0.557 | 0.448 | 0.279 |
+
+Headroom captured:
+
+| model | VSR | YT L1 | YT L2 | YT L3 |
+|---|---|---|---|---|
+| Qwen2.5-VL-3B | 0.519 | 0.206 | 0.166 | 0.006 |
+| Qwen2.5-VL-7B | 0.610 | 0.269 | 0.189 | 0.037 |
+| Qwen3.5-4B | 0.627 | 0.373 | 0.181 | 0.061 |
+| Qwen3.5-4B-Base | 0.655 | 0.409 | 0.264 | 0.038 |
+
+1. **All four models now agree on the ordering**: VSR > L1 > L2 > L3, easiest to
+   hardest, with Spearman 1.00 for all six model pairs. The earlier split — where
+   Qwen2.5-VL found L3 hardest and Qwen3.5 found L2 hardest — was a property of
+   the level-3 v1 annotations, and did not survive their rebuild (see the level-3
+   caveat below).
+2. **Only VSR is comfortably solved** (0.52-0.66 of headroom). It is also the
+   easiest question type — a binary relation often inferable from object
+   identity without metric depth.
+3. **Level 3 is now at the floor for everyone**: 0.006-0.061 of headroom, a
+   0.054 spread that is the narrowest of the four splits. With each path given
+   its own start point the split no longer separates the models; it only shows
+   that none of them can trace depth monotonicity along a path.
+4. **Level 1 is the sharpest discriminator**, 0.206 to 0.409, a 0.203 spread.
+   It is the only Youtube level where model strength still shows through.
+5. **Level 2 sits at 0.17-0.26 for everyone**, a narrow band that tracks neither
+   size nor family. Nothing in its option text helps, so this is what four-way
+   depth ranking actually costs these models: clearly above guessing, nowhere
+   near solved.
+6. **The base model wins on three splits of four.** Qwen3.5-4B-Base (likelihood
+   scoring) beats Qwen3.5-4B (generation) on VSR, L1 and L2, but not on L3
+   (0.038 vs 0.061) — where both are close enough to the floor that the ordering
+   carries little weight. 4B-Instruct is also the only model producing
+   unparsable answers (up to 0.7% on L1), which are scored wrong.
+7. **The stereo intervention barely moves the aggregate.** Accuracy ranges over
+   the six baselines are 0.012-0.036 with no consistent sign. Whether the same
+   *items* flip is a separate question — see `../intervention_analysis`.
+8. **Agreement on item difficulty is highest on L1, lowest on L3.** Excess mass
+   at "0 or 4 models correct" over the independent null: +0.358 (L1), +0.267
+   (VSR), +0.234 (L2), +0.160 (L3). On L3 the failures are close to independent
+   noise, which is what near-chance accuracy looks like.
+
+## Caveat on the level-2 numbers
+
+Level 2 was rebuilt three times before this run, because two option
+constructions leaked and one of them also leaked through the letters:
+
+| construction | strongest text-only heuristic |
+|---|---|
+| distractors at Kendall distance 1 / 2 / >=3 from the gold | 0.388 (pick the medoid) |
+| all three at the same distance | 0.750 |
+| prefix ladder, distractors assigned to letters in fixed order | 0.759 (run the ladder, avoid D) |
+| prefix ladder, letters shuffled | 0.500 (run the ladder) |
+| **random distractors (current)** | **0.252** |
+
+Under the leaking constructions these same models scored 0.40-0.47 on level 2;
+with random distractors they score 0.37-0.45 against a real 0.25 floor. The
+0.02-0.10 drop is the part of the old score that came from the option text
+rather than the picture. Any level-2 result from before `rerun_l2_v5.log` is
+contaminated and has been deleted.
+
+## Caveat on the level-3 numbers
+
+Level 3 was rebuilt on 2026-08-31 (`rerun_l3_v2.log`), and unlike level 2 the
+change was to the task, not to a leak:
+
+| version | n | paths | these models scored |
+|---|---|---|---|
+| v1 | 795 | all four start at a shared reference point R | 0.27 - 0.50 |
+| **v2 (current)** | **836** | **each path has its own start point** | **0.26 - 0.30** |
+
+Removing the shared start removed whatever local cue the models were reading off
+it, and every model fell to the floor. The v1 runs are kept under
+`result/_archive_l3_v1/`; they are not comparable to anything current.
+
+The v2 annotations check out — all 836 gold paths are monotone (`worst_step` >=
+0), every distractor violates monotonicity (worst is -0.0007), and the median
+margin between the gold and the strongest distractor is 0.069 — so the collapse
+is difficulty, not a broken split. A little signal survives: accuracy over
+quartiles of that margin rises by +0.088 (7B), +0.083 (3.5-4B) and +0.044
+(Base), reaching 0.30-0.345 on the most separable quarter. Only 2.5-VL-3B is
+flat (-0.005), i.e. reading nothing at all. Both subtypes are hard, with
+`near_to_far` (0.280-0.334) consistently easier than `far_to_near`
+(0.233-0.264).

analysis/dataset_level_analysis/summary_by_model_task.csv

@@ -0,0 +1,17 @@
+model,task,n,scoring,chance,floor,acc_b0,acc_mean,acc_min,acc_max,acc_range,acc_std,slope_per_baseline,delta_b025_minus_b0,headroom_captured_mean,above_floor
+Qwen_Qwen2.5-VL-3B-Instruct,vsr,2195,generate,0.5,0.5,0.7713,0.7595,0.7481,0.7713,0.0232,0.008,-0.0929,-0.0232,0.5191,True
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level1,821,generate,0.25,0.25,0.3995,0.4046,0.3995,0.4117,0.0122,0.005,-0.0077,0.0012,0.2061,True
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level2,764,generate,0.25,0.25,0.3626,0.3743,0.3626,0.3809,0.0183,0.0063,0.0344,0.0079,0.1658,True
+Qwen_Qwen2.5-VL-3B-Instruct,youtube_level3,836,generate,0.25,0.25,0.2667,0.2548,0.2392,0.2667,0.0275,0.0102,0.0287,-0.0012,0.0064,True
+Qwen_Qwen2.5-VL-7B-Instruct,vsr,2195,generate,0.5,0.5,0.8223,0.805,0.7882,0.8223,0.0342,0.0101,-0.1047,-0.0342,0.61,True
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level1,821,generate,0.25,0.25,0.447,0.4519,0.4421,0.4665,0.0244,0.0077,0.0097,0.0049,0.2692,True
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level2,764,generate,0.25,0.25,0.4031,0.3918,0.377,0.4031,0.0262,0.0109,-0.1017,-0.0262,0.1891,True
+Qwen_Qwen2.5-VL-7B-Instruct,youtube_level3,836,generate,0.25,0.25,0.2799,0.2779,0.2679,0.2871,0.0191,0.0061,-0.0273,-0.006,0.0372,True
+Qwen_Qwen3.5-4B,vsr,2195,generate,0.5,0.5,0.8214,0.8134,0.8046,0.8214,0.0169,0.0062,-0.0695,-0.0169,0.6269,True
+Qwen_Qwen3.5-4B,youtube_level1,821,generate,0.25,0.25,0.5274,0.5296,0.5189,0.5408,0.0219,0.0078,0.0564,0.0073,0.3729,True
+Qwen_Qwen3.5-4B,youtube_level2,764,generate,0.25,0.25,0.3914,0.3859,0.3796,0.3914,0.0118,0.0042,-0.0337,-0.0065,0.1812,True
+Qwen_Qwen3.5-4B,youtube_level3,836,generate,0.25,0.25,0.2931,0.2957,0.2883,0.3038,0.0156,0.005,-0.0144,0.0,0.0609,True
+Qwen_Qwen3.5-4B-Base,vsr,2195,likelihood,0.5,0.5,0.8319,0.8273,0.8164,0.8323,0.0159,0.0057,-0.0594,-0.0155,0.6547,True
+Qwen_Qwen3.5-4B-Base,youtube_level1,821,likelihood,0.25,0.25,0.542,0.557,0.542,0.5725,0.0305,0.0107,0.0807,0.0134,0.4094,True
+Qwen_Qwen3.5-4B-Base,youtube_level2,764,likelihood,0.25,0.25,0.4424,0.4483,0.4398,0.4542,0.0144,0.0053,-0.0097,-0.0026,0.2644,True
+Qwen_Qwen3.5-4B-Base,youtube_level3,836,likelihood,0.25,0.25,0.3026,0.2789,0.2667,0.3026,0.0359,0.0117,-0.0622,-0.0203,0.0385,True

analysis/intervention_analysis/Qwen_Qwen2.5-VL-3B-Instruct__vsr_failure_overlap.csv

@@ -0,0 +1,16 @@
+baseline_a,baseline_b,failures_a,failures_b,failures_shared,failure_jaccard,failure_recall_a_in_b,successes_a,successes_b,successes_shared,success_jaccard,answer_agreement
+0.00,0.05,502,510,429,0.7358490566037735,0.8545816733067729,1693,1685,1612,0.912797281993205,0.9298405466970388
+0.00,0.10,502,528,428,0.7109634551495017,0.852589641434263,1693,1667,1593,0.9015280135823429,0.9207289293849659
+0.00,0.15,502,531,420,0.6851549755301795,0.8366533864541833,1693,1664,1582,0.8912676056338028,0.9120728929384966
+0.00,0.20,502,543,413,0.6534810126582279,0.8227091633466136,1693,1652,1563,0.877104377104377,0.9002277904328019
+0.00,0.25,502,553,408,0.6306027820710973,0.8127490039840638,1693,1642,1548,0.8662562954672636,0.8911161731207289
+0.05,0.10,510,528,433,0.715702479338843,0.8490196078431372,1685,1667,1590,0.9023836549375709,0.9216400911161731
+0.05,0.15,510,531,434,0.71499176276771,0.8509803921568627,1685,1664,1588,0.9017603634298694,0.9211845102505695
+0.05,0.20,510,543,429,0.6875,0.8411764705882353,1685,1652,1571,0.8895809739524349,0.9111617312072893
+0.05,0.25,510,553,423,0.6609375,0.8294117647058824,1685,1642,1555,0.8775395033860045,0.9011389521640091
+0.10,0.15,528,531,445,0.7247557003257329,0.8428030303030303,1667,1664,1581,0.9034285714285715,0.9230068337129841
+0.10,0.20,528,543,441,0.7,0.8352272727272727,1667,1652,1565,0.8922462941847207,0.9138952164009112
+0.10,0.25,528,553,430,0.6605222734254992,0.8143939393939394,1667,1642,1544,0.8747875354107649,0.8993166287015946
+0.15,0.20,531,543,447,0.7129186602870813,0.8418079096045198,1664,1652,1568,0.897025171624714,0.9179954441913439
+0.15,0.25,531,553,445,0.6964006259780907,0.8380414312617702,1664,1642,1556,0.8891428571428571,0.911617312072893
+0.20,0.25,543,553,460,0.7232704402515723,0.8471454880294659,1652,1642,1559,0.8985590778097983,0.9198177676537586

analysis/intervention_analysis/Qwen_Qwen2.5-VL-3B-Instruct__youtube_level1_failure_overlap.csv

@@ -0,0 +1,16 @@
+baseline_a,baseline_b,failures_a,failures_b,failures_shared,failure_jaccard,failure_recall_a_in_b,successes_a,successes_b,successes_shared,success_jaccard,answer_agreement
+0.00,0.05,493,488,450,0.847457627118644,0.9127789046653144,328,333,290,0.7816711590296496,0.8514007308160779
+0.00,0.10,493,483,451,0.8590476190476191,0.9148073022312373,328,338,296,0.8,0.8440925700365408
+0.00,0.15,493,484,450,0.8538899430740038,0.9127789046653144,328,337,294,0.7924528301886793,0.8367844092570037
+0.00,0.20,493,493,445,0.822550831792976,0.9026369168356998,328,328,280,0.7446808510638298,0.8185140073081608
+0.00,0.25,493,492,441,0.8106617647058824,0.8945233265720081,328,329,277,0.7289473684210527,0.8063337393422655
+0.05,0.10,488,483,449,0.8601532567049809,0.9200819672131147,333,338,299,0.803763440860215,0.8440925700365408
+0.05,0.15,488,484,447,0.8514285714285714,0.9159836065573771,333,337,296,0.7914438502673797,0.8392204628501827
+0.05,0.20,488,493,454,0.8614800759013282,0.930327868852459,333,328,294,0.8010899182561307,0.833130328867235
+0.05,0.25,488,492,442,0.8215613382899628,0.9057377049180327,333,329,283,0.7467018469656992,0.8063337393422655
+0.10,0.15,483,484,446,0.8560460652591171,0.9233954451345756,338,337,300,0.8,0.8453105968331304
+0.10,0.20,483,493,450,0.8555133079847909,0.9316770186335404,338,328,295,0.7951482479784366,0.8550548112058465
+0.10,0.25,483,492,445,0.839622641509434,0.9213250517598344,338,329,291,0.773936170212766,0.8270401948842875
+0.15,0.20,484,493,452,0.8609523809523809,0.9338842975206612,337,328,296,0.8021680216802168,0.8514007308160779
+0.15,0.25,484,492,441,0.8242990654205608,0.9111570247933884,337,329,286,0.7526315789473684,0.8136419001218027
+0.20,0.25,493,492,459,0.8726235741444867,0.9310344827586207,328,329,295,0.8149171270718232,0.8660170523751523

analysis/intervention_analysis/Qwen_Qwen2.5-VL-3B-Instruct__youtube_level2_failure_overlap.csv

@@ -0,0 +1,16 @@
+baseline_a,baseline_b,failures_a,failures_b,failures_shared,failure_jaccard,failure_recall_a_in_b,successes_a,successes_b,successes_shared,success_jaccard,answer_agreement
+0.00,0.05,487,478,450,0.8737864077669902,0.9240246406570842,277,286,249,0.7929936305732485,0.8612565445026178
+0.00,0.10,487,475,441,0.8464491362763915,0.9055441478439425,277,289,243,0.7523219814241486,0.8272251308900523
+0.00,0.15,487,474,437,0.833969465648855,0.8973305954825462,277,290,240,0.7339449541284404,0.8206806282722513
+0.00,0.20,487,473,437,0.8355640535372849,0.8973305954825462,277,291,241,0.7370030581039755,0.8259162303664922
+0.00,0.25,487,481,435,0.8161350844277674,0.893223819301848,277,283,231,0.7021276595744681,0.7892670157068062
+0.05,0.10,478,475,444,0.8722986247544204,0.9288702928870293,286,289,255,0.796875,0.8547120418848168
+0.05,0.15,478,474,438,0.8521400778210116,0.9163179916317992,286,290,250,0.7668711656441718,0.8416230366492147
+0.05,0.20,478,473,441,0.8647058823529412,0.9225941422594143,286,291,254,0.7863777089783281,0.8586387434554974
+0.05,0.25,478,481,435,0.8301526717557252,0.9100418410041841,286,283,240,0.729483282674772,0.8141361256544503
+0.10,0.15,475,474,436,0.8499025341130604,0.9178947368421052,289,290,251,0.7652439024390244,0.8390052356020943
+0.10,0.20,475,473,435,0.847953216374269,0.9157894736842105,289,291,251,0.7629179331306991,0.8390052356020943
+0.10,0.25,475,481,436,0.8384615384615385,0.9178947368421052,289,283,244,0.7439024390243902,0.8180628272251309
+0.15,0.20,474,473,436,0.8532289628180039,0.919831223628692,290,291,253,0.7713414634146342,0.8534031413612565
+0.15,0.25,474,481,438,0.8471953578336557,0.9240506329113924,290,283,247,0.7576687116564417,0.837696335078534
+0.20,0.25,473,481,439,0.8524271844660194,0.9281183932346723,291,283,249,0.7661538461538462,0.8298429319371727

analysis/intervention_analysis/Qwen_Qwen2.5-VL-3B-Instruct__youtube_level3_failure_overlap.csv

@@ -0,0 +1,16 @@
+baseline_a,baseline_b,failures_a,failures_b,failures_shared,failure_jaccard,failure_recall_a_in_b,successes_a,successes_b,successes_shared,success_jaccard,answer_agreement
+0.00,0.05,613,631,569,0.8429629629629629,0.9282218597063622,223,205,161,0.602996254681648,0.7547846889952153
+0.00,0.10,613,636,570,0.8394698085419735,0.9298531810766721,223,200,157,0.5902255639097744,0.7619617224880383
+0.00,0.15,613,625,562,0.8313609467455622,0.9168026101141925,223,211,160,0.583941605839416,0.7356459330143541
+0.00,0.20,613,619,561,0.8360655737704918,0.9151712887438825,223,217,165,0.6,0.7452153110047847
+0.00,0.25,613,614,551,0.8150887573964497,0.8988580750407831,223,222,160,0.5614035087719298,0.7260765550239234
+0.05,0.10,631,636,588,0.865979381443299,0.9318541996830428,205,200,157,0.6330645161290323,0.7811004784688995
+0.05,0.15,631,625,580,0.8579881656804734,0.919175911251981,205,211,160,0.625,0.7667464114832536
+0.05,0.20,631,619,571,0.8409425625920471,0.9049128367670365,205,217,157,0.5924528301886792,0.7667464114832536
+0.05,0.25,631,614,569,0.841715976331361,0.901743264659271,205,222,160,0.599250936329588,0.7464114832535885
+0.10,0.15,636,625,582,0.8571428571428571,0.9150943396226415,200,211,157,0.6181102362204725,0.7739234449760766
+0.10,0.20,636,619,582,0.8647845468053492,0.9150943396226415,200,217,163,0.6417322834645669,0.7918660287081339
+0.10,0.25,636,614,578,0.8601190476190477,0.9088050314465409,200,222,164,0.6356589147286822,0.7715311004784688
+0.15,0.20,625,619,574,0.8567164179104477,0.9184,211,217,166,0.6335877862595419,0.7799043062200957
+0.15,0.25,625,614,573,0.8603603603603603,0.9168,211,222,170,0.6463878326996197,0.7811004784688995
+0.20,0.25,619,614,578,0.8824427480916031,0.9337641357027464,217,222,181,0.7015503875968992,0.8014354066985646

analysis/intervention_analysis/Qwen_Qwen2.5-VL-7B-Instruct__vsr_failure_overlap.csv

@@ -0,0 +1,16 @@
+baseline_a,baseline_b,failures_a,failures_b,failures_shared,failure_jaccard,failure_recall_a_in_b,successes_a,successes_b,successes_shared,success_jaccard,answer_agreement
+0.00,0.05,390,422,340,0.7203389830508474,0.8717948717948718,1805,1773,1723,0.9288409703504044,0.939863325740319
+0.00,0.10,390,433,331,0.6727642276422764,0.8487179487179487,1805,1762,1703,0.9136266094420601,0.9266514806378132
+0.00,0.15,390,424,335,0.6993736951983298,0.8589743589743589,1805,1771,1716,0.9225806451612903,0.9343963553530752
+0.00,0.20,390,434,323,0.6447105788423154,0.8282051282051283,1805,1761,1694,0.9049145299145299,0.9189066059225512
+0.00,0.25,390,465,325,0.6132075471698113,0.8333333333333334,1805,1730,1665,0.8903743315508021,0.9066059225512528
+0.05,0.10,422,433,362,0.7342799188640974,0.8578199052132701,1773,1762,1702,0.928532460447354,0.9403189066059225
+0.05,0.15,422,424,355,0.7230142566191446,0.8412322274881516,1773,1771,1704,0.9260869565217391,0.9380410022779043
+0.05,0.20,422,434,345,0.675146771037182,0.8175355450236966,1773,1761,1684,0.9102702702702703,0.924373576309795
+0.05,0.25,422,465,348,0.6456400742115028,0.8246445497630331,1773,1730,1656,0.8965890633459664,0.9129840546697039
+0.10,0.15,433,424,359,0.7208835341365462,0.8290993071593533,1762,1771,1697,0.9242919389978214,0.9366742596810934
+0.10,0.20,433,434,357,0.7,0.8244803695150116,1762,1761,1685,0.9167573449401524,0.9302961275626423
+0.10,0.25,433,465,360,0.6691449814126395,0.8314087759815243,1762,1730,1657,0.9029972752043597,0.9189066059225512
+0.15,0.20,424,434,361,0.7263581488933601,0.8514150943396226,1771,1761,1698,0.925845147219193,0.9380410022779043
+0.15,0.25,424,465,363,0.6901140684410646,0.8561320754716981,1771,1730,1669,0.9110262008733624,0.9257403189066059
+0.20,0.25,434,465,376,0.7189292543021033,0.8663594470046083,1761,1730,1672,0.9191863661352392,0.9330296127562643

analysis/intervention_analysis/Qwen_Qwen2.5-VL-7B-Instruct__youtube_level1_failure_overlap.csv

@@ -0,0 +1,16 @@
+baseline_a,baseline_b,failures_a,failures_b,failures_shared,failure_jaccard,failure_recall_a_in_b,successes_a,successes_b,successes_shared,success_jaccard,answer_agreement
+0.00,0.05,454,453,417,0.8510204081632653,0.9185022026431718,367,368,331,0.8193069306930693,0.8781973203410475
+0.00,0.10,454,447,413,0.8463114754098361,0.9096916299559471,367,374,333,0.8161764705882353,0.853836784409257
+0.00,0.15,454,438,405,0.8316221765913757,0.8920704845814978,367,383,334,0.8028846153846154,0.8501827040194885
+0.00,0.20,454,458,409,0.8131212723658051,0.9008810572687225,367,363,318,0.7718446601941747,0.8270401948842875
+0.00,0.25,454,450,394,0.7725490196078432,0.8678414096916299,367,371,311,0.7283372365339579,0.7978075517661388
+0.05,0.10,453,447,419,0.8711018711018711,0.9249448123620309,368,374,340,0.845771144278607,0.8830694275274056
+0.05,0.15,453,438,412,0.860125260960334,0.9094922737306843,368,383,342,0.8361858190709046,0.8842874543239951
+0.05,0.20,453,458,416,0.8404040404040404,0.9183222958057395,368,363,326,0.8049382716049382,0.8623629719853837
+0.05,0.25,453,450,408,0.8242424242424242,0.9006622516556292,368,371,326,0.7893462469733656,0.8428745432399513
+0.10,0.15,447,438,414,0.8789808917197452,0.9261744966442953,374,383,350,0.85995085995086,0.8842874543239951
+0.10,0.20,447,458,420,0.865979381443299,0.9395973154362416,374,363,336,0.8379052369077307,0.8721071863580999
+0.10,0.25,447,450,411,0.845679012345679,0.9194630872483222,374,371,335,0.8170731707317073,0.8501827040194885
+0.15,0.20,438,458,416,0.8666666666666667,0.9497716894977168,383,363,341,0.8419753086419753,0.8952496954933008
+0.15,0.25,438,450,406,0.8423236514522822,0.9269406392694064,383,371,339,0.8168674698795181,0.8611449451887941
+0.20,0.25,458,450,423,0.8721649484536083,0.9235807860262009,363,371,336,0.8442211055276382,0.879415347137637

analysis/intervention_analysis/Qwen_Qwen2.5-VL-7B-Instruct__youtube_level2_failure_overlap.csv

@@ -0,0 +1,16 @@
+baseline_a,baseline_b,failures_a,failures_b,failures_shared,failure_jaccard,failure_recall_a_in_b,successes_a,successes_b,successes_shared,success_jaccard,answer_agreement
+0.00,0.05,456,463,429,0.8755102040816326,0.9407894736842105,308,301,274,0.817910447761194,0.856020942408377
+0.00,0.10,456,460,428,0.8770491803278688,0.9385964912280702,308,304,276,0.8214285714285714,0.8678010471204188
+0.00,0.15,456,457,415,0.8333333333333334,0.9100877192982456,308,307,266,0.7621776504297995,0.8272251308900523
+0.00,0.20,456,476,421,0.8238747553816047,0.9232456140350878,308,288,253,0.7376093294460642,0.8102094240837696
+0.00,0.25,456,476,418,0.8132295719844358,0.9166666666666666,308,288,250,0.7225433526011561,0.7997382198952879
+0.05,0.10,463,460,436,0.8952772073921971,0.9416846652267818,301,304,277,0.8445121951219512,0.8835078534031413
+0.05,0.15,463,457,423,0.8511066398390342,0.9136069114470843,301,307,267,0.782991202346041,0.8481675392670157
+0.05,0.20,463,476,439,0.878,0.9481641468682506,301,288,264,0.8123076923076923,0.8586387434554974
+0.05,0.25,463,476,433,0.8557312252964426,0.9352051835853131,301,288,258,0.7794561933534743,0.849476439790576
+0.10,0.15,460,457,426,0.8676171079429735,0.9260869565217391,304,307,273,0.8076923076923077,0.8691099476439791
+0.10,0.20,460,476,439,0.8832997987927566,0.9543478260869566,304,288,267,0.8215384615384616,0.8717277486910995
+0.10,0.25,460,476,434,0.8645418326693227,0.9434782608695652,304,288,262,0.793939393939394,0.8455497382198953
+0.15,0.20,457,476,430,0.8548707753479126,0.9409190371991247,307,288,261,0.781437125748503,0.8638743455497382
+0.15,0.25,457,476,427,0.8438735177865613,0.9343544857768052,307,288,258,0.7655786350148368,0.8337696335078534
+0.20,0.25,476,476,451,0.9001996007984032,0.9474789915966386,288,288,263,0.8402555910543131,0.8887434554973822

analysis/intervention_analysis/Qwen_Qwen2.5-VL-7B-Instruct__youtube_level3_failure_overlap.csv

@@ -0,0 +1,16 @@
+baseline_a,baseline_b,failures_a,failures_b,failures_shared,failure_jaccard,failure_recall_a_in_b,successes_a,successes_b,successes_shared,success_jaccard,answer_agreement
+0.00,0.05,602,596,555,0.8631415241057543,0.9219269102990033,234,240,193,0.6868327402135231,0.8026315789473685
+0.00,0.10,602,612,567,0.8763523956723338,0.9418604651162791,234,224,189,0.7026022304832714,0.7954545454545454
+0.00,0.15,602,600,556,0.8606811145510835,0.9235880398671097,234,236,190,0.6785714285714286,0.7715311004784688
+0.00,0.20,602,605,551,0.8399390243902439,0.915282392026578,234,231,180,0.631578947368421,0.7547846889952153
+0.00,0.25,602,607,556,0.8514548238897397,0.9235880398671097,234,229,183,0.6535714285714286,0.7679425837320574
+0.05,0.10,596,612,561,0.8670788253477589,0.9412751677852349,240,224,189,0.6872727272727273,0.8205741626794258
+0.05,0.15,596,600,559,0.8775510204081632,0.9379194630872483,240,236,199,0.7184115523465704,0.8098086124401914
+0.05,0.20,596,605,554,0.8562596599690881,0.9295302013422819,240,231,189,0.6702127659574468,0.7954545454545454
+0.05,0.25,596,607,552,0.847926267281106,0.9261744966442953,240,229,185,0.6514084507042254,0.7715311004784688
+0.10,0.15,612,600,565,0.8732612055641422,0.923202614379085,224,236,189,0.6974169741697417,0.8169856459330144
+0.10,0.20,612,605,566,0.869431643625192,0.9248366013071896,224,231,185,0.6851851851851852,0.8169856459330144
+0.10,0.25,612,607,566,0.8667687595712098,0.9248366013071896,224,229,183,0.6777777777777778,0.8002392344497608
+0.15,0.20,600,605,565,0.8828125,0.9416666666666667,236,231,196,0.7232472324723247,0.8217703349282297
+0.15,0.25,600,607,560,0.865533230293663,0.9333333333333333,236,229,189,0.6847826086956522,0.7942583732057417
+0.20,0.25,605,607,564,0.8703703703703703,0.9322314049586777,231,229,188,0.6911764705882353,0.8014354066985646

analysis/intervention_analysis/Qwen_Qwen3.5-4B-Base__vsr_failure_overlap.csv

@@ -0,0 +1,16 @@
+baseline_a,baseline_b,failures_a,failures_b,failures_shared,failure_jaccard,failure_recall_a_in_b,successes_a,successes_b,successes_shared,success_jaccard,answer_agreement
+0.00,0.05,369,370,319,0.7595238095238095,0.8644986449864499,1826,1825,1775,0.9461620469083155,0.9539863325740319
+0.00,0.10,369,368,310,0.7259953161592506,0.8401084010840109,1826,1827,1768,0.9379310344827586,0.9466970387243736
+0.00,0.15,369,378,308,0.7015945330296127,0.8346883468834688,1826,1817,1756,0.9305776364599894,0.9403189066059225
+0.00,0.20,369,386,291,0.6271551724137931,0.7886178861788617,1826,1809,1731,0.9091386554621849,0.9211845102505695
+0.00,0.25,369,403,292,0.6083333333333333,0.7913279132791328,1826,1792,1715,0.9012086179716238,0.9143507972665148
+0.05,0.10,370,368,319,0.7613365155131265,0.8621621621621621,1825,1827,1776,0.9466950959488273,0.9544419134396356
+0.05,0.15,370,378,310,0.7077625570776256,0.8378378378378378,1825,1817,1757,0.9320954907161804,0.9416856492027335
+0.05,0.20,370,386,302,0.6651982378854625,0.8162162162162162,1825,1809,1741,0.9197041732699419,0.9307517084282461
+0.05,0.25,370,403,298,0.6273684210526316,0.8054054054054054,1825,1792,1720,0.9066947812335266,0.9193621867881548
+0.10,0.15,368,378,310,0.7110091743119266,0.842391304347826,1827,1817,1759,0.9331564986737401,0.9425968109339408
+0.10,0.20,368,386,316,0.7214611872146118,0.8586956521739131,1827,1809,1757,0.9350718467269824,0.9444191343963554
+0.10,0.25,368,403,310,0.6724511930585684,0.842391304347826,1827,1792,1734,0.919893899204244,0.9312072892938497
+0.15,0.20,378,386,323,0.7324263038548753,0.8544973544973545,1817,1809,1754,0.936965811965812,0.9462414578587699
+0.15,0.25,378,403,325,0.7127192982456141,0.8597883597883598,1817,1792,1739,0.9299465240641711,0.9403189066059225
+0.20,0.25,386,403,336,0.7417218543046358,0.8704663212435233,1809,1792,1742,0.9370629370629371,0.9466970387243736

analysis/intervention_analysis/Qwen_Qwen3.5-4B-Base__youtube_level1_failure_overlap.csv

@@ -0,0 +1,16 @@
+baseline_a,baseline_b,failures_a,failures_b,failures_shared,failure_jaccard,failure_recall_a_in_b,successes_a,successes_b,successes_shared,success_jaccard,answer_agreement
+0.00,0.05,376,373,341,0.8357843137254902,0.9069148936170213,445,448,413,0.8604166666666667,0.8915956151035322
+0.00,0.10,376,361,332,0.8197530864197531,0.8829787234042553,445,460,416,0.8507157464212679,0.8830694275274056
+0.00,0.15,376,351,322,0.7950617283950617,0.8563829787234043,445,470,416,0.8336673346693386,0.8672350791717418
+0.00,0.20,376,356,319,0.7723970944309927,0.848404255319149,445,465,408,0.8127490039840638,0.8501827040194885
+0.00,0.25,376,365,320,0.7600950118764845,0.851063829787234,445,456,400,0.7984031936127745,0.8428745432399513
+0.05,0.10,373,361,338,0.8535353535353535,0.9061662198391421,448,460,425,0.8799171842650103,0.9037758830694276
+0.05,0.15,373,351,321,0.7965260545905707,0.8605898123324397,448,470,418,0.836,0.8672350791717418
+0.05,0.20,373,356,318,0.7737226277372263,0.8525469168900804,448,465,410,0.8151093439363817,0.8489646772228989
+0.05,0.25,373,365,319,0.7613365155131265,0.8552278820375335,448,456,402,0.8007968127490039,0.8453105968331304
+0.10,0.15,361,351,318,0.8071065989847716,0.8808864265927978,460,470,427,0.8489065606361829,0.876979293544458
+0.10,0.20,361,356,325,0.8290816326530612,0.9002770083102493,460,465,429,0.8649193548387096,0.8855054811205847
+0.10,0.25,361,365,318,0.7794117647058824,0.8808864265927978,460,456,413,0.8210735586481114,0.8587088915956151
+0.15,0.20,351,356,320,0.8268733850129198,0.9116809116809117,470,465,434,0.8662674650698603,0.8842874543239951
+0.15,0.25,351,365,325,0.8312020460358056,0.9259259259259259,470,456,430,0.8669354838709677,0.8964677222898904
+0.20,0.25,356,365,324,0.8161209068010076,0.9101123595505618,465,456,424,0.8531187122736419,0.8891595615103532

analysis/intervention_analysis/Qwen_Qwen3.5-4B-Base__youtube_level2_failure_overlap.csv

@@ -0,0 +1,16 @@
+baseline_a,baseline_b,failures_a,failures_b,failures_shared,failure_jaccard,failure_recall_a_in_b,successes_a,successes_b,successes_shared,success_jaccard,answer_agreement
+0.00,0.05,426,418,389,0.8549450549450549,0.9131455399061033,338,346,309,0.824,0.8691099476439791
+0.00,0.10,426,420,388,0.8471615720524017,0.9107981220657277,338,344,306,0.8138297872340425,0.862565445026178
+0.00,0.15,426,417,384,0.8366013071895425,0.9014084507042254,338,347,305,0.8026315789473685,0.8520942408376964
+0.00,0.20,426,420,382,0.8232758620689655,0.8967136150234741,338,344,300,0.7853403141361257,0.8219895287958116
+0.00,0.25,426,428,381,0.8054968287526427,0.8943661971830986,338,336,291,0.7597911227154047,0.8115183246073299
+0.05,0.10,418,420,388,0.8622222222222222,0.9282296650717703,346,344,314,0.8351063829787234,0.8808900523560209
+0.05,0.15,418,417,390,0.8764044943820225,0.9330143540669856,346,347,319,0.8529411764705882,0.8795811518324608
+0.05,0.20,418,420,384,0.8458149779735683,0.9186602870813397,346,344,310,0.8157894736842105,0.8416230366492147
+0.05,0.25,418,428,383,0.8272138228941684,0.916267942583732,346,336,301,0.7900262467191601,0.8350785340314136
+0.10,0.15,420,417,397,0.9022727272727272,0.9452380952380952,344,347,324,0.8828337874659401,0.9070680628272252
+0.10,0.20,420,420,391,0.8708240534521158,0.930952380952381,344,344,315,0.8445040214477212,0.8638743455497382
+0.10,0.25,420,428,386,0.8354978354978355,0.919047619047619,344,336,302,0.798941798941799,0.8520942408376964
+0.15,0.20,417,420,393,0.8851351351351351,0.9424460431654677,347,344,320,0.862533692722372,0.8808900523560209
+0.15,0.25,417,428,391,0.8612334801762115,0.9376498800959233,347,336,310,0.8310991957104558,0.8730366492146597
+0.20,0.25,420,428,395,0.8719646799116998,0.9404761904761905,344,336,311,0.8428184281842819,0.8848167539267016

analysis/intervention_analysis/Qwen_Qwen3.5-4B-Base__youtube_level3_failure_overlap.csv

@@ -0,0 +1,16 @@
+baseline_a,baseline_b,failures_a,failures_b,failures_shared,failure_jaccard,failure_recall_a_in_b,successes_a,successes_b,successes_shared,success_jaccard,answer_agreement
+0.00,0.05,583,609,545,0.8423493044822257,0.934819897084048,253,227,189,0.6494845360824743,0.7523923444976076
+0.00,0.10,583,604,541,0.8374613003095975,0.9279588336192109,253,232,190,0.6440677966101694,0.7547846889952153
+0.00,0.15,583,613,542,0.8287461773700305,0.9296740994854202,253,223,182,0.6190476190476191,0.7344497607655502
+0.00,0.20,583,608,534,0.8127853881278538,0.9159519725557461,253,228,179,0.5927152317880795,0.7236842105263158
+0.00,0.25,583,600,528,0.8061068702290076,0.9056603773584906,253,236,181,0.5876623376623377,0.715311004784689
+0.05,0.10,609,604,563,0.8661538461538462,0.9244663382594417,227,232,186,0.6813186813186813,0.7906698564593302
+0.05,0.15,609,613,567,0.8656488549618321,0.9310344827586207,227,223,181,0.6728624535315985,0.7727272727272727
+0.05,0.20,609,608,562,0.8580152671755725,0.922824302134647,227,228,181,0.6605839416058394,0.7691387559808612
+0.05,0.25,609,600,550,0.834597875569044,0.90311986863711,227,236,177,0.6188811188811189,0.7559808612440191
+0.10,0.15,604,613,569,0.8780864197530864,0.9420529801324503,232,223,188,0.704119850187266,0.819377990430622
+0.10,0.20,604,608,566,0.8761609907120743,0.9370860927152318,232,228,190,0.7037037037037037,0.8038277511961722
+0.10,0.25,604,600,560,0.8695652173913043,0.9271523178807947,232,236,192,0.6956521739130435,0.7954545454545454
+0.15,0.20,613,608,570,0.8755760368663594,0.9298531810766721,223,228,185,0.6954887218045113,0.8098086124401914
+0.15,0.25,613,600,563,0.8661538461538462,0.9184339314845025,223,236,186,0.6813186813186813,0.7894736842105263
+0.20,0.25,608,600,560,0.8641975308641975,0.9210526315789473,228,236,188,0.6811594202898551,0.7978468899521531

analysis/intervention_analysis/Qwen_Qwen3.5-4B__vsr_failure_overlap.csv

@@ -0,0 +1,16 @@
+baseline_a,baseline_b,failures_a,failures_b,failures_shared,failure_jaccard,failure_recall_a_in_b,successes_a,successes_b,successes_shared,success_jaccard,answer_agreement
+0.00,0.05,392,401,347,0.7780269058295964,0.8852040816326531,1803,1794,1749,0.9464285714285714,0.9548974943052392
+0.00,0.10,392,398,343,0.767337807606264,0.875,1803,1797,1748,0.9438444924406048,0.952619589977221
+0.00,0.15,392,414,334,0.7076271186440678,0.8520408163265306,1803,1781,1723,0.9258463191832348,0.937129840546697
+0.00,0.20,392,423,326,0.6666666666666666,0.8316326530612245,1803,1772,1706,0.9127875869448903,0.9257403189066059
+0.00,0.25,392,429,327,0.6619433198380567,0.8341836734693877,1803,1766,1701,0.9105995717344754,0.9239179954441914
+0.05,0.10,401,398,356,0.8036117381489842,0.8877805486284289,1794,1797,1752,0.9526916802610114,0.9603644646924829
+0.05,0.15,401,414,350,0.7526881720430108,0.8728179551122195,1794,1781,1730,0.9376693766937669,0.9476082004555809
+0.05,0.20,401,423,341,0.7060041407867494,0.8503740648379052,1794,1772,1712,0.9234088457389428,0.9353075170842825
+0.05,0.25,401,429,337,0.6835699797160243,0.8403990024937655,1794,1766,1702,0.9160387513455328,0.9289293849658314
+0.10,0.15,398,414,355,0.7768052516411379,0.8919597989949749,1797,1781,1738,0.9445652173913044,0.9535307517084283
+0.10,0.20,398,423,350,0.7430997876857749,0.8793969849246231,1797,1772,1724,0.9344173441734417,0.944874715261959
+0.10,0.25,398,429,341,0.7016460905349794,0.8567839195979899,1797,1766,1709,0.9217907227615966,0.9339407744874715
+0.15,0.20,414,423,359,0.7510460251046025,0.8671497584541062,1781,1772,1717,0.9351851851851852,0.9457858769931663
+0.15,0.25,414,429,354,0.7239263803680982,0.855072463768116,1781,1766,1706,0.9266702878870179,0.9384965831435079
+0.20,0.25,423,429,372,0.775,0.8794326241134752,1772,1766,1715,0.9407569939659901,0.9507972665148063

analysis/intervention_analysis/Qwen_Qwen3.5-4B__youtube_level1_failure_overlap.csv

@@ -0,0 +1,16 @@
+baseline_a,baseline_b,failures_a,failures_b,failures_shared,failure_jaccard,failure_recall_a_in_b,successes_a,successes_b,successes_shared,success_jaccard,answer_agreement
+0.00,0.05,388,393,361,0.8595238095238096,0.9304123711340206,433,428,401,0.8717391304347826,0.8964677222898904
+0.00,0.10,388,395,354,0.8251748251748252,0.9123711340206185,433,426,392,0.8394004282655246,0.8745432399512789
+0.00,0.15,388,377,342,0.8085106382978723,0.8814432989690721,433,444,398,0.8308977035490606,0.8599269183922046
+0.00,0.20,388,382,339,0.7865429234338747,0.8737113402061856,433,439,390,0.8091286307053942,0.8404384896467723
+0.00,0.25,388,382,338,0.7824074074074074,0.8711340206185567,433,439,389,0.8053830227743272,0.8343483556638246
+0.05,0.10,393,395,363,0.8541176470588235,0.9236641221374046,428,426,396,0.8646288209606987,0.8855054811205847
+0.05,0.15,393,377,342,0.7990654205607477,0.8702290076335878,428,444,393,0.8204592901878914,0.8635809987819733
+0.05,0.20,393,382,343,0.7939814814814815,0.8727735368956743,428,439,389,0.8138075313807531,0.8453105968331304
+0.05,0.25,393,382,338,0.7734553775743707,0.8600508905852418,428,439,384,0.7950310559006211,0.8258221680876979
+0.10,0.15,395,377,349,0.8250591016548463,0.8835443037974684,426,444,398,0.8432203389830508,0.8781973203410475
+0.10,0.20,395,382,350,0.819672131147541,0.8860759493670886,426,439,394,0.8365180467091295,0.8684531059683313
+0.10,0.25,395,382,347,0.8069767441860465,0.8784810126582279,426,439,391,0.8248945147679325,0.8514007308160779
+0.15,0.20,377,382,341,0.8157894736842105,0.9045092838196287,444,439,403,0.8395833333333333,0.8721071863580999
+0.15,0.25,377,382,338,0.8028503562945368,0.896551724137931,444,439,400,0.8281573498964804,0.8611449451887941
+0.20,0.25,382,382,346,0.8277511961722488,0.9057591623036649,439,439,403,0.848421052631579,0.8781973203410475

analysis/intervention_analysis/Qwen_Qwen3.5-4B__youtube_level2_failure_overlap.csv

@@ -0,0 +1,16 @@
+baseline_a,baseline_b,failures_a,failures_b,failures_shared,failure_jaccard,failure_recall_a_in_b,successes_a,successes_b,successes_shared,success_jaccard,answer_agreement
+0.00,0.05,465,466,438,0.8884381338742393,0.9419354838709677,299,298,271,0.8312883435582822,0.8782722513089005
+0.00,0.10,465,472,438,0.8777555110220441,0.9419354838709677,299,292,265,0.8128834355828221,0.8586387434554974
+0.00,0.15,465,468,433,0.866,0.9311827956989247,299,296,264,0.797583081570997,0.8363874345549738
+0.00,0.20,465,474,434,0.8594059405940594,0.9333333333333333,299,290,259,0.7848484848484848,0.8285340314136126
+0.00,0.25,465,470,423,0.826171875,0.9096774193548387,299,294,252,0.7390029325513197,0.81282722513089
+0.05,0.10,466,472,441,0.8873239436619719,0.9463519313304721,298,292,267,0.826625386996904,0.8821989528795812
+0.05,0.15,466,468,438,0.8830645161290323,0.9399141630901288,298,296,268,0.8220858895705522,0.8664921465968587
+0.05,0.20,466,474,437,0.8687872763419483,0.9377682403433476,298,290,261,0.7981651376146789,0.856020942408377
+0.05,0.25,466,470,432,0.8571428571428571,0.927038626609442,298,294,260,0.7831325301204819,0.8599476439790575
+0.10,0.15,472,468,445,0.898989898989899,0.9427966101694916,292,296,269,0.8432601880877743,0.8821989528795812
+0.10,0.20,472,474,447,0.8957915831663327,0.9470338983050848,292,290,265,0.8359621451104101,0.8678010471204188
+0.10,0.25,472,470,438,0.8690476190476191,0.9279661016949152,292,294,260,0.7975460122699386,0.856020942408377
+0.15,0.20,468,474,446,0.8991935483870968,0.9529914529914529,296,290,268,0.8427672955974843,0.8704188481675392
+0.15,0.25,468,470,440,0.8835341365461847,0.9401709401709402,296,294,266,0.8209876543209876,0.8704188481675392
+0.20,0.25,474,470,443,0.8842315369261478,0.9345991561181435,290,294,263,0.8193146417445483,0.8520942408376964