claim5/claim5_summary.json

{
  "claim": "Claim 5 - Prompt Engineering baseline (Table 1 row 2, Section 5.4)",
  "target": {
    "paper_gsm8k_prompt_eng": 20.0,
    "paper_gsm8k_zero_shot": 83.6
  },
  "result": {
    "measured_flexible": 22.14,
    "ci95": [
      19.92,
      24.48
    ],
    "reproduces_20_0": true,
    "measured_strict": 18.57,
    "measured_offline": 22.29
  },
  "provenance": {
    "densesteer_commit": "7f0a93ddcae5dcfe310917734db39c9de3b30d51",
    "transformers": "5.13.1",
    "torch": "2.11.0+cu128",
    "python": "3.12.13",
    "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition, 580.82.07, 97887 MiB",
    "batch_size": 256,
    "gen_kwargs": "max_gen_toks=2048",
    "model": "Qwen/Qwen2.5-3B-Instruct",
    "task": "gsm8k_pe_concat",
    "system_instruction": "Qwen default persona",
    "runtime_min": {
      "full": 3.66,
      "A": 0.59,
      "B": 0.51,
      "C": 0.59
    }
  },
  "composition_note": "A.3 has no {problem} placeholder, no output-format instruction, and appears nowhere in the repo at commit 7f0a93d. Three compositions were tested at n=50: A (A.3 as system turn, replaces Qwen persona) 68.0%; B (A.3 prepended to A.1 in the user turn, persona retained) 36.0%; C (A.3 + problem, no format instruction) 52.0%. Placement alone moves the result 32pp. B was run at full scale as the only clean +A.3 delta and the closest match to the paper.",
  "findings": [
    "Reproduced: 22.14% vs paper 20.0%, 95% CI [19.92, 24.48] contains 20.0.",
    "Mechanism confirmed: steps 7.54 -> 1.33, tokens 293.6 -> 24.4, 0% truncation, 0% empty.",
    "Reinterpretation: the model COMPLIES with A.3 rather than failing it. A.3 forbids new reasoning (steps collapse) and demands plain text (latex_wrapped 1261 -> 53, strict_form 19 -> 1266).",
    "A.3 is a rewriting prompt with its input removed - every rule references an 'original solution' that does not exist at inference time.",
    "strict-match on the baseline scores 1.36% (19/1319) and latex_wrapped is 1261/1319: it measures LaTeX formatting, not correctness. Same generations give -60.96pp under flexible-extract and +17.21pp under strict-match.",
    "Offline boxed-first scorer corroborates flexible-extract (-62.24pp) and recovers 19 problems it misses on the baseline (84.53 vs 83.09).",
    "n=50 smoke gave 36.0% CI [22.9, 50.8], which excludes the true 22.14%. Smoke runs resolved mechanism but not magnitude."
  ],
  "mcnemar": [
    {
      "scoring": "flexible-extract",
      "n": 1319,
      "base_%": 83.09,
      "prompteng_%": 22.14,
      "delta_pp": -60.96,
      "both_right": 274,
      "broke": 822,
      "fixed": 18,
      "p": "1.57e-216",
      "ci95_pp": 2.78
    },
    {
      "scoring": "strict-match",
      "n": 1319,
      "base_%": 1.36,
      "prompteng_%": 18.57,
      "delta_pp": 17.21,
      "both_right": 4,
      "broke": 14,
      "fixed": 241,
      "p": "1.44e-54",
      "ci95_pp": 2.18
    },
    {
      "scoring": "offline boxed-first",
      "n": 1319,
      "base_%": 84.53,
      "prompteng_%": 22.29,
      "delta_pp": -62.24,
      "both_right": 278,
      "broke": 837,
      "fixed": 16,
      "p": "1.11e-223",
      "ci95_pp": 2.75
    }
  ],
  "diagnostics_smoke_n50": [
    {
      "variant": "BASELINE_50",
      "n": 50,
      "acc_flex": 84.0,
      "acc_strict": 0.0,
      "avg_steps": 7.2,
      "avg_density": 43.7,
      "avg_tokens": 293.4,
      "pct_boxed": 100.0,
      "pct_number": 100.0,
      "pct_trunc": 0.0,
      "pct_empty": 0.0
    },
    {
      "variant": "A_system",
      "n": 50,
      "acc_flex": 68.0,
      "acc_strict": 62.0,
      "avg_steps": 2.44,
      "avg_density": 73.76,
      "avg_tokens": 121.38,
      "pct_boxed": 100.0,
      "pct_number": 100.0,
      "pct_trunc": 0.0,
      "pct_empty": 0.0
    },
    {
      "variant": "B_concat",
      "n": 50,
      "acc_flex": 36.0,
      "acc_strict": 32.0,
      "avg_steps": 1.38,
      "avg_density": 16.19,
      "avg_tokens": 31.24,
      "pct_boxed": 100.0,
      "pct_number": 100.0,
      "pct_trunc": 0.0,
      "pct_empty": 0.0
    },
    {
      "variant": "C_only",
      "n": 50,
      "acc_flex": 52.0,
      "acc_strict": 0.0,
      "avg_steps": 2.46,
      "avg_density": 56.97,
      "avg_tokens": 112.54,
      "pct_boxed": 4.0,
      "pct_number": 100.0,
      "pct_trunc": 0.0,
      "pct_empty": 0.0
    }
  ],
  "diagnostics_full_n1319": [
    {
      "variant": "BASELINE",
      "n": 1319,
      "acc_flex": 83.1,
      "acc_strict": 1.4,
      "avg_steps": 7.54,
      "avg_density": 41.35,
      "avg_tokens": 293.6,
      "pct_boxed": 99.8,
      "pct_number": 100.0,
      "pct_trunc": 0.2,
      "pct_empty": 0.0
    },
    {
      "variant": "B_concat_full",
      "n": 1319,
      "acc_flex": 22.1,
      "acc_strict": 18.6,
      "avg_steps": 1.33,
      "avg_density": 14.75,
      "avg_tokens": 24.43,
      "pct_boxed": 100.0,
      "pct_number": 100.0,
      "pct_trunc": 0.0,
      "pct_empty": 0.0
    }
  ]
}