Upload Craftax ReMDM planner checkpoints and results (part 2)

.gitattributes

@@ -295,3 +295,26 @@ experiments/rl_finetuning/outputs/review_anchor_baseline_rl/figures/t_bin_grad_n
 experiments/rl_finetuning/outputs/review_anchor_baseline_rl/figures/t_distribution_analysis.png filter=lfs diff=lfs merge=lfs -text
 experiments/rl_finetuning/outputs/review_anchor_baseline_rl/figures/win_rate_and_effective_batch_size.png filter=lfs diff=lfs merge=lfs -text
 experiments/rl_finetuning/outputs/review_run1_bc_all/figures/curves_bc_all.png filter=lfs diff=lfs merge=lfs -text
+experiments/rl_finetuning/outputs/review_run1_bc_all/figures/per_layer_grad_heatmap_bc_all.png filter=lfs diff=lfs merge=lfs -text
+experiments/rl_finetuning/outputs/review_run1_bc_all/figures/representation_drift.png filter=lfs diff=lfs merge=lfs -text
+experiments/rl_finetuning/outputs/review_run1_bc_all/figures/t_bin_grad_norms_bc_all.png filter=lfs diff=lfs merge=lfs -text
+experiments/rl_finetuning/outputs/review_run1_bc_all/figures/t_distribution_analysis.png filter=lfs diff=lfs merge=lfs -text
+experiments/rl_finetuning/outputs/review_run1_bc_all/figures/win_rate_and_effective_batch_size.png filter=lfs diff=lfs merge=lfs -text
+experiments/rl_finetuning/outputs/review_run2_advclip_lr_matched/figures/curves_advantage_clip.png filter=lfs diff=lfs merge=lfs -text
+experiments/rl_finetuning/outputs/review_run2_advclip_lr_matched/figures/per_layer_grad_heatmap_advantage_clip.png filter=lfs diff=lfs merge=lfs -text
+experiments/rl_finetuning/outputs/review_run2_advclip_lr_matched/figures/representation_drift.png filter=lfs diff=lfs merge=lfs -text
+experiments/rl_finetuning/outputs/review_run2_advclip_lr_matched/figures/t_bin_grad_norms_advantage_clip.png filter=lfs diff=lfs merge=lfs -text
+experiments/rl_finetuning/outputs/review_run2_advclip_lr_matched/figures/t_distribution_analysis.png filter=lfs diff=lfs merge=lfs -text
+experiments/rl_finetuning/outputs/review_run2_advclip_lr_matched/figures/win_rate_and_effective_batch_size.png filter=lfs diff=lfs merge=lfs -text
+experiments/rl_finetuning/outputs/review_run4_baseline_lr1e-4/figures/curves_baseline_rl.png filter=lfs diff=lfs merge=lfs -text
+experiments/rl_finetuning/outputs/review_run4_baseline_lr1e-4/figures/per_layer_grad_heatmap_baseline_rl.png filter=lfs diff=lfs merge=lfs -text
+experiments/rl_finetuning/outputs/review_run4_baseline_lr1e-4/figures/representation_drift.png filter=lfs diff=lfs merge=lfs -text
+experiments/rl_finetuning/outputs/review_run4_baseline_lr1e-4/figures/t_bin_grad_norms_baseline_rl.png filter=lfs diff=lfs merge=lfs -text
+experiments/rl_finetuning/outputs/review_run4_baseline_lr1e-4/figures/t_distribution_analysis.png filter=lfs diff=lfs merge=lfs -text
+experiments/rl_finetuning/outputs/review_run4_baseline_lr1e-4/figures/win_rate_and_effective_batch_size.png filter=lfs diff=lfs merge=lfs -text
+experiments/rl_finetuning/outputs/review_run4_baseline_lr1e-5/figures/curves_baseline_rl.png filter=lfs diff=lfs merge=lfs -text
+experiments/rl_finetuning/outputs/review_run4_baseline_lr1e-5/figures/per_layer_grad_heatmap_baseline_rl.png filter=lfs diff=lfs merge=lfs -text
+experiments/rl_finetuning/outputs/review_run4_baseline_lr1e-5/figures/representation_drift.png filter=lfs diff=lfs merge=lfs -text
+experiments/rl_finetuning/outputs/review_run4_baseline_lr1e-5/figures/t_bin_grad_norms_baseline_rl.png filter=lfs diff=lfs merge=lfs -text
+experiments/rl_finetuning/outputs/review_run4_baseline_lr1e-5/figures/t_distribution_analysis.png filter=lfs diff=lfs merge=lfs -text
+experiments/rl_finetuning/outputs/review_run4_baseline_lr1e-5/figures/win_rate_and_effective_batch_size.png filter=lfs diff=lfs merge=lfs -text

experiments/rl_finetuning/outputs/review_run1_bc_all/tables/achievement_summary.csv

@@ -0,0 +1,23 @@
+Achievement,Pretrained,bc_all,delta_bc_all
+Achievements/collect_coal,0.6357,0.1616,-0.4741
+Achievements/collect_diamond,0.0,0.0,0.0
+Achievements/collect_drink,0.5006,0.2148,-0.2858
+Achievements/collect_iron,0.4313,0.0772,-0.3541
+Achievements/collect_sapling,0.8907,0.9468,0.0561
+Achievements/collect_stone,0.9154,0.3702,-0.5452
+Achievements/collect_wood,0.9953,0.8827,-0.1126
+Achievements/defeat_skeleton,0.208,0.0496,-0.1584
+Achievements/defeat_zombie,0.6663,0.2506,-0.4157
+Achievements/eat_cow,0.4501,0.1475,-0.3025
+Achievements/eat_plant,0.0,0.0,0.0
+Achievements/make_iron_pickaxe,0.0517,0.003,-0.0488
+Achievements/make_iron_sword,0.2385,0.0181,-0.2205
+Achievements/make_stone_pickaxe,0.7203,0.1856,-0.5348
+Achievements/make_stone_sword,0.792,0.2562,-0.5358
+Achievements/make_wood_pickaxe,0.9553,0.4224,-0.5329
+Achievements/make_wood_sword,0.812,0.1741,-0.6379
+Achievements/place_furnace,0.879,0.3354,-0.5436
+Achievements/place_plant,0.6381,0.3167,-0.3213
+Achievements/place_stone,0.7286,0.2572,-0.4714
+Achievements/place_table,0.9835,0.5177,-0.4658
+Achievements/wake_up,0.3831,0.0476,-0.3354

experiments/rl_finetuning/outputs/review_run1_bc_all/tables/achievement_summary.tex

@@ -0,0 +1,33 @@
+\begin{table}[htbp]
+\centering
+\caption{Per-achievement final unlock rates and delta vs pretrained.}
+\label{tab:achievements}
+\begin{tabular}{lrrr}
+\toprule
+\textbf{Achievement} & \textbf{Pretrained} & \textbf{bc\_all} & \textbf{delta\_bc\_all} \\
+\midrule
+Achievements/collect\_coal & 0.6357 & 0.1616 & -0.4741 \\
+Achievements/collect\_diamond & 0.0000 & 0.0000 & 0.0000 \\
+Achievements/collect\_drink & 0.5006 & 0.2148 & -0.2858 \\
+Achievements/collect\_iron & 0.4313 & 0.0772 & -0.3541 \\
+Achievements/collect\_sapling & 0.8907 & 0.9468 & 0.0561 \\
+Achievements/collect\_stone & 0.9154 & 0.3702 & -0.5452 \\
+Achievements/collect\_wood & 0.9953 & 0.8827 & -0.1126 \\
+Achievements/defeat\_skeleton & 0.2080 & 0.0496 & -0.1584 \\
+Achievements/defeat\_zombie & 0.6663 & 0.2506 & -0.4157 \\
+Achievements/eat\_cow & 0.4501 & 0.1475 & -0.3025 \\
+Achievements/eat\_plant & 0.0000 & 0.0000 & 0.0000 \\
+Achievements/make\_iron\_pickaxe & 0.0517 & 0.0030 & -0.0488 \\
+Achievements/make\_iron\_sword & 0.2385 & 0.0181 & -0.2205 \\
+Achievements/make\_stone\_pickaxe & 0.7203 & 0.1856 & -0.5348 \\
+Achievements/make\_stone\_sword & 0.7920 & 0.2562 & -0.5358 \\
+Achievements/make\_wood\_pickaxe & 0.9553 & 0.4224 & -0.5329 \\
+Achievements/make\_wood\_sword & 0.8120 & 0.1741 & -0.6379 \\
+Achievements/place\_furnace & 0.8790 & 0.3354 & -0.5436 \\
+Achievements/place\_plant & 0.6381 & 0.3167 & -0.3213 \\
+Achievements/place\_stone & 0.7286 & 0.2572 & -0.4714 \\
+Achievements/place\_table & 0.9835 & 0.5177 & -0.4658 \\
+Achievements/wake\_up & 0.3831 & 0.0476 & -0.3354 \\
+\bottomrule
+\end{tabular}
+\end{table}

experiments/rl_finetuning/outputs/review_run1_bc_all/tables/forgetting_analysis.csv

@@ -0,0 +1,2 @@
+Method,First_Collapse_Iter,Min_Score,Min_Score_Iter,Recovery_Score,Recovered
+bc_all,150,4.6103,500,4.7348,N

experiments/rl_finetuning/outputs/review_run1_bc_all/tables/forgetting_analysis.tex

@@ -0,0 +1,12 @@
+\begin{table}[htbp]
+\centering
+\caption{Catastrophic forgetting timeline.}
+\label{tab:forgetting}
+\begin{tabular}{lrrrrr}
+\toprule
+\textbf{Method} & \textbf{First\_Collapse\_Iter} & \textbf{Min\_Score} & \textbf{Min\_Score\_Iter} & \textbf{Recovery\_Score} & \textbf{Recovered} \\
+\midrule
+bc\_all & 150 & 4.6103 & 500 & 4.7348 & N \\
+\bottomrule
+\end{tabular}
+\end{table}

experiments/rl_finetuning/outputs/review_run1_bc_all/tables/gradient_analysis.csv

@@ -0,0 +1,2 @@
+Method,Mean_Grad_Align,Final_Grad_Align,Trend,Mean_KL_Drift,Final_KL_Drift
+bc_all,0.0081,-0.1948,down,0.264528,0.367367

experiments/rl_finetuning/outputs/review_run1_bc_all/tables/gradient_analysis.tex

@@ -0,0 +1,12 @@
+\begin{table}[htbp]
+\centering
+\caption{Gradient alignment and representation drift analysis.}
+\label{tab:gradient}
+\begin{tabular}{lrrrrr}
+\toprule
+\textbf{Method} & \textbf{Mean\_Grad\_Align} & \textbf{Final\_Grad\_Align} & \textbf{Trend} & \textbf{Mean\_KL\_Drift} & \textbf{Final\_KL\_Drift} \\
+\midrule
+bc\_all & 0.0081 & -0.1948 & down & 0.2645 & 0.3674 \\
+\bottomrule
+\end{tabular}
+\end{table}

experiments/rl_finetuning/outputs/review_run1_bc_all/tables/group_summary.csv

@@ -0,0 +1,2 @@
+Group,N,Mean,Best,Worst,StdDev
+B,1,4.7348,4.7348,4.7348,0.0

experiments/rl_finetuning/outputs/review_run1_bc_all/tables/group_summary.tex

@@ -0,0 +1,12 @@
+\begin{table}[htbp]
+\centering
+\caption{Group summary statistics.}
+\label{tab:group_summary}
+\begin{tabular}{lrrrrr}
+\toprule
+\textbf{Group} & \textbf{N} & \textbf{Mean} & \textbf{Best} & \textbf{Worst} & \textbf{StdDev} \\
+\midrule
+B & 1 & 4.7348 & 4.7348 & 4.7348 & 0.0000 \\
+\bottomrule
+\end{tabular}
+\end{table}

experiments/rl_finetuning/outputs/review_run1_bc_all/tables/hypothesis_verdict.csv

@@ -0,0 +1,2 @@
+Ablation,Group,Hypothesis,Result,Conclusion
+bc_all,B,"If it degrades like baseline_rl: the self-generated rollouts, not the return wei...",COLLAPSE,Hypothesis REFUTED — intervention did not prevent collapse

experiments/rl_finetuning/outputs/review_run1_bc_all/tables/hypothesis_verdict.tex

@@ -0,0 +1,12 @@
+\begin{table}[htbp]
+\centering
+\caption{Hypothesis verdict per ablation.}
+\label{tab:hypothesis}
+\begin{tabular}{lrrrr}
+\toprule
+\textbf{Ablation} & \textbf{Group} & \textbf{Hypothesis} & \textbf{Result} & \textbf{Conclusion} \\
+\midrule
+bc\_all & B & If it degrades like baseline\_rl: the self-generated rollouts, not the return wei... & COLLAPSE & Hypothesis REFUTED — intervention did not prevent collapse \\
+\bottomrule
+\end{tabular}
+\end{table}

experiments/rl_finetuning/outputs/review_run1_bc_all/tables/main_results.csv

@@ -0,0 +1,2 @@
+Method,Group,Final_Score,Score_Std,Delta_vs_Pretrained,Delta_vs_Baseline_RL,Verdict
+bc_all,B,4.7348,0.1869,-7.2407,-7.2407,COLLAPSE

experiments/rl_finetuning/outputs/review_run1_bc_all/tables/main_results.tex

@@ -0,0 +1,12 @@
+\begin{table}[htbp]
+\centering
+\caption{Main ablation results.}
+\label{tab:main_results}
+\begin{tabular}{lrrrrrr}
+\toprule
+\textbf{Method} & \textbf{Group} & \textbf{Final\_Score} & \textbf{Score\_Std} & \textbf{Delta\_vs\_Pretrained} & \textbf{Delta\_vs\_Baseline\_RL} & \textbf{Verdict} \\
+\midrule
+bc\_all & B & 4.7348 & 0.1869 & -7.2407 & -7.2407 & COLLAPSE \\
+\bottomrule
+\end{tabular}
+\end{table}

experiments/rl_finetuning/outputs/review_run1_bc_all/tables/per_env.csv

@@ -0,0 +1,3 @@
+Method,Achievements/collect_coal,Achievements/collect_diamond,Achievements/collect_drink,Achievements/collect_iron,Achievements/collect_sapling,Achievements/collect_stone,Achievements/collect_wood,Achievements/defeat_skeleton,Achievements/defeat_zombie,Achievements/eat_cow,Achievements/eat_plant,Achievements/make_iron_pickaxe,Achievements/make_iron_sword,Achievements/make_stone_pickaxe,Achievements/make_stone_sword,Achievements/make_wood_pickaxe,Achievements/make_wood_sword,Achievements/place_furnace,Achievements/place_plant,Achievements/place_stone,Achievements/place_table,Achievements/wake_up
+pretrained,0.6357,0.0,0.5006,0.4313,0.8907,0.9154,0.9953,0.208,0.6663,0.4501,0.0,0.0517,0.2385,0.7203,0.792,0.9553,0.812,0.879,0.6381,0.7286,0.9835,0.3831
+bc_all,0.1616,0.0,0.2148,0.0772,0.9468,0.3702,0.8827,0.0496,0.2506,0.1475,0.0,0.003,0.0181,0.1856,0.2562,0.4224,0.1741,0.3354,0.3167,0.2572,0.5177,0.0476

experiments/rl_finetuning/outputs/review_run1_bc_all/tables/per_env.tex

@@ -0,0 +1,13 @@
+\begin{table}[htbp]
+\centering
+\caption{Per-environment (per-achievement) win rates at final eval.}
+\label{tab:per_env}
+\begin{tabular}{lrrrrrrrrrrrrrrrrrrrrrr}
+\toprule
+\textbf{Method} & \textbf{Achievements/collect\_coal} & \textbf{Achievements/collect\_diamond} & \textbf{Achievements/collect\_drink} & \textbf{Achievements/collect\_iron} & \textbf{Achievements/collect\_sapling} & \textbf{Achievements/collect\_stone} & \textbf{Achievements/collect\_wood} & \textbf{Achievements/defeat\_skeleton} & \textbf{Achievements/defeat\_zombie} & \textbf{Achievements/eat\_cow} & \textbf{Achievements/eat\_plant} & \textbf{Achievements/make\_iron\_pickaxe} & \textbf{Achievements/make\_iron\_sword} & \textbf{Achievements/make\_stone\_pickaxe} & \textbf{Achievements/make\_stone\_sword} & \textbf{Achievements/make\_wood\_pickaxe} & \textbf{Achievements/make\_wood\_sword} & \textbf{Achievements/place\_furnace} & \textbf{Achievements/place\_plant} & \textbf{Achievements/place\_stone} & \textbf{Achievements/place\_table} & \textbf{Achievements/wake\_up} \\
+\midrule
+pretrained & 0.6357 & 0.0000 & 0.5006 & 0.4313 & 0.8907 & 0.9154 & 0.9953 & 0.2080 & 0.6663 & 0.4501 & 0.0000 & 0.0517 & 0.2385 & 0.7203 & 0.7920 & 0.9553 & 0.8120 & 0.8790 & 0.6381 & 0.7286 & 0.9835 & 0.3831 \\
+bc\_all & 0.1616 & 0.0000 & 0.2148 & 0.0772 & 0.9468 & 0.3702 & 0.8827 & 0.0496 & 0.2506 & 0.1475 & 0.0000 & 0.0030 & 0.0181 & 0.1856 & 0.2562 & 0.4224 & 0.1741 & 0.3354 & 0.3167 & 0.2572 & 0.5177 & 0.0476 \\
+\bottomrule
+\end{tabular}
+\end{table}

experiments/rl_finetuning/outputs/review_run1_bc_all/tables/repr_drift.csv

@@ -0,0 +1,2 @@
+Method,KL_mean,KL_low_t,KL_mid_t,KL_high_t
+bc_all,0.367367,0.268196,0.310345,0.564531

experiments/rl_finetuning/outputs/review_run1_bc_all/tables/repr_drift.tex

@@ -0,0 +1,12 @@
+\begin{table}[htbp]
+\centering
+\caption{Representation drift (KL divergence) at final iteration.}
+\label{tab:repr_drift}
+\begin{tabular}{lrrrr}
+\toprule
+\textbf{Method} & \textbf{KL\_mean} & \textbf{KL\_low\_t} & \textbf{KL\_mid\_t} & \textbf{KL\_high\_t} \\
+\midrule
+bc\_all & 0.3674 & 0.2682 & 0.3103 & 0.5645 \\
+\bottomrule
+\end{tabular}
+\end{table}

experiments/rl_finetuning/outputs/review_run1_bc_all/tables/t_distribution.csv

@@ -0,0 +1,2 @@
+Method,HighLow_Ratio,LowHigh_Cos_Sim,Dominant_Regime
+bc_all,3.18,0.1141,high-t

experiments/rl_finetuning/outputs/review_run1_bc_all/tables/t_distribution.tex

@@ -0,0 +1,12 @@
+\begin{table}[htbp]
+\centering
+\caption{Timestep distribution analysis.}
+\label{tab:t_dist}
+\begin{tabular}{lrrr}
+\toprule
+\textbf{Method} & \textbf{HighLow\_Ratio} & \textbf{LowHigh\_Cos\_Sim} & \textbf{Dominant\_Regime} \\
+\midrule
+bc\_all & 3.1800 & 0.1141 & high-t \\
+\bottomrule
+\end{tabular}
+\end{table}

experiments/rl_finetuning/outputs/review_run2_advclip_lr_matched/diagnosis.md

@@ -0,0 +1,59 @@
+# RL Fine-Tuning Ablation Suite — Diagnosis Report
+
+**Pretrained baseline score:** 11.9754
+**Baseline RL score:** nan
+
+---
+
+## Primary Failure Mode
+
+**ALL ablations collapsed.** This is the strongest evidence for a fundamental
+incompatibility between the RL fine-tuning signal and the model.
+
+**Most likely failure mode:** Catastrophic Forgetting
+> The pretrained representations are corrupted by RL gradients.
+
+**Evidence strength:** 0% (0/0 supporting ablations succeeded)
+
+---
+
+## Hypothesis Rankings (by Evidence Strength)
+
+| Hypothesis | Evidence | Supporting Ablations |
+|---|---|---|
+| Catastrophic Forgetting | 0% (0/0) | — |
+| Gradient Conflict | 0% (0/0) | — |
+| Signal Sparsity | 0% (0/0) | — |
+| Distributional Shift | 0% (0/0) | — |
+| Mode Collapse | 0% (0/1) | — |
+| t-Bias | 0% (0/0) | — |
+
+---
+
+## Evidence Details per Ablation
+
+### advantage_clip  [COLLAPSE]
+- **Score:** 4.9246  (delta vs pretrained: -7.0509, delta vs baseline_rl: -7.0509)
+- **Hypothesis tested:** If clipping helps: large advantage magnitudes destabilise training
+- **Mean grad alignment:** -0.0182
+- **Final KL drift:** 0.329165
+
+---
+
+## Recommendations
+
+### Catastrophic Forgetting
+Implement a strong parameter regularisation regime (EWC + LLRD) or use LoRA to restrict the parameter update space.
+
+### Gradient Conflict
+Apply PCGrad in the full training pipeline, and investigate whether the t-distribution of RL batches is biased.
+
+---
+
+## Next Experiments
+
+Based on the pattern of successes and failures above, the highest-priority
+follow-up experiments are:
+
+3. **Increase num_envs and num_seeds**: current results may be high-variance. Confirm findings with num_seeds=3.
+4. **Profile the reward signal**: plot the return histogram over training to determine if rewards are collapsing to zero.

experiments/rl_finetuning/outputs/review_run2_advclip_lr_matched/tables/achievement_summary.csv

@@ -0,0 +1,23 @@
+Achievement,Pretrained,advantage_clip,delta_advantage_clip
+Achievements/collect_coal,0.6357,0.1778,-0.4579
+Achievements/collect_diamond,0.0,0.0,0.0
+Achievements/collect_drink,0.5006,0.2297,-0.2709
+Achievements/collect_iron,0.4313,0.0795,-0.3518
+Achievements/collect_sapling,0.8907,0.9577,0.067
+Achievements/collect_stone,0.9154,0.3811,-0.5343
+Achievements/collect_wood,0.9953,0.8964,-0.0989
+Achievements/defeat_skeleton,0.208,0.05,-0.158
+Achievements/defeat_zombie,0.6663,0.2753,-0.391
+Achievements/eat_cow,0.4501,0.1431,-0.307
+Achievements/eat_plant,0.0,0.0,0.0
+Achievements/make_iron_pickaxe,0.0517,0.002,-0.0497
+Achievements/make_iron_sword,0.2385,0.0245,-0.214
+Achievements/make_stone_pickaxe,0.7203,0.2008,-0.5195
+Achievements/make_stone_sword,0.792,0.2725,-0.5195
+Achievements/make_wood_pickaxe,0.9553,0.4327,-0.5227
+Achievements/make_wood_sword,0.812,0.1952,-0.6168
+Achievements/place_furnace,0.879,0.3435,-0.5355
+Achievements/place_plant,0.6381,0.3179,-0.3202
+Achievements/place_stone,0.7286,0.2638,-0.4647
+Achievements/place_table,0.9835,0.5229,-0.4607
+Achievements/wake_up,0.3831,0.0583,-0.3248