ABPrune-Llama32-3B-final

.gitattributes

@@ -48,3 +48,4 @@ runs/vast-paper_revision-abprune4-48731700/20260826-1406_llama32-1b-abprune4-wik
 runs/vast-paper_revision-fusegpt-epochs-48746991/20260826-1712_llama32-1b-fusegpt-epoch1/removed_4/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 runs/vast-paper_revision-fusegpt-epochs-48746991/20260826-1712_llama32-1b-fusegpt-epoch5/removed_4/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 runs/vast-paper_revision-abprune-recoverability-48992755/20260828-180214_mlp-only-recoverability/models/qwen3_4b_final/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+runs/vast-paper_revision-abprune-recoverability-48992755/20260828-180214_mlp-only-recoverability/models/llama32_3b_final/tokenizer.json filter=lfs diff=lfs merge=lfs -text

runs/vast-paper_revision-abprune-recoverability-48992755/20260828-180214_mlp-only-recoverability/models/llama32_3b_final/config.json

@@ -0,0 +1,36 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "dtype": "bfloat16",
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 8,
+  "pad_token_id": null,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_parameters": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_theta": 500000.0,
+    "rope_type": "llama3"
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.3.0",
+  "use_cache": false,
+  "vocab_size": 128256
+}

runs/vast-paper_revision-abprune-recoverability-48992755/20260828-180214_mlp-only-recoverability/models/llama32_3b_final/generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "5.3.0"
+}

runs/vast-paper_revision-abprune-recoverability-48992755/20260828-180214_mlp-only-recoverability/models/llama32_3b_final/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:beffc865a546f0aa9d958a532c08b04c804c46fdaf8df619e620624040efcf8f
+size 5620169400

runs/vast-paper_revision-abprune-recoverability-48992755/20260828-180214_mlp-only-recoverability/models/llama32_3b_final/progressive_metadata.json

@@ -0,0 +1,283 @@
+{
+  "base_model": "meta-llama/Llama-3.2-3B",
+  "num_progressive": 4,
+  "layer_path": null,
+  "resume_from_cycle": 0,
+  "profile_no_save": false,
+  "save_full_model_cycles": [
+    1,
+    2,
+    3,
+    4
+  ],
+  "num_sequences": 128,
+  "seq_len": 2048,
+  "lora": {
+    "enabled": false,
+    "ran": false,
+    "seq_len": 2048,
+    "batch_size": 1,
+    "epochs": 0.0,
+    "rank": 8,
+    "alpha": 16.0,
+    "dropout": 0.0,
+    "target_modules": [
+      "q_proj",
+      "k_proj",
+      "v_proj",
+      "o_proj",
+      "gate_proj",
+      "down_proj",
+      "up_proj"
+    ],
+    "respect_exclude_pairs": false,
+    "kl_enabled": false,
+    "kl_weight": 0.1,
+    "kl_temp": 4.0,
+    "post_ppl": null,
+    "ppl_over_lora": null,
+    "lr": 0.0001,
+    "weight_decay": 0.0,
+    "max_grad_norm": 1.0,
+    "grad_accum_steps": 1,
+    "log_steps": 100,
+    "eval_every": 2000,
+    "eval_max_batches": null
+  },
+  "artifacts": {
+    "final_pre_lora": {
+      "stage": "final_pre_lora",
+      "path": "/workspace/abprune_outputs/20260828-180214_mlp-only-recoverability/llama32_3b_mlp_only_final_pre_lora_hf",
+      "weight_bytes": 5637380696,
+      "post_ppl": {
+        "wikitext2": 13.358069060664373
+      }
+    },
+    "final_post_lora": {
+      "stage": "final_model",
+      "path": "/workspace/abprune_outputs/20260828-180214_mlp-only-recoverability/llama32_3b_mlp_only",
+      "weight_bytes": 30200453400,
+      "post_ppl": null
+    }
+  },
+  "eval": {
+    "datasets": [
+      "wikitext2"
+    ],
+    "configs": [
+      null
+    ],
+    "split": "test",
+    "num_samples": 0,
+    "seq_len": 2048,
+    "pre_ppl": {
+      "wikitext2": 7.813860440257022
+    },
+    "post_ppl": {
+      "wikitext2": 13.358069060664373
+    }
+  },
+  "cycles": [
+    {
+      "cycle": 1,
+      "layer_merged": 19,
+      "dwce_score": null,
+      "comm_post_ppl": null,
+      "merge_only_ppl": null,
+      "distill_post_ppl": {
+        "wikitext2": 9.10644612032909
+      },
+      "lora_post_ppl": null,
+      "post_ppl": {
+        "wikitext2": 9.10644612032909
+      },
+      "cycle_dir": "cycle_1",
+      "timing": {
+        "seconds": {
+          "selection": 7.664505392313004e-05,
+          "rebase": 0.0,
+          "fisher": 39.878104789182544,
+          "fit": 551.2670529559255,
+          "materialize_save_layer": 11.77531522512436,
+          "metadata_write": 0.000635005533695221,
+          "teacher_snapshot": 0.058126578107476234,
+          "cycle_cleanup": 0.16294351872056723,
+          "cycle_total": 610.3934094570577
+        },
+        "tracked_component_total_seconds": 603.142254717648,
+        "cycle_total_seconds": 610.3934094570577,
+        "ratios_of_tracked_components": {
+          "selection": 1.2707624664600936e-07,
+          "rebase": 0.0,
+          "fisher": 0.0661172459353737,
+          "fit": 0.9139917633759432,
+          "materialize_save_layer": 0.019523280176476435,
+          "metadata_write": 1.052828795741544e-06,
+          "teacher_snapshot": 9.637291642696683e-05,
+          "cycle_cleanup": 0.0002701576907372321
+        },
+        "ratios_of_cycle_total": {
+          "selection": 1.255666472403519e-07,
+          "rebase": 0.0,
+          "fisher": 0.06533180760364687,
+          "fit": 0.903134018839219,
+          "materialize_save_layer": 0.01929135380999354,
+          "metadata_write": 1.0403217398104864e-06,
+          "teacher_snapshot": 9.522805654009203e-05,
+          "cycle_cleanup": 0.0002669483585438853
+        }
+      }
+    },
+    {
+      "cycle": 2,
+      "layer_merged": 19,
+      "dwce_score": null,
+      "comm_post_ppl": null,
+      "merge_only_ppl": null,
+      "distill_post_ppl": {
+        "wikitext2": 10.395807204908728
+      },
+      "lora_post_ppl": null,
+      "post_ppl": {
+        "wikitext2": 10.395807204908728
+      },
+      "cycle_dir": "cycle_2",
+      "timing": {
+        "seconds": {
+          "selection": 2.1005980670452118e-05,
+          "rebase": 0.0,
+          "fisher": 36.03837434761226,
+          "fit": 544.6739859431982,
+          "materialize_save_layer": 11.367755050770938,
+          "metadata_write": 0.000926886685192585,
+          "teacher_snapshot": 0.31084288097918034,
+          "cycle_cleanup": 0.15985145792365074,
+          "cycle_total": 600.502872178331
+        },
+        "tracked_component_total_seconds": 592.5517575731501,
+        "cycle_total_seconds": 600.502872178331,
+        "ratios_of_tracked_components": {
+          "selection": 3.5450035211243033e-08,
+          "rebase": 0.0,
+          "fisher": 0.06081894769025869,
+          "fit": 0.9192006925672794,
+          "materialize_save_layer": 0.019184408628418585,
+          "metadata_write": 1.5642290708726174e-06,
+          "teacher_snapshot": 0.0005245835102274707,
+          "cycle_cleanup": 0.00026976792470979584
+        },
+        "ratios_of_cycle_total": {
+          "selection": 3.498064979148673e-08,
+          "rebase": 0.0,
+          "fisher": 0.060013658580653655,
+          "fit": 0.9070297765061258,
+          "materialize_save_layer": 0.01893039247178665,
+          "metadata_write": 1.5435174886512915e-06,
+          "teacher_snapshot": 0.0005176376256979325,
+          "cycle_cleanup": 0.0002661959922752538
+        }
+      }
+    },
+    {
+      "cycle": 3,
+      "layer_merged": 19,
+      "dwce_score": null,
+      "comm_post_ppl": null,
+      "merge_only_ppl": null,
+      "distill_post_ppl": {
+        "wikitext2": 11.985243243632604
+      },
+      "lora_post_ppl": null,
+      "post_ppl": {
+        "wikitext2": 11.985243243632604
+      },
+      "cycle_dir": "cycle_3",
+      "timing": {
+        "seconds": {
+          "selection": 2.8845854103565216e-05,
+          "rebase": 0.0,
+          "fisher": 30.80826369393617,
+          "fit": 529.9406791003421,
+          "materialize_save_layer": 14.179267317987978,
+          "metadata_write": 0.0008358554914593697,
+          "teacher_snapshot": 0.3201894089579582,
+          "cycle_cleanup": 0.12816663831472397,
+          "cycle_total": 583.6494344500825
+        },
+        "tracked_component_total_seconds": 575.3774308608845,
+        "cycle_total_seconds": 583.6494344500825,
+        "ratios_of_tracked_components": {
+          "selection": 5.0133794890782925e-08,
+          "rebase": 0.0,
+          "fisher": 0.053544442380787496,
+          "fit": 0.921031397264645,
+          "materialize_save_layer": 0.02464341935826861,
+          "metadata_write": 1.4527081644630304e-06,
+          "teacher_snapshot": 0.0005564858678570137,
+          "cycle_cleanup": 0.00022275228648256152
+        },
+        "ratios_of_cycle_total": {
+          "selection": 4.942325375633051e-08,
+          "rebase": 0.0,
+          "fisher": 0.05278556248917447,
+          "fit": 0.9079777137103798,
+          "materialize_save_layer": 0.02429415070254931,
+          "metadata_write": 1.4321190806034396e-06,
+          "teacher_snapshot": 0.0005485988507118873,
+          "cycle_cleanup": 0.00021959524116644307
+        }
+      }
+    },
+    {
+      "cycle": 4,
+      "layer_merged": 19,
+      "dwce_score": null,
+      "comm_post_ppl": null,
+      "merge_only_ppl": null,
+      "distill_post_ppl": {
+        "wikitext2": 13.358069060664373
+      },
+      "lora_post_ppl": null,
+      "post_ppl": {
+        "wikitext2": 13.358069060664373
+      },
+      "cycle_dir": "cycle_4",
+      "timing": {
+        "seconds": {
+          "selection": 2.8915703296661377e-05,
+          "rebase": 0.0,
+          "fisher": 35.686987621709704,
+          "fit": 504.39452768024057,
+          "materialize_save_layer": 10.595756338909268,
+          "metadata_write": 0.0005818037316203117,
+          "teacher_snapshot": 0.16267723310738802,
+          "cycle_cleanup": 0.0980784185230732,
+          "cycle_total": 557.4720219066367
+        },
+        "tracked_component_total_seconds": 550.9386380119249,
+        "cycle_total_seconds": 557.4720219066367,
+        "ratios_of_tracked_components": {
+          "selection": 5.248443529211234e-08,
+          "rebase": 0.0,
+          "fisher": 0.06477488627497073,
+          "fit": 0.9155185221721971,
+          "materialize_save_layer": 0.019232189590376716,
+          "metadata_write": 1.0560227427863187e-06,
+          "teacher_snapshot": 0.0002952728704859268,
+          "cycle_cleanup": 0.00017802058479142338
+        },
+        "ratios_of_cycle_total": {
+          "selection": 5.1869335429185126e-08,
+          "rebase": 0.0,
+          "fisher": 0.0640157464757692,
+          "fit": 0.9047889541705372,
+          "materialize_save_layer": 0.01900679482114675,
+          "metadata_write": 1.043646512753155e-06,
+          "teacher_snapshot": 0.0002918123721276771,
+          "cycle_cleanup": 0.0001759342436372511
+        }
+      }
+    }
+  ],
+  "final_num_layers": 24
+}

runs/vast-paper_revision-abprune-recoverability-48992755/20260828-180214_mlp-only-recoverability/models/llama32_3b_final/run_args.txt

@@ -0,0 +1,10 @@
+git_commit=149726d2f34de695886cf0f2aa64ff80b4d8c64f
+start_time=2026-08-28T09:02:26+00:00
+HEAD_PERMUTE=0
+HEAD_PERMUTE_SELECT=0
+HEAD_PERMUTE_MERGE=0
+command:
+/workspace/abprune_runtime/envs/abprune/bin/python /workspace/paper_revision/external/abprune_update/src/fuse_layers_order.py --model meta-llama/Llama-3.2-3B --dataset wikitext2 --dataset_config wikitext-2-raw-v1 --target_tokens 2097152 --num_samples 128 --seq_len 2048 --batch_size 2 --distill_batch_size 1 --distill_seq_len 2048 --distill_epochs 1.0 --eval_batch_size 4 --eval_seq_len 2048 --eval_num_samples 0 --distill_kl_weight 0.02 --distill_kl_temp 4.0 --distill_hidden_mse_weight 1.0 --distill_attn_mse_weight 0.0 --distill_mlp_mse_weight 1.0 --reparam_eta 0 --reparam_gamma 0 --reparam_attn_reg_scale 1.0 --reparam_mlp_reg_scale 1.0 --reparam_param_subset mlp --reparam_train_subset same --distill_weight_decay 0.0 --distill_max_grad_norm 1.0 --distill_grad_accum_steps 1 --distill_eval_every 2000 --lora_eval_every 2000 --lora_epochs 0 --fisher_mode param --save_full_model_cycles 1\,2\,3\,4 --distill_method reparam --redistrib_teacher_source previous_cycle --exclude_pairs 0\,1\,-1 --num_progressive 4 --output_dir /workspace/abprune_outputs/20260828-180214_mlp-only-recoverability/llama32_3b_mlp_only --dtype bfloat16 --comm_mu_auto --comm_skip_post_reselect --no_head_permute_select --no_head_permute_merge --layer_order 19 19 19 19 --distill_log_steps 100 
+end_time=2026-08-28T09:42:58+00:00
+elapsed_seconds=2432
+exit_code=0

runs/vast-paper_revision-abprune-recoverability-48992755/20260828-180214_mlp-only-recoverability/models/llama32_3b_final/stage_metrics.json

@@ -0,0 +1,6 @@
+{
+  "stage": "final_model",
+  "path": "/workspace/abprune_outputs/20260828-180214_mlp-only-recoverability/llama32_3b_mlp_only",
+  "weight_bytes": 30200453400,
+  "post_ppl": null
+}

runs/vast-paper_revision-abprune-recoverability-48992755/20260828-180214_mlp-only-recoverability/models/llama32_3b_final/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

runs/vast-paper_revision-abprune-recoverability-48992755/20260828-180214_mlp-only-recoverability/models/llama32_3b_final/tokenizer_config.json

@@ -0,0 +1,14 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|end_of_text|>",
+  "is_local": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|end_of_text|>",
+  "tokenizer_class": "TokenizersBackend"
+}