Upload-contextual-Fisher-4B-artifacts

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/manifest.json

@@ -0,0 +1,161 @@
+{
+  "format": "abprune_compact_manifest_v1",
+  "model": "Qwen/Qwen3-4B-Base",
+  "base_model_revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539",
+  "run_id": "qwen3_4b_anchor_input_no_rho_contextual_fisher_v1",
+  "target_prune_count": 5,
+  "layer_order": [
+    25,
+    25,
+    25,
+    25,
+    25
+  ],
+  "stages": [
+    {
+      "format": "abprune_compact_stage_v1",
+      "model": "Qwen/Qwen3-4B-Base",
+      "base_model_revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539",
+      "run_id": "qwen3_4b_anchor_input_no_rho_contextual_fisher_v1",
+      "target_prune_count": 5,
+      "stage": 1,
+      "removed_count": 1,
+      "anchor_index": 25,
+      "layer_order": [
+        25,
+        25,
+        25,
+        25,
+        25
+      ],
+      "packaged_stage_range": [
+        1,
+        5
+      ],
+      "removed_original_layers": [
+        26
+      ],
+      "source_cycle": "cycle_1",
+      "anchor_state_file": "anchor_layer.pt",
+      "sha256": "2ea21e63d8385b7c6b32cb47cb5a93198822649e15ac1ec7e1a401aba2f6ce2f"
+    },
+    {
+      "format": "abprune_compact_stage_v1",
+      "model": "Qwen/Qwen3-4B-Base",
+      "base_model_revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539",
+      "run_id": "qwen3_4b_anchor_input_no_rho_contextual_fisher_v1",
+      "target_prune_count": 5,
+      "stage": 2,
+      "removed_count": 2,
+      "anchor_index": 25,
+      "layer_order": [
+        25,
+        25,
+        25,
+        25,
+        25
+      ],
+      "packaged_stage_range": [
+        1,
+        5
+      ],
+      "removed_original_layers": [
+        26,
+        27
+      ],
+      "source_cycle": "cycle_2",
+      "anchor_state_file": "anchor_layer.pt",
+      "sha256": "b4c0450e441f2366f09cc6b5b72fc17d1f845cb9e8be7add342281cc27b40a6c"
+    },
+    {
+      "format": "abprune_compact_stage_v1",
+      "model": "Qwen/Qwen3-4B-Base",
+      "base_model_revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539",
+      "run_id": "qwen3_4b_anchor_input_no_rho_contextual_fisher_v1",
+      "target_prune_count": 5,
+      "stage": 3,
+      "removed_count": 3,
+      "anchor_index": 25,
+      "layer_order": [
+        25,
+        25,
+        25,
+        25,
+        25
+      ],
+      "packaged_stage_range": [
+        1,
+        5
+      ],
+      "removed_original_layers": [
+        26,
+        27,
+        28
+      ],
+      "source_cycle": "cycle_3",
+      "anchor_state_file": "anchor_layer.pt",
+      "sha256": "0c3aacef2687a4fdda11ff7fa75ceed7e55be1269fe40618b1e782f79d6745af"
+    },
+    {
+      "format": "abprune_compact_stage_v1",
+      "model": "Qwen/Qwen3-4B-Base",
+      "base_model_revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539",
+      "run_id": "qwen3_4b_anchor_input_no_rho_contextual_fisher_v1",
+      "target_prune_count": 5,
+      "stage": 4,
+      "removed_count": 4,
+      "anchor_index": 25,
+      "layer_order": [
+        25,
+        25,
+        25,
+        25,
+        25
+      ],
+      "packaged_stage_range": [
+        1,
+        5
+      ],
+      "removed_original_layers": [
+        26,
+        27,
+        28,
+        29
+      ],
+      "source_cycle": "cycle_4",
+      "anchor_state_file": "anchor_layer.pt",
+      "sha256": "36cdbc27d39227c16d250dd572efe3be8e4f38bacb89beac45d94102cf45ae11"
+    },
+    {
+      "format": "abprune_compact_stage_v1",
+      "model": "Qwen/Qwen3-4B-Base",
+      "base_model_revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539",
+      "run_id": "qwen3_4b_anchor_input_no_rho_contextual_fisher_v1",
+      "target_prune_count": 5,
+      "stage": 5,
+      "removed_count": 5,
+      "anchor_index": 25,
+      "layer_order": [
+        25,
+        25,
+        25,
+        25,
+        25
+      ],
+      "packaged_stage_range": [
+        1,
+        5
+      ],
+      "removed_original_layers": [
+        26,
+        27,
+        28,
+        29,
+        30
+      ],
+      "source_cycle": "cycle_5",
+      "anchor_state_file": "anchor_layer.pt",
+      "sha256": "a7812d906d202c1bd843be4aff0faabad90d2da61812e26b0f164a62e7faaed8"
+    }
+  ]
+}

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_01/anchor_layer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ea21e63d8385b7c6b32cb47cb5a93198822649e15ac1ec7e1a401aba2f6ce2f
+size 201866630

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_01/metadata.json

@@ -0,0 +1,27 @@
+{
+  "format": "abprune_compact_stage_v1",
+  "model": "Qwen/Qwen3-4B-Base",
+  "base_model_revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539",
+  "run_id": "qwen3_4b_anchor_input_no_rho_contextual_fisher_v1",
+  "target_prune_count": 5,
+  "stage": 1,
+  "removed_count": 1,
+  "anchor_index": 25,
+  "layer_order": [
+    25,
+    25,
+    25,
+    25,
+    25
+  ],
+  "packaged_stage_range": [
+    1,
+    5
+  ],
+  "removed_original_layers": [
+    26
+  ],
+  "source_cycle": "cycle_1",
+  "anchor_state_file": "anchor_layer.pt",
+  "sha256": "2ea21e63d8385b7c6b32cb47cb5a93198822649e15ac1ec7e1a401aba2f6ce2f"
+}

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_02/anchor_layer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4c0450e441f2366f09cc6b5b72fc17d1f845cb9e8be7add342281cc27b40a6c
+size 201866630

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_02/metadata.json

@@ -0,0 +1,28 @@
+{
+  "format": "abprune_compact_stage_v1",
+  "model": "Qwen/Qwen3-4B-Base",
+  "base_model_revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539",
+  "run_id": "qwen3_4b_anchor_input_no_rho_contextual_fisher_v1",
+  "target_prune_count": 5,
+  "stage": 2,
+  "removed_count": 2,
+  "anchor_index": 25,
+  "layer_order": [
+    25,
+    25,
+    25,
+    25,
+    25
+  ],
+  "packaged_stage_range": [
+    1,
+    5
+  ],
+  "removed_original_layers": [
+    26,
+    27
+  ],
+  "source_cycle": "cycle_2",
+  "anchor_state_file": "anchor_layer.pt",
+  "sha256": "b4c0450e441f2366f09cc6b5b72fc17d1f845cb9e8be7add342281cc27b40a6c"
+}

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_03/anchor_layer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c3aacef2687a4fdda11ff7fa75ceed7e55be1269fe40618b1e782f79d6745af
+size 201866630

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_03/metadata.json

@@ -0,0 +1,29 @@
+{
+  "format": "abprune_compact_stage_v1",
+  "model": "Qwen/Qwen3-4B-Base",
+  "base_model_revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539",
+  "run_id": "qwen3_4b_anchor_input_no_rho_contextual_fisher_v1",
+  "target_prune_count": 5,
+  "stage": 3,
+  "removed_count": 3,
+  "anchor_index": 25,
+  "layer_order": [
+    25,
+    25,
+    25,
+    25,
+    25
+  ],
+  "packaged_stage_range": [
+    1,
+    5
+  ],
+  "removed_original_layers": [
+    26,
+    27,
+    28
+  ],
+  "source_cycle": "cycle_3",
+  "anchor_state_file": "anchor_layer.pt",
+  "sha256": "0c3aacef2687a4fdda11ff7fa75ceed7e55be1269fe40618b1e782f79d6745af"
+}

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_04/anchor_layer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36cdbc27d39227c16d250dd572efe3be8e4f38bacb89beac45d94102cf45ae11
+size 201866630

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_04/metadata.json

@@ -0,0 +1,30 @@
+{
+  "format": "abprune_compact_stage_v1",
+  "model": "Qwen/Qwen3-4B-Base",
+  "base_model_revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539",
+  "run_id": "qwen3_4b_anchor_input_no_rho_contextual_fisher_v1",
+  "target_prune_count": 5,
+  "stage": 4,
+  "removed_count": 4,
+  "anchor_index": 25,
+  "layer_order": [
+    25,
+    25,
+    25,
+    25,
+    25
+  ],
+  "packaged_stage_range": [
+    1,
+    5
+  ],
+  "removed_original_layers": [
+    26,
+    27,
+    28,
+    29
+  ],
+  "source_cycle": "cycle_4",
+  "anchor_state_file": "anchor_layer.pt",
+  "sha256": "36cdbc27d39227c16d250dd572efe3be8e4f38bacb89beac45d94102cf45ae11"
+}

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_05/anchor_layer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7812d906d202c1bd843be4aff0faabad90d2da61812e26b0f164a62e7faaed8
+size 201866630

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_05/metadata.json

@@ -0,0 +1,31 @@
+{
+  "format": "abprune_compact_stage_v1",
+  "model": "Qwen/Qwen3-4B-Base",
+  "base_model_revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539",
+  "run_id": "qwen3_4b_anchor_input_no_rho_contextual_fisher_v1",
+  "target_prune_count": 5,
+  "stage": 5,
+  "removed_count": 5,
+  "anchor_index": 25,
+  "layer_order": [
+    25,
+    25,
+    25,
+    25,
+    25
+  ],
+  "packaged_stage_range": [
+    1,
+    5
+  ],
+  "removed_original_layers": [
+    26,
+    27,
+    28,
+    29,
+    30
+  ],
+  "source_cycle": "cycle_5",
+  "anchor_state_file": "anchor_layer.pt",
+  "sha256": "a7812d906d202c1bd843be4aff0faabad90d2da61812e26b0f164a62e7faaed8"
+}

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/eval/stage_01_ppl.json

@@ -0,0 +1,25 @@
+{
+  "model": "/workspace/models/qwen3-4b-base-906bfd4",
+  "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539",
+  "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_input_no_rho/compact/stage_01",
+  "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test",
+  "protocol": "HCHC experiment.py: filter empty rows, join with double newline, then tokenize the complete stream once",
+  "sequence_length": 2048,
+  "batch_size": 4,
+  "dtype": "bfloat16",
+  "metrics": {
+    "nll": 2.145418803445224,
+    "perplexity": 8.54561942281943,
+    "tokens": 296815,
+    "sequences": 145,
+    "source_tokens": 298938
+  },
+  "anchor": 25,
+  "cycles": 1,
+  "original_layers": 36,
+  "decoder_layers": 35,
+  "post_merge_residual_scale": null,
+  "post_merge_residual_scales": [],
+  "parameterization": "unscaled compact ABPrune",
+  "linear_patch": null
+}

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/eval/stage_02_ppl.json

@@ -0,0 +1,25 @@
+{
+  "model": "/workspace/models/qwen3-4b-base-906bfd4",
+  "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539",
+  "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_input_no_rho/compact/stage_02",
+  "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test",
+  "protocol": "HCHC experiment.py: filter empty rows, join with double newline, then tokenize the complete stream once",
+  "sequence_length": 2048,
+  "batch_size": 4,
+  "dtype": "bfloat16",
+  "metrics": {
+    "nll": 2.2925252815772748,
+    "perplexity": 9.899906192368073,
+    "tokens": 296815,
+    "sequences": 145,
+    "source_tokens": 298938
+  },
+  "anchor": 25,
+  "cycles": 2,
+  "original_layers": 36,
+  "decoder_layers": 34,
+  "post_merge_residual_scale": null,
+  "post_merge_residual_scales": [],
+  "parameterization": "unscaled compact ABPrune",
+  "linear_patch": null
+}

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/eval/stage_03_ppl.json

@@ -0,0 +1,25 @@
+{
+  "model": "/workspace/models/qwen3-4b-base-906bfd4",
+  "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539",
+  "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_input_no_rho/compact/stage_03",
+  "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test",
+  "protocol": "HCHC experiment.py: filter empty rows, join with double newline, then tokenize the complete stream once",
+  "sequence_length": 2048,
+  "batch_size": 4,
+  "dtype": "bfloat16",
+  "metrics": {
+    "nll": 2.4010539860560978,
+    "perplexity": 11.03480077974187,
+    "tokens": 296815,
+    "sequences": 145,
+    "source_tokens": 298938
+  },
+  "anchor": 25,
+  "cycles": 3,
+  "original_layers": 36,
+  "decoder_layers": 33,
+  "post_merge_residual_scale": null,
+  "post_merge_residual_scales": [],
+  "parameterization": "unscaled compact ABPrune",
+  "linear_patch": null
+}

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/eval/stage_04_ppl.json

@@ -0,0 +1,25 @@
+{
+  "model": "/workspace/models/qwen3-4b-base-906bfd4",
+  "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539",
+  "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_input_no_rho/compact/stage_04",
+  "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test",
+  "protocol": "HCHC experiment.py: filter empty rows, join with double newline, then tokenize the complete stream once",
+  "sequence_length": 2048,
+  "batch_size": 4,
+  "dtype": "bfloat16",
+  "metrics": {
+    "nll": 2.586987911421677,
+    "perplexity": 13.289681558091948,
+    "tokens": 296815,
+    "sequences": 145,
+    "source_tokens": 298938
+  },
+  "anchor": 25,
+  "cycles": 4,
+  "original_layers": 36,
+  "decoder_layers": 32,
+  "post_merge_residual_scale": null,
+  "post_merge_residual_scales": [],
+  "parameterization": "unscaled compact ABPrune",
+  "linear_patch": null
+}

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/eval/stage_05_ppl.json

@@ -0,0 +1,25 @@
+{
+  "model": "/workspace/models/qwen3-4b-base-906bfd4",
+  "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539",
+  "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_input_no_rho/compact/stage_05",
+  "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test",
+  "protocol": "HCHC experiment.py: filter empty rows, join with double newline, then tokenize the complete stream once",
+  "sequence_length": 2048,
+  "batch_size": 4,
+  "dtype": "bfloat16",
+  "metrics": {
+    "nll": 2.848904904003801,
+    "perplexity": 17.268860422166572,
+    "tokens": 296815,
+    "sequences": 145,
+    "source_tokens": 298938
+  },
+  "anchor": 25,
+  "cycles": 5,
+  "original_layers": 36,
+  "decoder_layers": 31,
+  "post_merge_residual_scale": null,
+  "post_merge_residual_scales": [],
+  "parameterization": "unscaled compact ABPrune",
+  "linear_patch": null
+}

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/eval/stage_05_zs6.json

@@ -0,0 +1,126 @@
+{
+  "model": "/workspace/models/qwen3-4b-base-906bfd4",
+  "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539",
+  "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_input_no_rho/compact/stage_05",
+  "cycle": 5,
+  "anchor": 25,
+  "post_merge_residual_scale": null,
+  "post_merge_residual_scales": [],
+  "tasks": [
+    "hellaswag",
+    "piqa",
+    "winogrande",
+    "arc_challenge",
+    "arc_easy",
+    "openbookqa"
+  ],
+  "preferred_metrics": {
+    "hellaswag": "acc_norm,none",
+    "piqa": "acc_norm,none",
+    "winogrande": "acc,none",
+    "arc_challenge": "acc_norm,none",
+    "arc_easy": "acc_norm,none",
+    "openbookqa": "acc_norm,none"
+  },
+  "task_scores": {
+    "hellaswag": 0.6079466241784505,
+    "piqa": 0.6931447225244831,
+    "winogrande": 0.6708760852407262,
+    "arc_challenge": 0.386518771331058,
+    "arc_easy": 0.5740740740740741,
+    "openbookqa": 0.324
+  },
+  "task_mean": 0.5427600462247986,
+  "six_task_mean": 0.5427600462247986,
+  "boolq_excluded": true,
+  "num_fewshot": 0,
+  "batch_size": "auto:4",
+  "linear_patch": null,
+  "lm_eval_results": {
+    "hellaswag": {
+      "name": "hellaswag",
+      "alias": "hellaswag",
+      "sample_len": 10042,
+      "acc,none": 0.4193387771360287,
+      "acc_stderr,none": 0.004924424018073678,
+      "acc_norm,none": 0.6079466241784505,
+      "acc_norm_stderr,none": 0.0048721072620824665
+    },
+    "piqa": {
+      "name": "piqa",
+      "alias": "piqa",
+      "sample_len": 1838,
+      "acc,none": 0.7002176278563657,
+      "acc_stderr,none": 0.010689686967138092,
+      "acc_norm,none": 0.6931447225244831,
+      "acc_norm_stderr,none": 0.01076029507058038
+    },
+    "winogrande": {
+      "name": "winogrande",
+      "alias": "winogrande",
+      "sample_len": 1267,
+      "acc,none": 0.6708760852407262,
+      "acc_stderr,none": 0.013206387089091474
+    },
+    "arc_challenge": {
+      "name": "arc_challenge",
+      "alias": "arc_challenge",
+      "sample_len": 1172,
+      "acc,none": 0.3361774744027304,
+      "acc_stderr,none": 0.013804855026205756,
+      "acc_norm,none": 0.386518771331058,
+      "acc_norm_stderr,none": 0.014230084761910473
+    },
+    "arc_easy": {
+      "name": "arc_easy",
+      "alias": "arc_easy",
+      "sample_len": 2376,
+      "acc,none": 0.5989057239057239,
+      "acc_stderr,none": 0.010057051106534367,
+      "acc_norm,none": 0.5740740740740741,
+      "acc_norm_stderr,none": 0.010146568651002255
+    },
+    "openbookqa": {
+      "name": "openbookqa",
+      "alias": "openbookqa",
+      "sample_len": 500,
+      "acc,none": 0.206,
+      "acc_stderr,none": 0.01810479403733356,
+      "acc_norm,none": 0.324,
+      "acc_norm_stderr,none": 0.02095055731247745
+    }
+  },
+  "lm_eval_versions": {
+    "arc_challenge": 1.0,
+    "arc_easy": 1.0,
+    "hellaswag": 1.0,
+    "openbookqa": 1.0,
+    "piqa": 1.0,
+    "winogrande": 1.0
+  },
+  "lm_eval_config": {
+    "model": "/workspace/models/qwen3-4b-base-906bfd4",
+    "model_args": null,
+    "model_num_parameters": 3517814016,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "",
+    "batch_size": "auto:4",
+    "batch_sizes": [
+      64,
+      64,
+      64,
+      64,
+      64
+    ],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  }
+}

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/logs/train.log

@@ -0,0 +1,166 @@
+
+Qwen3ForCausalLM(
+  (model): Qwen3Model(
+    (embed_tokens): Embedding(151936, 2560)
+    (layers): ModuleList(
+      (0-35): 36 x Qwen3DecoderLayer(
+        (self_attn): Qwen3Attention(
+          (q_proj): Linear(in_features=2560, out_features=4096, bias=False)
+          (k_proj): Linear(in_features=2560, out_features=1024, bias=False)
+          (v_proj): Linear(in_features=2560, out_features=1024, bias=False)
+          (o_proj): Linear(in_features=4096, out_features=2560, bias=False)
+          (q_norm): Qwen3RMSNorm((128,), eps=1e-06)
+          (k_norm): Qwen3RMSNorm((128,), eps=1e-06)
+        )
+        (mlp): Qwen3MLP(
+          (gate_proj): Linear(in_features=2560, out_features=9728, bias=False)
+          (up_proj): Linear(in_features=2560, out_features=9728, bias=False)
+          (down_proj): Linear(in_features=9728, out_features=2560, bias=False)
+          (act_fn): SiLUActivation()
+        )
+        (input_layernorm): Qwen3RMSNorm((2560,), eps=1e-06)
+        (post_attention_layernorm): Qwen3RMSNorm((2560,), eps=1e-06)
+      )
+    )
+    (norm): Qwen3RMSNorm((2560,), eps=1e-06)
+    (rotary_emb): Qwen3RotaryEmbedding()
+  )
+  (lm_head): Linear(in_features=2560, out_features=151936, bias=False)
+)
+[progressive] Cycle 1/5
+[fuse] Head permutation disabled; merging with original head order.
+/opt/conda/lib/python3.11/site-packages/transformers/models/qwen3/modeling_qwen3.py:143: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#results-reproducibility (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:233.)
+  freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
+/opt/conda/lib/python3.11/site-packages/torch/nn/modules/linear.py:125: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#results-reproducibility (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:233.)
+  return F.linear(input, self.weight, self.bias)
+/opt/conda/lib/python3.11/site-packages/torch/autograd/graph.py:829: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#results-reproducibility (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:233.)
+  return Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
+/opt/conda/lib/python3.11/site-packages/torch/autograd/graph.py:829: UserWarning: Flash Attention defaults to a non-deterministic algorithm. To explicitly enable determinism call torch.use_deterministic_algorithms(True, warn_only=False). (Triggered internally at /pytorch/aten/src/ATen/native/transformers/cuda/attention_backward.cu:110.)
+  return Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
+[fisher-input] mode=anchor_input cos=0.9340915456414223 norm_ratio=0.9107728470116854
+
+[reparam] fisher_prior_inputs fa_mean=2.9234447159372697e-09 fb_mean=2.255867078323111e-09 active_ratio=1.0 rel_contrast=0.36681776883309347 prior_mean=0.5411659374571683 prior_active_mean=0.5411659280459086
+[reparam] Cycle 1: training U + gates for pair 25-26 (epochs=1.0, skip_train=False, hidden_mse_w=1.0, attn_mse_w=0.0, mlp_mse_w=1.0, eta=0.0, gamma=0.0, attn_reg_scale=1.0, mlp_reg_scale=1.0, param_subset=mlp, fisher_init_subset=same, train_subset=same, lambda_init=fisher_prior).
+[reparam] subset=mlp gates=3 (attn=0, mlp=3, other=0) train_subset=mlp (attn=0, mlp=3, other=0) elems=74711040 (~285.0 MiB), U_elems=74711040 (~285.0 MiB; +optimizer state)
+[reparam] lambda_init mean=0.541165950842071 std=0.2155650558184483 min=0.0006457812269218266 max=0.9998443126678467 mean_abs_from_0.5=0.18340888441654674
+[reparam] first_grad phase=joint step=1 gate_l2=6.677313e-03 gate_mean_abs=1.0369183559975537e-07 gate_max_abs=3.420014e-03 u_l2=7.087303e-01 u_mean_abs=8.06867079645918e-06 u_max_abs=1.947609e-01 rho_l2=0.000000e+00
+[reparam] epoch=1 step=1 phase=joint loss=1.995709e+00 mse=1.315175e+00 mlp_mse=6.757779e-01 kl=1.486308e-02 lambda_mean=0.5411658303779469 lambda_std=0.21556509434620413 lambda_abs0.5=0.1834089111863521
+[reparam] epoch=1 step=100 phase=joint loss=1.709446e+00 mse=1.090492e+00 mlp_mse=6.154334e-01 kl=1.100188e-02 lambda_mean=0.5411567018743146 lambda_std=0.21557913118733563 lambda_abs0.5=0.18342588324295847
+[reparam] epoch=1 step=200 phase=joint loss=1.461789e+00 mse=9.137141e-01 mlp_mse=5.451486e-01 kl=9.144066e-03 lambda_mean=0.541145552250377 lambda_std=0.21560418408116053 lambda_abs0.5=0.18345907780162074
+[reparam] epoch=1 step=300 phase=joint loss=1.356519e+00 mse=8.302782e-01 mlp_mse=5.237296e-01 kl=7.847066e-03 lambda_mean=0.541133853845429 lambda_std=0.2156325349448108 lambda_abs0.5=0.18349779363264118
+[reparam] epoch=1 step=400 phase=joint loss=1.188608e+00 mse=7.273744e-01 mlp_mse=4.587298e-01 kl=7.824773e-03 lambda_mean=0.5411215665047628 lambda_std=0.21566127303443094 lambda_abs0.5=0.1835378546463816
+[reparam] epoch=1 step=500 phase=joint loss=1.273348e+00 mse=7.721003e-01 mlp_mse=4.979949e-01 kl=1.016577e-02 lambda_mean=0.5411094665527344 lambda_std=0.21569094826291602 lambda_abs0.5=0.18357972262198466
+[timing] Cycle 1 component wall-clock:
+[timing]   selection: 0.000s (0.00% of cycle)
+[timing]   rebase: 0.000s (0.00% of cycle)
+[timing]   fisher: 18.807s (6.95% of cycle)
+[timing]   fit: 251.186s (92.83% of cycle)
+[timing]   materialize_save_layer: 0.303s (0.11% of cycle)
+[timing]   metadata_write: 0.002s (0.00% of cycle)
+[timing]   teacher_snapshot: 0.095s (0.04% of cycle)
+[timing]   cycle_cleanup: 0.177s (0.07% of cycle)
+[timing]   cycle_total: 270.575s
+[timing]   tracked_components_total: 270.570s
+[progressive] Cycle 2/5
+[fuse] Head permutation disabled; merging with original head order.
+[fisher-input] mode=anchor_input cos=0.9501073732972145 norm_ratio=0.8818922247737646
+[reparam] fisher_prior_inputs fa_mean=2.3329901846648824e-09 fb_mean=3.0635342746041206e-09 active_ratio=1.0 rel_contrast=0.3967977958813048 prior_mean=0.40642471313476564 prior_active_mean=0.40642471114794415
+[reparam] Cycle 2: training U + gates for pair 25-26 (epochs=1.0, skip_train=False, hidden_mse_w=1.0, attn_mse_w=0.0, mlp_mse_w=1.0, eta=0.0, gamma=0.0, attn_reg_scale=1.0, mlp_reg_scale=1.0, param_subset=mlp, fisher_init_subset=same, train_subset=same, lambda_init=fisher_prior).
+[reparam] subset=mlp gates=3 (attn=0, mlp=3, other=0) train_subset=mlp (attn=0, mlp=3, other=0) elems=74711040 (~285.0 MiB), U_elems=74711040 (~285.0 MiB; +optimizer state)
+[reparam] lambda_init mean=0.4064247265196683 std=0.21451395279119853 min=0.0001658098481129855 max=0.9995361566543579 mean_abs_from_0.5=0.1983989113255551
+[reparam] first_grad phase=joint step=1 gate_l2=6.575216e-03 gate_mean_abs=1.034208364951171e-07 gate_max_abs=1.666576e-03 u_l2=9.438444e-01 u_mean_abs=1.0214174337881177e-05 u_max_abs=2.991875e-01 rho_l2=0.000000e+00
+[reparam] epoch=1 step=1 phase=joint loss=2.022338e+00 mse=1.438241e+00 mlp_mse=5.806731e-01 kl=1.070173e-02 lambda_mean=0.40642451236122534 lambda_std=0.2145140621592356 lambda_abs0.5=0.1983990384821306
+[reparam] epoch=1 step=100 phase=joint loss=1.769435e+00 mse=1.118005e+00 mlp_mse=6.469180e-01 kl=1.410097e-02 lambda_mean=0.4063954938922012 lambda_std=0.21452795095136692 lambda_abs0.5=0.19842904743395354
+[reparam] epoch=1 step=200 phase=joint loss=1.214479e+00 mse=7.512352e-01 mlp_mse=4.595519e-01 kl=1.153888e-02 lambda_mean=0.4063581500137061 lambda_std=0.21454938423127834 lambda_abs0.5=0.19847566035755893
+[reparam] epoch=1 step=300 phase=joint loss=1.277933e+00 mse=8.076477e-01 mlp_mse=4.675763e-01 kl=8.465483e-03 lambda_mean=0.4063199762712445 lambda_std=0.21457156939667468 lambda_abs0.5=0.19852596082185445
+[reparam] epoch=1 step=400 phase=joint loss=1.118311e+00 mse=6.829595e-01 mlp_mse=4.332641e-01 kl=6.524324e-03 lambda_mean=0.40628162852504796 lambda_std=0.21459161115936864 lambda_abs0.5=0.19857431378280907
+[reparam] epoch=1 step=500 phase=joint loss=1.116184e+00 mse=6.761870e-01 mlp_mse=4.375716e-01 kl=7.580361e-03 lambda_mean=0.4062451011256168 lambda_std=0.21461139477535568 lambda_abs0.5=0.19862278051543655
+[timing] Cycle 2 component wall-clock:
+[timing]   selection: 0.000s (0.00% of cycle)
+[timing]   rebase: 0.000s (0.00% of cycle)
+[timing]   fisher: 15.864s (6.13% of cycle)
+[timing]   fit: 242.408s (93.61% of cycle)
+[timing]   materialize_save_layer: 0.234s (0.09% of cycle)
+[timing]   metadata_write: 0.002s (0.00% of cycle)
+[timing]   teacher_snapshot: 0.262s (0.10% of cycle)
+[timing]   cycle_cleanup: 0.173s (0.07% of cycle)
+[timing]   cycle_total: 258.944s
+[timing]   tracked_components_total: 258.944s
+[progressive] Cycle 3/5
+[fuse] Head permutation disabled; merging with original head order.
+[fisher-input] mode=anchor_input cos=0.9345197342336178 norm_ratio=0.8426651768386364
+[reparam] fisher_prior_inputs fa_mean=3.4907529560880728e-09 fb_mean=3.608339152618244e-09 active_ratio=1.0 rel_contrast=0.4211514255456757 prior_mean=0.4212010299950315 prior_active_mean=0.42120103041330975
+[reparam] Cycle 3: training U + gates for pair 25-26 (epochs=1.0, skip_train=False, hidden_mse_w=1.0, attn_mse_w=0.0, mlp_mse_w=1.0, eta=0.0, gamma=0.0, attn_reg_scale=1.0, mlp_reg_scale=1.0, param_subset=mlp, fisher_init_subset=same, train_subset=same, lambda_init=fisher_prior).
+[reparam] subset=mlp gates=3 (attn=0, mlp=3, other=0) train_subset=mlp (attn=0, mlp=3, other=0) elems=74711040 (~285.0 MiB), U_elems=74711040 (~285.0 MiB; +optimizer state)
+[reparam] lambda_init mean=0.4212010299950315 std=0.23356970592726928 min=9.999999747378752e-05 max=0.9997745156288147 mean_abs_from_0.5=0.2105757194652892
+[reparam] first_grad phase=joint step=1 gate_l2=1.332312e-02 gate_mean_abs=1.342424519335557e-07 gate_max_abs=4.482283e-03 u_l2=1.417598e+00 u_mean_abs=1.4800271182729486e-05 u_max_abs=3.731053e-01 rho_l2=0.000000e+00
+[reparam] epoch=1 step=1 phase=joint loss=3.123611e+00 mse=2.197948e+00 mlp_mse=9.202895e-01 kl=1.678985e-02 lambda_mean=0.42120086937619927 lambda_std=0.23356978067761727 lambda_abs0.5=0.21057581985205934
+[reparam] epoch=1 step=100 phase=joint loss=2.040690e+00 mse=1.271280e+00 mlp_mse=7.651801e-01 kl=1.322137e-02 lambda_mean=0.42117737887198464 lambda_std=0.23358772741411782 lambda_abs0.5=0.21060681928668107
+[reparam] epoch=1 step=200 phase=joint loss=2.035768e+00 mse=1.258906e+00 mlp_mse=7.719739e-01 kl=1.527641e-02 lambda_mean=0.42114482678865134 lambda_std=0.2336135410572164 lambda_abs0.5=0.21065459000436884
+[reparam] epoch=1 step=300 phase=joint loss=1.819170e+00 mse=1.081086e+00 mlp_mse=7.350052e-01 kl=9.620363e-03 lambda_mean=0.42111123068290846 lambda_std=0.23364108583148438 lambda_abs0.5=0.2107074135228207
+[reparam] epoch=1 step=400 phase=joint loss=1.871875e+00 mse=1.131923e+00 mlp_mse=7.354233e-01 kl=1.415256e-02 lambda_mean=0.42107802273934347 lambda_std=0.23366809489269982 lambda_abs0.5=0.21076126098632814
+[reparam] epoch=1 step=500 phase=joint loss=1.541098e+00 mse=9.294675e-01 mlp_mse=6.095661e-01 kl=6.450965e-03 lambda_mean=0.4210467823764734 lambda_std=0.23369364168124818 lambda_abs0.5=0.2108133215653269
+[timing] Cycle 3 component wall-clock:
+[timing]   selection: 0.000s (0.00% of cycle)
+[timing]   rebase: 0.000s (0.00% of cycle)
+[timing]   fisher: 15.649s (6.13% of cycle)
+[timing]   fit: 238.865s (93.59% of cycle)
+[timing]   materialize_save_layer: 0.282s (0.11% of cycle)
+[timing]   metadata_write: 0.002s (0.00% of cycle)
+[timing]   teacher_snapshot: 0.258s (0.10% of cycle)
+[timing]   cycle_cleanup: 0.174s (0.07% of cycle)
+[timing]   cycle_total: 255.230s
+[timing]   tracked_components_total: 255.229s
+[progressive] Cycle 4/5
+[fuse] Head permutation disabled; merging with original head order.
+[fisher-input] mode=anchor_input cos=0.9170798286795616 norm_ratio=0.7955950330942869
+[reparam] fisher_prior_inputs fa_mean=5.5289961022808395e-09 fb_mean=3.692328295743975e-09 active_ratio=1.0 rel_contrast=0.4200789736028303 prior_mean=0.458057323255037 prior_active_mean=0.45805733402570087
+[reparam] Cycle 4: training U + gates for pair 25-26 (epochs=1.0, skip_train=False, hidden_mse_w=1.0, attn_mse_w=0.0, mlp_mse_w=1.0, eta=0.0, gamma=0.0, attn_reg_scale=1.0, mlp_reg_scale=1.0, param_subset=mlp, fisher_init_subset=same, train_subset=same, lambda_init=fisher_prior).
+[reparam] subset=mlp gates=3 (attn=0, mlp=3, other=0) train_subset=mlp (attn=0, mlp=3, other=0) elems=74711040 (~285.0 MiB), U_elems=74711040 (~285.0 MiB; +optimizer state)
+[reparam] lambda_init mean=0.4580573098701343 std=0.24263101371239107 min=0.00014809299318585545 max=0.9998855590820312 mean_abs_from_0.5=0.2100394934938665
+[reparam] first_grad phase=joint step=1 gate_l2=1.489211e-02 gate_mean_abs=1.6281908501058065e-07 gate_max_abs=4.719686e-03 u_l2=2.608332e+00 u_mean_abs=1.8988910392580324e-05 u_max_abs=6.668255e-01 rho_l2=0.000000e+00
+[reparam] epoch=1 step=1 phase=joint loss=3.498965e+00 mse=2.630174e+00 mlp_mse=8.635214e-01 kl=1.646959e-02 lambda_mean=0.4580572563305236 lambda_std=0.24263107341427997 lambda_abs0.5=0.21003955372592858
+[reparam] epoch=1 step=100 phase=joint loss=2.621434e+00 mse=1.597691e+00 mlp_mse=1.018658e+00 kl=1.588896e-02 lambda_mean=0.4580450626841763 lambda_std=0.2426497619700872 lambda_abs0.5=0.21006600229363692
+[reparam] epoch=1 step=200 phase=joint loss=2.379503e+00 mse=1.416892e+00 mlp_mse=9.591293e-01 kl=1.088076e-02 lambda_mean=0.45802510579427086 lambda_std=0.24267714422033224 lambda_abs0.5=0.21010857297663102
+[reparam] epoch=1 step=300 phase=joint loss=2.493998e+00 mse=1.612908e+00 mlp_mse=8.754276e-01 kl=1.769663e-02 lambda_mean=0.458003502561335 lambda_std=0.24270550529423363 lambda_abs0.5=0.21015445642304
+[reparam] epoch=1 step=400 phase=joint loss=2.970123e+00 mse=1.733482e+00 mlp_mse=1.231520e+00 kl=1.600275e-02 lambda_mean=0.45798159147563733 lambda_std=0.24273394569685802 lambda_abs0.5=0.21020175197668242
+[reparam] epoch=1 step=500 phase=joint loss=2.139623e+00 mse=1.302018e+00 mlp_mse=8.334249e-01 kl=1.306289e-02 lambda_mean=0.4579596402352316 lambda_std=0.2427601821774139 lambda_abs0.5=0.21024641070449562
+[timing] Cycle 4 component wall-clock:
+[timing]   selection: 0.000s (0.00% of cycle)
+[timing]   rebase: 0.000s (0.00% of cycle)
+[timing]   fisher: 15.491s (6.37% of cycle)
+[timing]   fit: 226.950s (93.30% of cycle)
+[timing]   materialize_save_layer: 0.363s (0.15% of cycle)
+[timing]   metadata_write: 0.002s (0.00% of cycle)
+[timing]   teacher_snapshot: 0.258s (0.11% of cycle)
+[timing]   cycle_cleanup: 0.174s (0.07% of cycle)
+[timing]   cycle_total: 243.238s
+[timing]   tracked_components_total: 243.237s
+[progressive] Cycle 5/5
+[fuse] Head permutation disabled; merging with original head order.
+[fisher-input] mode=anchor_input cos=0.907120268791914 norm_ratio=0.784327195957303
+[reparam] fisher_prior_inputs fa_mean=8.622785835062754e-09 fb_mean=5.1461533825714835e-09 active_ratio=1.0 rel_contrast=0.4221493771201686 prior_mean=0.4628780431914748 prior_active_mean=0.4628780384858449
+[reparam] Cycle 5: training U + gates for pair 25-26 (epochs=1.0, skip_train=False, hidden_mse_w=1.0, attn_mse_w=0.0, mlp_mse_w=1.0, eta=0.0, gamma=0.0, attn_reg_scale=1.0, mlp_reg_scale=1.0, param_subset=mlp, fisher_init_subset=same, train_subset=same, lambda_init=fisher_prior).
+[reparam] subset=mlp gates=3 (attn=0, mlp=3, other=0) train_subset=mlp (attn=0, mlp=3, other=0) elems=74711040 (~285.0 MiB), U_elems=74711040 (~285.0 MiB; +optimizer state)
+[reparam] lambda_init mean=0.4628780431914748 std=0.24472377510990143 min=9.999999747378752e-05 max=0.9997746348381042 mean_abs_from_0.5=0.2110746885600843
+[reparam] first_grad phase=joint step=1 gate_l2=2.406353e-02 gate_mean_abs=3.2932629549402027e-07 gate_max_abs=5.017741e-03 u_l2=5.286914e+00 u_mean_abs=4.273248150160438e-05 u_max_abs=9.866973e-01 rho_l2=0.000000e+00
+[reparam] epoch=1 step=1 phase=joint loss=7.221016e+00 mse=5.248845e+00 mlp_mse=1.961453e+00 kl=3.349205e-02 lambda_mean=0.4628780298065721 lambda_std=0.24472377307955573 lambda_abs0.5=0.211074722022341
+[reparam] epoch=1 step=100 phase=joint loss=4.795070e+00 mse=2.882280e+00 mlp_mse=1.904266e+00 kl=2.663786e-02 lambda_mean=0.4628679911295573 lambda_std=0.2447413103171458 lambda_abs0.5=0.21109960455643503
+[reparam] epoch=1 step=200 phase=joint loss=5.219382e+00 mse=2.953206e+00 mlp_mse=2.253151e+00 kl=4.070204e-02 lambda_mean=0.4628468563682155 lambda_std=0.2447717489816577 lambda_abs0.5=0.21114643163848343
+[reparam] epoch=1 step=300 phase=joint loss=3.699711e+00 mse=2.201450e+00 mlp_mse=1.492145e+00 kl=1.911399e-02 lambda_mean=0.4628230981659471 lambda_std=0.2448012642142387 lambda_abs0.5=0.21119349964878015
+[reparam] epoch=1 step=400 phase=joint loss=3.306127e+00 mse=1.924341e+00 mlp_mse=1.377654e+00 kl=1.291327e-02 lambda_mean=0.4628004241407963 lambda_std=0.24483003625899244 lambda_abs0.5=0.2112407550477145
+[reparam] epoch=1 step=500 phase=joint loss=3.403753e+00 mse=1.903926e+00 mlp_mse=1.495226e+00 kl=1.438092e-02 lambda_mean=0.4627775894968133 lambda_std=0.24485888773422773 lambda_abs0.5=0.21128915485582853
+[timing] Cycle 5 component wall-clock:
+[timing]   selection: 0.000s (0.00% of cycle)
+[timing]   rebase: 0.000s (0.00% of cycle)
+[timing]   fisher: 15.340s (6.55% of cycle)
+[timing]   fit: 218.057s (93.13% of cycle)
+[timing]   materialize_save_layer: 0.331s (0.14% of cycle)
+[timing]   metadata_write: 0.002s (0.00% of cycle)
+[timing]   teacher_snapshot: 0.247s (0.11% of cycle)
+[timing]   cycle_cleanup: 0.174s (0.07% of cycle)
+[timing]   cycle_total: 234.152s
+[timing]   tracked_components_total: 234.151s
+
+
+[progressive] Completed 5 cycles. Final model saved to /workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_input_no_rho/run.

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/metadata/cycle_01.json

@@ -0,0 +1,714 @@
+{
+  "cycle": 1,
+  "layer_merged": 25,
+  "num_layers_before": 36,
+  "num_layers_after": 35,
+  "fused_layer_state": "fused_layer.pt",
+  "profile_no_save": false,
+  "dwce_score": null,
+  "dwce_scores": null,
+  "dwce_meta": {
+    "method": "ordered",
+    "scope": "absolute",
+    "order_position": 0,
+    "requested_index": 25,
+    "resolved_index": 25,
+    "num_pairs": 35,
+    "exclude_pairs": [
+      0,
+      1,
+      34
+    ],
+    "start_index_ignored": 0
+  },
+  "fisher_num_batches": 32,
+  "fisher_input": {
+    "mode": "anchor_input",
+    "tokens": 65536,
+    "context_vs_native_source_mlp_input_cosine": 0.9340915456414223,
+    "context_vs_native_source_mlp_input_norm_ratio": 0.9107728470116854,
+    "native_source_mlp_input_global_rms": 0.8161880495888352,
+    "context_source_mlp_input_global_rms": 0.741906545817173,
+    "context_detached": true,
+    "anchor_attention_included": false,
+    "normalization": "anchor.post_attention_layernorm(anchor_block_input)"
+  },
+  "merge_method": "reparam",
+  "merged_params": 3,
+  "num_sequences": 64,
+  "teacher_source": "base_model",
+  "teacher_cycle": 0,
+  "eval": {
+    "datasets": [
+      "wikitext2"
+    ],
+    "configs": [
+      null
+    ],
+    "split": "test",
+    "num_samples": 0,
+    "seq_len": 1024,
+    "post_ppl": null,
+    "merge_only_ppl": null
+  },
+  "comm": {
+    "enabled": false
+  },
+  "distill": {
+    "enabled": true,
+    "method": "reparam",
+    "calib_samples": 256,
+    "inst_samples": 0,
+    "seq_len": 1024,
+    "batch_size": 1,
+    "epochs": 1.0,
+    "lr": 0.0001,
+    "kl_weight": 0.02,
+    "kl_temp": 4.0,
+    "kl_chunk_size": 16384,
+    "hidden_mse_weight": 1.0,
+    "attn_mse_weight": 0.0,
+    "mlp_mse_weight": 1.0,
+    "reparam_eta": 0.0,
+    "reparam_gamma": 0.0,
+    "reparam_attn_reg_scale": 1.0,
+    "reparam_mlp_reg_scale": 1.0,
+    "reparam_param_subset": "mlp",
+    "reparam_train_subset": "same",
+    "reparam_skip_train": false,
+    "reparam_stats": {
+      "enabled": true,
+      "epochs": 1.0,
+      "requested_epochs": 1.0,
+      "skip_train": false,
+      "trained": true,
+      "lr": 0.0001,
+      "hidden_mse_weight": 1.0,
+      "attn_mse_weight": 0.0,
+      "mlp_mse_weight": 1.0,
+      "eta": 0.0,
+      "gamma": 0.0,
+      "attn_reg_scale": 1.0,
+      "mlp_reg_scale": 1.0,
+      "param_subset": "mlp",
+      "fisher_init_subset": "mlp",
+      "fit_only_parameter_count": 0,
+      "fit_only_parameter_names": [],
+      "train_subset": "mlp",
+      "anchor_preserving_init": false,
+      "num_gates": 3,
+      "num_attn_gates": 0,
+      "num_mlp_gates": 3,
+      "num_other_gates": 0,
+      "num_trainable_attn_gates": 0,
+      "num_trainable_mlp_gates": 3,
+      "num_trainable_other_gates": 0,
+      "lambda_init_summary": {
+        "num_tensors": 3,
+        "num_elements": 74711040,
+        "mean": 0.541165950842071,
+        "std": 0.2155650558184483,
+        "min": 0.0006457812269218266,
+        "max": 0.9998443126678467,
+        "mean_abs_from_0_5": 0.18340888441654674,
+        "per_tensor": {
+          "mlp.gate_proj.weight": {
+            "mean": 0.5323492351331209,
+            "std": 0.2249757872766237,
+            "min": 0.0015450043138116598,
+            "max": 0.9990851879119873,
+            "mean_abs_from_0_5": 0.1914050453587582
+          },
+          "mlp.up_proj.weight": {
+            "mean": 0.5398846274928043,
+            "std": 0.20355052891350855,
+            "min": 0.0006457812269218266,
+            "max": 0.9998443126678467,
+            "mean_abs_from_0_5": 0.17201584263851769
+          },
+          "mlp.down_proj.weight": {
+            "mean": 0.5512639899002878,
+            "std": 0.21720177377288416,
+            "min": 0.0008037883089855313,
+            "max": 0.9998044371604919,
+            "mean_abs_from_0_5": 0.1868057652523643
+          }
+        }
+      },
+      "lambda_final_summary": {
+        "num_tensors": 3,
+        "num_elements": 74711040,
+        "mean": 0.5411081414473684,
+        "std": 0.2156944432104828,
+        "min": 0.0006452285451814532,
+        "max": 0.9998443126678467,
+        "mean_abs_from_0_5": 0.18358466834352727,
+        "per_tensor": {
+          "mlp.gate_proj.weight": {
+            "mean": 0.5322986803556743,
+            "std": 0.22508105888439325,
+            "min": 0.0015426566824316978,
+            "max": 0.9990851879119873,
+            "mean_abs_from_0_5": 0.19155781394556948
+          },
+          "mlp.up_proj.weight": {
+            "mean": 0.5398464002107319,
+            "std": 0.20367742406294012,
+            "min": 0.0006452285451814532,
+            "max": 0.9998443126678467,
+            "mean_abs_from_0_5": 0.172193748072574
+          },
+          "mlp.down_proj.weight": {
+            "mean": 0.551179343775699,
+            "std": 0.21736070169695754,
+            "min": 0.0008039805688895285,
+            "max": 0.9998044371604919,
+            "mean_abs_from_0_5": 0.18700244301243832
+          }
+        }
+      },
+      "lambda_delta_from_init": {
+        "num_tensors": 3,
+        "num_elements": 74711040,
+        "mean_abs": 0.0007151240580960324,
+        "rms": 0.0009971574718633711,
+        "max_abs": 0.014105618000030518,
+        "per_tensor": {
+          "mlp.gate_proj.weight": {
+            "mean_abs": 0.0006646090432217246,
+            "rms": 0.0009496650343290378,
+            "max_abs": 0.014105618000030518
+          },
+          "mlp.up_proj.weight": {
+            "mean_abs": 0.0007358902379086144,
+            "rms": 0.001009327299234651,
+            "max_abs": 0.013269424438476562
+          },
+          "mlp.down_proj.weight": {
+            "mean_abs": 0.0007448728931577582,
+            "rms": 0.001030710335968046,
+            "max_abs": 0.011881321668624878
+          }
+        }
+      },
+      "first_grad_summary": {
+        "joint": {
+          "step": 1,
+          "phase": "joint",
+          "gate": {
+            "num_tensors": 3,
+            "num_tensors_without_grad": 0,
+            "num_elements": 74711040,
+            "l2": 0.006677313205845488,
+            "mean_abs": 1.0369183559975537e-07,
+            "max_abs": 0.003420014400035143
+          },
+          "u": {
+            "num_tensors": 3,
+            "num_tensors_without_grad": 0,
+            "num_elements": 74711040,
+            "l2": 0.7087302685233066,
+            "mean_abs": 8.06867079645918e-06,
+            "max_abs": 0.19476085901260376
+          },
+          "rho": {
+            "num_tensors": 0,
+            "num_tensors_without_grad": 0,
+            "num_elements": 0,
+            "l2": 0.0,
+            "mean_abs": null,
+            "max_abs": 0.0
+          }
+        }
+      },
+      "u_delta_summary": {
+        "num_tensors": 3,
+        "num_elements": 74711040,
+        "mean": 2.471904480715462e-07,
+        "std": 0.005323753836930941,
+        "rms": 0.0053237538426696655,
+        "mean_abs": 0.004189868454347577,
+        "min": -0.050679028034210205,
+        "max": 0.050065264105796814,
+        "per_tensor": {
+          "mlp.gate_proj.weight": {
+            "mean": -1.0140172794069115e-06,
+            "std": 0.005231255157344916,
+            "rms": 0.005231255255622577,
+            "mean_abs": 0.004095561880814402,
+            "min": -0.04558738321065903,
+            "max": 0.04795807600021362
+          },
+          "mlp.up_proj.weight": {
+            "mean": 8.562219016695101e-07,
+            "std": 0.0052267596254550455,
+            "rms": 0.005226759695586063,
+            "mean_abs": 0.004135956262287341,
+            "min": -0.0421622209250927,
+            "max": 0.050065264105796814
+          },
+          "mlp.down_proj.weight": {
+            "mean": 8.993667219520399e-07,
+            "std": 0.005508358708206814,
+            "rms": 0.005508358781628003,
+            "mean_abs": 0.004338087219940989,
+            "min": -0.050679028034210205,
+            "max": 0.04775146394968033
+          }
+        }
+      },
+      "rho_objective": "none",
+      "rho_schedule": "joint",
+      "rho_epochs": 0.0,
+      "rho_lr": null,
+      "rho_bounds": null,
+      "rho_after_phase": null,
+      "rho_final": {},
+      "u_fisher_orthogonality": {},
+      "lambda_init": "fisher_prior",
+      "skip_fisher_merge_init": false,
+      "requested_skip_merge_cycles": [],
+      "fisher_prior_input_summary": {
+        "num_tensors": 3,
+        "num_elements": 74711040,
+        "fisher_mode": "param",
+        "eps": 0.0,
+        "param_subset": "mlp",
+        "fa_mean": 2.9234447159372697e-09,
+        "fb_mean": 2.255867078323111e-09,
+        "denom_mean": 5.179312043573621e-09,
+        "active_ratio": 1.0,
+        "mean_abs_fa_minus_fb": 2.468631745805875e-09,
+        "mean_relative_contrast": 0.36681776883309347,
+        "prior_mean": 0.5411659374571683,
+        "prior_active_mean": 0.5411659280459086,
+        "distributions": {
+          "fa": {
+            "quantile_method": "concatenated_per_tensor_even_stride_samples",
+            "quantile_sample_count": 393216,
+            "quantiles": {
+              "p0": 5.326228537374256e-17,
+              "p1": 1.2528050818971792e-10,
+              "p10": 4.1363135139249607e-10,
+              "p25": 6.691387999246956e-10,
+              "p50": 1.151335027316236e-09,
+              "p75": 2.1231931768284085e-09,
+              "p90": 4.122304275711031e-09,
+              "p95": 6.674993002775409e-09,
+              "p99": 1.9887647084715354e-08,
+              "p99_9": 1.2767549151249113e-07,
+              "p100": 2.3973718725756044e-06
+            }
+          },
+          "fb": {
+            "quantile_method": "concatenated_per_tensor_even_stride_samples",
+            "quantile_sample_count": 393216,
+            "quantiles": {
+              "p0": 8.710958297968149e-17,
+              "p1": 1.118948891876137e-10,
+              "p10": 3.5977149037691447e-10,
+              "p25": 5.674885006357044e-10,
+              "p50": 9.325938910009768e-10,
+              "p75": 1.6385217627146176e-09,
+              "p90": 3.070551368722363e-09,
+              "p95": 4.798474506628736e-09,
+              "p99": 1.5672222630769284e-08,
+              "p99_9": 1.867880996542226e-07,
+              "p100": 2.7776613933383487e-06
+            }
+          },
+          "denom": {
+            "quantile_method": "concatenated_per_tensor_even_stride_samples",
+            "quantile_sample_count": 393216,
+            "quantiles": {
+              "p0": 2.372053300105808e-16,
+              "p1": 4.418666266214899e-10,
+              "p10": 1.0292593444205522e-09,
+              "p25": 1.5058829738734403e-09,
+              "p50": 2.3361337309069086e-09,
+              "p75": 3.859721431354046e-09,
+              "p90": 7.040350080700364e-09,
+              "p95": 1.0910539138819786e-08,
+              "p99": 3.29047047387121e-08,
+              "p99_9": 3.043814160719194e-07,
+              "p100": 3.3931987672985997e-06
+            }
+          }
+        },
+        "per_tensor": {
+          "mlp.gate_proj.weight": {
+            "num_elements": 24903680,
+            "fa_mean": 2.1206360363641197e-09,
+            "fb_mean": 1.7711508797602667e-09,
+            "denom_mean": 3.891787065712331e-09,
+            "active_ratio": 1.0,
+            "mean_abs_fa_minus_fb": 1.7996274879662548e-09,
+            "mean_relative_contrast": 0.3828100907175164,
+            "prior_mean": 0.5323491949784128,
+            "prior_active_mean": 0.532349169254303,
+            "fa_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 2.1206360363641197e-09,
+              "std": 3.4110394880657845e-08,
+              "min": 4.4447809061708095e-17,
+              "max": 2.2837210053694434e-05,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 5.326228537374256e-17,
+                "p1": 7.972757176677447e-11,
+                "p10": 2.7219945741840945e-10,
+                "p25": 4.943463416395844e-10,
+                "p50": 8.849019295098515e-10,
+                "p75": 1.5215690929437642e-09,
+                "p90": 2.596466153192978e-09,
+                "p95": 3.800813885845855e-09,
+                "p99": 9.916343302052155e-09,
+                "p99_9": 7.440273463998892e-08,
+                "p100": 5.840683456881379e-07
+              }
+            },
+            "fb_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 1.7711508797602667e-09,
+              "std": 2.8168813761068194e-08,
+              "min": 3.842841574415971e-17,
+              "max": 1.7336304154014215e-05,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 8.710958297968149e-17,
+                "p1": 7.107184285537471e-11,
+                "p10": 2.3066989984776853e-10,
+                "p25": 4.178216661543388e-10,
+                "p50": 7.483191843959958e-10,
+                "p75": 1.3012557698033334e-09,
+                "p90": 2.2561017498645697e-09,
+                "p95": 3.3291462919748938e-09,
+                "p99": 8.38778824174824e-09,
+                "p99_9": 5.196224250880732e-08,
+                "p100": 1.0523916671445477e-06
+              }
+            },
+            "denom_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 3.891787065712331e-09,
+              "std": 5.219215889713716e-08,
+              "min": 1.4157123391951164e-16,
+              "max": 2.329333256056998e-05,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 2.372053300105808e-16,
+                "p1": 3.1434430058929763e-10,
+                "p10": 7.80574382908128e-10,
+                "p25": 1.1740619587641277e-09,
+                "p50": 1.7899193238690714e-09,
+                "p75": 2.7866069451931708e-09,
+                "p90": 4.4687205047466705e-09,
+                "p95": 6.439061284169156e-09,
+                "p99": 1.650193581781423e-08,
+                "p99_9": 1.204171979907187e-07,
+                "p100": 1.0598926110105822e-06
+              }
+            }
+          },
+          "mlp.up_proj.weight": {
+            "num_elements": 24903680,
+            "fa_mean": 3.3876750948318083e-09,
+            "fb_mean": 2.5868554362923054e-09,
+            "denom_mean": 5.974530531124114e-09,
+            "active_ratio": 1.0,
+            "mean_abs_fa_minus_fb": 2.7293217978072596e-09,
+            "mean_relative_contrast": 0.34403168527703537,
+            "prior_mean": 0.5398845471833882,
+            "prior_active_mean": 0.5398845672607422,
+            "fa_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 3.3876750948318083e-09,
+              "std": 8.641646276917719e-08,
+              "min": 7.670331338407448e-17,
+              "max": 0.00015834042278584093,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 7.441954712215438e-16,
+                "p1": 1.817083150612575e-10,
+                "p10": 4.30987912025671e-10,
+                "p25": 6.360905135949224e-10,
+                "p50": 1.0568520503184686e-09,
+                "p75": 1.961866225030917e-09,
+                "p90": 3.776234880348284e-09,
+                "p95": 5.987847551125469e-09,
+                "p99": 1.8428140791115766e-08,
+                "p99_9": 1.0239917713761315e-07,
+                "p100": 2.3973718725756044e-06
+              }
+            },
+            "fb_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 2.5868554362923054e-09,
+              "std": 5.687513337167567e-08,
+              "min": 1.3457810343720053e-16,
+              "max": 6.675625627394766e-05,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 2.698398154023945e-16,
+                "p1": 1.7747431302339578e-10,
+                "p10": 3.821233329759366e-10,
+                "p25": 5.458912211153688e-10,
+                "p50": 8.709082344182661e-10,
+                "p75": 1.5500513095290103e-09,
+                "p90": 2.9184350491817668e-09,
+                "p95": 4.603996739405147e-09,
+                "p99": 1.354797873887037e-08,
+                "p99_9": 7.411497193743344e-08,
+                "p100": 2.7776613933383487e-06
+              }
+            },
+            "denom_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 5.974530531124114e-09,
+              "std": 1.186903574537433e-07,
+              "min": 6.938408977544925e-16,
+              "max": 0.00016521777433808893,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 1.900548810126676e-15,
+                "p1": 5.383404833025907e-10,
+                "p10": 1.0451411958101176e-09,
+                "p25": 1.4416527971405912e-09,
+                "p50": 2.1695814034217165e-09,
+                "p75": 3.580021168403391e-09,
+                "p90": 6.39527364398873e-09,
+                "p95": 9.87610970781816e-09,
+                "p99": 2.834804391227408e-08,
+                "p99_9": 1.9823336572244443e-07,
+                "p100": 3.3931987672985997e-06
+              }
+            }
+          },
+          "mlp.down_proj.weight": {
+            "num_elements": 24903680,
+            "fa_mean": 3.262023016615881e-09,
+            "fb_mean": 2.4095949189167616e-09,
+            "denom_mean": 5.67161853388442e-09,
+            "active_ratio": 1.0,
+            "mean_abs_fa_minus_fb": 2.876945951644111e-09,
+            "mean_relative_contrast": 0.3736115305047286,
+            "prior_mean": 0.5512640702097039,
+            "prior_active_mean": 0.5512640476226807,
+            "fa_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 3.262023016615881e-09,
+              "std": 1.3292665964089707e-08,
+              "min": 4.4929303333329784e-11,
+              "max": 6.302336714725243e-06,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 9.201431977468033e-11,
+                "p1": 3.714796248388552e-10,
+                "p10": 6.563575238871522e-10,
+                "p25": 9.641075715549619e-10,
+                "p50": 1.6296080040945071e-09,
+                "p75": 3.036724427474269e-09,
+                "p90": 6.1237934723124e-09,
+                "p95": 9.691755842311522e-09,
+                "p99": 3.259335912275674e-08,
+                "p99_9": 2.6606900860315363e-07,
+                "p100": 1.9585156678658677e-06
+              }
+            },
+            "fb_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 2.4095949189167616e-09,
+              "std": 6.17222588983802e-09,
+              "min": 4.2460219368267715e-11,
+              "max": 3.1653239602746908e-06,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 8.674327228730405e-11,
+                "p1": 3.306336038289004e-10,
+                "p10": 5.541075376314097e-10,
+                "p25": 7.717937955398213e-10,
+                "p50": 1.19429888201239e-09,
+                "p75": 2.1084156642814378e-09,
+                "p90": 4.060415115247906e-09,
+                "p95": 6.428912069367243e-09,
+                "p99": 2.6638982575377668e-08,
+                "p99_9": 2.554081390826468e-07,
+                "p100": 1.4529259715345688e-06
+              }
+            },
+            "denom_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 5.67161853388442e-09,
+              "std": 1.5043915061010418e-08,
+              "min": 1.627987605834491e-10,
+              "max": 6.304765975073678e-06,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 2.031227824828008e-10,
+                "p1": 9.693505997887542e-10,
+                "p10": 1.5807277709001255e-09,
+                "p25": 2.164176837737841e-09,
+                "p50": 3.2246059156193496e-09,
+                "p75": 5.379296119656374e-09,
+                "p90": 1.0116525395176268e-08,
+                "p95": 1.491388879060196e-08,
+                "p99": 4.8380218942156716e-08,
+                "p99_9": 3.8704141047674057e-07,
+                "p100": 1.9607687136158347e-06
+              }
+            }
+          }
+        }
+      }
+    },
+    "reparam_gate_summary": {
+      "num_tensors": 3,
+      "num_elements": 74711040,
+      "global_mean": 0.5411081414473684,
+      "per_tensor_mean": {
+        "mlp.gate_proj.weight": 0.5322986841201782,
+        "mlp.up_proj.weight": 0.5398464202880859,
+        "mlp.down_proj.weight": 0.5511793494224548
+      }
+    },
+    "post_merge_nll_scale": null,
+    "merge_only_ppl": null,
+    "post_ppl": null,
+    "weight_decay": 0.0,
+    "max_grad_norm": 1.0,
+    "grad_accum_steps": 1,
+    "instruction_dataset": null,
+    "instruction_config": null,
+    "instruction_split": "train"
+  },
+  "lora": {
+    "enabled": false,
+    "seq_len": 1024,
+    "batch_size": 1,
+    "epochs": 0.0,
+    "rank": 8,
+    "alpha": 16.0,
+    "dropout": 0.0,
+    "target_modules": [
+      "q_proj",
+      "k_proj",
+      "v_proj",
+      "o_proj",
+      "gate_proj",
+      "down_proj",
+      "up_proj"
+    ],
+    "respect_exclude_pairs": false,
+    "kl_enabled": false,
+    "kl_weight": 0.1,
+    "kl_temp": 4.0,
+    "post_ppl": null,
+    "lr": 0.0001,
+    "weight_decay": 0.0,
+    "max_grad_norm": 1.0,
+    "grad_accum_steps": 1,
+    "log_steps": 100,
+    "eval_every": 2000,
+    "eval_max_batches": null
+  },
+  "norm_policy": "hybrid",
+  "full_model_saved": false,
+  "timing": {
+    "seconds": {
+      "selection": 2.9177055694162846e-05,
+      "rebase": 0.0,
+      "fisher": 18.806512578041293,
+      "fit": 251.1861999729881,
+      "materialize_save_layer": 0.30324649706017226,
+      "metadata_write": 0.0018380529945716262,
+      "teacher_snapshot": 0.09547231101896614,
+      "cycle_cleanup": 0.1771488080266863,
+      "cycle_total": 270.5753599590389
+    },
+    "tracked_component_total_seconds": 270.5704473971855,
+    "cycle_total_seconds": 270.5753599590389,
+    "ratios_of_tracked_components": {
+      "selection": 1.0783533817103172e-07,
+      "rebase": 0.0,
+      "fisher": 0.0695068983289005,
+      "fit": 0.9283578542643197,
+      "materialize_save_layer": 0.0011207672529550863,
+      "metadata_write": 6.7932511190826595e-06,
+      "teacher_snapshot": 0.0003528556497480931,
+      "cycle_cleanup": 0.0006547234176193665
+    },
+    "ratios_of_cycle_total": {
+      "selection": 1.0783338031437829e-07,
+      "rebase": 0.0,
+      "fisher": 0.06950563636278011,
+      "fit": 0.9283409990141526,
+      "materialize_save_layer": 0.0011207469043229925,
+      "metadata_write": 6.793127780925359e-06,
+      "teacher_snapshot": 0.00035284924330663083,
+      "cycle_cleanup": 0.0006547115304715996
+    }
+  }
+}

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/metadata/cycle_02.json

@@ -0,0 +1,714 @@
+{
+  "cycle": 2,
+  "layer_merged": 25,
+  "num_layers_before": 35,
+  "num_layers_after": 34,
+  "fused_layer_state": "fused_layer.pt",
+  "profile_no_save": false,
+  "dwce_score": null,
+  "dwce_scores": null,
+  "dwce_meta": {
+    "method": "ordered",
+    "scope": "absolute",
+    "order_position": 1,
+    "requested_index": 25,
+    "resolved_index": 25,
+    "num_pairs": 34,
+    "exclude_pairs": [
+      0,
+      1,
+      33
+    ],
+    "start_index_ignored": 25
+  },
+  "fisher_num_batches": 32,
+  "fisher_input": {
+    "mode": "anchor_input",
+    "tokens": 65536,
+    "context_vs_native_source_mlp_input_cosine": 0.9501073732972145,
+    "context_vs_native_source_mlp_input_norm_ratio": 0.8818922247737646,
+    "native_source_mlp_input_global_rms": 0.8439129886118018,
+    "context_source_mlp_input_global_rms": 0.741906545817173,
+    "context_detached": true,
+    "anchor_attention_included": false,
+    "normalization": "anchor.post_attention_layernorm(anchor_block_input)"
+  },
+  "merge_method": "reparam",
+  "merged_params": 3,
+  "num_sequences": 64,
+  "teacher_source": "previous_cycle_memory",
+  "teacher_cycle": 1,
+  "eval": {
+    "datasets": [
+      "wikitext2"
+    ],
+    "configs": [
+      null
+    ],
+    "split": "test",
+    "num_samples": 0,
+    "seq_len": 1024,
+    "post_ppl": null,
+    "merge_only_ppl": null
+  },
+  "comm": {
+    "enabled": false
+  },
+  "distill": {
+    "enabled": true,
+    "method": "reparam",
+    "calib_samples": 256,
+    "inst_samples": 0,
+    "seq_len": 1024,
+    "batch_size": 1,
+    "epochs": 1.0,
+    "lr": 0.0001,
+    "kl_weight": 0.02,
+    "kl_temp": 4.0,
+    "kl_chunk_size": 16384,
+    "hidden_mse_weight": 1.0,
+    "attn_mse_weight": 0.0,
+    "mlp_mse_weight": 1.0,
+    "reparam_eta": 0.0,
+    "reparam_gamma": 0.0,
+    "reparam_attn_reg_scale": 1.0,
+    "reparam_mlp_reg_scale": 1.0,
+    "reparam_param_subset": "mlp",
+    "reparam_train_subset": "same",
+    "reparam_skip_train": false,
+    "reparam_stats": {
+      "enabled": true,
+      "epochs": 1.0,
+      "requested_epochs": 1.0,
+      "skip_train": false,
+      "trained": true,
+      "lr": 0.0001,
+      "hidden_mse_weight": 1.0,
+      "attn_mse_weight": 0.0,
+      "mlp_mse_weight": 1.0,
+      "eta": 0.0,
+      "gamma": 0.0,
+      "attn_reg_scale": 1.0,
+      "mlp_reg_scale": 1.0,
+      "param_subset": "mlp",
+      "fisher_init_subset": "mlp",
+      "fit_only_parameter_count": 0,
+      "fit_only_parameter_names": [],
+      "train_subset": "mlp",
+      "anchor_preserving_init": false,
+      "num_gates": 3,
+      "num_attn_gates": 0,
+      "num_mlp_gates": 3,
+      "num_other_gates": 0,
+      "num_trainable_attn_gates": 0,
+      "num_trainable_mlp_gates": 3,
+      "num_trainable_other_gates": 0,
+      "lambda_init_summary": {
+        "num_tensors": 3,
+        "num_elements": 74711040,
+        "mean": 0.4064247265196683,
+        "std": 0.21451395279119853,
+        "min": 0.0001658098481129855,
+        "max": 0.9995361566543579,
+        "mean_abs_from_0_5": 0.1983989113255551,
+        "per_tensor": {
+          "mlp.gate_proj.weight": {
+            "mean": 0.37069509405838813,
+            "std": 0.21537235729257143,
+            "min": 0.00028491884586401284,
+            "max": 0.9991639852523804,
+            "mean_abs_from_0_5": 0.2160420066431949
+          },
+          "mlp.up_proj.weight": {
+            "mean": 0.3934421740080181,
+            "std": 0.19275160279907425,
+            "min": 0.0006863977177999914,
+            "max": 0.9995361566543579,
+            "mean_abs_from_0_5": 0.18594432630037006
+          },
+          "mlp.down_proj.weight": {
+            "mean": 0.45513691149259866,
+            "std": 0.2251493834958686,
+            "min": 0.0001658098481129855,
+            "max": 0.9993318915367126,
+            "mean_abs_from_0_5": 0.19321040103310033
+          }
+        }
+      },
+      "lambda_final_summary": {
+        "num_tensors": 3,
+        "num_elements": 74711040,
+        "mean": 0.40624111242461625,
+        "std": 0.214613612600965,
+        "min": 0.0001658085675444454,
+        "max": 0.9995357990264893,
+        "mean_abs_from_0_5": 0.19862831517269736,
+        "per_tensor": {
+          "mlp.gate_proj.weight": {
+            "mean": 0.3705450359143709,
+            "std": 0.2154568008397774,
+            "min": 0.00028493572608567774,
+            "max": 0.9991644620895386,
+            "mean_abs_from_0_5": 0.21625145360043174
+          },
+          "mlp.up_proj.weight": {
+            "mean": 0.39327557212428044,
+            "std": 0.19286355379943085,
+            "min": 0.0006864079041406512,
+            "max": 0.9995357990264893,
+            "mean_abs_from_0_5": 0.18618782445004112
+          },
+          "mlp.down_proj.weight": {
+            "mean": 0.45490272923519737,
+            "std": 0.22527484694152852,
+            "min": 0.0001658085675444454,
+            "max": 0.9993322491645813,
+            "mean_abs_from_0_5": 0.19344566746761924
+          }
+        }
+      },
+      "lambda_delta_from_init": {
+        "num_tensors": 3,
+        "num_elements": 74711040,
+        "mean_abs": 0.0006964494261825294,
+        "rms": 0.0009861191196736771,
+        "max_abs": 0.014759182929992676,
+        "per_tensor": {
+          "mlp.gate_proj.weight": {
+            "mean_abs": 0.0006317402971418281,
+            "rms": 0.000922039612135761,
+            "max_abs": 0.014759182929992676
+          },
+          "mlp.up_proj.weight": {
+            "mean_abs": 0.0007219902778926649,
+            "rms": 0.0009998261438843832,
+            "max_abs": 0.014169573783874512
+          },
+          "mlp.down_proj.weight": {
+            "mean_abs": 0.0007356177035130953,
+            "rms": 0.001033190877919363,
+            "max_abs": 0.01385641098022461
+          }
+        }
+      },
+      "first_grad_summary": {
+        "joint": {
+          "step": 1,
+          "phase": "joint",
+          "gate": {
+            "num_tensors": 3,
+            "num_tensors_without_grad": 0,
+            "num_elements": 74711040,
+            "l2": 0.006575216417202265,
+            "mean_abs": 1.034208364951171e-07,
+            "max_abs": 0.001666576019488275
+          },
+          "u": {
+            "num_tensors": 3,
+            "num_tensors_without_grad": 0,
+            "num_elements": 74711040,
+            "l2": 0.9438444464126213,
+            "mean_abs": 1.0214174337881177e-05,
+            "max_abs": 0.29918748140335083
+          },
+          "rho": {
+            "num_tensors": 0,
+            "num_tensors_without_grad": 0,
+            "num_elements": 0,
+            "l2": 0.0,
+            "mean_abs": null,
+            "max_abs": 0.0
+          }
+        }
+      },
+      "u_delta_summary": {
+        "num_tensors": 3,
+        "num_elements": 74711040,
+        "mean": -3.7678185515432503e-06,
+        "std": 0.005333629756010723,
+        "rms": 0.005333631086854399,
+        "mean_abs": 0.0041970564607988325,
+        "min": -0.06592461466789246,
+        "max": 0.05918984115123749,
+        "per_tensor": {
+          "mlp.gate_proj.weight": {
+            "mean": -9.529558500569117e-06,
+            "std": 0.005266276303331785,
+            "rms": 0.005266284925402262,
+            "mean_abs": 0.004146255317487215,
+            "min": -0.05786953121423721,
+            "max": 0.050258371978998184
+          },
+          "mlp.up_proj.weight": {
+            "mean": -6.029762229637096e-08,
+            "std": 0.005232326139197076,
+            "rms": 0.005232326139544513,
+            "mean_abs": 0.004144843001114695,
+            "min": -0.046780675649642944,
+            "max": 0.05495802313089371
+          },
+          "mlp.down_proj.weight": {
+            "mean": -1.7135995317642627e-06,
+            "std": 0.005498351119151601,
+            "rms": 0.005498351386179226,
+            "mean_abs": 0.004300071063794588,
+            "min": -0.06592461466789246,
+            "max": 0.05918984115123749
+          }
+        }
+      },
+      "rho_objective": "none",
+      "rho_schedule": "joint",
+      "rho_epochs": 0.0,
+      "rho_lr": null,
+      "rho_bounds": null,
+      "rho_after_phase": null,
+      "rho_final": {},
+      "u_fisher_orthogonality": {},
+      "lambda_init": "fisher_prior",
+      "skip_fisher_merge_init": false,
+      "requested_skip_merge_cycles": [],
+      "fisher_prior_input_summary": {
+        "num_tensors": 3,
+        "num_elements": 74711040,
+        "fisher_mode": "param",
+        "eps": 0.0,
+        "param_subset": "mlp",
+        "fa_mean": 2.3329901846648824e-09,
+        "fb_mean": 3.0635342746041206e-09,
+        "denom_mean": 5.396524908032836e-09,
+        "active_ratio": 1.0,
+        "mean_abs_fa_minus_fb": 2.7332246470011837e-09,
+        "mean_relative_contrast": 0.3967977958813048,
+        "prior_mean": 0.40642471313476564,
+        "prior_active_mean": 0.40642471114794415,
+        "distributions": {
+          "fa": {
+            "quantile_method": "concatenated_per_tensor_even_stride_samples",
+            "quantile_sample_count": 393216,
+            "quantiles": {
+              "p0": 2.7453715903412387e-17,
+              "p1": 8.800300071998279e-11,
+              "p10": 2.8287375219981925e-10,
+              "p25": 4.6106912732213345e-10,
+              "p50": 8.007523533137828e-10,
+              "p75": 1.5761332239350168e-09,
+              "p90": 3.2756926060528713e-09,
+              "p95": 5.3239848085695485e-09,
+              "p99": 2.317673875040782e-08,
+              "p99_9": 2.4605895987406257e-07,
+              "p100": 2.041214884229703e-06
+            }
+          },
+          "fb": {
+            "quantile_method": "concatenated_per_tensor_even_stride_samples",
+            "quantile_sample_count": 393216,
+            "quantiles": {
+              "p0": 9.45427272257565e-17,
+              "p1": 1.8665750889379495e-10,
+              "p10": 5.292299931625166e-10,
+              "p25": 8.140411678070336e-10,
+              "p50": 1.3065410975343639e-09,
+              "p75": 2.260567066869612e-09,
+              "p90": 4.237653783434325e-09,
+              "p95": 6.5942904470261965e-09,
+              "p99": 2.6404272546187713e-08,
+              "p99_9": 1.806278504545844e-07,
+              "p100": 1.193124262499623e-06
+            }
+          },
+          "denom": {
+            "quantile_method": "concatenated_per_tensor_even_stride_samples",
+            "quantile_sample_count": 393216,
+            "quantiles": {
+              "p0": 4.048886574000316e-16,
+              "p1": 4.589846003266729e-10,
+              "p10": 1.052270492962748e-09,
+              "p25": 1.5327821234478733e-09,
+              "p50": 2.3853892194836135e-09,
+              "p75": 4.033706257899894e-09,
+              "p90": 7.2411334706146135e-09,
+              "p95": 1.1215436579448124e-08,
+              "p99": 5.440258732392067e-08,
+              "p99_9": 3.8030532323318766e-07,
+              "p100": 3.234339146729326e-06
+            }
+          }
+        },
+        "per_tensor": {
+          "mlp.gate_proj.weight": {
+            "num_elements": 24903680,
+            "fa_mean": 1.5215001188775396e-09,
+            "fb_mean": 2.433113733882477e-09,
+            "denom_mean": 3.95461400234796e-09,
+            "active_ratio": 1.0,
+            "mean_abs_fa_minus_fb": 2.0093418578127103e-09,
+            "mean_relative_contrast": 0.43208397313168173,
+            "prior_mean": 0.37069509405838813,
+            "prior_active_mean": 0.3706950843334198,
+            "fa_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 1.5215001188775396e-09,
+              "std": 2.8865351836064804e-08,
+              "min": 1.5798755961791253e-17,
+              "max": 3.023528552148491e-05,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 2.7453715903412387e-17,
+                "p1": 5.383182094531591e-11,
+                "p10": 1.7551146647143412e-10,
+                "p25": 3.080868615779053e-10,
+                "p50": 5.526146207301963e-10,
+                "p75": 1.0270205796913956e-09,
+                "p90": 1.925444692574274e-09,
+                "p95": 2.958600031632841e-09,
+                "p99": 8.03926969439317e-09,
+                "p99_9": 5.0021313313663995e-08,
+                "p100": 6.889340511406772e-07
+              }
+            },
+            "fb_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 2.433113733882477e-09,
+              "std": 3.1765033841484075e-08,
+              "min": 5.118014604049348e-17,
+              "max": 2.8242542612133548e-05,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 9.45427272257565e-17,
+                "p1": 1.2251744063718206e-10,
+                "p10": 3.7222364079880776e-10,
+                "p25": 6.349160086571715e-10,
+                "p50": 1.10203801639841e-09,
+                "p75": 1.8901853415798087e-09,
+                "p90": 3.254403191377264e-09,
+                "p95": 4.82538631274565e-09,
+                "p99": 1.1802093524693191e-08,
+                "p99_9": 6.372916772079407e-08,
+                "p100": 5.957631969977228e-07
+              }
+            },
+            "denom_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 3.95461400234796e-09,
+              "std": 4.9700039684611095e-08,
+              "min": 1.1946107995777343e-16,
+              "max": 3.0317112759803422e-05,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 4.048886574000316e-16,
+                "p1": 3.2757968559948836e-10,
+                "p10": 7.902159482320315e-10,
+                "p25": 1.1861218673914209e-09,
+                "p50": 1.835636309621691e-09,
+                "p75": 2.925039765955262e-09,
+                "p90": 4.879169068772171e-09,
+                "p95": 7.175397609415768e-09,
+                "p99": 1.7596196499880534e-08,
+                "p99_9": 1.022413869122829e-07,
+                "p100": 7.006891564742546e-07
+              }
+            }
+          },
+          "mlp.up_proj.weight": {
+            "num_elements": 24903680,
+            "fa_mean": 2.2250455793638187e-09,
+            "fb_mean": 3.3638406493034718e-09,
+            "denom_mean": 5.588886827019068e-09,
+            "active_ratio": 1.0,
+            "mean_abs_fa_minus_fb": 2.6430687907314248e-09,
+            "mean_relative_contrast": 0.37188861244603205,
+            "prior_mean": 0.39344213385331,
+            "prior_active_mean": 0.3934421241283417,
+            "fa_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 2.2250455793638187e-09,
+              "std": 5.303644378495179e-08,
+              "min": 2.0082707810591133e-16,
+              "max": 8.031377365114167e-05,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 4.372049231014737e-16,
+                "p1": 1.5466086744631014e-10,
+                "p10": 3.1708349834680405e-10,
+                "p25": 4.475782799939765e-10,
+                "p50": 7.05520086796696e-10,
+                "p75": 1.2839318497270824e-09,
+                "p90": 2.539314092331324e-09,
+                "p95": 4.105263684550664e-09,
+                "p99": 1.3281240995866028e-08,
+                "p99_9": 6.281263864593711e-08,
+                "p100": 2.041214884229703e-06
+              }
+            },
+            "fb_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 3.3638406493034718e-09,
+              "std": 6.852858222462656e-08,
+              "min": 2.038748879385895e-16,
+              "max": 7.564180850749835e-05,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 5.093726331333525e-16,
+                "p1": 2.4593260761207603e-10,
+                "p10": 5.425060400909842e-10,
+                "p25": 7.695817316744069e-10,
+                "p50": 1.2012056904708857e-09,
+                "p75": 2.1004162853444086e-09,
+                "p90": 3.972078221892161e-09,
+                "p95": 6.275899799845774e-09,
+                "p99": 1.9911867710220577e-08,
+                "p99_9": 9.005135126471941e-08,
+                "p100": 1.193124262499623e-06
+              }
+            },
+            "denom_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 5.588886827019068e-09,
+              "std": 1.0347559117750533e-07,
+              "min": 8.566257012610737e-16,
+              "max": 0.0001292934757657349,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 1.5408257758890362e-15,
+                "p1": 5.594407714859528e-10,
+                "p10": 1.0695031527063747e-09,
+                "p25": 1.4380779900236007e-09,
+                "p50": 2.1161083996190655e-09,
+                "p75": 3.4873259835421777e-09,
+                "p90": 6.32809982192839e-09,
+                "p95": 9.881541807033045e-09,
+                "p99": 2.999603410103191e-08,
+                "p99_9": 1.4827676864115347e-07,
+                "p100": 3.234339146729326e-06
+              }
+            }
+          },
+          "mlp.down_proj.weight": {
+            "num_elements": 24903680,
+            "fa_mean": 3.2524248557532894e-09,
+            "fb_mean": 3.393648440626413e-09,
+            "denom_mean": 6.64607389473148e-09,
+            "active_ratio": 1.0,
+            "mean_abs_fa_minus_fb": 3.5472632924594156e-09,
+            "mean_relative_contrast": 0.38642080206620066,
+            "prior_mean": 0.45513691149259866,
+            "prior_active_mean": 0.4551369249820709,
+            "fa_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 3.2524248557532894e-09,
+              "std": 1.1486712981317914e-08,
+              "min": 2.998382098162722e-11,
+              "max": 1.2815156878787093e-05,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 7.099346110983618e-11,
+                "p1": 2.929453624123113e-10,
+                "p10": 5.22057452823077e-10,
+                "p25": 7.564069370857851e-10,
+                "p50": 1.2962776407832166e-09,
+                "p75": 2.6015287701852685e-09,
+                "p90": 5.215481824194512e-09,
+                "p95": 8.517121230511293e-09,
+                "p99": 5.1338147954993474e-08,
+                "p99_9": 3.505568884065724e-07,
+                "p100": 1.3563462744059507e-06
+              }
+            },
+            "fb_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 3.393648440626413e-09,
+              "std": 1.0468527427652187e-08,
+              "min": 3.501308271092185e-11,
+              "max": 4.42080045104376e-06,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 8.949042201722435e-11,
+                "p1": 4.6213197157918273e-10,
+                "p10": 7.801047585687115e-10,
+                "p25": 1.0709344522297215e-09,
+                "p50": 1.6280852221939313e-09,
+                "p75": 2.841281876442281e-09,
+                "p90": 5.48243850317931e-09,
+                "p95": 8.733022305307259e-09,
+                "p99": 7.965197568182703e-08,
+                "p99_9": 2.755199091097893e-07,
+                "p100": 8.375918127967452e-07
+              }
+            },
+            "denom_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 6.64607389473148e-09,
+              "std": 1.6735628488079415e-08,
+              "min": 1.6841017469459985e-10,
+              "max": 1.6790576410130598e-05,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 2.2261475995932756e-10,
+                "p1": 1.062598675716231e-09,
+                "p10": 1.6788760381913903e-09,
+                "p25": 2.279081590117471e-09,
+                "p50": 3.447163221537153e-09,
+                "p75": 5.653203682953745e-09,
+                "p90": 1.0188144550227207e-08,
+                "p95": 1.7312565603333496e-08,
+                "p99": 1.0933062810636329e-07,
+                "p99_9": 4.976928380528989e-07,
+                "p100": 1.5853232753215707e-06
+              }
+            }
+          }
+        }
+      }
+    },
+    "reparam_gate_summary": {
+      "num_tensors": 3,
+      "num_elements": 74711040,
+      "global_mean": 0.40624111242461625,
+      "per_tensor_mean": {
+        "mlp.gate_proj.weight": 0.37054505944252014,
+        "mlp.up_proj.weight": 0.39327558875083923,
+        "mlp.down_proj.weight": 0.4549027383327484
+      }
+    },
+    "post_merge_nll_scale": null,
+    "merge_only_ppl": null,
+    "post_ppl": null,
+    "weight_decay": 0.0,
+    "max_grad_norm": 1.0,
+    "grad_accum_steps": 1,
+    "instruction_dataset": null,
+    "instruction_config": null,
+    "instruction_split": "train"
+  },
+  "lora": {
+    "enabled": false,
+    "seq_len": 1024,
+    "batch_size": 1,
+    "epochs": 0.0,
+    "rank": 8,
+    "alpha": 16.0,
+    "dropout": 0.0,
+    "target_modules": [
+      "q_proj",
+      "k_proj",
+      "v_proj",
+      "o_proj",
+      "gate_proj",
+      "down_proj",
+      "up_proj"
+    ],
+    "respect_exclude_pairs": false,
+    "kl_enabled": false,
+    "kl_weight": 0.1,
+    "kl_temp": 4.0,
+    "post_ppl": null,
+    "lr": 0.0001,
+    "weight_decay": 0.0,
+    "max_grad_norm": 1.0,
+    "grad_accum_steps": 1,
+    "log_steps": 100,
+    "eval_every": 2000,
+    "eval_max_batches": null
+  },
+  "norm_policy": "hybrid",
+  "full_model_saved": false,
+  "timing": {
+    "seconds": {
+      "selection": 2.9968912713229656e-05,
+      "rebase": 0.0,
+      "fisher": 15.864232624066062,
+      "fit": 242.4078334759688,
+      "materialize_save_layer": 0.23381438199430704,
+      "metadata_write": 0.002036107936874032,
+      "teacher_snapshot": 0.26235245703719556,
+      "cycle_cleanup": 0.17335443303454667,
+      "cycle_total": 258.94427959702443
+    },
+    "tracked_component_total_seconds": 258.9436534489505,
+    "cycle_total_seconds": 258.94427959702443,
+    "ratios_of_tracked_components": {
+      "selection": 1.1573526639508037e-07,
+      "rebase": 0.0,
+      "fisher": 0.06126519191633179,
+      "fit": 0.9361412424952842,
+      "materialize_save_layer": 0.0009029546732660217,
+      "metadata_write": 7.86313126332498e-06,
+      "teacher_snapshot": 0.001013164267758032,
+      "cycle_cleanup": 0.0006694677808302518
+    },
+    "ratios_of_cycle_total": {
+      "selection": 1.1573498653790703e-07,
+      "rebase": 0.0,
+      "fisher": 0.06126504377217515,
+      "fit": 0.936138978830542,
+      "materialize_save_layer": 0.0009029524898490703,
+      "metadata_write": 7.863112249641793e-06,
+      "teacher_snapshot": 0.0010131618178454262,
+      "cycle_cleanup": 0.0006694661620033668
+    }
+  }
+}

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/metadata/cycle_03.json

@@ -0,0 +1,714 @@
+{
+  "cycle": 3,
+  "layer_merged": 25,
+  "num_layers_before": 34,
+  "num_layers_after": 33,
+  "fused_layer_state": "fused_layer.pt",
+  "profile_no_save": false,
+  "dwce_score": null,
+  "dwce_scores": null,
+  "dwce_meta": {
+    "method": "ordered",
+    "scope": "absolute",
+    "order_position": 2,
+    "requested_index": 25,
+    "resolved_index": 25,
+    "num_pairs": 33,
+    "exclude_pairs": [
+      0,
+      1,
+      32
+    ],
+    "start_index_ignored": 25
+  },
+  "fisher_num_batches": 32,
+  "fisher_input": {
+    "mode": "anchor_input",
+    "tokens": 65536,
+    "context_vs_native_source_mlp_input_cosine": 0.9345197342336178,
+    "context_vs_native_source_mlp_input_norm_ratio": 0.8426651768386364,
+    "native_source_mlp_input_global_rms": 0.8828791755630592,
+    "context_source_mlp_input_global_rms": 0.741906545817173,
+    "context_detached": true,
+    "anchor_attention_included": false,
+    "normalization": "anchor.post_attention_layernorm(anchor_block_input)"
+  },
+  "merge_method": "reparam",
+  "merged_params": 3,
+  "num_sequences": 64,
+  "teacher_source": "previous_cycle_memory",
+  "teacher_cycle": 2,
+  "eval": {
+    "datasets": [
+      "wikitext2"
+    ],
+    "configs": [
+      null
+    ],
+    "split": "test",
+    "num_samples": 0,
+    "seq_len": 1024,
+    "post_ppl": null,
+    "merge_only_ppl": null
+  },
+  "comm": {
+    "enabled": false
+  },
+  "distill": {
+    "enabled": true,
+    "method": "reparam",
+    "calib_samples": 256,
+    "inst_samples": 0,
+    "seq_len": 1024,
+    "batch_size": 1,
+    "epochs": 1.0,
+    "lr": 0.0001,
+    "kl_weight": 0.02,
+    "kl_temp": 4.0,
+    "kl_chunk_size": 16384,
+    "hidden_mse_weight": 1.0,
+    "attn_mse_weight": 0.0,
+    "mlp_mse_weight": 1.0,
+    "reparam_eta": 0.0,
+    "reparam_gamma": 0.0,
+    "reparam_attn_reg_scale": 1.0,
+    "reparam_mlp_reg_scale": 1.0,
+    "reparam_param_subset": "mlp",
+    "reparam_train_subset": "same",
+    "reparam_skip_train": false,
+    "reparam_stats": {
+      "enabled": true,
+      "epochs": 1.0,
+      "requested_epochs": 1.0,
+      "skip_train": false,
+      "trained": true,
+      "lr": 0.0001,
+      "hidden_mse_weight": 1.0,
+      "attn_mse_weight": 0.0,
+      "mlp_mse_weight": 1.0,
+      "eta": 0.0,
+      "gamma": 0.0,
+      "attn_reg_scale": 1.0,
+      "mlp_reg_scale": 1.0,
+      "param_subset": "mlp",
+      "fisher_init_subset": "mlp",
+      "fit_only_parameter_count": 0,
+      "fit_only_parameter_names": [],
+      "train_subset": "mlp",
+      "anchor_preserving_init": false,
+      "num_gates": 3,
+      "num_attn_gates": 0,
+      "num_mlp_gates": 3,
+      "num_other_gates": 0,
+      "num_trainable_attn_gates": 0,
+      "num_trainable_mlp_gates": 3,
+      "num_trainable_other_gates": 0,
+      "lambda_init_summary": {
+        "num_tensors": 3,
+        "num_elements": 74711040,
+        "mean": 0.4212010299950315,
+        "std": 0.23356970592726928,
+        "min": 9.999999747378752e-05,
+        "max": 0.9997745156288147,
+        "mean_abs_from_0_5": 0.2105757194652892,
+        "per_tensor": {
+          "mlp.gate_proj.weight": {
+            "mean": 0.3716253180252878,
+            "std": 0.23303592978720136,
+            "min": 9.999999747378752e-05,
+            "max": 0.9995793700218201,
+            "mean_abs_from_0_5": 0.2299541674162212
+          },
+          "mlp.up_proj.weight": {
+            "mean": 0.39956765425832647,
+            "std": 0.20997761321987776,
+            "min": 0.00029567029559984803,
+            "max": 0.9992896318435669,
+            "mean_abs_from_0_5": 0.19897422790527344
+          },
+          "mlp.down_proj.weight": {
+            "mean": 0.49241011770148024,
+            "std": 0.23931483786111962,
+            "min": 0.00010188115265918896,
+            "max": 0.9997745156288147,
+            "mean_abs_from_0_5": 0.20279876307437295
+          }
+        }
+      },
+      "lambda_final_summary": {
+        "num_tensors": 3,
+        "num_elements": 74711040,
+        "mean": 0.4210428873697917,
+        "std": 0.2336966499990058,
+        "min": 9.989592217607424e-05,
+        "max": 0.9997745156288147,
+        "mean_abs_from_0_5": 0.21081958569978412,
+        "per_tensor": {
+          "mlp.gate_proj.weight": {
+            "mean": 0.3715029666298314,
+            "std": 0.2331458652922106,
+            "min": 9.989592217607424e-05,
+            "max": 0.9995793700218201,
+            "mean_abs_from_0_5": 0.2301787727757504
+          },
+          "mlp.up_proj.weight": {
+            "mean": 0.39941912199321544,
+            "std": 0.21011744922164038,
+            "min": 0.000295877835014835,
+            "max": 0.9992902278900146,
+            "mean_abs_from_0_5": 0.199235695286801
+          },
+          "mlp.down_proj.weight": {
+            "mean": 0.4922065734863281,
+            "std": 0.2394785470440631,
+            "min": 0.00010187706357100978,
+            "max": 0.9997745156288147,
+            "mean_abs_from_0_5": 0.203044289036801
+          }
+        }
+      },
+      "lambda_delta_from_init": {
+        "num_tensors": 3,
+        "num_elements": 74711040,
+        "mean_abs": 0.0007153441770034924,
+        "rms": 0.001024013293275524,
+        "max_abs": 0.017594516277313232,
+        "per_tensor": {
+          "mlp.gate_proj.weight": {
+            "mean_abs": 0.0006471755473237288,
+            "rms": 0.0009584510435273058,
+            "max_abs": 0.017594516277313232
+          },
+          "mlp.up_proj.weight": {
+            "mean_abs": 0.0007402507882369192,
+            "rms": 0.0010366001368305436,
+            "max_abs": 0.012343227863311768
+          },
+          "mlp.down_proj.weight": {
+            "mean_abs": 0.0007586061954498291,
+            "rms": 0.0010736113951981274,
+            "max_abs": 0.015336185693740845
+          }
+        }
+      },
+      "first_grad_summary": {
+        "joint": {
+          "step": 1,
+          "phase": "joint",
+          "gate": {
+            "num_tensors": 3,
+            "num_tensors_without_grad": 0,
+            "num_elements": 74711040,
+            "l2": 0.0133231244228809,
+            "mean_abs": 1.342424519335557e-07,
+            "max_abs": 0.004482282791286707
+          },
+          "u": {
+            "num_tensors": 3,
+            "num_tensors_without_grad": 0,
+            "num_elements": 74711040,
+            "l2": 1.417598118960336,
+            "mean_abs": 1.4800271182729486e-05,
+            "max_abs": 0.37310534715652466
+          },
+          "rho": {
+            "num_tensors": 0,
+            "num_tensors_without_grad": 0,
+            "num_elements": 0,
+            "l2": 0.0,
+            "mean_abs": null,
+            "max_abs": 0.0
+          }
+        }
+      },
+      "u_delta_summary": {
+        "num_tensors": 3,
+        "num_elements": 74711040,
+        "mean": -5.5260280772161325e-06,
+        "std": 0.005574702612524797,
+        "rms": 0.005574705351413392,
+        "mean_abs": 0.004377101597033049,
+        "min": -0.060992807149887085,
+        "max": 0.05952872335910797,
+        "per_tensor": {
+          "mlp.gate_proj.weight": {
+            "mean": -1.493488732529314e-05,
+            "std": 0.005506612586837063,
+            "rms": 0.005506632839800724,
+            "mean_abs": 0.004327193059419331,
+            "min": -0.057948097586631775,
+            "max": 0.054732732474803925
+          },
+          "mlp.up_proj.weight": {
+            "mean": -1.5638511333810656e-06,
+            "std": 0.0054651628651773065,
+            "rms": 0.0054651630889245555,
+            "mean_abs": 0.00431675534499319,
+            "min": -0.05090612173080444,
+            "max": 0.05136364698410034
+          },
+          "mlp.down_proj.weight": {
+            "mean": -7.93457729741931e-08,
+            "std": 0.005748130690756814,
+            "rms": 0.005748130691304449,
+            "mean_abs": 0.004487356386686626,
+            "min": -0.060992807149887085,
+            "max": 0.05952872335910797
+          }
+        }
+      },
+      "rho_objective": "none",
+      "rho_schedule": "joint",
+      "rho_epochs": 0.0,
+      "rho_lr": null,
+      "rho_bounds": null,
+      "rho_after_phase": null,
+      "rho_final": {},
+      "u_fisher_orthogonality": {},
+      "lambda_init": "fisher_prior",
+      "skip_fisher_merge_init": false,
+      "requested_skip_merge_cycles": [],
+      "fisher_prior_input_summary": {
+        "num_tensors": 3,
+        "num_elements": 74711040,
+        "fisher_mode": "param",
+        "eps": 0.0,
+        "param_subset": "mlp",
+        "fa_mean": 3.4907529560880728e-09,
+        "fb_mean": 3.608339152618244e-09,
+        "denom_mean": 7.099091909255724e-09,
+        "active_ratio": 1.0,
+        "mean_abs_fa_minus_fb": 3.850215977619978e-09,
+        "mean_relative_contrast": 0.4211514255456757,
+        "prior_mean": 0.4212010299950315,
+        "prior_active_mean": 0.42120103041330975,
+        "distributions": {
+          "fa": {
+            "quantile_method": "concatenated_per_tensor_even_stride_samples",
+            "quantile_sample_count": 393216,
+            "quantiles": {
+              "p0": 4.9110387946653096e-17,
+              "p1": 6.744973329864123e-11,
+              "p10": 2.553908751146139e-10,
+              "p25": 4.581742762965746e-10,
+              "p50": 8.972063647583184e-10,
+              "p75": 2.0415968915443727e-09,
+              "p90": 4.9818309477700495e-09,
+              "p95": 9.49643208514317e-09,
+              "p99": 4.527190711201001e-08,
+              "p99_9": 3.0338031820065225e-07,
+              "p100": 3.212454430467915e-06
+            }
+          },
+          "fb": {
+            "quantile_method": "concatenated_per_tensor_even_stride_samples",
+            "quantile_sample_count": 393216,
+            "quantiles": {
+              "p0": 9.340282601954413e-17,
+              "p1": 2.0569211611753957e-10,
+              "p10": 5.807981873218182e-10,
+              "p25": 8.882524715758677e-10,
+              "p50": 1.4196552822198782e-09,
+              "p75": 2.4382225127794754e-09,
+              "p90": 4.579676193827709e-09,
+              "p95": 7.493645703959828e-09,
+              "p99": 2.654263120405176e-08,
+              "p99_9": 1.2998445697576244e-07,
+              "p100": 9.553566997055896e-06
+            }
+          },
+          "denom": {
+            "quantile_method": "concatenated_per_tensor_even_stride_samples",
+            "quantile_sample_count": 393216,
+            "quantiles": {
+              "p0": 3.8577411486559704e-16,
+              "p1": 4.6532025455015e-10,
+              "p10": 1.0994267718444917e-09,
+              "p25": 1.6480441455968275e-09,
+              "p50": 2.668073317835251e-09,
+              "p75": 4.757259475240971e-09,
+              "p90": 9.842624493217045e-09,
+              "p95": 1.6633984856184725e-08,
+              "p99": 6.6978380175442e-08,
+              "p99_9": 3.9643850868742447e-07,
+              "p100": 1.276602142752381e-05
+            }
+          }
+        },
+        "per_tensor": {
+          "mlp.gate_proj.weight": {
+            "num_elements": 24903680,
+            "fa_mean": 2.0008303037780264e-09,
+            "fb_mean": 2.7869559385420104e-09,
+            "denom_mean": 4.787786242320037e-09,
+            "active_ratio": 1.0,
+            "mean_abs_fa_minus_fb": 2.567744379695413e-09,
+            "mean_relative_contrast": 0.45990829467773436,
+            "prior_mean": 0.3716253180252878,
+            "prior_active_mean": 0.37162530422210693,
+            "fa_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 2.0008303037780264e-09,
+              "std": 6.175828655001273e-08,
+              "min": 9.702156914589622e-18,
+              "max": 0.00011491585610201582,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 4.9110387946653096e-17,
+                "p1": 4.1023684449470466e-11,
+                "p10": 1.4101385337195183e-10,
+                "p25": 2.7586616324626334e-10,
+                "p50": 5.822460846793831e-10,
+                "p75": 1.2434536733607615e-09,
+                "p90": 2.5538253733969896e-09,
+                "p95": 4.056313063216521e-09,
+                "p99": 1.105847147186978e-08,
+                "p99_9": 5.7735228153887874e-08,
+                "p100": 9.19702301871439e-07
+              }
+            },
+            "fb_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 2.7869559385420104e-09,
+              "std": 4.9403207810468744e-08,
+              "min": 3.322607834846671e-17,
+              "max": 7.65280710766092e-05,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 9.340282601954413e-17,
+                "p1": 1.3895187778167895e-10,
+                "p10": 3.9794345596533276e-10,
+                "p25": 6.751451619990689e-10,
+                "p50": 1.1621330564537402e-09,
+                "p75": 2.0308341674990515e-09,
+                "p90": 3.541798854200806e-09,
+                "p95": 5.277855485985583e-09,
+                "p99": 1.3128763853842429e-08,
+                "p99_9": 7.189348139036156e-08,
+                "p100": 6.565408625647251e-07
+              }
+            },
+            "denom_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 4.787786242320037e-09,
+              "std": 8.665642188720831e-08,
+              "min": 9.186405832195869e-17,
+              "max": 0.00011531391646713018,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 3.8577411486559704e-16,
+                "p1": 3.3819735900664227e-10,
+                "p10": 8.056342815088158e-10,
+                "p25": 1.2402761040419819e-09,
+                "p50": 1.996588672170674e-09,
+                "p75": 3.330656417332989e-09,
+                "p90": 5.7661093677552344e-09,
+                "p95": 8.559139175190467e-09,
+                "p99": 2.132503240659389e-08,
+                "p99_9": 1.2814045646791783e-07,
+                "p100": 9.568631185175036e-07
+              }
+            }
+          },
+          "mlp.up_proj.weight": {
+            "num_elements": 24903680,
+            "fa_mean": 3.115581205891335e-09,
+            "fb_mean": 3.8725344989200634e-09,
+            "denom_mean": 6.988116303163176e-09,
+            "active_ratio": 1.0,
+            "mean_abs_fa_minus_fb": 3.500850640959566e-09,
+            "mean_relative_contrast": 0.3979484558105469,
+            "prior_mean": 0.39956765425832647,
+            "prior_active_mean": 0.3995676636695862,
+            "fa_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 3.115581205891335e-09,
+              "std": 1.226629581059088e-07,
+              "min": 1.6615440644987718e-16,
+              "max": 0.00020774986478500068,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 4.085887620114344e-16,
+                "p1": 1.3930817610585677e-10,
+                "p10": 2.89188784030614e-10,
+                "p25": 4.278759846432223e-10,
+                "p50": 7.494264098184544e-10,
+                "p75": 1.5956500565295073e-09,
+                "p90": 3.455201014190834e-09,
+                "p95": 5.954158055487824e-09,
+                "p99": 2.0365645170272728e-08,
+                "p99_9": 9.852370652652098e-08,
+                "p100": 3.212454430467915e-06
+              }
+            },
+            "fb_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 3.8725344989200634e-09,
+              "std": 1.1675086797049761e-07,
+              "min": 2.5377377091490775e-16,
+              "max": 0.0002655751013662666,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 4.901030041602316e-16,
+                "p1": 2.7961960524791607e-10,
+                "p10": 5.961968696510667e-10,
+                "p25": 8.271405227411321e-10,
+                "p50": 1.2557110906641356e-09,
+                "p75": 2.1960000484710918e-09,
+                "p90": 4.277086240733752e-09,
+                "p95": 6.862456825018626e-09,
+                "p99": 2.416306621455533e-08,
+                "p99_9": 1.1487016848832354e-07,
+                "p100": 9.553566997055896e-06
+              }
+            },
+            "denom_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 6.988116303163176e-09,
+              "std": 2.0452136492854775e-07,
+              "min": 6.376559318136939e-16,
+              "max": 0.0003713919431902468,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 1.4977149752473445e-15,
+                "p1": 5.791266355359426e-10,
+                "p10": 1.1107601505244702e-09,
+                "p25": 1.5103936990001898e-09,
+                "p50": 2.2976265334762047e-09,
+                "p75": 3.9733558665489e-09,
+                "p90": 7.602896978653462e-09,
+                "p95": 1.2159005358114428e-08,
+                "p99": 4.033581291196242e-08,
+                "p99_9": 1.9562816078178002e-07,
+                "p100": 1.276602142752381e-05
+              }
+            }
+          },
+          "mlp.down_proj.weight": {
+            "num_elements": 24903680,
+            "fa_mean": 5.355847358594857e-09,
+            "fb_mean": 4.165527020392658e-09,
+            "denom_mean": 9.52137318228396e-09,
+            "active_ratio": 1.0,
+            "mean_abs_fa_minus_fb": 5.482052912204955e-09,
+            "mean_relative_contrast": 0.4055975261487459,
+            "prior_mean": 0.49241011770148024,
+            "prior_active_mean": 0.4924101233482361,
+            "fa_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 5.355847358594857e-09,
+              "std": 2.6445244403639477e-08,
+              "min": 3.722554903218267e-11,
+              "max": 4.158949013799429e-05,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 8.586910349439592e-11,
+                "p1": 3.085957600568179e-10,
+                "p10": 5.49720158282696e-10,
+                "p25": 8.290371722452505e-10,
+                "p50": 1.6164970473298013e-09,
+                "p75": 3.847333118756069e-09,
+                "p90": 1.0387013027468583e-08,
+                "p95": 2.0887048535200847e-08,
+                "p99": 1.4719044827415928e-07,
+                "p99_9": 4.180859036750917e-07,
+                "p100": 1.7244606169697363e-06
+              }
+            },
+            "fb_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 4.165527020392658e-09,
+              "std": 2.1144822345388555e-08,
+              "min": 6.904850302635879e-11,
+              "max": 2.2764666937291622e-05,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 1.5472133296778878e-10,
+                "p1": 5.662488811175592e-10,
+                "p10": 9.134333289750884e-10,
+                "p25": 1.2362679768784801e-09,
+                "p50": 1.831772844518298e-09,
+                "p75": 3.060613984473548e-09,
+                "p90": 6.204539548804178e-09,
+                "p95": 1.133181992685195e-08,
+                "p99": 3.686090366272765e-08,
+                "p99_9": 1.4781498691718298e-07,
+                "p100": 9.631440889279475e-07
+              }
+            },
+            "denom_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 9.52137318228396e-09,
+              "std": 3.7715831763966836e-08,
+              "min": 2.1520285553577878e-10,
+              "max": 6.435415707528591e-05,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 3.187021202499807e-10,
+                "p1": 1.211141742452071e-09,
+                "p10": 1.900277712962861e-09,
+                "p25": 2.5887223475962173e-09,
+                "p50": 4.024935051916145e-09,
+                "p75": 7.699403781202818e-09,
+                "p90": 1.7037542932030192e-08,
+                "p95": 3.1331630623299134e-08,
+                "p99": 1.570304704046066e-07,
+                "p99_9": 5.062567538516305e-07,
+                "p100": 1.9648232409963384e-06
+              }
+            }
+          }
+        }
+      }
+    },
+    "reparam_gate_summary": {
+      "num_tensors": 3,
+      "num_elements": 74711040,
+      "global_mean": 0.4210428873697917,
+      "per_tensor_mean": {
+        "mlp.gate_proj.weight": 0.37150299549102783,
+        "mlp.up_proj.weight": 0.3994191288948059,
+        "mlp.down_proj.weight": 0.4922066032886505
+      }
+    },
+    "post_merge_nll_scale": null,
+    "merge_only_ppl": null,
+    "post_ppl": null,
+    "weight_decay": 0.0,
+    "max_grad_norm": 1.0,
+    "grad_accum_steps": 1,
+    "instruction_dataset": null,
+    "instruction_config": null,
+    "instruction_split": "train"
+  },
+  "lora": {
+    "enabled": false,
+    "seq_len": 1024,
+    "batch_size": 1,
+    "epochs": 0.0,
+    "rank": 8,
+    "alpha": 16.0,
+    "dropout": 0.0,
+    "target_modules": [
+      "q_proj",
+      "k_proj",
+      "v_proj",
+      "o_proj",
+      "gate_proj",
+      "down_proj",
+      "up_proj"
+    ],
+    "respect_exclude_pairs": false,
+    "kl_enabled": false,
+    "kl_weight": 0.1,
+    "kl_temp": 4.0,
+    "post_ppl": null,
+    "lr": 0.0001,
+    "weight_decay": 0.0,
+    "max_grad_norm": 1.0,
+    "grad_accum_steps": 1,
+    "log_steps": 100,
+    "eval_every": 2000,
+    "eval_max_batches": null
+  },
+  "norm_policy": "hybrid",
+  "full_model_saved": false,
+  "timing": {
+    "seconds": {
+      "selection": 2.824899274855852e-05,
+      "rebase": 0.0,
+      "fisher": 15.648925730027258,
+      "fit": 238.8646190930158,
+      "materialize_save_layer": 0.2815707069821656,
+      "metadata_write": 0.0019015349680557847,
+      "teacher_snapshot": 0.2577912079868838,
+      "cycle_cleanup": 0.1744942250661552,
+      "cycle_total": 255.22997705591843
+    },
+    "tracked_component_total_seconds": 255.22933074703906,
+    "cycle_total_seconds": 255.22997705591843,
+    "ratios_of_tracked_components": {
+      "selection": 1.1068082444081025e-07,
+      "rebase": 0.0,
+      "fisher": 0.06131319501651283,
+      "fit": 0.9358823235318415,
+      "materialize_save_layer": 0.001103206697122259,
+      "metadata_write": 7.450299550173642e-06,
+      "teacher_snapshot": 0.0010100375502782,
+      "cycle_cleanup": 0.0006836762238705965
+    },
+    "ratios_of_cycle_total": {
+      "selection": 1.1068054416809136e-07,
+      "rebase": 0.0,
+      "fisher": 0.06131303975550932,
+      "fit": 0.9358799536336707,
+      "materialize_save_layer": 0.0011032039035151272,
+      "metadata_write": 7.450280684071749e-06,
+      "teacher_snapshot": 0.0010100349925996517,
+      "cycle_cleanup": 0.0006836744926240588
+    }
+  }
+}

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/metadata/cycle_04.json

@@ -0,0 +1,714 @@
+{
+  "cycle": 4,
+  "layer_merged": 25,
+  "num_layers_before": 33,
+  "num_layers_after": 32,
+  "fused_layer_state": "fused_layer.pt",
+  "profile_no_save": false,
+  "dwce_score": null,
+  "dwce_scores": null,
+  "dwce_meta": {
+    "method": "ordered",
+    "scope": "absolute",
+    "order_position": 3,
+    "requested_index": 25,
+    "resolved_index": 25,
+    "num_pairs": 32,
+    "exclude_pairs": [
+      0,
+      1,
+      31
+    ],
+    "start_index_ignored": 25
+  },
+  "fisher_num_batches": 32,
+  "fisher_input": {
+    "mode": "anchor_input",
+    "tokens": 65536,
+    "context_vs_native_source_mlp_input_cosine": 0.9170798286795616,
+    "context_vs_native_source_mlp_input_norm_ratio": 0.7955950330942869,
+    "native_source_mlp_input_global_rms": 0.9356920004316579,
+    "context_source_mlp_input_global_rms": 0.741906545817173,
+    "context_detached": true,
+    "anchor_attention_included": false,
+    "normalization": "anchor.post_attention_layernorm(anchor_block_input)"
+  },
+  "merge_method": "reparam",
+  "merged_params": 3,
+  "num_sequences": 64,
+  "teacher_source": "previous_cycle_memory",
+  "teacher_cycle": 3,
+  "eval": {
+    "datasets": [
+      "wikitext2"
+    ],
+    "configs": [
+      null
+    ],
+    "split": "test",
+    "num_samples": 0,
+    "seq_len": 1024,
+    "post_ppl": null,
+    "merge_only_ppl": null
+  },
+  "comm": {
+    "enabled": false
+  },
+  "distill": {
+    "enabled": true,
+    "method": "reparam",
+    "calib_samples": 256,
+    "inst_samples": 0,
+    "seq_len": 1024,
+    "batch_size": 1,
+    "epochs": 1.0,
+    "lr": 0.0001,
+    "kl_weight": 0.02,
+    "kl_temp": 4.0,
+    "kl_chunk_size": 16384,
+    "hidden_mse_weight": 1.0,
+    "attn_mse_weight": 0.0,
+    "mlp_mse_weight": 1.0,
+    "reparam_eta": 0.0,
+    "reparam_gamma": 0.0,
+    "reparam_attn_reg_scale": 1.0,
+    "reparam_mlp_reg_scale": 1.0,
+    "reparam_param_subset": "mlp",
+    "reparam_train_subset": "same",
+    "reparam_skip_train": false,
+    "reparam_stats": {
+      "enabled": true,
+      "epochs": 1.0,
+      "requested_epochs": 1.0,
+      "skip_train": false,
+      "trained": true,
+      "lr": 0.0001,
+      "hidden_mse_weight": 1.0,
+      "attn_mse_weight": 0.0,
+      "mlp_mse_weight": 1.0,
+      "eta": 0.0,
+      "gamma": 0.0,
+      "attn_reg_scale": 1.0,
+      "mlp_reg_scale": 1.0,
+      "param_subset": "mlp",
+      "fisher_init_subset": "mlp",
+      "fit_only_parameter_count": 0,
+      "fit_only_parameter_names": [],
+      "train_subset": "mlp",
+      "anchor_preserving_init": false,
+      "num_gates": 3,
+      "num_attn_gates": 0,
+      "num_mlp_gates": 3,
+      "num_other_gates": 0,
+      "num_trainable_attn_gates": 0,
+      "num_trainable_mlp_gates": 3,
+      "num_trainable_other_gates": 0,
+      "lambda_init_summary": {
+        "num_tensors": 3,
+        "num_elements": 74711040,
+        "mean": 0.4580573098701343,
+        "std": 0.24263101371239107,
+        "min": 0.00014809299318585545,
+        "max": 0.9998855590820312,
+        "mean_abs_from_0_5": 0.2100394934938665,
+        "per_tensor": {
+          "mlp.gate_proj.weight": {
+            "mean": 0.407879638671875,
+            "std": 0.24441356969159123,
+            "min": 0.00014809299318585545,
+            "max": 0.999763548374176,
+            "mean_abs_from_0_5": 0.22474003842002468
+          },
+          "mlp.up_proj.weight": {
+            "mean": 0.42505790308902136,
+            "std": 0.21936847342470758,
+            "min": 0.00016644679999444634,
+            "max": 0.9994775652885437,
+            "mean_abs_from_0_5": 0.19783714695980675
+          },
+          "mlp.down_proj.weight": {
+            "mean": 0.5412343878495066,
+            "std": 0.2412959187952874,
+            "min": 0.000617141486145556,
+            "max": 0.9998855590820312,
+            "mean_abs_from_0_5": 0.20754129510176808
+          }
+        }
+      },
+      "lambda_final_summary": {
+        "num_tensors": 3,
+        "num_elements": 74711040,
+        "mean": 0.4579571372584293,
+        "std": 0.24276338768358452,
+        "min": 0.0001480335631640628,
+        "max": 0.999885082244873,
+        "mean_abs_from_0_5": 0.21025187174479168,
+        "per_tensor": {
+          "mlp.gate_proj.weight": {
+            "mean": 0.40780892623098275,
+            "std": 0.2445381049150893,
+            "min": 0.0001480335631640628,
+            "max": 0.9997634291648865,
+            "mean_abs_from_0_5": 0.2249459517629523
+          },
+          "mlp.up_proj.weight": {
+            "mean": 0.42495578966642683,
+            "std": 0.21952058614349726,
+            "min": 0.00016639918612781912,
+            "max": 0.9994774460792542,
+            "mean_abs_from_0_5": 0.19808281346371298
+          },
+          "mlp.down_proj.weight": {
+            "mean": 0.5411066958778783,
+            "std": 0.24144612979261557,
+            "min": 0.0006164578371681273,
+            "max": 0.999885082244873,
+            "mean_abs_from_0_5": 0.2077268500077097
+          }
+        }
+      },
+      "lambda_delta_from_init": {
+        "num_tensors": 3,
+        "num_elements": 74711040,
+        "mean_abs": 0.0007044661463352672,
+        "rms": 0.0009954130078559018,
+        "max_abs": 0.014476001262664795,
+        "per_tensor": {
+          "mlp.gate_proj.weight": {
+            "mean_abs": 0.0006585721906862761,
+            "rms": 0.0009558323586327739,
+            "max_abs": 0.014476001262664795
+          },
+          "mlp.up_proj.weight": {
+            "mean_abs": 0.0007559581806785182,
+            "rms": 0.001046725696129258,
+            "max_abs": 0.012083232402801514
+          },
+          "mlp.down_proj.weight": {
+            "mean_abs": 0.0006988680676410072,
+            "rms": 0.0009814738854801131,
+            "max_abs": 0.012299299240112305
+          }
+        }
+      },
+      "first_grad_summary": {
+        "joint": {
+          "step": 1,
+          "phase": "joint",
+          "gate": {
+            "num_tensors": 3,
+            "num_tensors_without_grad": 0,
+            "num_elements": 74711040,
+            "l2": 0.014892112273215585,
+            "mean_abs": 1.6281908501058065e-07,
+            "max_abs": 0.004719685763120651
+          },
+          "u": {
+            "num_tensors": 3,
+            "num_tensors_without_grad": 0,
+            "num_elements": 74711040,
+            "l2": 2.6083320086499184,
+            "mean_abs": 1.8988910392580324e-05,
+            "max_abs": 0.666825532913208
+          },
+          "rho": {
+            "num_tensors": 0,
+            "num_tensors_without_grad": 0,
+            "num_elements": 0,
+            "l2": 0.0,
+            "mean_abs": null,
+            "max_abs": 0.0
+          }
+        }
+      },
+      "u_delta_summary": {
+        "num_tensors": 3,
+        "num_elements": 74711040,
+        "mean": -2.833513255545608e-06,
+        "std": 0.005553694112230594,
+        "rms": 0.0055536948350645024,
+        "mean_abs": 0.004375604893031873,
+        "min": -0.05110279470682144,
+        "max": 0.052179597318172455,
+        "per_tensor": {
+          "mlp.gate_proj.weight": {
+            "mean": -9.59915403080614e-06,
+            "std": 0.0056053621298696775,
+            "rms": 0.005605370349114787,
+            "mean_abs": 0.00443419845480668,
+            "min": -0.04592924565076828,
+            "max": 0.05161784589290619
+          },
+          "mlp.up_proj.weight": {
+            "mean": 1.130695148420177e-06,
+            "std": 0.005637471292762231,
+            "rms": 0.0056374714061527414,
+            "mean_abs": 0.004469261671367445,
+            "min": -0.04231437295675278,
+            "max": 0.04117022827267647
+          },
+          "mlp.down_proj.weight": {
+            "mean": -3.208088425086125e-08,
+            "std": 0.005415654933257122,
+            "rms": 0.005415654933352142,
+            "mean_abs": 0.004223354552921496,
+            "min": -0.05110279470682144,
+            "max": 0.052179597318172455
+          }
+        }
+      },
+      "rho_objective": "none",
+      "rho_schedule": "joint",
+      "rho_epochs": 0.0,
+      "rho_lr": null,
+      "rho_bounds": null,
+      "rho_after_phase": null,
+      "rho_final": {},
+      "u_fisher_orthogonality": {},
+      "lambda_init": "fisher_prior",
+      "skip_fisher_merge_init": false,
+      "requested_skip_merge_cycles": [],
+      "fisher_prior_input_summary": {
+        "num_tensors": 3,
+        "num_elements": 74711040,
+        "fisher_mode": "param",
+        "eps": 0.0,
+        "param_subset": "mlp",
+        "fa_mean": 5.5289961022808395e-09,
+        "fb_mean": 3.692328295743975e-09,
+        "denom_mean": 9.221324497750111e-09,
+        "active_ratio": 1.0,
+        "mean_abs_fa_minus_fb": 5.451618945746879e-09,
+        "mean_relative_contrast": 0.4200789736028303,
+        "prior_mean": 0.458057323255037,
+        "prior_active_mean": 0.45805733402570087,
+        "distributions": {
+          "fa": {
+            "quantile_method": "concatenated_per_tensor_even_stride_samples",
+            "quantile_sample_count": 393216,
+            "quantiles": {
+              "p0": 4.496545207159888e-17,
+              "p1": 8.008665258740777e-11,
+              "p10": 3.0179608812019865e-10,
+              "p25": 5.636144884135774e-10,
+              "p50": 1.1933728449875503e-09,
+              "p75": 2.8763742498938427e-09,
+              "p90": 7.203484475581945e-09,
+              "p95": 1.3144569876999412e-08,
+              "p99": 6.477724667774964e-08,
+              "p99_9": 3.066589897571248e-07,
+              "p100": 1.736645026539918e-05
+            }
+          },
+          "fb": {
+            "quantile_method": "concatenated_per_tensor_even_stride_samples",
+            "quantile_sample_count": 393216,
+            "quantiles": {
+              "p0": 1.1246857151528297e-16,
+              "p1": 2.244470581613811e-10,
+              "p10": 6.206291036647826e-10,
+              "p25": 9.581282434112381e-10,
+              "p50": 1.5343447623550333e-09,
+              "p75": 2.585785585651479e-09,
+              "p90": 4.8078234726744995e-09,
+              "p95": 7.47101314146903e-09,
+              "p99": 2.2268785926371493e-08,
+              "p99_9": 2.907799796503241e-07,
+              "p100": 4.478436039789813e-06
+            }
+          },
+          "denom": {
+            "quantile_method": "concatenated_per_tensor_even_stride_samples",
+            "quantile_sample_count": 393216,
+            "quantiles": {
+              "p0": 3.7603117132936365e-16,
+              "p1": 5.239008671331646e-10,
+              "p10": 1.228398494035332e-09,
+              "p25": 1.8775123677983174e-09,
+              "p50": 3.172766271930527e-09,
+              "p75": 5.925352652980109e-09,
+              "p90": 1.1810028510694792e-08,
+              "p95": 1.9583909605103145e-08,
+              "p99": 8.239430115963842e-08,
+              "p99_9": 5.784818313259166e-07,
+              "p100": 1.7384478269377723e-05
+            }
+          }
+        },
+        "per_tensor": {
+          "mlp.gate_proj.weight": {
+            "num_elements": 24903680,
+            "fa_mean": 3.0919385321076813e-09,
+            "fb_mean": 3.119360096542024e-09,
+            "denom_mean": 6.211298329473816e-09,
+            "active_ratio": 1.0,
+            "mean_abs_fa_minus_fb": 3.524305731460979e-09,
+            "mean_relative_contrast": 0.44948007684004937,
+            "prior_mean": 0.407879638671875,
+            "prior_active_mean": 0.40787965059280396,
+            "fa_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 3.0919385321076813e-09,
+              "std": 1.9860889408716278e-07,
+              "min": 5.501347470699285e-18,
+              "max": 0.0005819218931719661,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 4.496545207159888e-17,
+                "p1": 4.799038250125065e-11,
+                "p10": 1.715660530310359e-10,
+                "p25": 3.413606897151311e-10,
+                "p50": 7.467454432585896e-10,
+                "p75": 1.6379795297893907e-09,
+                "p90": 3.4665845749515256e-09,
+                "p95": 5.607261766016336e-09,
+                "p99": 1.6615141262832367e-08,
+                "p99_9": 1.0001848949059422e-07,
+                "p100": 2.8657714210567065e-06
+              }
+            },
+            "fb_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 3.119360096542024e-09,
+              "std": 7.386643173549125e-08,
+              "min": 4.941216990389204e-17,
+              "max": 0.00012477046402636915,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 1.1246857151528297e-16,
+                "p1": 1.5077908366301074e-10,
+                "p10": 4.202491410421061e-10,
+                "p25": 7.016863201592116e-10,
+                "p50": 1.214516709424629e-09,
+                "p75": 2.172074076156605e-09,
+                "p90": 3.973810613899786e-09,
+                "p95": 6.033880506350897e-09,
+                "p99": 1.531950388766745e-08,
+                "p99_9": 6.981503020142554e-08,
+                "p100": 4.420937784743728e-06
+              }
+            },
+            "denom_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 6.211298329473816e-09,
+              "std": 2.182978333493113e-07,
+              "min": 1.4687982762797076e-16,
+              "max": 0.0005836660275235772,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 3.7603117132936365e-16,
+                "p1": 3.765399103627942e-10,
+                "p10": 9.018272240091108e-10,
+                "p25": 1.3917711427779977e-09,
+                "p50": 2.2726802662020873e-09,
+                "p75": 3.920083369024496e-09,
+                "p90": 7.084533848455976e-09,
+                "p95": 1.0762750690673784e-08,
+                "p99": 2.8655620099016232e-08,
+                "p99_9": 1.6804818869786686e-07,
+                "p100": 4.70151871923008e-06
+              }
+            }
+          },
+          "mlp.up_proj.weight": {
+            "num_elements": 24903680,
+            "fa_mean": 4.5894380132506885e-09,
+            "fb_mean": 3.998328685683143e-09,
+            "denom_mean": 8.587766100582054e-09,
+            "active_ratio": 1.0,
+            "mean_abs_fa_minus_fb": 4.621264045119077e-09,
+            "mean_relative_contrast": 0.3956742939196135,
+            "prior_mean": 0.4250578629343133,
+            "prior_active_mean": 0.42505785822868347,
+            "fa_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 4.5894380132506885e-09,
+              "std": 3.063204609119636e-07,
+              "min": 1.6562937837147752e-16,
+              "max": 0.0005579429562203586,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 3.9307590944784353e-16,
+                "p1": 1.5726865643106436e-10,
+                "p10": 3.2332847510474494e-10,
+                "p25": 4.901125061351763e-10,
+                "p50": 9.001318579393569e-10,
+                "p75": 1.9986625687806736e-09,
+                "p90": 4.5706372020504205e-09,
+                "p95": 8.249176453034579e-09,
+                "p99": 2.9520059285914613e-08,
+                "p99_9": 1.567699428051128e-07,
+                "p100": 1.736645026539918e-05
+              }
+            },
+            "fb_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 3.998328685683143e-09,
+              "std": 1.1989905179977847e-07,
+              "min": 2.676295888056528e-16,
+              "max": 0.0003172223223373294,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 5.345012948817498e-16,
+                "p1": 3.225757438940491e-10,
+                "p10": 6.486755577128633e-10,
+                "p25": 8.898415337910137e-10,
+                "p50": 1.3302192680697544e-09,
+                "p75": 2.294165746263843e-09,
+                "p90": 4.5327106512615956e-09,
+                "p95": 7.184062678078362e-09,
+                "p99": 2.299171697472957e-08,
+                "p99_9": 1.0017304674647676e-07,
+                "p100": 2.937207682407461e-06
+              }
+            },
+            "denom_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 8.587766100582054e-09,
+              "std": 3.5673787374520573e-07,
+              "min": 1.1316722281902404e-15,
+              "max": 0.0006367954192683101,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 1.783842916588084e-15,
+                "p1": 6.641797112294512e-10,
+                "p10": 1.2318223108209736e-09,
+                "p25": 1.6786774192922849e-09,
+                "p50": 2.557653866475107e-09,
+                "p75": 4.562751954040323e-09,
+                "p90": 9.025880487456561e-09,
+                "p95": 1.4564435879549364e-08,
+                "p99": 4.835067102249013e-08,
+                "p99_9": 2.4352050331799546e-07,
+                "p100": 1.7384478269377723e-05
+              }
+            }
+          },
+          "mlp.down_proj.weight": {
+            "num_elements": 24903680,
+            "fa_mean": 8.905611761484149e-09,
+            "fb_mean": 3.959296105006759e-09,
+            "denom_mean": 1.2864909063194462e-08,
+            "active_ratio": 1.0,
+            "mean_abs_fa_minus_fb": 8.20928706066058e-09,
+            "mean_relative_contrast": 0.4150825500488281,
+            "prior_mean": 0.5412344681589227,
+            "prior_active_mean": 0.5412344932556152,
+            "fa_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 8.905611761484149e-09,
+              "std": 1.0304853339768152e-07,
+              "min": 5.623263660270439e-11,
+              "max": 0.00031076997402124107,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 1.2515000147317323e-10,
+                "p1": 4.179714074847851e-10,
+                "p10": 7.708610971768337e-10,
+                "p25": 1.2210690236713617e-09,
+                "p50": 2.4853239466438026e-09,
+                "p75": 5.94486859739618e-09,
+                "p90": 1.4117514268718878e-08,
+                "p95": 3.2351145762277156e-08,
+                "p99": 1.1885208550665993e-07,
+                "p99_9": 4.0515749333280837e-07,
+                "p100": 5.607094408333069e-06
+              }
+            },
+            "fb_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 3.959296105006759e-09,
+              "std": 5.3970827248152716e-08,
+              "min": 9.058090388869289e-11,
+              "max": 0.0001465905806981027,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 1.4529331904267195e-10,
+                "p1": 6.393576779117893e-10,
+                "p10": 1.0530327720914556e-09,
+                "p25": 1.4058580966036516e-09,
+                "p50": 2.0048078752665788e-09,
+                "p75": 3.167273110449287e-09,
+                "p90": 5.88521720246149e-09,
+                "p95": 9.004961221137364e-09,
+                "p99": 3.396770154040496e-08,
+                "p99_9": 4.071422949891712e-07,
+                "p100": 4.478436039789813e-06
+              }
+            },
+            "denom_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 1.2864909063194462e-08,
+              "std": 1.2612532833667148e-07,
+              "min": 2.528115494726535e-10,
+              "max": 0.0003108055389020592,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 4.693783695053355e-10,
+                "p1": 1.4953719373878016e-09,
+                "p10": 2.3944108917817175e-09,
+                "p25": 3.3202118832065253e-09,
+                "p50": 5.231484578871459e-09,
+                "p75": 9.534314671100219e-09,
+                "p90": 1.928477644241866e-08,
+                "p95": 3.9661031792093127e-08,
+                "p99": 1.4153383176562784e-07,
+                "p99_9": 7.74018360516493e-07,
+                "p100": 1.0085530448122881e-05
+              }
+            }
+          }
+        }
+      }
+    },
+    "reparam_gate_summary": {
+      "num_tensors": 3,
+      "num_elements": 74711040,
+      "global_mean": 0.4579571372584293,
+      "per_tensor_mean": {
+        "mlp.gate_proj.weight": 0.40780892968177795,
+        "mlp.up_proj.weight": 0.424955815076828,
+        "mlp.down_proj.weight": 0.5411067008972168
+      }
+    },
+    "post_merge_nll_scale": null,
+    "merge_only_ppl": null,
+    "post_ppl": null,
+    "weight_decay": 0.0,
+    "max_grad_norm": 1.0,
+    "grad_accum_steps": 1,
+    "instruction_dataset": null,
+    "instruction_config": null,
+    "instruction_split": "train"
+  },
+  "lora": {
+    "enabled": false,
+    "seq_len": 1024,
+    "batch_size": 1,
+    "epochs": 0.0,
+    "rank": 8,
+    "alpha": 16.0,
+    "dropout": 0.0,
+    "target_modules": [
+      "q_proj",
+      "k_proj",
+      "v_proj",
+      "o_proj",
+      "gate_proj",
+      "down_proj",
+      "up_proj"
+    ],
+    "respect_exclude_pairs": false,
+    "kl_enabled": false,
+    "kl_weight": 0.1,
+    "kl_temp": 4.0,
+    "post_ppl": null,
+    "lr": 0.0001,
+    "weight_decay": 0.0,
+    "max_grad_norm": 1.0,
+    "grad_accum_steps": 1,
+    "log_steps": 100,
+    "eval_every": 2000,
+    "eval_max_batches": null
+  },
+  "norm_policy": "hybrid",
+  "full_model_saved": false,
+  "timing": {
+    "seconds": {
+      "selection": 3.2701995223760605e-05,
+      "rebase": 0.0,
+      "fisher": 15.490823901025578,
+      "fit": 226.9504180450458,
+      "materialize_save_layer": 0.3626433659810573,
+      "metadata_write": 0.0018653030274435878,
+      "teacher_snapshot": 0.2577098779147491,
+      "cycle_cleanup": 0.17373662791214883,
+      "cycle_total": 243.23791539203376
+    },
+    "tracked_component_total_seconds": 243.237229822902,
+    "cycle_total_seconds": 243.23791539203376,
+    "ratios_of_tracked_components": {
+      "selection": 1.344448596441035e-07,
+      "rebase": 0.0,
+      "fisher": 0.06368607269661908,
+      "fit": 0.9330414517970196,
+      "materialize_save_layer": 0.0014909040291451,
+      "metadata_write": 7.668657585032076e-06,
+      "teacher_snapshot": 0.0010595001353303705,
+      "cycle_cleanup": 0.0007142682394411592
+    },
+    "ratios_of_cycle_total": {
+      "selection": 1.3444448070957126e-07,
+      "rebase": 0.0,
+      "fisher": 0.0636858931966201,
+      "fit": 0.9330388220079221,
+      "materialize_save_layer": 0.0014908998270132937,
+      "metadata_write": 7.668635970823972e-06,
+      "teacher_snapshot": 0.0010594971491158787,
+      "cycle_cleanup": 0.0007142662262671194
+    }
+  }
+}

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/metadata/cycle_05.json

@@ -0,0 +1,714 @@
+{
+  "cycle": 5,
+  "layer_merged": 25,
+  "num_layers_before": 32,
+  "num_layers_after": 31,
+  "fused_layer_state": "fused_layer.pt",
+  "profile_no_save": false,
+  "dwce_score": null,
+  "dwce_scores": null,
+  "dwce_meta": {
+    "method": "ordered",
+    "scope": "absolute",
+    "order_position": 4,
+    "requested_index": 25,
+    "resolved_index": 25,
+    "num_pairs": 31,
+    "exclude_pairs": [
+      0,
+      1,
+      30
+    ],
+    "start_index_ignored": 25
+  },
+  "fisher_num_batches": 32,
+  "fisher_input": {
+    "mode": "anchor_input",
+    "tokens": 65536,
+    "context_vs_native_source_mlp_input_cosine": 0.907120268791914,
+    "context_vs_native_source_mlp_input_norm_ratio": 0.784327195957303,
+    "native_source_mlp_input_global_rms": 0.9506513367731548,
+    "context_source_mlp_input_global_rms": 0.741906545817173,
+    "context_detached": true,
+    "anchor_attention_included": false,
+    "normalization": "anchor.post_attention_layernorm(anchor_block_input)"
+  },
+  "merge_method": "reparam",
+  "merged_params": 3,
+  "num_sequences": 64,
+  "teacher_source": "previous_cycle_memory",
+  "teacher_cycle": 4,
+  "eval": {
+    "datasets": [
+      "wikitext2"
+    ],
+    "configs": [
+      null
+    ],
+    "split": "test",
+    "num_samples": 0,
+    "seq_len": 1024,
+    "post_ppl": null,
+    "merge_only_ppl": null
+  },
+  "comm": {
+    "enabled": false
+  },
+  "distill": {
+    "enabled": true,
+    "method": "reparam",
+    "calib_samples": 256,
+    "inst_samples": 0,
+    "seq_len": 1024,
+    "batch_size": 1,
+    "epochs": 1.0,
+    "lr": 0.0001,
+    "kl_weight": 0.02,
+    "kl_temp": 4.0,
+    "kl_chunk_size": 16384,
+    "hidden_mse_weight": 1.0,
+    "attn_mse_weight": 0.0,
+    "mlp_mse_weight": 1.0,
+    "reparam_eta": 0.0,
+    "reparam_gamma": 0.0,
+    "reparam_attn_reg_scale": 1.0,
+    "reparam_mlp_reg_scale": 1.0,
+    "reparam_param_subset": "mlp",
+    "reparam_train_subset": "same",
+    "reparam_skip_train": false,
+    "reparam_stats": {
+      "enabled": true,
+      "epochs": 1.0,
+      "requested_epochs": 1.0,
+      "skip_train": false,
+      "trained": true,
+      "lr": 0.0001,
+      "hidden_mse_weight": 1.0,
+      "attn_mse_weight": 0.0,
+      "mlp_mse_weight": 1.0,
+      "eta": 0.0,
+      "gamma": 0.0,
+      "attn_reg_scale": 1.0,
+      "mlp_reg_scale": 1.0,
+      "param_subset": "mlp",
+      "fisher_init_subset": "mlp",
+      "fit_only_parameter_count": 0,
+      "fit_only_parameter_names": [],
+      "train_subset": "mlp",
+      "anchor_preserving_init": false,
+      "num_gates": 3,
+      "num_attn_gates": 0,
+      "num_mlp_gates": 3,
+      "num_other_gates": 0,
+      "num_trainable_attn_gates": 0,
+      "num_trainable_mlp_gates": 3,
+      "num_trainable_other_gates": 0,
+      "lambda_init_summary": {
+        "num_tensors": 3,
+        "num_elements": 74711040,
+        "mean": 0.4628780431914748,
+        "std": 0.24472377510990143,
+        "min": 9.999999747378752e-05,
+        "max": 0.9997746348381042,
+        "mean_abs_from_0_5": 0.2110746885600843,
+        "per_tensor": {
+          "mlp.gate_proj.weight": {
+            "mean": 0.41098191110711346,
+            "std": 0.24769645370469148,
+            "min": 9.999999747378752e-05,
+            "max": 0.9995620846748352,
+            "mean_abs_from_0_5": 0.22703522130062706
+          },
+          "mlp.up_proj.weight": {
+            "mean": 0.41958778782894735,
+            "std": 0.22082401101706,
+            "min": 9.999999747378752e-05,
+            "max": 0.9997394680976868,
+            "mean_abs_from_0_5": 0.2010550850316098
+          },
+          "mlp.down_proj.weight": {
+            "mean": 0.5580644306383635,
+            "std": 0.23648403371627943,
+            "min": 0.0005670467508025467,
+            "max": 0.9997746348381042,
+            "mean_abs_from_0_5": 0.20513375934801603
+          }
+        }
+      },
+      "lambda_final_summary": {
+        "num_tensors": 3,
+        "num_elements": 74711040,
+        "mean": 0.46277491251627606,
+        "std": 0.24486259419477333,
+        "min": 9.998407767852768e-05,
+        "max": 0.9997747540473938,
+        "mean_abs_from_0_5": 0.2112954390676398,
+        "per_tensor": {
+          "mlp.gate_proj.weight": {
+            "mean": 0.41090706273129113,
+            "std": 0.24783250992305905,
+            "min": 9.998407767852768e-05,
+            "max": 0.9995643496513367,
+            "mean_abs_from_0_5": 0.22725866217362253
+          },
+          "mlp.up_proj.weight": {
+            "mean": 0.4194740696957237,
+            "std": 0.22098494680414246,
+            "min": 9.99880867311731e-05,
+            "max": 0.9997411370277405,
+            "mean_abs_from_0_5": 0.20131926285593132
+          },
+          "mlp.down_proj.weight": {
+            "mean": 0.5579436051218133,
+            "std": 0.23663358497063522,
+            "min": 0.0005670535028912127,
+            "max": 0.9997747540473938,
+            "mean_abs_from_0_5": 0.20530839217336555
+          }
+        }
+      },
+      "lambda_delta_from_init": {
+        "num_tensors": 3,
+        "num_elements": 74711040,
+        "mean_abs": 0.0007425977472673383,
+        "rms": 0.0010488923253869355,
+        "max_abs": 0.014951139688491821,
+        "per_tensor": {
+          "mlp.gate_proj.weight": {
+            "mean_abs": 0.0006981567332619115,
+            "rms": 0.0010138403019686283,
+            "max_abs": 0.014951139688491821
+          },
+          "mlp.up_proj.weight": {
+            "mean_abs": 0.0008036805610907705,
+            "rms": 0.0011122846666812294,
+            "max_abs": 0.011961936950683594
+          },
+          "mlp.down_proj.weight": {
+            "mean_abs": 0.0007259559474493328,
+            "rms": 0.0010175834084421907,
+            "max_abs": 0.012071788311004639
+          }
+        }
+      },
+      "first_grad_summary": {
+        "joint": {
+          "step": 1,
+          "phase": "joint",
+          "gate": {
+            "num_tensors": 3,
+            "num_tensors_without_grad": 0,
+            "num_elements": 74711040,
+            "l2": 0.024063533060815562,
+            "mean_abs": 3.2932629549402027e-07,
+            "max_abs": 0.005017740651965141
+          },
+          "u": {
+            "num_tensors": 3,
+            "num_tensors_without_grad": 0,
+            "num_elements": 74711040,
+            "l2": 5.286914000267545,
+            "mean_abs": 4.273248150160438e-05,
+            "max_abs": 0.986697256565094
+          },
+          "rho": {
+            "num_tensors": 0,
+            "num_tensors_without_grad": 0,
+            "num_elements": 0,
+            "l2": 0.0,
+            "mean_abs": null,
+            "max_abs": 0.0
+          }
+        }
+      },
+      "u_delta_summary": {
+        "num_tensors": 3,
+        "num_elements": 74711040,
+        "mean": -3.3053514223929758e-06,
+        "std": 0.005952320056232756,
+        "rms": 0.005952320973971309,
+        "mean_abs": 0.00468159558480246,
+        "min": -0.04747747629880905,
+        "max": 0.05788468196988106,
+        "per_tensor": {
+          "mlp.gate_proj.weight": {
+            "mean": -1.0741236129481542e-05,
+            "std": 0.006081615494397862,
+            "rms": 0.006081624979876164,
+            "mean_abs": 0.004804118683463649,
+            "min": -0.04747747629880905,
+            "max": 0.05788468196988106
+          },
+          "mlp.up_proj.weight": {
+            "mean": 5.44674893669588e-07,
+            "std": 0.006125061930111616,
+            "rms": 0.006125061954329392,
+            "mean_abs": 0.00485493634876452,
+            "min": -0.04183783754706383,
+            "max": 0.05026666074991226
+          },
+          "mlp.down_proj.weight": {
+            "mean": 2.8050696863302666e-07,
+            "std": 0.005638069573233815,
+            "rms": 0.00563806958021175,
+            "mean_abs": 0.004385731722179212,
+            "min": -0.04429596662521362,
+            "max": 0.044422950595617294
+          }
+        }
+      },
+      "rho_objective": "none",
+      "rho_schedule": "joint",
+      "rho_epochs": 0.0,
+      "rho_lr": null,
+      "rho_bounds": null,
+      "rho_after_phase": null,
+      "rho_final": {},
+      "u_fisher_orthogonality": {},
+      "lambda_init": "fisher_prior",
+      "skip_fisher_merge_init": false,
+      "requested_skip_merge_cycles": [],
+      "fisher_prior_input_summary": {
+        "num_tensors": 3,
+        "num_elements": 74711040,
+        "fisher_mode": "param",
+        "eps": 0.0,
+        "param_subset": "mlp",
+        "fa_mean": 8.622785835062754e-09,
+        "fb_mean": 5.1461533825714835e-09,
+        "denom_mean": 1.3768939616535423e-08,
+        "active_ratio": 1.0,
+        "mean_abs_fa_minus_fb": 8.650126123380656e-09,
+        "mean_relative_contrast": 0.4221493771201686,
+        "prior_mean": 0.4628780431914748,
+        "prior_active_mean": 0.4628780384858449,
+        "distributions": {
+          "fa": {
+            "quantile_method": "concatenated_per_tensor_even_stride_samples",
+            "quantile_sample_count": 393216,
+            "quantiles": {
+              "p0": 3.417065243355326e-17,
+              "p1": 9.783148596786972e-11,
+              "p10": 3.662524172831638e-10,
+              "p25": 6.899040783103771e-10,
+              "p50": 1.532819982053013e-09,
+              "p75": 3.864754738458487e-09,
+              "p90": 1.0186454346694518e-08,
+              "p95": 1.9240395943143085e-08,
+              "p99": 6.438692423671455e-08,
+              "p99_9": 1.2079875659765094e-06,
+              "p100": 4.065752727910876e-05
+            }
+          },
+          "fb": {
+            "quantile_method": "concatenated_per_tensor_even_stride_samples",
+            "quantile_sample_count": 393216,
+            "quantiles": {
+              "p0": 2.4815992770808187e-16,
+              "p1": 2.874530058427638e-10,
+              "p10": 7.61757656952966e-10,
+              "p25": 1.184452647073897e-09,
+              "p50": 1.895775758598006e-09,
+              "p75": 3.1585358772900918e-09,
+              "p90": 5.791106261199275e-09,
+              "p95": 9.546111456870676e-09,
+              "p99": 3.334456266657071e-08,
+              "p99_9": 5.067554411652964e-07,
+              "p100": 1.583154698892031e-05
+            }
+          },
+          "denom": {
+            "quantile_method": "concatenated_per_tensor_even_stride_samples",
+            "quantile_sample_count": 393216,
+            "quantiles": {
+              "p0": 7.567213415059812e-16,
+              "p1": 6.487239079255858e-10,
+              "p10": 1.4966138328631473e-09,
+              "p25": 2.3197497256433053e-09,
+              "p50": 4.035459522100382e-09,
+              "p75": 7.684739955493569e-09,
+              "p90": 1.613611644302182e-08,
+              "p95": 2.7976545524666108e-08,
+              "p99": 8.43180103515806e-08,
+              "p99_9": 1.7316402818323695e-06,
+              "p100": 4.5034703362034634e-05
+            }
+          }
+        },
+        "per_tensor": {
+          "mlp.gate_proj.weight": {
+            "num_elements": 24903680,
+            "fa_mean": 4.907347398617084e-09,
+            "fb_mean": 4.269799989599146e-09,
+            "denom_mean": 9.17714738821623e-09,
+            "active_ratio": 1.0,
+            "mean_abs_fa_minus_fb": 5.653319141473682e-09,
+            "mean_relative_contrast": 0.4540704426012541,
+            "prior_mean": 0.41098191110711346,
+            "prior_active_mean": 0.4109819233417511,
+            "fa_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 4.907347398617084e-09,
+              "std": 6.291769437301842e-07,
+              "min": 2.411236117803506e-17,
+              "max": 0.002025436609983444,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 3.417065243355326e-17,
+                "p1": 5.889760901212071e-11,
+                "p10": 2.1112755987928722e-10,
+                "p25": 4.21434193098591e-10,
+                "p50": 9.178747206739502e-10,
+                "p75": 2.08083461572528e-09,
+                "p90": 4.6115338214747226e-09,
+                "p95": 7.832427151299726e-09,
+                "p99": 2.497264794953935e-08,
+                "p99_9": 1.5654329388326005e-07,
+                "p100": 1.2314380001043901e-05
+              }
+            },
+            "fb_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 4.269799989599146e-09,
+              "std": 2.0916531608841202e-07,
+              "min": 6.916006808974621e-17,
+              "max": 0.000630074879154563,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 2.4815992770808187e-16,
+                "p1": 1.9312543231286838e-10,
+                "p10": 5.221494903118185e-10,
+                "p25": 8.575110621578119e-10,
+                "p50": 1.4848418050661394e-09,
+                "p75": 2.655764719250442e-09,
+                "p90": 4.9315036498853715e-09,
+                "p95": 7.68441044129986e-09,
+                "p99": 2.0961627100746227e-08,
+                "p99_9": 1.0189430099671881e-07,
+                "p100": 1.7572012893651845e-06
+              }
+            },
+            "denom_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 9.17714738821623e-09,
+              "std": 6.75172711967096e-07,
+              "min": 2.2913347660016175e-16,
+              "max": 0.0020792637951672077,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 7.567213415059812e-16,
+                "p1": 4.669871711548979e-10,
+                "p10": 1.1004260835889568e-09,
+                "p25": 1.7092529613904617e-09,
+                "p50": 2.8011202246602807e-09,
+                "p75": 4.92724261391686e-09,
+                "p90": 9.311075466200691e-09,
+                "p95": 1.4682214555250539e-08,
+                "p99": 4.198140501898706e-08,
+                "p99_9": 2.1717121967412822e-07,
+                "p100": 1.2488766515161842e-05
+              }
+            }
+          },
+          "mlp.up_proj.weight": {
+            "num_elements": 24903680,
+            "fa_mean": 6.5929647893139385e-09,
+            "fb_mean": 5.813624164530713e-09,
+            "denom_mean": 1.2406590748899985e-08,
+            "active_ratio": 1.0,
+            "mean_abs_fa_minus_fb": 7.208766299682841e-09,
+            "mean_relative_contrast": 0.4021101700632196,
+            "prior_mean": 0.41958778782894735,
+            "prior_active_mean": 0.41958779096603394,
+            "fa_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 6.5929647893139385e-09,
+              "std": 8.043095183289685e-07,
+              "min": 2.2091895107768955e-16,
+              "max": 0.0020796749740839005,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 3.647036673768339e-16,
+                "p1": 1.850097297584341e-10,
+                "p10": 3.8499564647409557e-10,
+                "p25": 5.844560391210507e-10,
+                "p50": 1.0634768621287094e-09,
+                "p75": 2.411384203426792e-09,
+                "p90": 5.802638369800661e-09,
+                "p95": 1.0598502520053898e-08,
+                "p99": 3.7318439893851973e-08,
+                "p99_9": 1.9354403946181264e-07,
+                "p100": 3.6849531170446426e-05
+              }
+            },
+            "fb_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 5.813624164530713e-09,
+              "std": 6.817022105063282e-07,
+              "min": 3.5946979785619036e-16,
+              "max": 0.002239371184259653,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 1.2604057778216066e-15,
+                "p1": 3.970711814904604e-10,
+                "p10": 7.892239639595289e-10,
+                "p25": 1.0947321937848642e-09,
+                "p50": 1.643124414307806e-09,
+                "p75": 2.8265279006234323e-09,
+                "p90": 5.582325268704835e-09,
+                "p95": 8.901376524761417e-09,
+                "p99": 2.8506606852829464e-08,
+                "p99_9": 1.352745613303341e-07,
+                "p100": 5.7007678151421715e-06
+              }
+            },
+            "denom_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 1.2406590748899985e-08,
+              "std": 1.1319317920368254e-06,
+              "min": 1.0626624422741828e-15,
+              "max": 0.0025341485161334276,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 1.919731565283144e-15,
+                "p1": 8.005450191639341e-10,
+                "p10": 1.4820462634901332e-09,
+                "p25": 2.0357808772075714e-09,
+                "p50": 3.1079221418650604e-09,
+                "p75": 5.665689251088679e-09,
+                "p90": 1.144762507010455e-08,
+                "p95": 1.892171574979784e-08,
+                "p99": 6.133333130264873e-08,
+                "p99_9": 3.0054937383283686e-07,
+                "p100": 3.693044345709495e-05
+              }
+            }
+          },
+          "mlp.down_proj.weight": {
+            "num_elements": 24903680,
+            "fa_mean": 1.436804531725724e-08,
+            "fb_mean": 5.355035993584591e-09,
+            "denom_mean": 1.9723080712490055e-08,
+            "active_ratio": 1.0,
+            "mean_abs_fa_minus_fb": 1.3088292928985443e-08,
+            "mean_relative_contrast": 0.41026751869603206,
+            "prior_mean": 0.5580644306383635,
+            "prior_active_mean": 0.5580644011497498,
+            "fa_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 1.436804531725724e-08,
+              "std": 6.086725498298996e-07,
+              "min": 6.9433729599222e-11,
+              "max": 0.0019417931325733662,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 1.5160861455143504e-10,
+                "p1": 5.501819000386377e-10,
+                "p10": 1.0612246637009548e-09,
+                "p25": 1.7264490947965783e-09,
+                "p50": 3.5203502335434678e-09,
+                "p75": 8.53321413529784e-09,
+                "p90": 2.1111652870331454e-08,
+                "p95": 3.6367097777656454e-08,
+                "p99": 1.332044377022612e-07,
+                "p99_9": 1.9547701413102914e-06,
+                "p100": 4.065752727910876e-05
+              }
+            },
+            "fb_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 5.355035993584591e-09,
+              "std": 2.711604937775122e-07,
+              "min": 1.3461134984460443e-10,
+              "max": 0.0012914687395095825,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 1.9344695290079983e-10,
+                "p1": 8.390951822256909e-10,
+                "p10": 1.325392906537104e-09,
+                "p25": 1.7528184459436602e-09,
+                "p50": 2.466499893216678e-09,
+                "p75": 3.764478506695923e-09,
+                "p90": 6.959535170381059e-09,
+                "p95": 1.2539932647825935e-08,
+                "p99": 4.831620259437841e-08,
+                "p99_9": 7.917637958598789e-07,
+                "p100": 1.583154698892031e-05
+              }
+            },
+            "denom_distribution": {
+              "num_elements": 24903680,
+              "finite_count": 24903680,
+              "finite_ratio": 1.0,
+              "zero_count": 0,
+              "zero_ratio": 0.0,
+              "le_eps_count": 0,
+              "le_eps_ratio": 0.0,
+              "mean": 1.9723080712490055e-08,
+              "std": 6.955251473822031e-07,
+              "min": 4.4098724671925993e-10,
+              "max": 0.001942638773471117,
+              "quantile_method": "deterministic_even_stride_sample",
+              "quantile_sample_count": 131072,
+              "quantiles": {
+                "p0": 5.996579899303356e-10,
+                "p1": 2.056612879997033e-09,
+                "p10": 3.2314095843588575e-09,
+                "p25": 4.432463729386882e-09,
+                "p50": 6.919648853909166e-09,
+                "p75": 1.3050116542956403e-08,
+                "p90": 2.8529145268407774e-08,
+                "p95": 4.629985284054783e-08,
+                "p99": 1.5639753314644622e-07,
+                "p99_9": 2.6720822461356875e-06,
+                "p100": 4.5034703362034634e-05
+              }
+            }
+          }
+        }
+      }
+    },
+    "reparam_gate_summary": {
+      "num_tensors": 3,
+      "num_elements": 74711040,
+      "global_mean": 0.46277491251627606,
+      "per_tensor_mean": {
+        "mlp.gate_proj.weight": 0.4109070897102356,
+        "mlp.up_proj.weight": 0.4194740951061249,
+        "mlp.down_proj.weight": 0.5579436421394348
+      }
+    },
+    "post_merge_nll_scale": null,
+    "merge_only_ppl": null,
+    "post_ppl": null,
+    "weight_decay": 0.0,
+    "max_grad_norm": 1.0,
+    "grad_accum_steps": 1,
+    "instruction_dataset": null,
+    "instruction_config": null,
+    "instruction_split": "train"
+  },
+  "lora": {
+    "enabled": false,
+    "seq_len": 1024,
+    "batch_size": 1,
+    "epochs": 0.0,
+    "rank": 8,
+    "alpha": 16.0,
+    "dropout": 0.0,
+    "target_modules": [
+      "q_proj",
+      "k_proj",
+      "v_proj",
+      "o_proj",
+      "gate_proj",
+      "down_proj",
+      "up_proj"
+    ],
+    "respect_exclude_pairs": false,
+    "kl_enabled": false,
+    "kl_weight": 0.1,
+    "kl_temp": 4.0,
+    "post_ppl": null,
+    "lr": 0.0001,
+    "weight_decay": 0.0,
+    "max_grad_norm": 1.0,
+    "grad_accum_steps": 1,
+    "log_steps": 100,
+    "eval_every": 2000,
+    "eval_max_batches": null
+  },
+  "norm_policy": "hybrid",
+  "full_model_saved": false,
+  "timing": {
+    "seconds": {
+      "selection": 2.8526061214506626e-05,
+      "rebase": 0.0,
+      "fisher": 15.339991886983626,
+      "fit": 218.0570480369497,
+      "materialize_save_layer": 0.33142108493484557,
+      "metadata_write": 0.0018587949452921748,
+      "teacher_snapshot": 0.24661824794020504,
+      "cycle_cleanup": 0.17409992404282093,
+      "cycle_total": 234.15170551894698
+    },
+    "tracked_component_total_seconds": 234.1510665018577,
+    "cycle_total_seconds": 234.15170551894698,
+    "ratios_of_tracked_components": {
+      "selection": 1.2182759464083117e-07,
+      "rebase": 0.0,
+      "fisher": 0.0655132266367957,
+      "fit": 0.9312665165042914,
+      "materialize_save_layer": 0.0014154156540312668,
+      "metadata_write": 7.938443215578638e-06,
+      "teacher_snapshot": 0.001053244179599961,
+      "cycle_cleanup": 0.0007435367544714542
+    },
+    "ratios_of_cycle_total": {
+      "selection": 1.2182726216443625e-07,
+      "rebase": 0.0,
+      "fisher": 0.06551304784642002,
+      "fit": 0.9312639750100179,
+      "materialize_save_layer": 0.0014154117912586709,
+      "metadata_write": 7.938421550987872e-06,
+      "teacher_snapshot": 0.0010532413052197448,
+      "cycle_cleanup": 0.000743534725305399
+    }
+  }
+}

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/protocol.json

@@ -0,0 +1,21 @@
+{
+  "model": "Qwen/Qwen3-4B-Base",
+  "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539",
+  "model_key": "4b",
+  "anchor": 25,
+  "removed_span": [26, 31],
+  "removed_blocks": 5,
+  "dense_blocks": 36,
+  "fisher_input_mode": "anchor_input",
+  "fisher_mode": "param",
+  "fisher_eps": 0.0,
+  "rho_variant": "no_rho",
+  "rho_schedule": "none",
+  "dataset": "DKYoon/SlimPajama-6B",
+  "target_tokens": 524288,
+  "calibration_sequences": 64,
+  "sequence_length": 1024,
+  "distill_epochs": 1.0,
+  "seed": 42,
+  "gpu": 0
+}

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_rho/compact/manifest.json

@@ -0,0 +1,161 @@
+{
+  "format": "abprune_compact_manifest_v1",
+  "model": "Qwen/Qwen3-4B-Base",
+  "base_model_revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539",
+  "run_id": "qwen3_4b_anchor_input_rho_contextual_fisher_v1",
+  "target_prune_count": 5,
+  "layer_order": [
+    25,
+    25,
+    25,
+    25,
+    25
+  ],
+  "stages": [
+    {
+      "format": "abprune_compact_stage_v1",
+      "model": "Qwen/Qwen3-4B-Base",
+      "base_model_revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539",
+      "run_id": "qwen3_4b_anchor_input_rho_contextual_fisher_v1",
+      "target_prune_count": 5,
+      "stage": 1,
+      "removed_count": 1,
+      "anchor_index": 25,
+      "layer_order": [
+        25,
+        25,
+        25,
+        25,
+        25
+      ],
+      "packaged_stage_range": [
+        1,
+        5
+      ],
+      "removed_original_layers": [
+        26
+      ],
+      "source_cycle": "cycle_1",
+      "anchor_state_file": "anchor_layer.pt",
+      "sha256": "b85fe2e31883e0c4c6c6881d5733ad1cb16cc828677ae95bfd9fc9dc6838a7f2"
+    },
+    {
+      "format": "abprune_compact_stage_v1",
+      "model": "Qwen/Qwen3-4B-Base",
+      "base_model_revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539",
+      "run_id": "qwen3_4b_anchor_input_rho_contextual_fisher_v1",
+      "target_prune_count": 5,
+      "stage": 2,
+      "removed_count": 2,
+      "anchor_index": 25,
+      "layer_order": [
+        25,
+        25,
+        25,
+        25,
+        25
+      ],
+      "packaged_stage_range": [
+        1,
+        5
+      ],
+      "removed_original_layers": [
+        26,
+        27
+      ],
+      "source_cycle": "cycle_2",
+      "anchor_state_file": "anchor_layer.pt",
+      "sha256": "62d25e32d3361f51f061100b532d978ae9e3c2ee68aef4b526d9768616416892"
+    },
+    {
+      "format": "abprune_compact_stage_v1",
+      "model": "Qwen/Qwen3-4B-Base",
+      "base_model_revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539",
+      "run_id": "qwen3_4b_anchor_input_rho_contextual_fisher_v1",
+      "target_prune_count": 5,
+      "stage": 3,
+      "removed_count": 3,
+      "anchor_index": 25,
+      "layer_order": [
+        25,
+        25,
+        25,
+        25,
+        25
+      ],
+      "packaged_stage_range": [
+        1,
+        5
+      ],
+      "removed_original_layers": [
+        26,
+        27,
+        28
+      ],
+      "source_cycle": "cycle_3",
+      "anchor_state_file": "anchor_layer.pt",
+      "sha256": "589ee811db70a31db1a4e82a21eefe617b08f0c89200fd4f7c5d3aedf797479e"
+    },
+    {
+      "format": "abprune_compact_stage_v1",
+      "model": "Qwen/Qwen3-4B-Base",
+      "base_model_revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539",
+      "run_id": "qwen3_4b_anchor_input_rho_contextual_fisher_v1",
+      "target_prune_count": 5,
+      "stage": 4,
+      "removed_count": 4,
+      "anchor_index": 25,
+      "layer_order": [
+        25,
+        25,
+        25,
+        25,
+        25
+      ],
+      "packaged_stage_range": [
+        1,
+        5
+      ],
+      "removed_original_layers": [
+        26,
+        27,
+        28,
+        29
+      ],
+      "source_cycle": "cycle_4",
+      "anchor_state_file": "anchor_layer.pt",
+      "sha256": "824da6009a2af44c4fd1bacca61a2c74b8d582ce57c66889aee021a2efbacd3b"
+    },
+    {
+      "format": "abprune_compact_stage_v1",
+      "model": "Qwen/Qwen3-4B-Base",
+      "base_model_revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539",
+      "run_id": "qwen3_4b_anchor_input_rho_contextual_fisher_v1",
+      "target_prune_count": 5,
+      "stage": 5,
+      "removed_count": 5,
+      "anchor_index": 25,
+      "layer_order": [
+        25,
+        25,
+        25,
+        25,
+        25
+      ],
+      "packaged_stage_range": [
+        1,
+        5
+      ],
+      "removed_original_layers": [
+        26,
+        27,
+        28,
+        29,
+        30
+      ],
+      "source_cycle": "cycle_5",
+      "anchor_state_file": "anchor_layer.pt",
+      "sha256": "310b58c5ed33748481809ecc404a0af8d41c67b3589e8a4cd2f627e58990b9a1"
+    }
+  ]
+}

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_rho/compact/stage_01/anchor_layer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b85fe2e31883e0c4c6c6881d5733ad1cb16cc828677ae95bfd9fc9dc6838a7f2
+size 201866630

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_rho/compact/stage_01/metadata.json

@@ -0,0 +1,27 @@
+{
+  "format": "abprune_compact_stage_v1",
+  "model": "Qwen/Qwen3-4B-Base",
+  "base_model_revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539",
+  "run_id": "qwen3_4b_anchor_input_rho_contextual_fisher_v1",
+  "target_prune_count": 5,
+  "stage": 1,
+  "removed_count": 1,
+  "anchor_index": 25,
+  "layer_order": [
+    25,
+    25,
+    25,
+    25,
+    25
+  ],
+  "packaged_stage_range": [
+    1,
+    5
+  ],
+  "removed_original_layers": [
+    26
+  ],
+  "source_cycle": "cycle_1",
+  "anchor_state_file": "anchor_layer.pt",
+  "sha256": "b85fe2e31883e0c4c6c6881d5733ad1cb16cc828677ae95bfd9fc9dc6838a7f2"
+}

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_rho/compact/stage_02/anchor_layer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62d25e32d3361f51f061100b532d978ae9e3c2ee68aef4b526d9768616416892
+size 201866630

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_rho/compact/stage_02/metadata.json

@@ -0,0 +1,28 @@
+{
+  "format": "abprune_compact_stage_v1",
+  "model": "Qwen/Qwen3-4B-Base",
+  "base_model_revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539",
+  "run_id": "qwen3_4b_anchor_input_rho_contextual_fisher_v1",
+  "target_prune_count": 5,
+  "stage": 2,
+  "removed_count": 2,
+  "anchor_index": 25,
+  "layer_order": [
+    25,
+    25,
+    25,
+    25,
+    25
+  ],
+  "packaged_stage_range": [
+    1,
+    5
+  ],
+  "removed_original_layers": [
+    26,
+    27
+  ],
+  "source_cycle": "cycle_2",
+  "anchor_state_file": "anchor_layer.pt",
+  "sha256": "62d25e32d3361f51f061100b532d978ae9e3c2ee68aef4b526d9768616416892"
+}

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_rho/compact/stage_03/anchor_layer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:589ee811db70a31db1a4e82a21eefe617b08f0c89200fd4f7c5d3aedf797479e
+size 201866630

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_rho/compact/stage_03/metadata.json

@@ -0,0 +1,29 @@
+{
+  "format": "abprune_compact_stage_v1",
+  "model": "Qwen/Qwen3-4B-Base",
+  "base_model_revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539",
+  "run_id": "qwen3_4b_anchor_input_rho_contextual_fisher_v1",
+  "target_prune_count": 5,
+  "stage": 3,
+  "removed_count": 3,
+  "anchor_index": 25,
+  "layer_order": [
+    25,
+    25,
+    25,
+    25,
+    25
+  ],
+  "packaged_stage_range": [
+    1,
+    5
+  ],
+  "removed_original_layers": [
+    26,
+    27,
+    28
+  ],
+  "source_cycle": "cycle_3",
+  "anchor_state_file": "anchor_layer.pt",
+  "sha256": "589ee811db70a31db1a4e82a21eefe617b08f0c89200fd4f7c5d3aedf797479e"
+}

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_rho/compact/stage_04/anchor_layer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:824da6009a2af44c4fd1bacca61a2c74b8d582ce57c66889aee021a2efbacd3b
+size 201866630

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_rho/compact/stage_04/metadata.json

@@ -0,0 +1,30 @@
+{
+  "format": "abprune_compact_stage_v1",
+  "model": "Qwen/Qwen3-4B-Base",
+  "base_model_revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539",
+  "run_id": "qwen3_4b_anchor_input_rho_contextual_fisher_v1",
+  "target_prune_count": 5,
+  "stage": 4,
+  "removed_count": 4,
+  "anchor_index": 25,
+  "layer_order": [
+    25,
+    25,
+    25,
+    25,
+    25
+  ],
+  "packaged_stage_range": [
+    1,
+    5
+  ],
+  "removed_original_layers": [
+    26,
+    27,
+    28,
+    29
+  ],
+  "source_cycle": "cycle_4",
+  "anchor_state_file": "anchor_layer.pt",
+  "sha256": "824da6009a2af44c4fd1bacca61a2c74b8d582ce57c66889aee021a2efbacd3b"
+}

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_rho/compact/stage_05/anchor_layer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:310b58c5ed33748481809ecc404a0af8d41c67b3589e8a4cd2f627e58990b9a1
+size 201866630

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_rho/compact/stage_05/metadata.json

@@ -0,0 +1,31 @@
+{
+  "format": "abprune_compact_stage_v1",
+  "model": "Qwen/Qwen3-4B-Base",
+  "base_model_revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539",
+  "run_id": "qwen3_4b_anchor_input_rho_contextual_fisher_v1",
+  "target_prune_count": 5,
+  "stage": 5,
+  "removed_count": 5,
+  "anchor_index": 25,
+  "layer_order": [
+    25,
+    25,
+    25,
+    25,
+    25
+  ],
+  "packaged_stage_range": [
+    1,
+    5
+  ],
+  "removed_original_layers": [
+    26,
+    27,
+    28,
+    29,
+    30
+  ],
+  "source_cycle": "cycle_5",
+  "anchor_state_file": "anchor_layer.pt",
+  "sha256": "310b58c5ed33748481809ecc404a0af8d41c67b3589e8a4cd2f627e58990b9a1"
+}

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_rho/eval/stage_01_ppl.json

@@ -0,0 +1,25 @@
+{
+  "model": "/workspace/models/qwen3-4b-base-906bfd4",
+  "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539",
+  "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_input_rho/compact/stage_01",
+  "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test",
+  "protocol": "HCHC experiment.py: filter empty rows, join with double newline, then tokenize the complete stream once",
+  "sequence_length": 2048,
+  "batch_size": 4,
+  "dtype": "bfloat16",
+  "metrics": {
+    "nll": 2.132754957264867,
+    "perplexity": 8.438081372774594,
+    "tokens": 296815,
+    "sequences": 145,
+    "source_tokens": 298938
+  },
+  "anchor": 25,
+  "cycles": 1,
+  "original_layers": 36,
+  "decoder_layers": 35,
+  "post_merge_residual_scale": null,
+  "post_merge_residual_scales": [],
+  "parameterization": "unscaled compact ABPrune",
+  "linear_patch": null
+}

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_rho/eval/stage_02_ppl.json

@@ -0,0 +1,25 @@
+{
+  "model": "/workspace/models/qwen3-4b-base-906bfd4",
+  "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539",
+  "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_input_rho/compact/stage_02",
+  "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test",
+  "protocol": "HCHC experiment.py: filter empty rows, join with double newline, then tokenize the complete stream once",
+  "sequence_length": 2048,
+  "batch_size": 4,
+  "dtype": "bfloat16",
+  "metrics": {
+    "nll": 2.2798083058718976,
+    "perplexity": 9.77480645692066,
+    "tokens": 296815,
+    "sequences": 145,
+    "source_tokens": 298938
+  },
+  "anchor": 25,
+  "cycles": 2,
+  "original_layers": 36,
+  "decoder_layers": 34,
+  "post_merge_residual_scale": null,
+  "post_merge_residual_scales": [],
+  "parameterization": "unscaled compact ABPrune",
+  "linear_patch": null
+}

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_rho/eval/stage_03_ppl.json

@@ -0,0 +1,25 @@
+{
+  "model": "/workspace/models/qwen3-4b-base-906bfd4",
+  "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539",
+  "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_input_rho/compact/stage_03",
+  "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test",
+  "protocol": "HCHC experiment.py: filter empty rows, join with double newline, then tokenize the complete stream once",
+  "sequence_length": 2048,
+  "batch_size": 4,
+  "dtype": "bfloat16",
+  "metrics": {
+    "nll": 2.372043491231984,
+    "perplexity": 10.719274662057657,
+    "tokens": 296815,
+    "sequences": 145,
+    "source_tokens": 298938
+  },
+  "anchor": 25,
+  "cycles": 3,
+  "original_layers": 36,
+  "decoder_layers": 33,
+  "post_merge_residual_scale": null,
+  "post_merge_residual_scales": [],
+  "parameterization": "unscaled compact ABPrune",
+  "linear_patch": null
+}

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_rho/eval/stage_04_ppl.json

@@ -0,0 +1,25 @@
+{
+  "model": "/workspace/models/qwen3-4b-base-906bfd4",
+  "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539",
+  "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_input_rho/compact/stage_04",
+  "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test",
+  "protocol": "HCHC experiment.py: filter empty rows, join with double newline, then tokenize the complete stream once",
+  "sequence_length": 2048,
+  "batch_size": 4,
+  "dtype": "bfloat16",
+  "metrics": {
+    "nll": 2.552255313149814,
+    "perplexity": 12.836020408000905,
+    "tokens": 296815,
+    "sequences": 145,
+    "source_tokens": 298938
+  },
+  "anchor": 25,
+  "cycles": 4,
+  "original_layers": 36,
+  "decoder_layers": 32,
+  "post_merge_residual_scale": null,
+  "post_merge_residual_scales": [],
+  "parameterization": "unscaled compact ABPrune",
+  "linear_patch": null
+}

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_rho/eval/stage_05_ppl.json

@@ -0,0 +1,25 @@
+{
+  "model": "/workspace/models/qwen3-4b-base-906bfd4",
+  "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539",
+  "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_input_rho/compact/stage_05",
+  "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test",
+  "protocol": "HCHC experiment.py: filter empty rows, join with double newline, then tokenize the complete stream once",
+  "sequence_length": 2048,
+  "batch_size": 4,
+  "dtype": "bfloat16",
+  "metrics": {
+    "nll": 2.793590261196268,
+    "perplexity": 16.339577972080622,
+    "tokens": 296815,
+    "sequences": 145,
+    "source_tokens": 298938
+  },
+  "anchor": 25,
+  "cycles": 5,
+  "original_layers": 36,
+  "decoder_layers": 31,
+  "post_merge_residual_scale": null,
+  "post_merge_residual_scales": [],
+  "parameterization": "unscaled compact ABPrune",
+  "linear_patch": null
+}

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_rho/eval/stage_05_zs6.json

@@ -0,0 +1,126 @@
+{
+  "model": "/workspace/models/qwen3-4b-base-906bfd4",
+  "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539",
+  "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_input_rho/compact/stage_05",
+  "cycle": 5,
+  "anchor": 25,
+  "post_merge_residual_scale": null,
+  "post_merge_residual_scales": [],
+  "tasks": [
+    "hellaswag",
+    "piqa",
+    "winogrande",
+    "arc_challenge",
+    "arc_easy",
+    "openbookqa"
+  ],
+  "preferred_metrics": {
+    "hellaswag": "acc_norm,none",
+    "piqa": "acc_norm,none",
+    "winogrande": "acc,none",
+    "arc_challenge": "acc_norm,none",
+    "arc_easy": "acc_norm,none",
+    "openbookqa": "acc_norm,none"
+  },
+  "task_scores": {
+    "hellaswag": 0.617307309300936,
+    "piqa": 0.70620239390642,
+    "winogrande": 0.6677190213101816,
+    "arc_challenge": 0.3873720136518771,
+    "arc_easy": 0.5833333333333334,
+    "openbookqa": 0.326
+  },
+  "task_mean": 0.5479890119171247,
+  "six_task_mean": 0.5479890119171247,
+  "boolq_excluded": true,
+  "num_fewshot": 0,
+  "batch_size": "auto:4",
+  "linear_patch": null,
+  "lm_eval_results": {
+    "hellaswag": {
+      "name": "hellaswag",
+      "alias": "hellaswag",
+      "sample_len": 10042,
+      "acc,none": 0.42690699063931487,
+      "acc_stderr,none": 0.004936176784631955,
+      "acc_norm,none": 0.617307309300936,
+      "acc_norm_stderr,none": 0.004850508945116094
+    },
+    "piqa": {
+      "name": "piqa",
+      "alias": "piqa",
+      "sample_len": 1838,
+      "acc,none": 0.6964091403699674,
+      "acc_stderr,none": 0.010728079893076368,
+      "acc_norm,none": 0.70620239390642,
+      "acc_norm_stderr,none": 0.010627574080514807
+    },
+    "winogrande": {
+      "name": "winogrande",
+      "alias": "winogrande",
+      "sample_len": 1267,
+      "acc,none": 0.6677190213101816,
+      "acc_stderr,none": 0.013238316554236528
+    },
+    "arc_challenge": {
+      "name": "arc_challenge",
+      "alias": "arc_challenge",
+      "sample_len": 1172,
+      "acc,none": 0.35409556313993173,
+      "acc_stderr,none": 0.013975454122756562,
+      "acc_norm,none": 0.3873720136518771,
+      "acc_norm_stderr,none": 0.014235872487909865
+    },
+    "arc_easy": {
+      "name": "arc_easy",
+      "alias": "arc_easy",
+      "sample_len": 2376,
+      "acc,none": 0.6111111111111112,
+      "acc_stderr,none": 0.010003248335313759,
+      "acc_norm,none": 0.5833333333333334,
+      "acc_norm_stderr,none": 0.01011628297778126
+    },
+    "openbookqa": {
+      "name": "openbookqa",
+      "alias": "openbookqa",
+      "sample_len": 500,
+      "acc,none": 0.218,
+      "acc_stderr,none": 0.018483378223178863,
+      "acc_norm,none": 0.326,
+      "acc_norm_stderr,none": 0.02098400956239356
+    }
+  },
+  "lm_eval_versions": {
+    "arc_challenge": 1.0,
+    "arc_easy": 1.0,
+    "hellaswag": 1.0,
+    "openbookqa": 1.0,
+    "piqa": 1.0,
+    "winogrande": 1.0
+  },
+  "lm_eval_config": {
+    "model": "/workspace/models/qwen3-4b-base-906bfd4",
+    "model_args": null,
+    "model_num_parameters": 3517814016,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "",
+    "batch_size": "auto:4",
+    "batch_sizes": [
+      64,
+      64,
+      64,
+      64,
+      64
+    ],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  }
+}

runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_rho/logs/train.log

@@ -0,0 +1,206 @@
+
+Qwen3ForCausalLM(
+  (model): Qwen3Model(
+    (embed_tokens): Embedding(151936, 2560)
+    (layers): ModuleList(
+      (0-35): 36 x Qwen3DecoderLayer(
+        (self_attn): Qwen3Attention(
+          (q_proj): Linear(in_features=2560, out_features=4096, bias=False)
+          (k_proj): Linear(in_features=2560, out_features=1024, bias=False)
+          (v_proj): Linear(in_features=2560, out_features=1024, bias=False)
+          (o_proj): Linear(in_features=4096, out_features=2560, bias=False)
+          (q_norm): Qwen3RMSNorm((128,), eps=1e-06)
+          (k_norm): Qwen3RMSNorm((128,), eps=1e-06)
+        )
+        (mlp): Qwen3MLP(
+          (gate_proj): Linear(in_features=2560, out_features=9728, bias=False)
+          (up_proj): Linear(in_features=2560, out_features=9728, bias=False)
+          (down_proj): Linear(in_features=9728, out_features=2560, bias=False)
+          (act_fn): SiLUActivation()
+        )
+        (input_layernorm): Qwen3RMSNorm((2560,), eps=1e-06)
+        (post_attention_layernorm): Qwen3RMSNorm((2560,), eps=1e-06)
+      )
+    )
+    (norm): Qwen3RMSNorm((2560,), eps=1e-06)
+    (rotary_emb): Qwen3RotaryEmbedding()
+  )
+  (lm_head): Linear(in_features=2560, out_features=151936, bias=False)
+)
+[progressive] Cycle 1/5
+[fuse] Head permutation disabled; merging with original head order.
+/opt/conda/lib/python3.11/site-packages/transformers/models/qwen3/modeling_qwen3.py:143: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#results-reproducibility (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:233.)
+  freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
+/opt/conda/lib/python3.11/site-packages/torch/nn/modules/linear.py:125: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#results-reproducibility (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:233.)
+  return F.linear(input, self.weight, self.bias)
+/opt/conda/lib/python3.11/site-packages/torch/autograd/graph.py:829: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#results-reproducibility (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:233.)
+  return Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
+/opt/conda/lib/python3.11/site-packages/torch/autograd/graph.py:829: UserWarning: Flash Attention defaults to a non-deterministic algorithm. To explicitly enable determinism call torch.use_deterministic_algorithms(True, warn_only=False). (Triggered internally at /pytorch/aten/src/ATen/native/transformers/cuda/attention_backward.cu:110.)
+  return Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
+[fisher-input] mode=anchor_input cos=0.9340915456414223 norm_ratio=0.9107728470116854
+
+[reparam] fisher_prior_inputs fa_mean=2.923395501503572e-09 fb_mean=2.2558122294101752e-09 active_ratio=1.0 rel_contrast=0.36682527776350055 prior_mean=0.5411720677425987 prior_active_mean=0.5411720871925354
+[reparam] Cycle 1: training U + gates for pair 25-26 (epochs=1.0, skip_train=False, hidden_mse_w=1.0, attn_mse_w=0.0, mlp_mse_w=1.0, eta=0.0, gamma=0.0, attn_reg_scale=1.0, mlp_reg_scale=1.0, param_subset=mlp, fisher_init_subset=same, train_subset=same, lambda_init=fisher_prior).
+[reparam] subset=mlp gates=3 (attn=0, mlp=3, other=0) train_subset=mlp (attn=0, mlp=3, other=0) elems=74711040 (~285.0 MiB), U_elems=74711040 (~285.0 MiB; +optimizer state)
+[reparam] lambda_init mean=0.5411720677425987 std=0.21556873075972857 min=0.0006477722781710327 max=0.9998457431793213 mean_abs_from_0.5=0.18341263888175027
+[reparam-rho] objective=kl_only schedule=sequential rho_epochs=1.0 lr=0.002 bounds=[-1.0, 2.0] projections=['down_proj', 'gate_proj', 'up_proj']
+[reparam] first_grad phase=rho step=1 gate_l2=0.000000e+00 gate_mean_abs=None gate_max_abs=0.000000e+00 u_l2=0.000000e+00 u_mean_abs=None u_max_abs=0.000000e+00 rho_l2=2.846146e-04
+[reparam] epoch=1 step=1 phase=rho loss=1.995709e+00 rho_loss=4.756396e-03 mse=1.315180e+00 mlp_mse=6.757720e-01 kl=1.486374e-02 lambda_mean=0.5411720677425987 lambda_std=0.21556873075972857 lambda_abs0.5=0.18341263888175027 rho={'gate_proj': -0.0019999074283987284, 'up_proj': -0.0019993276800960302, 'down_proj': -0.0019998897332698107}
+[reparam] epoch=1 step=100 phase=rho loss=2.130657e+00 rho_loss=4.203226e-03 mse=1.388810e+00 mlp_mse=7.376432e-01 kl=1.313508e-02 lambda_mean=0.5411720677425987 lambda_std=0.21556873075972857 lambda_abs0.5=0.18341263888175027 rho={'gate_proj': 0.061190564185380936, 'up_proj': 0.0909985825419426, 'down_proj': 0.053413525223731995}
+[reparam] epoch=1 step=200 phase=rho loss=2.040770e+00 rho_loss=3.480032e-03 mse=1.268025e+00 mlp_mse=7.692655e-01 kl=1.087510e-02 lambda_mean=0.5411720677425987 lambda_std=0.21556873075972857 lambda_abs0.5=0.18341263888175027 rho={'gate_proj': 0.25019198656082153, 'up_proj': 0.3110003173351288, 'down_proj': 0.25986307859420776}
+[reparam] epoch=1 step=300 phase=rho loss=2.002578e+00 rho_loss=2.821855e-03 mse=1.102674e+00 mlp_mse=8.970817e-01 kl=8.818296e-03 lambda_mean=0.5411720677425987 lambda_std=0.21556873075972857 lambda_abs0.5=0.18341263888175027 rho={'gate_proj': 0.5632190108299255, 'up_proj': 0.6403189897537231, 'down_proj': 0.5960606336593628}
+[reparam] epoch=1 step=400 phase=rho loss=1.978167e+00 rho_loss=3.026540e-03 mse=9.277866e-01 mlp_mse=1.047353e+00 kl=9.457937e-03 lambda_mean=0.5411720677425987 lambda_std=0.21556873075972857 lambda_abs0.5=0.18341263888175027 rho={'gate_proj': 0.7838373780250549, 'up_proj': 0.8668608069419861, 'down_proj': 0.8250398635864258}
+[reparam] epoch=1 step=500 phase=rho loss=2.060200e+00 rho_loss=3.466367e-03 mse=8.964794e-01 mlp_mse=1.160254e+00 kl=1.083240e-02 lambda_mean=0.5411720677425987 lambda_std=0.21556873075972857 lambda_abs0.5=0.18341263888175027 rho={'gate_proj': 0.9013062715530396, 'up_proj': 0.9889785647392273, 'down_proj': 0.953629195690155}
+[reparam-rho] rho phase complete values={'gate_proj': 0.908737301826477, 'up_proj': 0.9964994192123413, 'down_proj': 0.9617472887039185}
+[reparam] first_grad phase=u step=513 gate_l2=0.000000e+00 gate_mean_abs=None gate_max_abs=0.000000e+00 u_l2=1.358751e+00 u_mean_abs=2.418074005267076e-05 u_max_abs=4.083209e-01 rho_l2=0.000000e+00
+[reparam] epoch=2 step=600 phase=u loss=1.594410e+00 mse=7.355697e-01 mlp_mse=8.564382e-01 kl=7.506134e-03 lambda_mean=0.5411720677425987 lambda_std=0.21556873075972857 lambda_abs0.5=0.18341263888175027 rho={'gate_proj': 0.908737301826477, 'up_proj': 0.9964994192123413, 'down_proj': 0.9617472887039185}
+[reparam] epoch=2 step=700 phase=u loss=1.641499e+00 mse=7.659873e-01 mlp_mse=8.727433e-01 kl=8.650020e-03 lambda_mean=0.5411720677425987 lambda_std=0.21556873075972857 lambda_abs0.5=0.18341263888175027 rho={'gate_proj': 0.908737301826477, 'up_proj': 0.9964994192123413, 'down_proj': 0.9617472887039185}
+[reparam] epoch=2 step=800 phase=u loss=1.935805e+00 mse=9.197030e-01 mlp_mse=1.012461e+00 kl=1.137690e-02 lambda_mean=0.5411720677425987 lambda_std=0.21556873075972857 lambda_abs0.5=0.18341263888175027 rho={'gate_proj': 0.908737301826477, 'up_proj': 0.9964994192123413, 'down_proj': 0.9617472887039185}
+[reparam] epoch=2 step=900 phase=u loss=1.773922e+00 mse=8.680795e-01 mlp_mse=9.021410e-01 kl=1.156808e-02 lambda_mean=0.5411720677425987 lambda_std=0.21556873075972857 lambda_abs0.5=0.18341263888175027 rho={'gate_proj': 0.908737301826477, 'up_proj': 0.9964994192123413, 'down_proj': 0.9617472887039185}
+[reparam] epoch=2 step=1000 phase=u loss=1.284495e+00 mse=6.190402e-01 mlp_mse=6.630780e-01 kl=7.427331e-03 lambda_mean=0.5411720677425987 lambda_std=0.21556873075972857 lambda_abs0.5=0.18341263888175027 rho={'gate_proj': 0.908737301826477, 'up_proj': 0.9964994192123413, 'down_proj': 0.9617472887039185}
+[timing] Cycle 1 component wall-clock:
+[timing]   selection: 0.000s (0.00% of cycle)
+[timing]   rebase: 0.000s (0.00% of cycle)
+[timing]   fisher: 16.974s (3.23% of cycle)
+[timing]   fit: 508.183s (96.66% of cycle)
+[timing]   materialize_save_layer: 0.290s (0.06% of cycle)
+[timing]   metadata_write: 0.002s (0.00% of cycle)
+[timing]   teacher_snapshot: 0.106s (0.02% of cycle)
+[timing]   cycle_cleanup: 0.180s (0.03% of cycle)
+[timing]   cycle_total: 525.735s
+[timing]   tracked_components_total: 525.735s
+[progressive] Cycle 2/5
+[fuse] Head permutation disabled; merging with original head order.
+[fisher-input] mode=anchor_input cos=0.9459883216768503 norm_ratio=0.88246039301157
+[reparam] fisher_prior_inputs fa_mean=2.3743550907072443e-09 fb_mean=2.9281574841246636e-09 active_ratio=1.0 rel_contrast=0.34459433973881237 prior_mean=0.4550785131621779 prior_active_mean=0.45507851243019104
+[reparam] Cycle 2: training U + gates for pair 25-26 (epochs=1.0, skip_train=False, hidden_mse_w=1.0, attn_mse_w=0.0, mlp_mse_w=1.0, eta=0.0, gamma=0.0, attn_reg_scale=1.0, mlp_reg_scale=1.0, param_subset=mlp, fisher_init_subset=same, train_subset=same, lambda_init=fisher_prior).
+[reparam] subset=mlp gates=3 (attn=0, mlp=3, other=0) train_subset=mlp (attn=0, mlp=3, other=0) elems=74711040 (~285.0 MiB), U_elems=74711040 (~285.0 MiB; +optimizer state)
+[reparam] lambda_init mean=0.4550785131621779 std=0.20377763249639672 min=0.0002949995978269726 max=0.9995361566543579 mean_abs_from_0.5=0.1722971665231805
+[reparam-rho] objective=kl_only schedule=sequential rho_epochs=1.0 lr=0.002 bounds=[-1.0, 2.0] projections=['down_proj', 'gate_proj', 'up_proj']
+[reparam] first_grad phase=rho step=1 gate_l2=0.000000e+00 gate_mean_abs=None gate_max_abs=0.000000e+00 u_l2=0.000000e+00 u_mean_abs=None u_max_abs=0.000000e+00 rho_l2=7.212897e-04
+[reparam] epoch=1 step=1 phase=rho loss=2.139793e+00 rho_loss=4.265088e-03 mse=1.506416e+00 mlp_mse=6.291121e-01 kl=1.332840e-02 lambda_mean=0.4550785131621779 lambda_std=0.20377763249639672 lambda_abs0.5=0.1722971665231805 rho={'gate_proj': -0.0019999623764306307, 'up_proj': -0.001999924425035715, 'down_proj': -0.0019999502692371607}
+[reparam] epoch=1 step=100 phase=rho loss=1.846173e+00 rho_loss=4.108459e-03 mse=1.354352e+00 mlp_mse=4.877122e-01 kl=1.283893e-02 lambda_mean=0.4550785131621779 lambda_std=0.20377763249639672 lambda_abs0.5=0.1722971665231805 rho={'gate_proj': -0.1716773360967636, 'up_proj': -0.16872453689575195, 'down_proj': -0.18729351460933685}
+[reparam] epoch=1 step=200 phase=rho loss=2.377034e+00 rho_loss=7.421913e-03 mse=1.688458e+00 mlp_mse=6.811543e-01 kl=2.319348e-02 lambda_mean=0.4550785131621779 lambda_std=0.20377763249639672 lambda_abs0.5=0.1722971665231805 rho={'gate_proj': -0.21974655985832214, 'up_proj': -0.26743853092193604, 'down_proj': -0.31678247451782227}
+[reparam] epoch=1 step=300 phase=rho loss=1.890294e+00 rho_loss=4.523441e-03 mse=1.389674e+00 mlp_mse=4.960968e-01 kl=1.413575e-02 lambda_mean=0.4550785131621779 lambda_std=0.20377763249639672 lambda_abs0.5=0.1722971665231805 rho={'gate_proj': -0.2073226124048233, 'up_proj': -0.2791370749473572, 'down_proj': -0.36378031969070435}
+[reparam] epoch=1 step=400 phase=rho loss=2.296130e+00 rho_loss=5.729654e-03 mse=1.578289e+00 mlp_mse=7.121113e-01 kl=1.790517e-02 lambda_mean=0.4550785131621779 lambda_std=0.20377763249639672 lambda_abs0.5=0.1722971665231805 rho={'gate_proj': -0.1849168986082077, 'up_proj': -0.2626808285713196, 'down_proj': -0.38019105792045593}
+[reparam] epoch=1 step=500 phase=rho loss=1.761632e+00 rho_loss=3.170033e-03 mse=1.316754e+00 mlp_mse=4.417081e-01 kl=9.906352e-03 lambda_mean=0.4550785131621779 lambda_std=0.20377763249639672 lambda_abs0.5=0.1722971665231805 rho={'gate_proj': -0.19024714827537537, 'up_proj': -0.27119821310043335, 'down_proj': -0.38812050223350525}
+[reparam-rho] rho phase complete values={'gate_proj': -0.18974937498569489, 'up_proj': -0.27517983317375183, 'down_proj': -0.38872939348220825}
+[reparam] first_grad phase=u step=513 gate_l2=0.000000e+00 gate_mean_abs=None gate_max_abs=0.000000e+00 u_l2=2.079309e+00 u_mean_abs=2.7195367504630172e-05 u_max_abs=4.005861e-01 rho_l2=0.000000e+00
+[reparam] epoch=2 step=600 phase=u loss=2.489367e+00 mse=1.712602e+00 mlp_mse=7.676760e-01 kl=2.840116e-02 lambda_mean=0.4550785131621779 lambda_std=0.20377763249639672 lambda_abs0.5=0.1722971665231805 rho={'gate_proj': -0.18974937498569489, 'up_proj': -0.27517983317375183, 'down_proj': -0.38872939348220825}
+[reparam] epoch=2 step=700 phase=u loss=1.175084e+00 mse=7.425249e-01 mlp_mse=4.287537e-01 kl=1.189309e-02 lambda_mean=0.4550785131621779 lambda_std=0.20377763249639672 lambda_abs0.5=0.1722971665231805 rho={'gate_proj': -0.18974937498569489, 'up_proj': -0.27517983317375183, 'down_proj': -0.38872939348220825}
+[reparam] epoch=2 step=800 phase=u loss=1.041398e+00 mse=6.765918e-01 mlp_mse=3.628190e-01 kl=6.210998e-03 lambda_mean=0.4550785131621779 lambda_std=0.20377763249639672 lambda_abs0.5=0.1722971665231805 rho={'gate_proj': -0.18974937498569489, 'up_proj': -0.27517983317375183, 'down_proj': -0.38872939348220825}
+[reparam] epoch=2 step=900 phase=u loss=9.944851e-01 mse=6.480159e-01 mlp_mse=3.440630e-01 kl=7.519573e-03 lambda_mean=0.4550785131621779 lambda_std=0.20377763249639672 lambda_abs0.5=0.1722971665231805 rho={'gate_proj': -0.18974937498569489, 'up_proj': -0.27517983317375183, 'down_proj': -0.38872939348220825}
+[reparam] epoch=2 step=1000 phase=u loss=1.078392e+00 mse=6.980138e-01 mlp_mse=3.777351e-01 kl=8.260117e-03 lambda_mean=0.4550785131621779 lambda_std=0.20377763249639672 lambda_abs0.5=0.1722971665231805 rho={'gate_proj': -0.18974937498569489, 'up_proj': -0.27517983317375183, 'down_proj': -0.38872939348220825}
+[timing] Cycle 2 component wall-clock:
+[timing]   selection: 0.000s (0.00% of cycle)
+[timing]   rebase: 0.000s (0.00% of cycle)
+[timing]   fisher: 16.230s (3.19% of cycle)
+[timing]   fit: 492.164s (96.67% of cycle)
+[timing]   materialize_save_layer: 0.275s (0.05% of cycle)
+[timing]   metadata_write: 0.002s (0.00% of cycle)
+[timing]   teacher_snapshot: 0.272s (0.05% of cycle)
+[timing]   cycle_cleanup: 0.178s (0.04% of cycle)
+[timing]   cycle_total: 509.123s
+[timing]   tracked_components_total: 509.122s
+[progressive] Cycle 3/5
+[fuse] Head permutation disabled; merging with original head order.
+[fisher-input] mode=anchor_input cos=0.9242747537791729 norm_ratio=0.8410577848553658
+[reparam] fisher_prior_inputs fa_mean=3.74673801847517e-09 fb_mean=3.4374030140800945e-09 active_ratio=1.0 rel_contrast=0.3848759701377467 prior_mean=0.47654994496128017 prior_active_mean=0.4765499432881673
+[reparam] Cycle 3: training U + gates for pair 25-26 (epochs=1.0, skip_train=False, hidden_mse_w=1.0, attn_mse_w=0.0, mlp_mse_w=1.0, eta=0.0, gamma=0.0, attn_reg_scale=1.0, mlp_reg_scale=1.0, param_subset=mlp, fisher_init_subset=same, train_subset=same, lambda_init=fisher_prior).
+[reparam] subset=mlp gates=3 (attn=0, mlp=3, other=0) train_subset=mlp (attn=0, mlp=3, other=0) elems=74711040 (~285.0 MiB), U_elems=74711040 (~285.0 MiB; +optimizer state)
+[reparam] lambda_init mean=0.47654995834618286 std=0.22786270068066644 min=0.0001825599028961733 max=0.999645471572876 mean_abs_from_0.5=0.19243799845377604
+[reparam-rho] objective=kl_only schedule=sequential rho_epochs=1.0 lr=0.002 bounds=[-1.0, 2.0] projections=['down_proj', 'gate_proj', 'up_proj']
+[reparam] first_grad phase=rho step=1 gate_l2=0.000000e+00 gate_mean_abs=None gate_max_abs=0.000000e+00 u_l2=0.000000e+00 u_mean_abs=None u_max_abs=0.000000e+00 rho_l2=3.997404e-04
+[reparam] epoch=1 step=1 phase=rho loss=3.117546e+00 rho_loss=6.020769e-03 mse=2.115101e+00 mlp_mse=9.964248e-01 kl=1.881490e-02 lambda_mean=0.47654995834618286 lambda_std=0.22786270068066644 lambda_abs0.5=0.19243799845377604 rho={'gate_proj': -0.001999936532229185, 'up_proj': 0.0019999004434794188, 'down_proj': -0.001999856671318412}
+[reparam] epoch=1 step=100 phase=rho loss=3.821956e+00 rho_loss=7.813132e-03 mse=2.602064e+00 mlp_mse=1.212079e+00 kl=2.441604e-02 lambda_mean=0.47654995834618286 lambda_std=0.22786270068066644 lambda_abs0.5=0.19243799845377604 rho={'gate_proj': -0.09251413494348526, 'up_proj': 0.013608663342893124, 'down_proj': -0.06385193765163422}
+[reparam] epoch=1 step=200 phase=rho loss=3.476386e+00 rho_loss=1.167340e-02 mse=2.348960e+00 mlp_mse=1.115752e+00 kl=3.647939e-02 lambda_mean=0.47654995834618286 lambda_std=0.22786270068066644 lambda_abs0.5=0.19243799845377604 rho={'gate_proj': -0.17233502864837646, 'up_proj': -0.09057208150625229, 'down_proj': -0.16112695634365082}
+[reparam] epoch=1 step=300 phase=rho loss=3.576594e+00 rho_loss=7.000662e-03 mse=2.545300e+00 mlp_mse=1.024293e+00 kl=2.187707e-02 lambda_mean=0.47654995834618286 lambda_std=0.22786270068066644 lambda_abs0.5=0.19243799845377604 rho={'gate_proj': -0.23366567492485046, 'up_proj': -0.16860996186733246, 'down_proj': -0.2547451853752136}
+[reparam] epoch=1 step=400 phase=rho loss=3.145632e+00 rho_loss=6.533809e-03 mse=2.274325e+00 mlp_mse=8.647725e-01 kl=2.041815e-02 lambda_mean=0.47654995834618286 lambda_std=0.22786270068066644 lambda_abs0.5=0.19243799845377604 rho={'gate_proj': -0.22559776902198792, 'up_proj': -0.16930978000164032, 'down_proj': -0.2662374675273895}
+[reparam] epoch=1 step=500 phase=rho loss=3.241957e+00 rho_loss=6.286985e-03 mse=2.304701e+00 mlp_mse=9.309689e-01 kl=1.964683e-02 lambda_mean=0.47654995834618286 lambda_std=0.22786270068066644 lambda_abs0.5=0.19243799845377604 rho={'gate_proj': -0.21964842081069946, 'up_proj': -0.17884595692157745, 'down_proj': -0.2744521498680115}
+[reparam-rho] rho phase complete values={'gate_proj': -0.22315818071365356, 'up_proj': -0.18803080916404724, 'down_proj': -0.2787369191646576}
+[reparam] first_grad phase=u step=513 gate_l2=0.000000e+00 gate_mean_abs=None gate_max_abs=0.000000e+00 u_l2=2.371746e+00 u_mean_abs=3.869317619031981e-05 u_max_abs=6.250594e-01 rho_l2=0.000000e+00
+[reparam] epoch=2 step=600 phase=u loss=2.136091e+00 mse=1.347413e+00 mlp_mse=7.849607e-01 kl=1.161602e-02 lambda_mean=0.47654995834618286 lambda_std=0.22786270068066644 lambda_abs0.5=0.19243799845377604 rho={'gate_proj': -0.22315818071365356, 'up_proj': -0.18803080916404724, 'down_proj': -0.2787369191646576}
+[reparam] epoch=2 step=700 phase=u loss=2.095046e+00 mse=1.309404e+00 mlp_mse=7.805890e-01 kl=1.579065e-02 lambda_mean=0.47654995834618286 lambda_std=0.22786270068066644 lambda_abs0.5=0.19243799845377604 rho={'gate_proj': -0.22315818071365356, 'up_proj': -0.18803080916404724, 'down_proj': -0.2787369191646576}
+[reparam] epoch=2 step=800 phase=u loss=2.732887e+00 mse=1.720917e+00 mlp_mse=1.002140e+00 kl=3.071838e-02 lambda_mean=0.47654995834618286 lambda_std=0.22786270068066644 lambda_abs0.5=0.19243799845377604 rho={'gate_proj': -0.22315818071365356, 'up_proj': -0.18803080916404724, 'down_proj': -0.2787369191646576}
+[reparam] epoch=2 step=900 phase=u loss=1.570603e+00 mse=9.643101e-01 mlp_mse=6.033351e-01 kl=9.242858e-03 lambda_mean=0.47654995834618286 lambda_std=0.22786270068066644 lambda_abs0.5=0.19243799845377604 rho={'gate_proj': -0.22315818071365356, 'up_proj': -0.18803080916404724, 'down_proj': -0.2787369191646576}
+[reparam] epoch=2 step=1000 phase=u loss=1.607103e+00 mse=9.784901e-01 mlp_mse=6.258045e-01 kl=8.775534e-03 lambda_mean=0.47654995834618286 lambda_std=0.22786270068066644 lambda_abs0.5=0.19243799845377604 rho={'gate_proj': -0.22315818071365356, 'up_proj': -0.18803080916404724, 'down_proj': -0.2787369191646576}
+[timing] Cycle 3 component wall-clock:
+[timing]   selection: 0.000s (0.00% of cycle)
+[timing]   rebase: 0.000s (0.00% of cycle)
+[timing]   fisher: 15.877s (3.27% of cycle)
+[timing]   fit: 468.910s (96.59% of cycle)
+[timing]   materialize_save_layer: 0.237s (0.05% of cycle)
+[timing]   metadata_write: 0.002s (0.00% of cycle)
+[timing]   teacher_snapshot: 0.268s (0.06% of cycle)
+[timing]   cycle_cleanup: 0.178s (0.04% of cycle)
+[timing]   cycle_total: 485.473s
+[timing]   tracked_components_total: 485.472s
+[progressive] Cycle 4/5
+[fuse] Head permutation disabled; merging with original head order.
+[fisher-input] mode=anchor_input cos=0.9063979331403971 norm_ratio=0.7916548904031515
+[reparam] fisher_prior_inputs fa_mean=5.613525557061637e-09 fb_mean=3.4907165563549427e-09 active_ratio=1.0 rel_contrast=0.3861439219692297 prior_mean=0.5152522438450864 prior_active_mean=0.5152522325515747
+[reparam] Cycle 4: training U + gates for pair 25-26 (epochs=1.0, skip_train=False, hidden_mse_w=1.0, attn_mse_w=0.0, mlp_mse_w=1.0, eta=0.0, gamma=0.0, attn_reg_scale=1.0, mlp_reg_scale=1.0, param_subset=mlp, fisher_init_subset=same, train_subset=same, lambda_init=fisher_prior).
+[reparam] subset=mlp gates=3 (attn=0, mlp=3, other=0) train_subset=mlp (attn=0, mlp=3, other=0) elems=74711040 (~285.0 MiB), U_elems=74711040 (~285.0 MiB; +optimizer state)
+[reparam] lambda_init mean=0.515252257229989 std=0.22938150909477692 min=0.00025270492187701166 max=0.9998838901519775 mean_abs_from_0.5=0.1930719676770662
+[reparam-rho] objective=kl_only schedule=sequential rho_epochs=1.0 lr=0.002 bounds=[-1.0, 2.0] projections=['down_proj', 'gate_proj', 'up_proj']
+[reparam] first_grad phase=rho step=1 gate_l2=0.000000e+00 gate_mean_abs=None gate_max_abs=0.000000e+00 u_l2=0.000000e+00 u_mean_abs=None u_max_abs=0.000000e+00 rho_l2=6.639060e-04
+[reparam] epoch=1 step=1 phase=rho loss=4.500839e+00 rho_loss=6.559336e-03 mse=3.326269e+00 mlp_mse=1.168011e+00 kl=2.049792e-02 lambda_mean=0.515252257229989 lambda_std=0.22938150909477692 lambda_abs0.5=0.1930719676770662 rho={'gate_proj': 0.001999948639422655, 'up_proj': 0.001999962143599987, 'down_proj': 0.001999759813770652}
+[reparam] epoch=1 step=100 phase=rho loss=4.803223e+00 rho_loss=7.828027e-03 mse=3.533712e+00 mlp_mse=1.261682e+00 kl=2.446259e-02 lambda_mean=0.515252257229989 lambda_std=0.22938150909477692 lambda_abs0.5=0.1930719676770662 rho={'gate_proj': 0.16084375977516174, 'up_proj': 0.16108129918575287, 'down_proj': 0.09061072021722794}
+[reparam] epoch=1 step=200 phase=rho loss=4.913829e+00 rho_loss=6.075517e-03 mse=3.271103e+00 mlp_mse=1.636650e+00 kl=1.898599e-02 lambda_mean=0.515252257229989 lambda_std=0.22938150909477692 lambda_abs0.5=0.1930719676770662 rho={'gate_proj': 0.38290613889694214, 'up_proj': 0.3656698167324066, 'down_proj': 0.29921555519104004}
+[reparam] epoch=1 step=300 phase=rho loss=4.519518e+00 rho_loss=5.054371e-03 mse=2.930983e+00 mlp_mse=1.583480e+00 kl=1.579491e-02 lambda_mean=0.515252257229989 lambda_std=0.22938150909477692 lambda_abs0.5=0.1930719676770662 rho={'gate_proj': 0.6150959730148315, 'up_proj': 0.5777497291564941, 'down_proj': 0.5281074047088623}
+[reparam] epoch=1 step=400 phase=rho loss=4.084389e+00 rho_loss=1.636193e-02 mse=2.534635e+00 mlp_mse=1.533392e+00 kl=5.113104e-02 lambda_mean=0.515252257229989 lambda_std=0.22938150909477692 lambda_abs0.5=0.1930719676770662 rho={'gate_proj': 0.8129631280899048, 'up_proj': 0.784672200679779, 'down_proj': 0.7547294497489929}
+[reparam] epoch=1 step=500 phase=rho loss=4.841489e+00 rho_loss=6.658037e-03 mse=2.331696e+00 mlp_mse=2.503134e+00 kl=2.080637e-02 lambda_mean=0.515252257229989 lambda_std=0.22938150909477692 lambda_abs0.5=0.1930719676770662 rho={'gate_proj': 0.9365948438644409, 'up_proj': 0.9515950679779053, 'down_proj': 0.9314753413200378}
+[reparam-rho] rho phase complete values={'gate_proj': 0.9425783753395081, 'up_proj': 0.9663310647010803, 'down_proj': 0.9466240406036377}
+[reparam] first_grad phase=u step=513 gate_l2=0.000000e+00 gate_mean_abs=None gate_max_abs=0.000000e+00 u_l2=2.442667e+00 u_mean_abs=4.24195456792388e-05 u_max_abs=7.584000e-01 rho_l2=0.000000e+00
+[reparam] epoch=2 step=600 phase=u loss=3.356200e+00 mse=1.789572e+00 mlp_mse=1.557719e+00 kl=2.783983e-02 lambda_mean=0.515252257229989 lambda_std=0.22938150909477692 lambda_abs0.5=0.1930719676770662 rho={'gate_proj': 0.9425783753395081, 'up_proj': 0.9663310647010803, 'down_proj': 0.9466240406036377}
+[reparam] epoch=2 step=700 phase=u loss=3.255069e+00 mse=1.632663e+00 mlp_mse=1.617180e+00 kl=1.633444e-02 lambda_mean=0.515252257229989 lambda_std=0.22938150909477692 lambda_abs0.5=0.1930719676770662 rho={'gate_proj': 0.9425783753395081, 'up_proj': 0.9663310647010803, 'down_proj': 0.9466240406036377}
+[reparam] epoch=2 step=800 phase=u loss=2.867174e+00 mse=1.464978e+00 mlp_mse=1.397177e+00 kl=1.568307e-02 lambda_mean=0.515252257229989 lambda_std=0.22938150909477692 lambda_abs0.5=0.1930719676770662 rho={'gate_proj': 0.9425783753395081, 'up_proj': 0.9663310647010803, 'down_proj': 0.9466240406036377}
+[reparam] epoch=2 step=900 phase=u loss=3.041087e+00 mse=1.545682e+00 mlp_mse=1.490545e+00 kl=1.518722e-02 lambda_mean=0.515252257229989 lambda_std=0.22938150909477692 lambda_abs0.5=0.1930719676770662 rho={'gate_proj': 0.9425783753395081, 'up_proj': 0.9663310647010803, 'down_proj': 0.9466240406036377}
+[reparam] epoch=2 step=1000 phase=u loss=3.909813e+00 mse=2.050640e+00 mlp_mse=1.851655e+00 kl=2.349613e-02 lambda_mean=0.515252257229989 lambda_std=0.22938150909477692 lambda_abs0.5=0.1930719676770662 rho={'gate_proj': 0.9425783753395081, 'up_proj': 0.9663310647010803, 'down_proj': 0.9466240406036377}
+[timing] Cycle 4 component wall-clock:
+[timing]   selection: 0.000s (0.00% of cycle)
+[timing]   rebase: 0.000s (0.00% of cycle)
+[timing]   fisher: 15.741s (3.32% of cycle)
+[timing]   fit: 457.178s (96.53% of cycle)
+[timing]   materialize_save_layer: 0.268s (0.06% of cycle)
+[timing]   metadata_write: 0.002s (0.00% of cycle)
+[timing]   teacher_snapshot: 0.259s (0.05% of cycle)
+[timing]   cycle_cleanup: 0.177s (0.04% of cycle)
+[timing]   cycle_total: 473.626s
+[timing]   tracked_components_total: 473.625s
+[progressive] Cycle 5/5
+[fuse] Head permutation disabled; merging with original head order.
+[fisher-input] mode=anchor_input cos=0.9063154626637697 norm_ratio=0.7825618479400873
+[reparam] fisher_prior_inputs fa_mean=6.250133154887743e-09 fb_mean=5.004990231227886e-09 active_ratio=1.0 rel_contrast=0.37433193273711624 prior_mean=0.47306570755807975 prior_active_mean=0.47306570410728455
+[reparam] Cycle 5: training U + gates for pair 25-26 (epochs=1.0, skip_train=False, hidden_mse_w=1.0, attn_mse_w=0.0, mlp_mse_w=1.0, eta=0.0, gamma=0.0, attn_reg_scale=1.0, mlp_reg_scale=1.0, param_subset=mlp, fisher_init_subset=same, train_subset=same, lambda_init=fisher_prior).
+[reparam] subset=mlp gates=3 (attn=0, mlp=3, other=0) train_subset=mlp (attn=0, mlp=3, other=0) elems=74711040 (~285.0 MiB), U_elems=74711040 (~285.0 MiB; +optimizer state)
+[reparam] lambda_init mean=0.47306569417317706 std=0.22272197163507823 min=9.999999747378752e-05 max=0.9995927214622498 mean_abs_from_0.5=0.18716597306100946
+[reparam-rho] objective=kl_only schedule=sequential rho_epochs=1.0 lr=0.002 bounds=[-1.0, 2.0] projections=['down_proj', 'gate_proj', 'up_proj']
+[reparam] first_grad phase=rho step=1 gate_l2=0.000000e+00 gate_mean_abs=None gate_max_abs=0.000000e+00 u_l2=0.000000e+00 u_mean_abs=None u_max_abs=0.000000e+00 rho_l2=9.806296e-04
+[reparam] epoch=1 step=1 phase=rho loss=7.771140e+00 rho_loss=8.441261e-03 mse=6.150832e+00 mlp_mse=1.611866e+00 kl=2.637894e-02 lambda_mean=0.47306569417317706 lambda_std=0.22272197163507823 lambda_abs0.5=0.18716597306100946 rho={'gate_proj': 0.0019999195355921984, 'up_proj': 0.0019999786745756865, 'down_proj': 0.001999754225835204}
+[reparam] epoch=1 step=100 phase=rho loss=8.335280e+00 rho_loss=1.025439e-02 mse=6.495891e+00 mlp_mse=1.829135e+00 kl=3.204498e-02 lambda_mean=0.47306569417317706 lambda_std=0.22272197163507823 lambda_abs0.5=0.18716597306100946 rho={'gate_proj': 0.13291046023368835, 'up_proj': 0.1909928172826767, 'down_proj': 0.11599428206682205}
+[reparam] epoch=1 step=200 phase=rho loss=8.195517e+00 rho_loss=1.091045e-02 mse=6.099593e+00 mlp_mse=2.085014e+00 kl=3.409517e-02 lambda_mean=0.47306569417317706 lambda_std=0.22272197163507823 lambda_abs0.5=0.18716597306100946 rho={'gate_proj': 0.3523268401622772, 'up_proj': 0.4089561402797699, 'down_proj': 0.33066391944885254}
+[reparam] epoch=1 step=300 phase=rho loss=7.397445e+00 rho_loss=8.189706e-03 mse=5.604434e+00 mlp_mse=1.784821e+00 kl=2.559283e-02 lambda_mean=0.47306569417317706 lambda_std=0.22272197163507823 lambda_abs0.5=0.18716597306100946 rho={'gate_proj': 0.5773504972457886, 'up_proj': 0.6285123229026794, 'down_proj': 0.564320981502533}
+[reparam] epoch=1 step=400 phase=rho loss=6.770636e+00 rho_loss=8.979723e-03 mse=4.595263e+00 mlp_mse=2.166393e+00 kl=2.806164e-02 lambda_mean=0.47306569417317706 lambda_std=0.22272197163507823 lambda_abs0.5=0.18716597306100946 rho={'gate_proj': 0.7656285166740417, 'up_proj': 0.8290404677391052, 'down_proj': 0.7814235091209412}
+[reparam] epoch=1 step=500 phase=rho loss=5.061226e+00 rho_loss=9.013770e-03 mse=2.946798e+00 mlp_mse=2.105414e+00 kl=2.816803e-02 lambda_mean=0.47306569417317706 lambda_std=0.22272197163507823 lambda_abs0.5=0.18716597306100946 rho={'gate_proj': 0.8958165645599365, 'up_proj': 0.9979074001312256, 'down_proj': 0.9647148251533508}
+[reparam-rho] rho phase complete values={'gate_proj': 0.9063539505004883, 'up_proj': 1.0162287950515747, 'down_proj': 0.9849719405174255}
+[reparam] first_grad phase=u step=513 gate_l2=0.000000e+00 gate_mean_abs=None gate_max_abs=0.000000e+00 u_l2=4.686235e+00 u_mean_abs=7.453353650737227e-05 u_max_abs=1.210297e+00 rho_l2=0.000000e+00
+[reparam] epoch=2 step=600 phase=u loss=4.036732e+00 mse=2.093142e+00 mlp_mse=1.937756e+00 kl=1.823055e-02 lambda_mean=0.47306569417317706 lambda_std=0.22272197163507823 lambda_abs0.5=0.18716597306100946 rho={'gate_proj': 0.9063539505004883, 'up_proj': 1.0162287950515747, 'down_proj': 0.9849719405174255}
+[reparam] epoch=2 step=700 phase=u loss=4.540934e+00 mse=2.306481e+00 mlp_mse=2.227864e+00 kl=2.059061e-02 lambda_mean=0.47306569417317706 lambda_std=0.22272197163507823 lambda_abs0.5=0.18716597306100946 rho={'gate_proj': 0.9063539505004883, 'up_proj': 1.0162287950515747, 'down_proj': 0.9849719405174255}
+[reparam] epoch=2 step=800 phase=u loss=4.536172e+00 mse=2.285528e+00 mlp_mse=2.245308e+00 kl=1.667426e-02 lambda_mean=0.47306569417317706 lambda_std=0.22272197163507823 lambda_abs0.5=0.18716597306100946 rho={'gate_proj': 0.9063539505004883, 'up_proj': 1.0162287950515747, 'down_proj': 0.9849719405174255}
+[reparam] epoch=2 step=900 phase=u loss=3.896600e+00 mse=2.072172e+00 mlp_mse=1.820605e+00 kl=1.195010e-02 lambda_mean=0.47306569417317706 lambda_std=0.22272197163507823 lambda_abs0.5=0.18716597306100946 rho={'gate_proj': 0.9063539505004883, 'up_proj': 1.0162287950515747, 'down_proj': 0.9849719405174255}
+[reparam] epoch=2 step=1000 phase=u loss=4.293452e+00 mse=2.175881e+00 mlp_mse=2.111793e+00 kl=1.805609e-02 lambda_mean=0.47306569417317706 lambda_std=0.22272197163507823 lambda_abs0.5=0.18716597306100946 rho={'gate_proj': 0.9063539505004883, 'up_proj': 1.0162287950515747, 'down_proj': 0.9849719405174255}
+[timing] Cycle 5 component wall-clock:
+[timing]   selection: 0.000s (0.00% of cycle)
+[timing]   rebase: 0.000s (0.00% of cycle)
+[timing]   fisher: 15.254s (3.26% of cycle)
+[timing]   fit: 452.458s (96.61% of cycle)
+[timing]   materialize_save_layer: 0.169s (0.04% of cycle)
+[timing]   metadata_write: 0.002s (0.00% of cycle)
+[timing]   teacher_snapshot: 0.259s (0.06% of cycle)
+[timing]   cycle_cleanup: 0.179s (0.04% of cycle)
+[timing]   cycle_total: 468.322s
+[timing]   tracked_components_total: 468.321s
+
+
+[progressive] Completed 5 cycles. Final model saved to /workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_input_rho/run.