diff --git a/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/COMPACT_COMPLETE b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/COMPACT_COMPLETE new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/COMPLETE b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/COMPLETE new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/TRAIN_COMPLETE b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/TRAIN_COMPLETE new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/manifest.json b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..5f9d2d543cd84cde787ce2224d318774ecc4e327 --- /dev/null +++ b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/manifest.json @@ -0,0 +1,161 @@ +{ + "format": "abprune_compact_manifest_v1", + "model": "Qwen/Qwen3-4B-Base", + "base_model_revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539", + "run_id": "qwen3_4b_anchor_input_no_rho_contextual_fisher_v1", + "target_prune_count": 5, + "layer_order": [ + 25, + 25, + 25, + 25, + 25 + ], + "stages": [ + { + "format": "abprune_compact_stage_v1", + "model": "Qwen/Qwen3-4B-Base", + "base_model_revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539", + "run_id": "qwen3_4b_anchor_input_no_rho_contextual_fisher_v1", + "target_prune_count": 5, + "stage": 1, + "removed_count": 1, + "anchor_index": 25, + "layer_order": [ + 25, + 25, + 25, + 25, + 25 + ], + "packaged_stage_range": [ + 1, + 5 + ], + "removed_original_layers": [ + 26 + ], + "source_cycle": "cycle_1", + "anchor_state_file": "anchor_layer.pt", + "sha256": "2ea21e63d8385b7c6b32cb47cb5a93198822649e15ac1ec7e1a401aba2f6ce2f" + }, + { + "format": "abprune_compact_stage_v1", + "model": "Qwen/Qwen3-4B-Base", + "base_model_revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539", + "run_id": "qwen3_4b_anchor_input_no_rho_contextual_fisher_v1", + "target_prune_count": 5, + "stage": 2, + "removed_count": 2, + "anchor_index": 25, + "layer_order": [ + 25, + 25, + 25, + 25, + 25 + ], + "packaged_stage_range": [ + 1, + 5 + ], + "removed_original_layers": [ + 26, + 27 + ], + "source_cycle": "cycle_2", + "anchor_state_file": "anchor_layer.pt", + "sha256": "b4c0450e441f2366f09cc6b5b72fc17d1f845cb9e8be7add342281cc27b40a6c" + }, + { + "format": "abprune_compact_stage_v1", + "model": "Qwen/Qwen3-4B-Base", + "base_model_revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539", + "run_id": "qwen3_4b_anchor_input_no_rho_contextual_fisher_v1", + "target_prune_count": 5, + "stage": 3, + "removed_count": 3, + "anchor_index": 25, + "layer_order": [ + 25, + 25, + 25, + 25, + 25 + ], + "packaged_stage_range": [ + 1, + 5 + ], + "removed_original_layers": [ + 26, + 27, + 28 + ], + "source_cycle": "cycle_3", + "anchor_state_file": "anchor_layer.pt", + "sha256": "0c3aacef2687a4fdda11ff7fa75ceed7e55be1269fe40618b1e782f79d6745af" + }, + { + "format": "abprune_compact_stage_v1", + "model": "Qwen/Qwen3-4B-Base", + "base_model_revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539", + "run_id": "qwen3_4b_anchor_input_no_rho_contextual_fisher_v1", + "target_prune_count": 5, + "stage": 4, + "removed_count": 4, + "anchor_index": 25, + "layer_order": [ + 25, + 25, + 25, + 25, + 25 + ], + "packaged_stage_range": [ + 1, + 5 + ], + "removed_original_layers": [ + 26, + 27, + 28, + 29 + ], + "source_cycle": "cycle_4", + "anchor_state_file": "anchor_layer.pt", + "sha256": "36cdbc27d39227c16d250dd572efe3be8e4f38bacb89beac45d94102cf45ae11" + }, + { + "format": "abprune_compact_stage_v1", + "model": "Qwen/Qwen3-4B-Base", + "base_model_revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539", + "run_id": "qwen3_4b_anchor_input_no_rho_contextual_fisher_v1", + "target_prune_count": 5, + "stage": 5, + "removed_count": 5, + "anchor_index": 25, + "layer_order": [ + 25, + 25, + 25, + 25, + 25 + ], + "packaged_stage_range": [ + 1, + 5 + ], + "removed_original_layers": [ + 26, + 27, + 28, + 29, + 30 + ], + "source_cycle": "cycle_5", + "anchor_state_file": "anchor_layer.pt", + "sha256": "a7812d906d202c1bd843be4aff0faabad90d2da61812e26b0f164a62e7faaed8" + } + ] +} diff --git a/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_01/anchor_layer.pt b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_01/anchor_layer.pt new file mode 100644 index 0000000000000000000000000000000000000000..5e24e59673d7f015e80efb93612624512ffdaa06 --- /dev/null +++ b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_01/anchor_layer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ea21e63d8385b7c6b32cb47cb5a93198822649e15ac1ec7e1a401aba2f6ce2f +size 201866630 diff --git a/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_01/metadata.json b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_01/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..278d2bb7093a2f061ceeec7617b20680cfcf4d0a --- /dev/null +++ b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_01/metadata.json @@ -0,0 +1,27 @@ +{ + "format": "abprune_compact_stage_v1", + "model": "Qwen/Qwen3-4B-Base", + "base_model_revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539", + "run_id": "qwen3_4b_anchor_input_no_rho_contextual_fisher_v1", + "target_prune_count": 5, + "stage": 1, + "removed_count": 1, + "anchor_index": 25, + "layer_order": [ + 25, + 25, + 25, + 25, + 25 + ], + "packaged_stage_range": [ + 1, + 5 + ], + "removed_original_layers": [ + 26 + ], + "source_cycle": "cycle_1", + "anchor_state_file": "anchor_layer.pt", + "sha256": "2ea21e63d8385b7c6b32cb47cb5a93198822649e15ac1ec7e1a401aba2f6ce2f" +} diff --git a/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_02/anchor_layer.pt b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_02/anchor_layer.pt new file mode 100644 index 0000000000000000000000000000000000000000..4f194e14a154a0edbead41b2051eb8a075502df8 --- /dev/null +++ b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_02/anchor_layer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4c0450e441f2366f09cc6b5b72fc17d1f845cb9e8be7add342281cc27b40a6c +size 201866630 diff --git a/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_02/metadata.json b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_02/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..9e0a8d9013726a726e92e4f807c30bc3b4d8175a --- /dev/null +++ b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_02/metadata.json @@ -0,0 +1,28 @@ +{ + "format": "abprune_compact_stage_v1", + "model": "Qwen/Qwen3-4B-Base", + "base_model_revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539", + "run_id": "qwen3_4b_anchor_input_no_rho_contextual_fisher_v1", + "target_prune_count": 5, + "stage": 2, + "removed_count": 2, + "anchor_index": 25, + "layer_order": [ + 25, + 25, + 25, + 25, + 25 + ], + "packaged_stage_range": [ + 1, + 5 + ], + "removed_original_layers": [ + 26, + 27 + ], + "source_cycle": "cycle_2", + "anchor_state_file": "anchor_layer.pt", + "sha256": "b4c0450e441f2366f09cc6b5b72fc17d1f845cb9e8be7add342281cc27b40a6c" +} diff --git a/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_03/anchor_layer.pt b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_03/anchor_layer.pt new file mode 100644 index 0000000000000000000000000000000000000000..e8473615a17d0767f8d4e105bfcb3aaeaee168f1 --- /dev/null +++ b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_03/anchor_layer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c3aacef2687a4fdda11ff7fa75ceed7e55be1269fe40618b1e782f79d6745af +size 201866630 diff --git a/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_03/metadata.json b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_03/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..f659d58ea66c85d2622d541e07b5e27decd63faf --- /dev/null +++ b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_03/metadata.json @@ -0,0 +1,29 @@ +{ + "format": "abprune_compact_stage_v1", + "model": "Qwen/Qwen3-4B-Base", + "base_model_revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539", + "run_id": "qwen3_4b_anchor_input_no_rho_contextual_fisher_v1", + "target_prune_count": 5, + "stage": 3, + "removed_count": 3, + "anchor_index": 25, + "layer_order": [ + 25, + 25, + 25, + 25, + 25 + ], + "packaged_stage_range": [ + 1, + 5 + ], + "removed_original_layers": [ + 26, + 27, + 28 + ], + "source_cycle": "cycle_3", + "anchor_state_file": "anchor_layer.pt", + "sha256": "0c3aacef2687a4fdda11ff7fa75ceed7e55be1269fe40618b1e782f79d6745af" +} diff --git a/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_04/anchor_layer.pt b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_04/anchor_layer.pt new file mode 100644 index 0000000000000000000000000000000000000000..ad679ab6d5b6db67b18e3c4e7097c91e1925925b --- /dev/null +++ b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_04/anchor_layer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36cdbc27d39227c16d250dd572efe3be8e4f38bacb89beac45d94102cf45ae11 +size 201866630 diff --git a/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_04/metadata.json b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_04/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..016ee90500b9e3bc7b9d9fe111fd8d15981390e0 --- /dev/null +++ b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_04/metadata.json @@ -0,0 +1,30 @@ +{ + "format": "abprune_compact_stage_v1", + "model": "Qwen/Qwen3-4B-Base", + "base_model_revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539", + "run_id": "qwen3_4b_anchor_input_no_rho_contextual_fisher_v1", + "target_prune_count": 5, + "stage": 4, + "removed_count": 4, + "anchor_index": 25, + "layer_order": [ + 25, + 25, + 25, + 25, + 25 + ], + "packaged_stage_range": [ + 1, + 5 + ], + "removed_original_layers": [ + 26, + 27, + 28, + 29 + ], + "source_cycle": "cycle_4", + "anchor_state_file": "anchor_layer.pt", + "sha256": "36cdbc27d39227c16d250dd572efe3be8e4f38bacb89beac45d94102cf45ae11" +} diff --git a/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_05/anchor_layer.pt b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_05/anchor_layer.pt new file mode 100644 index 0000000000000000000000000000000000000000..f66c9abaa7a7deda7666b00f3a7dd5b9430ddcd3 --- /dev/null +++ b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_05/anchor_layer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7812d906d202c1bd843be4aff0faabad90d2da61812e26b0f164a62e7faaed8 +size 201866630 diff --git a/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_05/metadata.json b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_05/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..f5685e807debad5f4958377581061b6b048a9842 --- /dev/null +++ b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/compact/stage_05/metadata.json @@ -0,0 +1,31 @@ +{ + "format": "abprune_compact_stage_v1", + "model": "Qwen/Qwen3-4B-Base", + "base_model_revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539", + "run_id": "qwen3_4b_anchor_input_no_rho_contextual_fisher_v1", + "target_prune_count": 5, + "stage": 5, + "removed_count": 5, + "anchor_index": 25, + "layer_order": [ + 25, + 25, + 25, + 25, + 25 + ], + "packaged_stage_range": [ + 1, + 5 + ], + "removed_original_layers": [ + 26, + 27, + 28, + 29, + 30 + ], + "source_cycle": "cycle_5", + "anchor_state_file": "anchor_layer.pt", + "sha256": "a7812d906d202c1bd843be4aff0faabad90d2da61812e26b0f164a62e7faaed8" +} diff --git a/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/eval/stage_01_ppl.json b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/eval/stage_01_ppl.json new file mode 100644 index 0000000000000000000000000000000000000000..68f816729a7045482022602ca97f6d75cb495c15 --- /dev/null +++ b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/eval/stage_01_ppl.json @@ -0,0 +1,25 @@ +{ + "model": "/workspace/models/qwen3-4b-base-906bfd4", + "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539", + "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_input_no_rho/compact/stage_01", + "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test", + "protocol": "HCHC experiment.py: filter empty rows, join with double newline, then tokenize the complete stream once", + "sequence_length": 2048, + "batch_size": 4, + "dtype": "bfloat16", + "metrics": { + "nll": 2.145418803445224, + "perplexity": 8.54561942281943, + "tokens": 296815, + "sequences": 145, + "source_tokens": 298938 + }, + "anchor": 25, + "cycles": 1, + "original_layers": 36, + "decoder_layers": 35, + "post_merge_residual_scale": null, + "post_merge_residual_scales": [], + "parameterization": "unscaled compact ABPrune", + "linear_patch": null +} diff --git a/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/eval/stage_02_ppl.json b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/eval/stage_02_ppl.json new file mode 100644 index 0000000000000000000000000000000000000000..d818ffc8a8df0ca0cbb152aa72b4d3c358eb5c36 --- /dev/null +++ b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/eval/stage_02_ppl.json @@ -0,0 +1,25 @@ +{ + "model": "/workspace/models/qwen3-4b-base-906bfd4", + "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539", + "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_input_no_rho/compact/stage_02", + "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test", + "protocol": "HCHC experiment.py: filter empty rows, join with double newline, then tokenize the complete stream once", + "sequence_length": 2048, + "batch_size": 4, + "dtype": "bfloat16", + "metrics": { + "nll": 2.2925252815772748, + "perplexity": 9.899906192368073, + "tokens": 296815, + "sequences": 145, + "source_tokens": 298938 + }, + "anchor": 25, + "cycles": 2, + "original_layers": 36, + "decoder_layers": 34, + "post_merge_residual_scale": null, + "post_merge_residual_scales": [], + "parameterization": "unscaled compact ABPrune", + "linear_patch": null +} diff --git a/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/eval/stage_03_ppl.json b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/eval/stage_03_ppl.json new file mode 100644 index 0000000000000000000000000000000000000000..398b81d944bbd89162be521d3e4d034234fdde6a --- /dev/null +++ b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/eval/stage_03_ppl.json @@ -0,0 +1,25 @@ +{ + "model": "/workspace/models/qwen3-4b-base-906bfd4", + "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539", + "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_input_no_rho/compact/stage_03", + "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test", + "protocol": "HCHC experiment.py: filter empty rows, join with double newline, then tokenize the complete stream once", + "sequence_length": 2048, + "batch_size": 4, + "dtype": "bfloat16", + "metrics": { + "nll": 2.4010539860560978, + "perplexity": 11.03480077974187, + "tokens": 296815, + "sequences": 145, + "source_tokens": 298938 + }, + "anchor": 25, + "cycles": 3, + "original_layers": 36, + "decoder_layers": 33, + "post_merge_residual_scale": null, + "post_merge_residual_scales": [], + "parameterization": "unscaled compact ABPrune", + "linear_patch": null +} diff --git a/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/eval/stage_04_ppl.json b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/eval/stage_04_ppl.json new file mode 100644 index 0000000000000000000000000000000000000000..5c83670478489a402fc92ad49da63d28c8369981 --- /dev/null +++ b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/eval/stage_04_ppl.json @@ -0,0 +1,25 @@ +{ + "model": "/workspace/models/qwen3-4b-base-906bfd4", + "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539", + "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_input_no_rho/compact/stage_04", + "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test", + "protocol": "HCHC experiment.py: filter empty rows, join with double newline, then tokenize the complete stream once", + "sequence_length": 2048, + "batch_size": 4, + "dtype": "bfloat16", + "metrics": { + "nll": 2.586987911421677, + "perplexity": 13.289681558091948, + "tokens": 296815, + "sequences": 145, + "source_tokens": 298938 + }, + "anchor": 25, + "cycles": 4, + "original_layers": 36, + "decoder_layers": 32, + "post_merge_residual_scale": null, + "post_merge_residual_scales": [], + "parameterization": "unscaled compact ABPrune", + "linear_patch": null +} diff --git a/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/eval/stage_05_ppl.json b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/eval/stage_05_ppl.json new file mode 100644 index 0000000000000000000000000000000000000000..cd578fa7b6fa667e601a3a9a3b22b36c745a63fd --- /dev/null +++ b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/eval/stage_05_ppl.json @@ -0,0 +1,25 @@ +{ + "model": "/workspace/models/qwen3-4b-base-906bfd4", + "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539", + "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_input_no_rho/compact/stage_05", + "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test", + "protocol": "HCHC experiment.py: filter empty rows, join with double newline, then tokenize the complete stream once", + "sequence_length": 2048, + "batch_size": 4, + "dtype": "bfloat16", + "metrics": { + "nll": 2.848904904003801, + "perplexity": 17.268860422166572, + "tokens": 296815, + "sequences": 145, + "source_tokens": 298938 + }, + "anchor": 25, + "cycles": 5, + "original_layers": 36, + "decoder_layers": 31, + "post_merge_residual_scale": null, + "post_merge_residual_scales": [], + "parameterization": "unscaled compact ABPrune", + "linear_patch": null +} diff --git a/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/eval/stage_05_zs6.json b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/eval/stage_05_zs6.json new file mode 100644 index 0000000000000000000000000000000000000000..4ca3ec7d584c13de59442a1103da93bd30241251 --- /dev/null +++ b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/eval/stage_05_zs6.json @@ -0,0 +1,126 @@ +{ + "model": "/workspace/models/qwen3-4b-base-906bfd4", + "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539", + "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_input_no_rho/compact/stage_05", + "cycle": 5, + "anchor": 25, + "post_merge_residual_scale": null, + "post_merge_residual_scales": [], + "tasks": [ + "hellaswag", + "piqa", + "winogrande", + "arc_challenge", + "arc_easy", + "openbookqa" + ], + "preferred_metrics": { + "hellaswag": "acc_norm,none", + "piqa": "acc_norm,none", + "winogrande": "acc,none", + "arc_challenge": "acc_norm,none", + "arc_easy": "acc_norm,none", + "openbookqa": "acc_norm,none" + }, + "task_scores": { + "hellaswag": 0.6079466241784505, + "piqa": 0.6931447225244831, + "winogrande": 0.6708760852407262, + "arc_challenge": 0.386518771331058, + "arc_easy": 0.5740740740740741, + "openbookqa": 0.324 + }, + "task_mean": 0.5427600462247986, + "six_task_mean": 0.5427600462247986, + "boolq_excluded": true, + "num_fewshot": 0, + "batch_size": "auto:4", + "linear_patch": null, + "lm_eval_results": { + "hellaswag": { + "name": "hellaswag", + "alias": "hellaswag", + "sample_len": 10042, + "acc,none": 0.4193387771360287, + "acc_stderr,none": 0.004924424018073678, + "acc_norm,none": 0.6079466241784505, + "acc_norm_stderr,none": 0.0048721072620824665 + }, + "piqa": { + "name": "piqa", + "alias": "piqa", + "sample_len": 1838, + "acc,none": 0.7002176278563657, + "acc_stderr,none": 0.010689686967138092, + "acc_norm,none": 0.6931447225244831, + "acc_norm_stderr,none": 0.01076029507058038 + }, + "winogrande": { + "name": "winogrande", + "alias": "winogrande", + "sample_len": 1267, + "acc,none": 0.6708760852407262, + "acc_stderr,none": 0.013206387089091474 + }, + "arc_challenge": { + "name": "arc_challenge", + "alias": "arc_challenge", + "sample_len": 1172, + "acc,none": 0.3361774744027304, + "acc_stderr,none": 0.013804855026205756, + "acc_norm,none": 0.386518771331058, + "acc_norm_stderr,none": 0.014230084761910473 + }, + "arc_easy": { + "name": "arc_easy", + "alias": "arc_easy", + "sample_len": 2376, + "acc,none": 0.5989057239057239, + "acc_stderr,none": 0.010057051106534367, + "acc_norm,none": 0.5740740740740741, + "acc_norm_stderr,none": 0.010146568651002255 + }, + "openbookqa": { + "name": "openbookqa", + "alias": "openbookqa", + "sample_len": 500, + "acc,none": 0.206, + "acc_stderr,none": 0.01810479403733356, + "acc_norm,none": 0.324, + "acc_norm_stderr,none": 0.02095055731247745 + } + }, + "lm_eval_versions": { + "arc_challenge": 1.0, + "arc_easy": 1.0, + "hellaswag": 1.0, + "openbookqa": 1.0, + "piqa": 1.0, + "winogrande": 1.0 + }, + "lm_eval_config": { + "model": "/workspace/models/qwen3-4b-base-906bfd4", + "model_args": null, + "model_num_parameters": 3517814016, + "model_dtype": "torch.bfloat16", + "model_revision": "main", + "model_sha": "", + "batch_size": "auto:4", + "batch_sizes": [ + 64, + 64, + 64, + 64, + 64 + ], + "device": null, + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + } +} diff --git a/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/logs/eval.log b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/logs/eval.log new file mode 100644 index 0000000000000000000000000000000000000000..b48f35a55927a1f77587281a4eae3e3b2819be7a --- /dev/null +++ b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/arms/anchor_input_no_rho/logs/eval.log @@ -0,0 +1,196 @@ +`torch_dtype` is deprecated! Use `dtype` instead! + Loading weights: 0%| | 0/398 [00:00 131072). Running this sequence through the model will result in indexing errors +{ + "model": "/workspace/models/qwen3-4b-base-906bfd4", + "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539", + "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_input_no_rho/compact/stage_01", + "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test", + "protocol": "HCHC experiment.py: filter empty rows, join with double newline, then tokenize the complete stream once", + "sequence_length": 2048, + "batch_size": 4, + "dtype": "bfloat16", + "metrics": { + "nll": 2.145418803445224, + "perplexity": 8.54561942281943, + "tokens": 296815, + "sequences": 145, + "source_tokens": 298938 + }, + "anchor": 25, + "cycles": 1, + "original_layers": 36, + "decoder_layers": 35, + "post_merge_residual_scale": null, + "post_merge_residual_scales": [], + "parameterization": "unscaled compact ABPrune", + "linear_patch": null +} +`torch_dtype` is deprecated! Use `dtype` instead! + Loading weights: 0%| | 0/398 [00:00 131072). Running this sequence through the model will result in indexing errors +{ + "model": "/workspace/models/qwen3-4b-base-906bfd4", + "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539", + "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_input_no_rho/compact/stage_02", + "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test", + "protocol": "HCHC experiment.py: filter empty rows, join with double newline, then tokenize the complete stream once", + "sequence_length": 2048, + "batch_size": 4, + "dtype": "bfloat16", + "metrics": { + "nll": 2.2925252815772748, + "perplexity": 9.899906192368073, + "tokens": 296815, + "sequences": 145, + "source_tokens": 298938 + }, + "anchor": 25, + "cycles": 2, + "original_layers": 36, + "decoder_layers": 34, + "post_merge_residual_scale": null, + "post_merge_residual_scales": [], + "parameterization": "unscaled compact ABPrune", + "linear_patch": null +} +`torch_dtype` is deprecated! Use `dtype` instead! + Loading weights: 0%| | 0/398 [00:00 131072). Running this sequence through the model will result in indexing errors +{ + "model": "/workspace/models/qwen3-4b-base-906bfd4", + "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539", + "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_input_no_rho/compact/stage_03", + "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test", + "protocol": "HCHC experiment.py: filter empty rows, join with double newline, then tokenize the complete stream once", + "sequence_length": 2048, + "batch_size": 4, + "dtype": "bfloat16", + "metrics": { + "nll": 2.4010539860560978, + "perplexity": 11.03480077974187, + "tokens": 296815, + "sequences": 145, + "source_tokens": 298938 + }, + "anchor": 25, + "cycles": 3, + "original_layers": 36, + "decoder_layers": 33, + "post_merge_residual_scale": null, + "post_merge_residual_scales": [], + "parameterization": "unscaled compact ABPrune", + "linear_patch": null +} +`torch_dtype` is deprecated! Use `dtype` instead! + Loading weights: 0%| | 0/398 [00:00 131072). Running this sequence through the model will result in indexing errors +{ + "model": "/workspace/models/qwen3-4b-base-906bfd4", + "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539", + "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_input_no_rho/compact/stage_04", + "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test", + "protocol": "HCHC experiment.py: filter empty rows, join with double newline, then tokenize the complete stream once", + "sequence_length": 2048, + "batch_size": 4, + "dtype": "bfloat16", + "metrics": { + "nll": 2.586987911421677, + "perplexity": 13.289681558091948, + "tokens": 296815, + "sequences": 145, + "source_tokens": 298938 + }, + "anchor": 25, + "cycles": 4, + "original_layers": 36, + "decoder_layers": 32, + "post_merge_residual_scale": null, + "post_merge_residual_scales": [], + "parameterization": "unscaled compact ABPrune", + "linear_patch": null +} +`torch_dtype` is deprecated! Use `dtype` instead! + Loading weights: 0%| | 0/398 [00:00 131072). Running this sequence through the model will result in indexing errors +{ + "model": "/workspace/models/qwen3-4b-base-906bfd4", + "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539", + "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_input_no_rho/compact/stage_05", + "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test", + "protocol": "HCHC experiment.py: filter empty rows, join with double newline, then tokenize the complete stream once", + "sequence_length": 2048, + "batch_size": 4, + "dtype": "bfloat16", + "metrics": { + "nll": 2.848904904003801, + "perplexity": 17.268860422166572, + "tokens": 296815, + "sequences": 145, + "source_tokens": 298938 + }, + "anchor": 25, + "cycles": 5, + "original_layers": 36, + "decoder_layers": 31, + "post_merge_residual_scale": null, + "post_merge_residual_scales": [], + "parameterization": "unscaled compact ABPrune", + "linear_patch": null +} +`torch_dtype` is deprecated! Use `dtype` instead! + Loading weights: 0%| | 0/398 [00:00= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#results-reproducibility (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:233.) + freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2) +/opt/conda/lib/python3.11/site-packages/torch/nn/modules/linear.py:125: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#results-reproducibility (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:233.) + return F.linear(input, self.weight, self.bias) +/opt/conda/lib/python3.11/site-packages/torch/autograd/graph.py:829: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#results-reproducibility (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:233.) + return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass +/opt/conda/lib/python3.11/site-packages/torch/autograd/graph.py:829: UserWarning: Flash Attention defaults to a non-deterministic algorithm. To explicitly enable determinism call torch.use_deterministic_algorithms(True, warn_only=False). (Triggered internally at /pytorch/aten/src/ATen/native/transformers/cuda/attention_backward.cu:110.) + return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass +[fisher-input] mode=anchor_input cos=0.9340915456414223 norm_ratio=0.9107728470116854 + Loading weights: 0%| | 0/398 [00:00 131072). Running this sequence through the model will result in indexing errors +{ + "model": "/workspace/models/qwen3-4b-base-906bfd4", + "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539", + "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_input_rho/compact/stage_01", + "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test", + "protocol": "HCHC experiment.py: filter empty rows, join with double newline, then tokenize the complete stream once", + "sequence_length": 2048, + "batch_size": 4, + "dtype": "bfloat16", + "metrics": { + "nll": 2.132754957264867, + "perplexity": 8.438081372774594, + "tokens": 296815, + "sequences": 145, + "source_tokens": 298938 + }, + "anchor": 25, + "cycles": 1, + "original_layers": 36, + "decoder_layers": 35, + "post_merge_residual_scale": null, + "post_merge_residual_scales": [], + "parameterization": "unscaled compact ABPrune", + "linear_patch": null +} +`torch_dtype` is deprecated! Use `dtype` instead! + Loading weights: 0%| | 0/398 [00:00 131072). Running this sequence through the model will result in indexing errors +{ + "model": "/workspace/models/qwen3-4b-base-906bfd4", + "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539", + "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_input_rho/compact/stage_02", + "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test", + "protocol": "HCHC experiment.py: filter empty rows, join with double newline, then tokenize the complete stream once", + "sequence_length": 2048, + "batch_size": 4, + "dtype": "bfloat16", + "metrics": { + "nll": 2.2798083058718976, + "perplexity": 9.77480645692066, + "tokens": 296815, + "sequences": 145, + "source_tokens": 298938 + }, + "anchor": 25, + "cycles": 2, + "original_layers": 36, + "decoder_layers": 34, + "post_merge_residual_scale": null, + "post_merge_residual_scales": [], + "parameterization": "unscaled compact ABPrune", + "linear_patch": null +} +`torch_dtype` is deprecated! Use `dtype` instead! + Loading weights: 0%| | 0/398 [00:00 131072). Running this sequence through the model will result in indexing errors +{ + "model": "/workspace/models/qwen3-4b-base-906bfd4", + "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539", + "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_input_rho/compact/stage_03", + "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test", + "protocol": "HCHC experiment.py: filter empty rows, join with double newline, then tokenize the complete stream once", + "sequence_length": 2048, + "batch_size": 4, + "dtype": "bfloat16", + "metrics": { + "nll": 2.372043491231984, + "perplexity": 10.719274662057657, + "tokens": 296815, + "sequences": 145, + "source_tokens": 298938 + }, + "anchor": 25, + "cycles": 3, + "original_layers": 36, + "decoder_layers": 33, + "post_merge_residual_scale": null, + "post_merge_residual_scales": [], + "parameterization": "unscaled compact ABPrune", + "linear_patch": null +} +`torch_dtype` is deprecated! Use `dtype` instead! + Loading weights: 0%| | 0/398 [00:00 131072). Running this sequence through the model will result in indexing errors +{ + "model": "/workspace/models/qwen3-4b-base-906bfd4", + "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539", + "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_input_rho/compact/stage_04", + "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test", + "protocol": "HCHC experiment.py: filter empty rows, join with double newline, then tokenize the complete stream once", + "sequence_length": 2048, + "batch_size": 4, + "dtype": "bfloat16", + "metrics": { + "nll": 2.552255313149814, + "perplexity": 12.836020408000905, + "tokens": 296815, + "sequences": 145, + "source_tokens": 298938 + }, + "anchor": 25, + "cycles": 4, + "original_layers": 36, + "decoder_layers": 32, + "post_merge_residual_scale": null, + "post_merge_residual_scales": [], + "parameterization": "unscaled compact ABPrune", + "linear_patch": null +} +`torch_dtype` is deprecated! Use `dtype` instead! + Loading weights: 0%| | 0/398 [00:00 131072). Running this sequence through the model will result in indexing errors +{ + "model": "/workspace/models/qwen3-4b-base-906bfd4", + "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539", + "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_input_rho/compact/stage_05", + "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test", + "protocol": "HCHC experiment.py: filter empty rows, join with double newline, then tokenize the complete stream once", + "sequence_length": 2048, + "batch_size": 4, + "dtype": "bfloat16", + "metrics": { + "nll": 2.793590261196268, + "perplexity": 16.339577972080622, + "tokens": 296815, + "sequences": 145, + "source_tokens": 298938 + }, + "anchor": 25, + "cycles": 5, + "original_layers": 36, + "decoder_layers": 31, + "post_merge_residual_scale": null, + "post_merge_residual_scales": [], + "parameterization": "unscaled compact ABPrune", + "linear_patch": null +} +`torch_dtype` is deprecated! Use `dtype` instead! + Loading weights: 0%| | 0/398 [00:00= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#results-reproducibility (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:233.) + freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2) +/opt/conda/lib/python3.11/site-packages/torch/nn/modules/linear.py:125: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#results-reproducibility (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:233.) + return F.linear(input, self.weight, self.bias) +/opt/conda/lib/python3.11/site-packages/torch/autograd/graph.py:829: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#results-reproducibility (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:233.) + return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass +/opt/conda/lib/python3.11/site-packages/torch/autograd/graph.py:829: UserWarning: Flash Attention defaults to a non-deterministic algorithm. To explicitly enable determinism call torch.use_deterministic_algorithms(True, warn_only=False). (Triggered internally at /pytorch/aten/src/ATen/native/transformers/cuda/attention_backward.cu:110.) + return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass +[fisher-input] mode=anchor_input cos=0.9340915456414223 norm_ratio=0.9107728470116854 + Loading weights: 0%| | 0/398 [00:00 131072). Running this sequence through the model will result in indexing errors +{ + "model": "/workspace/models/qwen3-4b-base-906bfd4", + "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539", + "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_post_attention_no_rho/compact/stage_01", + "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test", + "protocol": "HCHC experiment.py: filter empty rows, join with double newline, then tokenize the complete stream once", + "sequence_length": 2048, + "batch_size": 4, + "dtype": "bfloat16", + "metrics": { + "nll": 2.145741819513255, + "perplexity": 8.548380241074604, + "tokens": 296815, + "sequences": 145, + "source_tokens": 298938 + }, + "anchor": 25, + "cycles": 1, + "original_layers": 36, + "decoder_layers": 35, + "post_merge_residual_scale": null, + "post_merge_residual_scales": [], + "parameterization": "unscaled compact ABPrune", + "linear_patch": null +} +`torch_dtype` is deprecated! Use `dtype` instead! + Loading weights: 0%| | 0/398 [00:00 131072). Running this sequence through the model will result in indexing errors +{ + "model": "/workspace/models/qwen3-4b-base-906bfd4", + "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539", + "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_post_attention_no_rho/compact/stage_02", + "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test", + "protocol": "HCHC experiment.py: filter empty rows, join with double newline, then tokenize the complete stream once", + "sequence_length": 2048, + "batch_size": 4, + "dtype": "bfloat16", + "metrics": { + "nll": 2.293003268077456, + "perplexity": 9.904639344982495, + "tokens": 296815, + "sequences": 145, + "source_tokens": 298938 + }, + "anchor": 25, + "cycles": 2, + "original_layers": 36, + "decoder_layers": 34, + "post_merge_residual_scale": null, + "post_merge_residual_scales": [], + "parameterization": "unscaled compact ABPrune", + "linear_patch": null +} +`torch_dtype` is deprecated! Use `dtype` instead! + Loading weights: 0%| | 0/398 [00:00 131072). Running this sequence through the model will result in indexing errors +{ + "model": "/workspace/models/qwen3-4b-base-906bfd4", + "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539", + "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_post_attention_no_rho/compact/stage_03", + "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test", + "protocol": "HCHC experiment.py: filter empty rows, join with double newline, then tokenize the complete stream once", + "sequence_length": 2048, + "batch_size": 4, + "dtype": "bfloat16", + "metrics": { + "nll": 2.4012237581713447, + "perplexity": 11.03667434024624, + "tokens": 296815, + "sequences": 145, + "source_tokens": 298938 + }, + "anchor": 25, + "cycles": 3, + "original_layers": 36, + "decoder_layers": 33, + "post_merge_residual_scale": null, + "post_merge_residual_scales": [], + "parameterization": "unscaled compact ABPrune", + "linear_patch": null +} +`torch_dtype` is deprecated! Use `dtype` instead! + Loading weights: 0%| | 0/398 [00:00 131072). Running this sequence through the model will result in indexing errors +{ + "model": "/workspace/models/qwen3-4b-base-906bfd4", + "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539", + "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_post_attention_no_rho/compact/stage_04", + "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test", + "protocol": "HCHC experiment.py: filter empty rows, join with double newline, then tokenize the complete stream once", + "sequence_length": 2048, + "batch_size": 4, + "dtype": "bfloat16", + "metrics": { + "nll": 2.5871801639425343, + "perplexity": 13.292236778488792, + "tokens": 296815, + "sequences": 145, + "source_tokens": 298938 + }, + "anchor": 25, + "cycles": 4, + "original_layers": 36, + "decoder_layers": 32, + "post_merge_residual_scale": null, + "post_merge_residual_scales": [], + "parameterization": "unscaled compact ABPrune", + "linear_patch": null +} +`torch_dtype` is deprecated! Use `dtype` instead! + Loading weights: 0%| | 0/398 [00:00 131072). Running this sequence through the model will result in indexing errors +{ + "model": "/workspace/models/qwen3-4b-base-906bfd4", + "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539", + "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_post_attention_no_rho/compact/stage_05", + "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test", + "protocol": "HCHC experiment.py: filter empty rows, join with double newline, then tokenize the complete stream once", + "sequence_length": 2048, + "batch_size": 4, + "dtype": "bfloat16", + "metrics": { + "nll": 2.849449394489157, + "perplexity": 17.278265712672614, + "tokens": 296815, + "sequences": 145, + "source_tokens": 298938 + }, + "anchor": 25, + "cycles": 5, + "original_layers": 36, + "decoder_layers": 31, + "post_merge_residual_scale": null, + "post_merge_residual_scales": [], + "parameterization": "unscaled compact ABPrune", + "linear_patch": null +} +`torch_dtype` is deprecated! Use `dtype` instead! + Loading weights: 0%| | 0/398 [00:00= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#results-reproducibility (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:233.) + freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2) +/opt/conda/lib/python3.11/site-packages/torch/nn/modules/linear.py:125: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#results-reproducibility (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:233.) + return F.linear(input, self.weight, self.bias) +/opt/conda/lib/python3.11/site-packages/torch/autograd/graph.py:829: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#results-reproducibility (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:233.) + return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass +/opt/conda/lib/python3.11/site-packages/torch/autograd/graph.py:829: UserWarning: Flash Attention defaults to a non-deterministic algorithm. To explicitly enable determinism call torch.use_deterministic_algorithms(True, warn_only=False). (Triggered internally at /pytorch/aten/src/ATen/native/transformers/cuda/attention_backward.cu:110.) + return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass +[fisher-input] mode=anchor_post_attention cos=0.9450913984328508 norm_ratio=0.9202500991523266 + Loading weights: 0%| | 0/398 [00:00 131072). Running this sequence through the model will result in indexing errors +{ + "model": "/workspace/models/qwen3-4b-base-906bfd4", + "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539", + "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_post_attention_rho/compact/stage_01", + "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test", + "protocol": "HCHC experiment.py: filter empty rows, join with double newline, then tokenize the complete stream once", + "sequence_length": 2048, + "batch_size": 4, + "dtype": "bfloat16", + "metrics": { + "nll": 2.1330115482724947, + "perplexity": 8.440246786377427, + "tokens": 296815, + "sequences": 145, + "source_tokens": 298938 + }, + "anchor": 25, + "cycles": 1, + "original_layers": 36, + "decoder_layers": 35, + "post_merge_residual_scale": null, + "post_merge_residual_scales": [], + "parameterization": "unscaled compact ABPrune", + "linear_patch": null +} +`torch_dtype` is deprecated! Use `dtype` instead! + Loading weights: 0%| | 0/398 [00:00 131072). Running this sequence through the model will result in indexing errors +{ + "model": "/workspace/models/qwen3-4b-base-906bfd4", + "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539", + "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_post_attention_rho/compact/stage_02", + "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test", + "protocol": "HCHC experiment.py: filter empty rows, join with double newline, then tokenize the complete stream once", + "sequence_length": 2048, + "batch_size": 4, + "dtype": "bfloat16", + "metrics": { + "nll": 2.270125824829628, + "perplexity": 9.68061879965598, + "tokens": 296815, + "sequences": 145, + "source_tokens": 298938 + }, + "anchor": 25, + "cycles": 2, + "original_layers": 36, + "decoder_layers": 34, + "post_merge_residual_scale": null, + "post_merge_residual_scales": [], + "parameterization": "unscaled compact ABPrune", + "linear_patch": null +} +`torch_dtype` is deprecated! Use `dtype` instead! + Loading weights: 0%| | 0/398 [00:00 131072). Running this sequence through the model will result in indexing errors +{ + "model": "/workspace/models/qwen3-4b-base-906bfd4", + "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539", + "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_post_attention_rho/compact/stage_03", + "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test", + "protocol": "HCHC experiment.py: filter empty rows, join with double newline, then tokenize the complete stream once", + "sequence_length": 2048, + "batch_size": 4, + "dtype": "bfloat16", + "metrics": { + "nll": 2.37049042274212, + "perplexity": 10.702639815217683, + "tokens": 296815, + "sequences": 145, + "source_tokens": 298938 + }, + "anchor": 25, + "cycles": 3, + "original_layers": 36, + "decoder_layers": 33, + "post_merge_residual_scale": null, + "post_merge_residual_scales": [], + "parameterization": "unscaled compact ABPrune", + "linear_patch": null +} +`torch_dtype` is deprecated! Use `dtype` instead! + Loading weights: 0%| | 0/398 [00:00 131072). Running this sequence through the model will result in indexing errors +{ + "model": "/workspace/models/qwen3-4b-base-906bfd4", + "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539", + "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_post_attention_rho/compact/stage_04", + "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test", + "protocol": "HCHC experiment.py: filter empty rows, join with double newline, then tokenize the complete stream once", + "sequence_length": 2048, + "batch_size": 4, + "dtype": "bfloat16", + "metrics": { + "nll": 2.5506233313987994, + "perplexity": 12.815089341144557, + "tokens": 296815, + "sequences": 145, + "source_tokens": 298938 + }, + "anchor": 25, + "cycles": 4, + "original_layers": 36, + "decoder_layers": 32, + "post_merge_residual_scale": null, + "post_merge_residual_scales": [], + "parameterization": "unscaled compact ABPrune", + "linear_patch": null +} +`torch_dtype` is deprecated! Use `dtype` instead! + Loading weights: 0%| | 0/398 [00:00 131072). Running this sequence through the model will result in indexing errors +{ + "model": "/workspace/models/qwen3-4b-base-906bfd4", + "revision": "906bfd4b4dc7f14ee4320094d8b41684abff8539", + "compact_stage": "/workspace/experiments/anchor_contextual_fisher_matrix_v1/4b/anchor_post_attention_rho/compact/stage_05", + "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test", + "protocol": "HCHC experiment.py: filter empty rows, join with double newline, then tokenize the complete stream once", + "sequence_length": 2048, + "batch_size": 4, + "dtype": "bfloat16", + "metrics": { + "nll": 2.7919276056618525, + "perplexity": 16.312433454579697, + "tokens": 296815, + "sequences": 145, + "source_tokens": 298938 + }, + "anchor": 25, + "cycles": 5, + "original_layers": 36, + "decoder_layers": 31, + "post_merge_residual_scale": null, + "post_merge_residual_scales": [], + "parameterization": "unscaled compact ABPrune", + "linear_patch": null +} +`torch_dtype` is deprecated! Use `dtype` instead! + Loading weights: 0%| | 0/398 [00:00= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#results-reproducibility (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:233.) + freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2) +/opt/conda/lib/python3.11/site-packages/torch/nn/modules/linear.py:125: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#results-reproducibility (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:233.) + return F.linear(input, self.weight, self.bias) +/opt/conda/lib/python3.11/site-packages/torch/autograd/graph.py:829: UserWarning: Deterministic behavior was enabled with either `torch.use_deterministic_algorithms(True)` or `at::Context::setDeterministicAlgorithms(true)`, but this operation is not deterministic because it uses CuBLAS and you have CUDA >= 10.2. To enable deterministic behavior in this case, you must set an environment variable before running your PyTorch application: CUBLAS_WORKSPACE_CONFIG=:4096:8 or CUBLAS_WORKSPACE_CONFIG=:16:8. For more information, go to https://docs.nvidia.com/cuda/cublas/index.html#results-reproducibility (Triggered internally at /pytorch/aten/src/ATen/Context.cpp:233.) + return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass +/opt/conda/lib/python3.11/site-packages/torch/autograd/graph.py:829: UserWarning: Flash Attention defaults to a non-deterministic algorithm. To explicitly enable determinism call torch.use_deterministic_algorithms(True, warn_only=False). (Triggered internally at /pytorch/aten/src/ATen/native/transformers/cuda/attention_backward.cu:110.) + return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass +[fisher-input] mode=anchor_post_attention cos=0.9450913984328508 norm_ratio=0.9202500991523266 + Loading weights: 0%| | 0/398 [00:00 None: +@@ -273,6 +274,16 @@ def parse_args() -> argparse.Namespace: + choices=["tensor", "param"], + help="Fisher approximation granularity", + ) ++ parser.add_argument( ++ "--fisher_input_mode", ++ default="native", ++ choices=["native", "anchor_input", "anchor_post_attention"], ++ help=( ++ "Input coordinate used for the absorbed/source MLP Fisher. " ++ "Contextual modes replace only the source MLP input while retaining " ++ "the global causal-LM objective." ++ ), ++ ) + parser.add_argument( + "--no_head_permute", + action="store_true", +@@ -2763,14 +2774,41 @@ def run_progressive( + _record_component("head_permutation", head_timing_start) + + fisher_timing_start = _timing_start() +- fisher_sums, num_batches, param_numels = compute_fisher( +- model, +- layer_a, +- layer_b, +- dataloader, +- fisher_mode=args.fisher_mode, +- device=args.device, +- ) ++ if args.fisher_input_mode == "native": ++ fisher_sums, num_batches, param_numels = compute_fisher( ++ model, ++ layer_a, ++ layer_b, ++ dataloader, ++ fisher_mode=args.fisher_mode, ++ device=args.device, ++ ) ++ fisher_input_diagnostics = { ++ "mode": "native", ++ "context_detached": False, ++ "anchor_attention_included": None, ++ } ++ else: ++ ( ++ fisher_sums, ++ num_batches, ++ param_numels, ++ fisher_input_diagnostics, ++ ) = compute_contextual_fisher( ++ model, ++ layer_a, ++ layer_b, ++ dataloader, ++ fisher_mode=args.fisher_mode, ++ device=args.device, ++ input_mode=args.fisher_input_mode, ++ ) ++ print( ++ "[fisher-input] " ++ f"mode={args.fisher_input_mode} " ++ f"cos={fisher_input_diagnostics.get('context_vs_native_source_mlp_input_cosine')} " ++ f"norm_ratio={fisher_input_diagnostics.get('context_vs_native_source_mlp_input_norm_ratio')}" ++ ) + _record_component("fisher", fisher_timing_start) + + fit_timing_start = _timing_start() +@@ -3116,6 +3154,7 @@ def run_progressive( + "dwce_scores": dwce_scores, + "dwce_meta": dwce_meta, + "fisher_num_batches": num_batches, ++ "fisher_input": fisher_input_diagnostics, + "merge_method": merge_method, + "merged_params": merged, + "num_sequences": num_sequences, diff --git a/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/code/scripts/eval_abprune_compact_zeroshot.py b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/code/scripts/eval_abprune_compact_zeroshot.py new file mode 100644 index 0000000000000000000000000000000000000000..0b7dc86d9d3aa1bbc45698fc2a199dc772697247 --- /dev/null +++ b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/code/scripts/eval_abprune_compact_zeroshot.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +"""Evaluate a compact ABPrune stage on the standard six zero-shot tasks.""" + +from __future__ import annotations + +import argparse +import json +from contextlib import ExitStack +from pathlib import Path + +import torch +from transformers import AutoTokenizer + +from fusegpt.abprune_residual_scale import residual_scale_hook +from fusegpt.abprune_compact import compact_scale_targets, read_compact_metadata +from fusegpt.abprune_stagewise_geometry import load_compact_model +from fusegpt.hadamard_channel_fusion import linear_patch_hook + + +TASKS = ( + "hellaswag", + "piqa", + "winogrande", + "arc_challenge", + "arc_easy", + "openbookqa", +) +PREFERRED_METRIC = { + "hellaswag": "acc_norm,none", + "piqa": "acc_norm,none", + "winogrande": "acc,none", + "arc_challenge": "acc_norm,none", + "arc_easy": "acc_norm,none", + "openbookqa": "acc_norm,none", +} + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--model", required=True) + parser.add_argument("--revision", required=True) + parser.add_argument("--compact-stage", type=Path, required=True) + parser.add_argument("--cycle", type=int, required=True) + parser.add_argument("--anchor", type=int, default=None) + parser.add_argument("--batch-size", default="auto:4") + parser.add_argument( + "--tasks", + nargs="+", + choices=TASKS, + default=None, + help="Optional subset of the standard six tasks. Defaults to all six.", + ) + parser.add_argument( + "--apply-post-merge-scale", + action="store_true", + help=( + "Read compact-stage metadata.json and apply its cumulative residual " + "scale during evaluation. Required for residual-scaled checkpoints " + "because that scale is not folded into the anchor weights." + ), + ) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--linear-patch", type=Path, default=None) + args = parser.parse_args() + selected_tasks = tuple(args.tasks) if args.tasks else TASKS + + # Import lazily so compact packaging and reconstruction do not require + # lm-eval in non-evaluation environments. + from lm_eval import evaluator + from lm_eval.models.huggingface import HFLM + + model = load_compact_model( + args.model, + args.compact_stage, + cycle=args.cycle, + anchor=args.anchor, + dtype=torch.bfloat16, + device=torch.device("cuda"), + attn="eager", + revision=args.revision, + ) + tokenizer = AutoTokenizer.from_pretrained(args.model, revision=args.revision) + lm = HFLM( + pretrained=model, + tokenizer=tokenizer, + backend="causal", + batch_size=args.batch_size, + ) + applied_scales = [] + scale_stack = ExitStack() + linear_patch = None + if args.apply_post_merge_scale: + metadata = read_compact_metadata(args.compact_stage) + targets = compact_scale_targets(metadata) + if not targets: + raise RuntimeError( + "--apply-post-merge-scale requires enabled residual scale metadata" + ) + for target in targets: + stored_scale = target["scale"]["cumulative_gamma"] + applied = ( + torch.tensor(stored_scale, device="cuda", dtype=torch.float32) + if isinstance(stored_scale, list) + else float(stored_scale) + ) + scale_stack.enter_context( + residual_scale_hook(model, int(target["student_index"]), applied) + ) + applied_scales.append( + { + "original_layer": int(target["original_layer"]), + "student_index": int(target["student_index"]), + "value": applied, + } + ) + if args.linear_patch is not None: + linear_patch = json.loads(args.linear_patch.read_text(encoding="utf-8")) + patch_scale = torch.tensor( + linear_patch["scale"], device="cuda", dtype=torch.float32 + ) + patch_anchor = int(linear_patch["anchor_student_index"]) + scale_stack.enter_context( + linear_patch_hook( + model.model.layers[patch_anchor], + patch_scale, + hadamard=linear_patch["mode"] == "hadamard", + ) + ) + + with scale_stack: + raw = evaluator.simple_evaluate( + model=lm, + tasks=list(selected_tasks), + num_fewshot=0, + batch_size=args.batch_size, + random_seed=0, + numpy_random_seed=1234, + torch_random_seed=1234, + fewshot_random_seed=1234, + ) + scores = { + task: float(raw["results"][task][PREFERRED_METRIC[task]]) + for task in selected_tasks + } + task_mean = sum(scores.values()) / len(scores) + report = { + "model": args.model, + "revision": args.revision, + "compact_stage": str(args.compact_stage), + "cycle": args.cycle, + "anchor": args.anchor, + "post_merge_residual_scale": ( + applied_scales[0]["value"].tolist() + if len(applied_scales) == 1 + and torch.is_tensor(applied_scales[0]["value"]) + else ( + applied_scales[0]["value"] + if len(applied_scales) == 1 + else None + ) + ), + "post_merge_residual_scales": [ + { + "original_layer": item["original_layer"], + "student_index": item["student_index"], + "parameterization": ( + "vector" if torch.is_tensor(item["value"]) else "scalar" + ), + "value": ( + item["value"].tolist() + if torch.is_tensor(item["value"]) + else item["value"] + ), + } + for item in applied_scales + ], + "tasks": list(selected_tasks), + "preferred_metrics": { + task: PREFERRED_METRIC[task] for task in selected_tasks + }, + "task_scores": scores, + "task_mean": task_mean, + "six_task_mean": task_mean if selected_tasks == TASKS else None, + "boolq_excluded": True, + "num_fewshot": 0, + "batch_size": args.batch_size, + "linear_patch": linear_patch, + "lm_eval_results": raw["results"], + "lm_eval_versions": raw.get("versions"), + "lm_eval_config": raw.get("config"), + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2, default=str) + "\n", encoding="utf-8") + print(json.dumps({"task_scores": scores, "task_mean": task_mean}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/code/scripts/evaluate_abprune_compact_ppl.py b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/code/scripts/evaluate_abprune_compact_ppl.py new file mode 100644 index 0000000000000000000000000000000000000000..f7d21685f93746ba55000d58c925dedb296bcce4 --- /dev/null +++ b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/code/scripts/evaluate_abprune_compact_ppl.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +"""Evaluate an unscaled or residual-scaled compact ABPrune checkpoint.""" + +from __future__ import annotations + +import argparse +import json +from contextlib import ExitStack +from pathlib import Path + +import torch + +from evaluate_hchc_wikitext2_ppl import evaluate_perplexity, packed_wikitext_tokens +from fusegpt.abprune_residual_scale import residual_scale_hook +from fusegpt.abprune_compact import compact_scale_targets, read_compact_metadata +from fusegpt.abprune_stagewise_geometry import load_compact_model +from fusegpt.hadamard_channel_fusion import linear_patch_hook + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--model", required=True) + parser.add_argument("--revision", required=True) + parser.add_argument("--compact-stage", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--anchor", type=int, default=None) + parser.add_argument("--cycles", type=int, required=True) + parser.add_argument("--original-layers", type=int, required=True) + parser.add_argument("--seq-len", type=int, default=2048) + parser.add_argument("--batch-size", type=int, default=4) + parser.add_argument("--device", default="cuda") + parser.add_argument("--apply-post-merge-scale", action="store_true") + parser.add_argument("--linear-patch", type=Path, default=None) + args = parser.parse_args() + + device = torch.device(args.device) + model = load_compact_model( + args.model, + args.compact_stage, + cycle=args.cycles, + anchor=args.anchor, + dtype=torch.bfloat16, + device=device, + attn="eager", + revision=args.revision, + ) + model.eval() + expected_depth = args.original_layers - args.cycles + if len(model.model.layers) != expected_depth: + raise RuntimeError( + f"Unexpected decoder depth: {len(model.model.layers)} != {expected_depth}" + ) + + from transformers import AutoTokenizer + + tokenizer = AutoTokenizer.from_pretrained(args.model, revision=args.revision) + applied_scales = [] + scale_stack = ExitStack() + linear_patch = None + if args.apply_post_merge_scale: + metadata = read_compact_metadata(args.compact_stage) + targets = compact_scale_targets(metadata) + if not targets: + raise RuntimeError( + "--apply-post-merge-scale requires enabled residual scale metadata" + ) + for target in targets: + stored_scale = target["scale"]["cumulative_gamma"] + applied = ( + torch.tensor(stored_scale, device=device, dtype=torch.float32) + if isinstance(stored_scale, list) + else float(stored_scale) + ) + scale_stack.enter_context( + residual_scale_hook(model, int(target["student_index"]), applied) + ) + applied_scales.append( + { + "original_layer": int(target["original_layer"]), + "student_index": int(target["student_index"]), + "value": applied, + } + ) + if args.linear_patch is not None: + linear_patch = json.loads(args.linear_patch.read_text(encoding="utf-8")) + patch_scale = torch.tensor( + linear_patch["scale"], device=device, dtype=torch.float32 + ) + patch_anchor = int(linear_patch["anchor_student_index"]) + scale_stack.enter_context( + linear_patch_hook( + model.model.layers[patch_anchor], + patch_scale, + hadamard=linear_patch["mode"] == "hadamard", + ) + ) + + with scale_stack: + tokens = packed_wikitext_tokens(tokenizer, "test") + metrics = evaluate_perplexity( + model=model, + tokens=tokens, + sequence_length=args.seq_len, + batch_size=args.batch_size, + ) + + report = { + "model": args.model, + "revision": args.revision, + "compact_stage": str(args.compact_stage), + "dataset": "Salesforce/wikitext:wikitext-2-raw-v1:test", + "protocol": ( + "HCHC experiment.py: filter empty rows, join with double newline, " + "then tokenize the complete stream once" + ), + "sequence_length": args.seq_len, + "batch_size": args.batch_size, + "dtype": "bfloat16", + "metrics": metrics, + "anchor": args.anchor, + "cycles": args.cycles, + "original_layers": args.original_layers, + "decoder_layers": len(model.model.layers), + "post_merge_residual_scale": ( + applied_scales[0]["value"].tolist() + if len(applied_scales) == 1 + and torch.is_tensor(applied_scales[0]["value"]) + else ( + applied_scales[0]["value"] + if len(applied_scales) == 1 + else None + ) + ), + "post_merge_residual_scales": [ + { + "original_layer": item["original_layer"], + "student_index": item["student_index"], + "parameterization": ( + "vector" if torch.is_tensor(item["value"]) else "scalar" + ), + "value": ( + item["value"].tolist() + if torch.is_tensor(item["value"]) + else item["value"] + ), + } + for item in applied_scales + ], + "parameterization": ( + "X_interface <- X_interface H D H.T" + if linear_patch and linear_patch["mode"] == "hadamard" + else ( + "X_interface <- X_interface D" + if linear_patch + else ( + "h_out = h_in + gamma * (block(h_in) - h_in)" + if applied_scales else "unscaled compact ABPrune" + ) + ) + ), + "linear_patch": linear_patch, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + print(json.dumps(report, indent=2), flush=True) + + +if __name__ == "__main__": + main() diff --git a/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/code/scripts/package_contextual_fisher_host.py b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/code/scripts/package_contextual_fisher_host.py new file mode 100644 index 0000000000000000000000000000000000000000..0c1faa73e3d7c43046ada21881f9e0586ec0963b --- /dev/null +++ b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/code/scripts/package_contextual_fisher_host.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +"""Package one host's contextual-Fisher arms without dense run checkpoints.""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +from pathlib import Path + + +VARIANTS = ( + "anchor_input_no_rho", + "anchor_input_rho", + "anchor_post_attention_no_rho", + "anchor_post_attention_rho", +) +MODEL_CYCLES = {"1.7b": 4, "4b": 5} + + +def link_or_copy(source: Path, destination: Path) -> None: + if not source.is_file(): + raise FileNotFoundError(source) + destination.parent.mkdir(parents=True, exist_ok=True) + try: + os.link(source, destination) + except OSError: + shutil.copy2(source, destination) + + +def copy_tree(source: Path, destination: Path) -> None: + if not source.is_dir(): + raise FileNotFoundError(source) + for path in source.rglob("*"): + if path.is_file(): + link_or_copy(path, destination / path.relative_to(source)) + + +def load(path: Path) -> dict: + return json.loads(path.read_text(encoding="utf-8")) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--experiment-root", type=Path, required=True) + parser.add_argument("--model-key", choices=tuple(MODEL_CYCLES), required=True) + parser.add_argument("--project-root", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--session-alias", required=True) + parser.add_argument("--contract", type=int, required=True) + parser.add_argument("--hardware", required=True) + parser.add_argument("--rate-usd-per-hour", type=float, required=True) + parser.add_argument("--git-commit", required=True) + parser.add_argument("--external-commit", required=True) + args = parser.parse_args() + + cycles = MODEL_CYCLES[args.model_key] + source_root = args.experiment_root / args.model_key + if args.output.exists() and any(args.output.iterdir()): + raise SystemExit(f"refusing to overwrite non-empty output: {args.output}") + args.output.mkdir(parents=True, exist_ok=True) + + rows = [] + compact_bytes = 0 + for variant in VARIANTS: + source = source_root / variant + if not (source / "COMPLETE").is_file(): + raise RuntimeError(f"incomplete arm: {source}") + for name in ("compact", "eval", "metadata", "logs"): + copy_tree(source / name, args.output / "arms" / variant / name) + for name in ("protocol.json", "TRAIN_COMPLETE", "COMPACT_COMPLETE", "COMPLETE"): + link_or_copy(source / name, args.output / "arms" / variant / name) + + ppls = [] + for stage in range(1, cycles + 1): + report = load(source / "eval" / f"stage_{stage:02d}_ppl.json") + if int(report["metrics"]["tokens"]) != 296815: + raise RuntimeError(f"wrong token count: {source} stage {stage}") + ppls.append(float(report["metrics"]["perplexity"])) + zs = load(source / "eval" / f"stage_{cycles:02d}_zs6.json") + rows.append( + { + "variant": variant, + "stage_ppl": ppls, + "final_ppl": ppls[-1], + "final_zs6": float(zs["six_task_mean"]), + "task_scores": zs["task_scores"], + } + ) + compact_bytes += sum( + path.stat().st_size for path in (source / "compact").rglob("*") if path.is_file() + ) + + code_paths = ( + "src/fusegpt/abprune_contextual_fisher.py", + "tests/test_abprune_contextual_fisher.py", + "patches/0003-Add-anchor-contextual-Fisher-input.patch", + "scripts/run_contextual_fisher_variant.sh", + "scripts/run_contextual_fisher_matrix_2gpu.sh", + "scripts/run_contextual_fisher_4b_queue.sh", + "scripts/summarize_contextual_fisher_matrix.py", + "scripts/package_contextual_fisher_host.py", + "scripts/evaluate_abprune_compact_ppl.py", + "scripts/eval_abprune_compact_zeroshot.py", + "scripts/write_tree_manifest.py", + "scripts/upload_verified_hf_tree.py", + "src/fusegpt/abprune_compact.py", + ) + for relative in code_paths: + link_or_copy(args.project_root / relative, args.output / "code" / relative) + + summary = { + "experiment": "anchor_contextual_fisher_matrix_v1", + "model_key": args.model_key, + "cycles": cycles, + "rows": rows, + } + (args.output / "summary.json").write_text( + json.dumps(summary, indent=2) + "\n", encoding="utf-8" + ) + lines = [ + f"# Anchor-contextual Fisher matrix: {args.model_key}", + "", + "| Variant | Final PPL | ZS6 |", + "|---|---:|---:|", + ] + for row in rows: + lines.append( + f"| {row['variant']} | {row['final_ppl']:.4f} | " + f"{100 * row['final_zs6']:.2f}% |" + ) + (args.output / "summary.md").write_text("\n".join(lines) + "\n", encoding="utf-8") + + provenance = { + "experiment": summary["experiment"], + "model_key": args.model_key, + "session_alias": args.session_alias, + "vast_contract": args.contract, + "hardware": args.hardware, + "rate_usd_per_hour": args.rate_usd_per_hour, + "git_commit": args.git_commit, + "external_abprune_commit": args.external_commit, + "full_models_omitted": True, + "compact_stage_count": cycles * len(VARIANTS), + "compact_bytes": compact_bytes, + "variants": list(VARIANTS), + } + (args.output / "provenance.json").write_text( + json.dumps(provenance, indent=2) + "\n", encoding="utf-8" + ) + print(json.dumps({"summary": summary, "provenance": provenance}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/code/scripts/run_contextual_fisher_4b_queue.sh b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/code/scripts/run_contextual_fisher_4b_queue.sh new file mode 100644 index 0000000000000000000000000000000000000000..7ba7d8a452c98e488864b54f239ddb3651d9970a --- /dev/null +++ b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/code/scripts/run_contextual_fisher_4b_queue.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 3 ]]; then + echo "Usage: $0 " >&2 + exit 2 +fi + +repo="$(cd "$1" && pwd)" +project="$(cd "$2" && pwd)" +root="$(mkdir -p "$3" && cd "$3" && pwd)" + +for mode in anchor_input anchor_post_attention; do + for rho_variant in no_rho rho; do + bash "$project/scripts/run_contextual_fisher_variant.sh" \ + "$repo" "$project" "$root" 4b "$mode" "$rho_variant" 0 + done +done + +touch "$root/4b/COMPLETE" +echo contextual_fisher_4b_queue=complete diff --git a/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/code/scripts/run_contextual_fisher_matrix_2gpu.sh b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/code/scripts/run_contextual_fisher_matrix_2gpu.sh new file mode 100644 index 0000000000000000000000000000000000000000..1aece2ea1f7698efb1c47b765e9fd6c4886192fd --- /dev/null +++ b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/code/scripts/run_contextual_fisher_matrix_2gpu.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 3 ]]; then + echo "Usage: $0 " >&2 + exit 2 +fi + +repo="$(cd "$1" && pwd)" +project="$(cd "$2" && pwd)" +root="$(mkdir -p "$3" && cd "$3" && pwd)" +mkdir -p "$root/logs" +export PYTHONPATH="$repo/src:$project/src:$project${PYTHONPATH:+:$PYTHONPATH}" + +apply_once() { + local marker="$1" + local patch="$2" + local target="$3" + if ! rg -q "$marker" "$repo/$target"; then + git -C "$repo" apply --check "$project/$patch" + git -C "$repo" apply "$project/$patch" + fi +} + +apply_once 'reparam_fisher_init_subset' \ + patches/abprune_fit_only_vo.patch src/fuse_layers.py +apply_once 'reparam_rho_objective' \ + patches/0001-Add-projection-rho-gradient-routing.patch src/fuse_layers.py +apply_once 'reparam_rho_schedule' \ + patches/0002-Add-sequential-rho-then-U.patch src/fuse_layers.py +apply_once 'fisher_input_mode' \ + patches/0003-Add-anchor-contextual-Fisher-input.patch src/fuse_layers.py + +python -m py_compile \ + "$repo/src/fuse_layers.py" \ + "$repo/src/fuse_layers_distill.py" \ + "$project/src/fusegpt/abprune_contextual_fisher.py" +python - < "$root/abprune_commit.txt" + +run_model_queue() { + local model_key="$1" + local gpu="$2" + local mode rho_variant + for mode in anchor_input anchor_post_attention; do + for rho_variant in no_rho rho; do + bash "$project/scripts/run_contextual_fisher_variant.sh" \ + "$repo" "$project" "$root" "$model_key" "$mode" "$rho_variant" "$gpu" + done + done +} + +run_model_queue 1.7b 0 > "$root/logs/queue_1_7b.log" 2>&1 & +pid_17b=$! +run_model_queue 4b 1 > "$root/logs/queue_4b.log" 2>&1 & +pid_4b=$! +printf 'pid_1_7b=%s\npid_4b=%s\n' "$pid_17b" "$pid_4b" | tee "$root/queue_pids.txt" +wait "$pid_17b" +wait "$pid_4b" + +python "$project/scripts/summarize_contextual_fisher_matrix.py" "$root" +touch "$root/COMPLETE" +echo contextual_fisher_matrix=complete diff --git a/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/code/scripts/run_contextual_fisher_variant.sh b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/code/scripts/run_contextual_fisher_variant.sh new file mode 100644 index 0000000000000000000000000000000000000000..480cb65af6c078faedef1e371e142e166e66a606 --- /dev/null +++ b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/code/scripts/run_contextual_fisher_variant.sh @@ -0,0 +1,173 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 7 ]]; then + echo "Usage: $0 <1.7b|4b> " >&2 + exit 2 +fi + +repo="$(cd "$1" && pwd)" +project="$(cd "$2" && pwd)" +experiment="$(mkdir -p "$3" && cd "$3" && pwd)" +model_key="$4" +fisher_input_mode="$5" +rho_variant="$6" +gpu_id="$7" + +case "$fisher_input_mode" in + anchor_input|anchor_post_attention) ;; + *) echo "invalid Fisher input mode: $fisher_input_mode" >&2; exit 2 ;; +esac +case "$rho_variant" in + no_rho|rho) ;; + *) echo "invalid rho variant: $rho_variant" >&2; exit 2 ;; +esac + +case "$model_key" in + 1.7b) + model_id=Qwen/Qwen3-1.7B-Base + revision=ea980cb0a6c2ae4b936e82123acc929f1cec04c1 + model=/workspace/models/qwen3-1.7b-base-ea980cb + anchor=12 + cycles=4 + num_layers=28 + final_stage=04 + ;; + 4b) + model_id=Qwen/Qwen3-4B-Base + revision=906bfd4b4dc7f14ee4320094d8b41684abff8539 + model=/workspace/models/qwen3-4b-base-906bfd4 + anchor=25 + cycles=5 + num_layers=36 + final_stage=05 + ;; + *) echo "invalid model key: $model_key" >&2; exit 2 ;; +esac + +variant="${fisher_input_mode}_${rho_variant}" +root="$experiment/$model_key/$variant" +out="$root/run" +layer_order=() +for ((stage=1; stage<=cycles; stage++)); do + layer_order+=("$anchor") +done + +export HF_HOME=/workspace/.hf_home +export HF_TOKEN_PATH=/root/.cache/huggingface/token +export HF_XET_HIGH_PERFORMANCE=1 +export TOKENIZERS_PARALLELISM=false +export PYTHONUNBUFFERED=1 +export PYTHONPATH="$repo/src:$project/src${PYTHONPATH:+:$PYTHONPATH}" +mkdir -p "$model" "$root/logs" "$root/compact" "$root/eval" "$root/metadata" + +if [[ ! -f "$model/config.json" ]]; then + MODEL_ID="$model_id" REVISION="$revision" MODEL_DIR="$model" python - <<'PY' +import os +from pathlib import Path +from huggingface_hub import snapshot_download + +token = Path('/root/.cache/huggingface/token').read_text(encoding='utf-8').strip() +snapshot_download( + repo_id=os.environ['MODEL_ID'], + revision=os.environ['REVISION'], + local_dir=os.environ['MODEL_DIR'], + token=token, +) +print('model_download_complete', os.environ['MODEL_ID']) +PY +fi + +cat > "$root/protocol.json" < "$root/logs/train.log" 2>&1 + touch "$root/TRAIN_COMPLETE" +fi + +if [[ ! -f "$root/COMPACT_COMPLETE" ]]; then + python -m fusegpt.abprune_compact \ + --run-root "$out" --output-root "$root/compact" \ + --model "$model_id" --base-revision "$revision" \ + --run-id "qwen3_${model_key}_${variant}_contextual_fisher_v1" \ + --cycles "$cycles" --layer-order "${layer_order[*]}" + for ((stage=1; stage<=cycles; stage++)); do + padded="$(printf '%02d' "$stage")" + cp "$out/cycle_${stage}/cycle_metadata.json" \ + "$root/metadata/cycle_${padded}.json" + done + touch "$root/COMPACT_COMPLETE" +fi + +for ((stage=1; stage<=cycles; stage++)); do + padded="$(printf '%02d' "$stage")" + if [[ ! -f "$root/eval/stage_${padded}_ppl.json" ]]; then + CUDA_VISIBLE_DEVICES="$gpu_id" python "$project/scripts/evaluate_abprune_compact_ppl.py" \ + --model "$model" --revision "$revision" \ + --compact-stage "$root/compact/stage_${padded}" \ + --anchor "$anchor" --cycles "$stage" --original-layers "$num_layers" \ + --seq-len 2048 --batch-size 4 \ + --output "$root/eval/stage_${padded}_ppl.json" \ + >> "$root/logs/eval.log" 2>&1 + fi +done + +if [[ ! -f "$root/eval/stage_${final_stage}_zs6.json" ]]; then + CUDA_VISIBLE_DEVICES="$gpu_id" python "$project/scripts/eval_abprune_compact_zeroshot.py" \ + --model "$model" --revision "$revision" \ + --compact-stage "$root/compact/stage_${final_stage}" \ + --cycle "$cycles" --anchor "$anchor" --batch-size auto:4 \ + --output "$root/eval/stage_${final_stage}_zs6.json" \ + >> "$root/logs/eval.log" 2>&1 +fi + +touch "$root/COMPLETE" +echo "contextual_fisher_variant_complete model=$model_key variant=$variant" diff --git a/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/code/scripts/summarize_contextual_fisher_matrix.py b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/code/scripts/summarize_contextual_fisher_matrix.py new file mode 100644 index 0000000000000000000000000000000000000000..626c7372014e17ad5024fdf21490ca9728d037d1 --- /dev/null +++ b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/code/scripts/summarize_contextual_fisher_matrix.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Summarize the two-model contextual-Fisher experiment matrix.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + + +MODELS = {"1.7b": 4, "4b": 5} +MODES = ("anchor_input", "anchor_post_attention") +RHO_VARIANTS = ("no_rho", "rho") + + +def load(path: Path): + return json.loads(path.read_text(encoding="utf-8")) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("root", type=Path) + args = parser.parse_args() + rows = [] + for model_key, cycles in MODELS.items(): + for mode in MODES: + for rho_variant in RHO_VARIANTS: + root = args.root / model_key / f"{mode}_{rho_variant}" + final = f"{cycles:02d}" + ppl = load(root / "eval" / f"stage_{final}_ppl.json") + zs = load(root / "eval" / f"stage_{final}_zs6.json") + stages = [] + for stage in range(1, cycles + 1): + padded = f"{stage:02d}" + metadata = load(root / "metadata" / f"cycle_{padded}.json") + stage_ppl = load(root / "eval" / f"stage_{padded}_ppl.json") + fisher_input = metadata["fisher_input"] + reparam = metadata["distill"]["reparam_stats"] + stages.append( + { + "stage": stage, + "ppl": stage_ppl["metrics"]["perplexity"], + "fisher_input": fisher_input, + "rho_final": reparam.get("rho_final"), + "u_fisher_orthogonality": reparam.get( + "u_fisher_orthogonality" + ), + } + ) + rows.append( + { + "model_key": model_key, + "model": ppl["model"], + "fisher_input_mode": mode, + "rho_variant": rho_variant, + "removed_blocks": cycles, + "final_ppl": ppl["metrics"]["perplexity"], + "final_zs6": zs["six_task_mean"], + "task_scores": zs["task_scores"], + "stages": stages, + } + ) + + summary = {"experiment": "anchor_contextual_fisher_matrix_v1", "rows": rows} + (args.root / "summary.json").write_text( + json.dumps(summary, indent=2) + "\n", encoding="utf-8" + ) + + lines = [ + "# Anchor-contextual Fisher matrix", + "", + "| Model | Fisher input | rho | PPL | ZS6 |", + "|---|---|---|---:|---:|", + ] + for row in rows: + lines.append( + f"| {row['model_key']} | {row['fisher_input_mode']} | " + f"{row['rho_variant']} | {row['final_ppl']:.4f} | " + f"{100 * row['final_zs6']:.2f}% |" + ) + (args.root / "summary.md").write_text("\n".join(lines) + "\n", encoding="utf-8") + print("\n".join(lines)) + + +if __name__ == "__main__": + main() diff --git a/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/code/scripts/upload_verified_hf_tree.py b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/code/scripts/upload_verified_hf_tree.py new file mode 100644 index 0000000000000000000000000000000000000000..2262ee982952a6d71eaf1866cadd0758ad839484 --- /dev/null +++ b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/code/scripts/upload_verified_hf_tree.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Upload an experiment artifact tree to Hugging Face and verify its manifest.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path + +from huggingface_hub import HfApi, hf_hub_download + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def scoped_token() -> str | None: + """Read an explicitly scoped host token without placing it on argv.""" + token_path = os.environ.get("HF_TOKEN_PATH") + if not token_path: + return None + return Path(token_path).read_text(encoding="utf-8").strip() + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("root", type=Path) + parser.add_argument("--repo-id", required=True) + parser.add_argument("--path-in-repo", required=True) + parser.add_argument("--repo-type", default="model") + parser.add_argument("--commit-message", required=True) + parser.add_argument("--manifest", default="manifest.json") + args = parser.parse_args() + + root = args.root.resolve() + manifest_path = root / args.manifest + if not manifest_path.is_file(): + raise SystemExit(f"missing manifest: {manifest_path}") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + expected = {item["path"]: item for item in manifest["files"]} + + token = scoped_token() + api = HfApi(token=token) + info = api.upload_folder( + repo_id=args.repo_id, + repo_type=args.repo_type, + folder_path=str(root), + path_in_repo=args.path_in_repo, + commit_message=args.commit_message, + ) + revision = info.oid + + tree = api.list_repo_tree( + repo_id=args.repo_id, + repo_type=args.repo_type, + path_in_repo=args.path_in_repo, + revision=revision, + recursive=True, + expand=True, + ) + prefix = args.path_in_repo.rstrip("/") + "/" + remote = { + item.path.removeprefix(prefix): int(item.size) + for item in tree + if hasattr(item, "size") + } + expected_with_manifest = { + **{name: int(item["size"]) for name, item in expected.items()}, + args.manifest: manifest_path.stat().st_size, + } + missing = sorted(name for name in expected_with_manifest if name not in remote) + mismatched = sorted( + name + for name, size in expected_with_manifest.items() + if name in remote and remote[name] != size + ) + if missing or mismatched: + raise RuntimeError({"missing": missing, "size_mismatch": mismatched}) + + downloaded_manifest = Path( + hf_hub_download( + repo_id=args.repo_id, + repo_type=args.repo_type, + revision=revision, + filename=f"{prefix}{args.manifest}", + token=token, + ) + ) + if sha256(downloaded_manifest) != sha256(manifest_path): + raise RuntimeError("remote manifest hash mismatch") + + report = { + "verified": True, + "repo_id": args.repo_id, + "path_in_repo": args.path_in_repo, + "revision": revision, + "manifest_files": len(expected), + "manifest_bytes": sum(int(item["size"]) for item in expected.values()), + "all_manifest_sizes_match": True, + "remote_manifest_hash_match": True, + } + print(json.dumps(report, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/code/scripts/write_tree_manifest.py b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/code/scripts/write_tree_manifest.py new file mode 100644 index 0000000000000000000000000000000000000000..dccd3a11590bf4d2ad26fc126ea464fd75d7b961 --- /dev/null +++ b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/code/scripts/write_tree_manifest.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Write a deterministic SHA256 manifest for an artifact directory.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("root", type=Path) + parser.add_argument("--output", default="manifest.json") + args = parser.parse_args() + root = args.root.resolve() + output = root / args.output + files = [] + for path in sorted(item for item in root.rglob("*") if item.is_file()): + if path == output: + continue + files.append( + { + "path": path.relative_to(root).as_posix(), + "size": path.stat().st_size, + "sha256": sha256(path), + } + ) + output.write_text( + json.dumps( + { + "schema": "sha256-tree-v1", + "files": files, + "total_files": len(files), + "total_bytes": sum(item["size"] for item in files), + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + print(f"manifest files={len(files)} bytes={sum(item['size'] for item in files)}") + + +if __name__ == "__main__": + main() diff --git a/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/code/src/fusegpt/abprune_compact.py b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/code/src/fusegpt/abprune_compact.py new file mode 100644 index 0000000000000000000000000000000000000000..3360c650b96e0bfd6153e0c48f8dca31f06d2da8 --- /dev/null +++ b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/code/src/fusegpt/abprune_compact.py @@ -0,0 +1,270 @@ +"""Package and reconstruct compact ABPrune stage checkpoints. + +ABPrune already writes ``fused_layer.pt`` for every cycle. This utility makes +those files portable by pairing them with an explicit, immutable stage record; +the dense base model plus that record is sufficient to reconstruct a student. +""" +from __future__ import annotations + +import argparse, hashlib, json, shutil +from pathlib import Path +from typing import Any + +import torch + + +def sha256(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(8 * 1024 * 1024), b""): + h.update(chunk) + return h.hexdigest() + + +def post_merge_scale(source_cycle: Path) -> dict | None: + """Return the deployable post-merge scale stored for a cycle, if present.""" + metadata_path = source_cycle / "cycle_metadata.json" + if not metadata_path.is_file(): + return None + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + distill = metadata.get("distill", {}) + scale = distill.get("post_merge_nll_scale") or {} + source = "post_merge_nll" + if not scale.get("enabled"): + scale = (distill.get("reparam_stats") or {}).get("residual_scale") or {} + source = "scale_aware_fit" + if not scale.get("enabled"): + return None + gamma = scale["gamma"] + cumulative_gamma = scale["cumulative_gamma"] + if not isinstance(gamma, list): + gamma = float(gamma) + else: + gamma = [float(value) for value in gamma] + if not isinstance(cumulative_gamma, list): + cumulative_gamma = float(cumulative_gamma) + else: + cumulative_gamma = [float(value) for value in cumulative_gamma] + result = { + "enabled": True, + "mode": scale.get("mode", "residual"), + "scale_parameterization": scale.get("scale_parameterization", "scalar"), + "gamma": gamma, + "cumulative_gamma": cumulative_gamma, + "source": source, + } + for field in ( + "optimization_mode", + "gamma_gradient_sources", + "theta_gradient_sources", + "nll_used", + ): + if field in scale: + result[field] = scale[field] + if "cumulative_gamma_statistics" in scale: + result["cumulative_gamma_statistics"] = scale["cumulative_gamma_statistics"] + if "raw" in scale and "ppl" in scale["raw"]: + result["calibration_raw_ppl"] = float(scale["raw"]["ppl"]) + if "fitted" in scale and "ppl" in scale["fitted"]: + result["calibration_fitted_ppl"] = float(scale["fitted"]["ppl"]) + return result + + +def read_compact_metadata(compact_stage: Path) -> dict[str, Any]: + return json.loads((compact_stage / "metadata.json").read_text(encoding="utf-8")) + + +def compact_scale_targets(metadata: dict[str, Any]) -> list[dict[str, Any]]: + """Return residual scales and current student indices for a compact stage.""" + if metadata.get("format") == "abprune_compact_stage_v2": + active = [int(value) for value in metadata["active_original_layers"]] + current_index = {original: index for index, original in enumerate(active)} + targets = [] + for state in metadata.get("retained_layer_states", []): + scale = state.get("post_merge_scale") or {} + if scale.get("enabled") and scale.get("mode") == "residual": + original = int(state["original_layer"]) + targets.append( + { + "original_layer": original, + "student_index": current_index[original], + "scale": scale, + } + ) + return targets + + scale = metadata.get("post_merge_scale") or {} + if scale.get("enabled") and scale.get("mode") == "residual": + return [ + { + "original_layer": int(metadata["anchor_index"]), + "student_index": int(metadata["anchor_index"]), + "scale": scale, + } + ] + return [] + + +def _package_topology_aware(args, source: Path, out: Path, order: list[int]) -> None: + """Package cycles whose recipients may switch between pair boundaries.""" + if int(getattr(args, "start_cycle", 1)) != 1: + raise ValueError("topology-aware compact packaging must start at cycle 1") + + removed: set[int] = set() + latest_states: dict[int, dict[str, Any]] = {} + stages: list[dict[str, Any]] = [] + original_layer_count: int | None = None + + for cycle in range(1, args.cycles + 1): + source_cycle = source / f"cycle_{cycle}" + src = source_cycle / "fused_layer.pt" + if not src.is_file(): + raise FileNotFoundError(src) + cycle_metadata_path = source_cycle / "cycle_metadata.json" + if not cycle_metadata_path.is_file(): + raise FileNotFoundError(cycle_metadata_path) + cycle_metadata = json.loads(cycle_metadata_path.read_text(encoding="utf-8")) + recipient = int(cycle_metadata["recipient_original_layer"]) + removed_original = int(cycle_metadata["removed_original_layer"]) + recipient_side = str(cycle_metadata["recipient_side"]) + if recipient_side not in {"left", "right"}: + raise ValueError(f"invalid recipient_side at cycle {cycle}: {recipient_side}") + if original_layer_count is None: + original_layer_count = int(cycle_metadata["num_layers_before"]) + if removed_original in removed: + raise ValueError(f"original layer {removed_original} removed more than once") + if recipient in removed: + raise ValueError(f"cycle {cycle} selects removed recipient {recipient}") + + removed.add(removed_original) + latest_states.pop(removed_original, None) + latest_states[recipient] = { + "original_layer": recipient, + "source_cycle": cycle, + "recipient_side": recipient_side, + "source_state": src, + "post_merge_scale": post_merge_scale(source_cycle), + } + + stage_dir = out / f"stage_{cycle:02d}" + stage_dir.mkdir(exist_ok=True) + retained_states = [] + for original, record in sorted(latest_states.items()): + state_name = f"layer_original_{original:04d}.pt" + state_path = stage_dir / state_name + shutil.copy2(record["source_state"], state_path) + packaged = { + "original_layer": original, + "state_file": state_name, + "sha256": sha256(state_path), + "source_cycle": int(record["source_cycle"]), + "recipient_side": record["recipient_side"], + } + if record["post_merge_scale"] is not None: + packaged["post_merge_scale"] = record["post_merge_scale"] + retained_states.append(packaged) + + assert original_layer_count is not None + active = [ + original + for original in range(original_layer_count) + if original not in removed + ] + meta = { + "format": "abprune_compact_stage_v2", + "model": args.model, + "base_model_revision": args.base_revision, + "run_id": args.run_id, + "target_prune_count": args.cycles, + "stage": cycle, + "removed_count": len(removed), + "original_layer_count": original_layer_count, + "layer_order": order, + "latest_pair_index": order[cycle - 1], + "latest_recipient_side": recipient_side, + "latest_recipient_original_layer": recipient, + "latest_removed_original_layer": removed_original, + "removed_original_layers": sorted(removed), + "active_original_layers": active, + "retained_layer_states": retained_states, + "source_cycle": f"cycle_{cycle}", + } + (stage_dir / "metadata.json").write_text( + json.dumps(meta, indent=2) + "\n", encoding="utf-8" + ) + stages.append(meta) + + manifest = { + "format": "abprune_compact_manifest_v2", + "model": args.model, + "base_model_revision": args.base_revision, + "run_id": args.run_id, + "target_prune_count": args.cycles, + "original_layer_count": original_layer_count, + "layer_order": order, + "stages": stages, + } + (out / "manifest.json").write_text( + json.dumps(manifest, indent=2) + "\n", encoding="utf-8" + ) + + +def package(args): + source = Path(args.run_root); out = Path(args.output_root); out.mkdir(parents=True, exist_ok=True) + order = [int(x) for x in args.layer_order.replace(',', ' ').split()] + start_cycle = int(getattr(args, "start_cycle", 1)) + expected_order = args.cycles - start_cycle + 1 + if not 1 <= start_cycle <= args.cycles: + raise ValueError("start_cycle must be between 1 and cycles") + if len(order) != expected_order: + raise ValueError("layer_order length must equal cycles - start_cycle + 1") + first_metadata_path = source / f"cycle_{start_cycle}" / "cycle_metadata.json" + if first_metadata_path.is_file(): + first_metadata = json.loads(first_metadata_path.read_text(encoding="utf-8")) + topology_fields = { + "recipient_side", + "recipient_original_layer", + "removed_original_layer", + } + if topology_fields.issubset(first_metadata): + _package_topology_aware(args, source, out, order) + return + stages = [] + for cycle in range(start_cycle, args.cycles + 1): + order_index = cycle - start_cycle + source_cycle = source / f"cycle_{cycle}" + src = source_cycle / "fused_layer.pt" + if not src.exists(): raise FileNotFoundError(src) + dst_dir = out / f"stage_{cycle:02d}"; dst_dir.mkdir(exist_ok=True) + dst = dst_dir / "anchor_layer.pt"; shutil.copy2(src, dst) + meta = { + "format": "abprune_compact_stage_v1", "model": args.model, + "base_model_revision": args.base_revision, "run_id": args.run_id, + "target_prune_count": args.cycles, "stage": cycle, + "removed_count": cycle, "anchor_index": order[order_index], + "layer_order": order, + "packaged_stage_range": [start_cycle, args.cycles], + "removed_original_layers": list(range(order[order_index] + 1, order[order_index] + cycle + 1)), + "source_cycle": f"cycle_{cycle}", "anchor_state_file": "anchor_layer.pt", + "sha256": sha256(dst), + } + scale = post_merge_scale(source_cycle) + if scale is not None: + meta["post_merge_scale"] = scale + (dst_dir / "metadata.json").write_text(json.dumps(meta, indent=2) + "\n", encoding="utf-8") + stages.append(meta) + manifest = {"format": "abprune_compact_manifest_v1", "model": args.model, + "base_model_revision": args.base_revision, "run_id": args.run_id, + "target_prune_count": args.cycles, "layer_order": order, "stages": stages} + (out / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + + +def main(): + p = argparse.ArgumentParser(); p.add_argument("--run-root", required=True); p.add_argument("--output-root", required=True) + p.add_argument("--model", required=True); p.add_argument("--base-revision", required=True); p.add_argument("--run-id", required=True) + p.add_argument("--cycles", type=int, required=True); p.add_argument("--layer-order", required=True) + p.add_argument("--start-cycle", type=int, default=1) + package(p.parse_args()) + + +if __name__ == "__main__": main() diff --git a/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/code/src/fusegpt/abprune_contextual_fisher.py b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/code/src/fusegpt/abprune_contextual_fisher.py new file mode 100644 index 0000000000000000000000000000000000000000..de7d9f7fe6a6b3354c67e1b6dcf06644bf1d5a49 --- /dev/null +++ b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/code/src/fusegpt/abprune_contextual_fisher.py @@ -0,0 +1,216 @@ +"""Counterfactual empirical Fisher collection for ABPrune MLP absorption. + +The anchor layer remains at its normal location and follows its native forward. +Only the absorbed/source MLP input is replaced while the model-wide causal-LM +loss is evaluated. This lets the source weights be scored in the coordinate +system in which they will be used after an MLP-only merge. +""" + +from __future__ import annotations + +import math +import os +from contextlib import ExitStack +from typing import Dict, List, Tuple + +import torch + +try: + from tqdm import tqdm +except Exception: # pragma: no cover - optional dependency + tqdm = None + + +_VALID_MODES = {"anchor_input", "anchor_post_attention"} + + +def _tqdm_enabled() -> bool: + value = os.environ.get("DISABLE_TQDM", os.environ.get("TQDM_DISABLE", "0")) + return value.strip().lower() not in {"1", "true", "yes", "on"} + + +def _mlp(layer: torch.nn.Module) -> torch.nn.Module: + module = getattr(layer, "mlp", None) + if not isinstance(module, torch.nn.Module): + raise TypeError("Contextual Fisher currently requires decoder_layer.mlp") + return module + + +def _post_attention_norm(layer: torch.nn.Module) -> torch.nn.Module: + module = getattr(layer, "post_attention_layernorm", None) + if not isinstance(module, torch.nn.Module): + raise TypeError( + "anchor_input Fisher requires decoder_layer.post_attention_layernorm" + ) + return module + + +def _allocate_fisher( + layers: Tuple[torch.nn.Module, torch.nn.Module], fisher_mode: str +) -> Tuple[List[Dict[str, object]], List[Dict[str, int]]]: + fisher_sums: List[Dict[str, object]] = [] + param_numels: List[Dict[str, int]] = [] + for layer in layers: + layer_sums: Dict[str, object] = {} + layer_numels: Dict[str, int] = {} + for name, param in layer.named_parameters(): + if not param.requires_grad: + continue + if fisher_mode == "param": + layer_sums[name] = torch.zeros_like( + param, dtype=torch.float32, device="cpu" + ) + else: + layer_sums[name] = 0.0 + layer_numels[name] = param.numel() + fisher_sums.append(layer_sums) + param_numels.append(layer_numels) + return fisher_sums, param_numels + + +class _InputDiagnostics: + def __init__(self) -> None: + self.tokens = 0 + self.cosine_sum = 0.0 + self.norm_ratio_sum = 0.0 + self.native_rms_sq_sum = 0.0 + self.context_rms_sq_sum = 0.0 + + @torch.no_grad() + def add(self, native: torch.Tensor, context: torch.Tensor) -> None: + if native.shape != context.shape: + raise RuntimeError( + f"Contextual Fisher shape mismatch: {native.shape} vs {context.shape}" + ) + native_f = native.detach().float().reshape(-1, native.shape[-1]) + context_f = context.detach().float().reshape(-1, context.shape[-1]) + native_norm = native_f.norm(dim=-1) + context_norm = context_f.norm(dim=-1) + denom = native_norm * context_norm + valid = denom > 0 + cosine = torch.zeros_like(denom) + cosine[valid] = (native_f[valid] * context_f[valid]).sum(dim=-1) / denom[valid] + ratio = context_norm / native_norm.clamp_min(torch.finfo(torch.float32).tiny) + count = int(native_f.shape[0]) + self.tokens += count + self.cosine_sum += float(cosine.sum().item()) + self.norm_ratio_sum += float(ratio.sum().item()) + self.native_rms_sq_sum += float(native_f.square().mean(dim=-1).sum().item()) + self.context_rms_sq_sum += float(context_f.square().mean(dim=-1).sum().item()) + + def summary(self, mode: str) -> Dict[str, object]: + if self.tokens == 0: + raise RuntimeError("No source MLP inputs were observed during Fisher") + inv = 1.0 / self.tokens + return { + "mode": mode, + "tokens": self.tokens, + "context_vs_native_source_mlp_input_cosine": self.cosine_sum * inv, + "context_vs_native_source_mlp_input_norm_ratio": self.norm_ratio_sum * inv, + "native_source_mlp_input_global_rms": math.sqrt( + self.native_rms_sq_sum * inv + ), + "context_source_mlp_input_global_rms": math.sqrt( + self.context_rms_sq_sum * inv + ), + "context_detached": True, + "anchor_attention_included": mode == "anchor_post_attention", + "normalization": ( + "anchor.post_attention_layernorm(anchor_block_input)" + if mode == "anchor_input" + else "native anchor MLP input after anchor attention and RMSNorm" + ), + } + + +def compute_contextual_fisher( + model: torch.nn.Module, + layer_a: torch.nn.Module, + layer_b: torch.nn.Module, + dataloader, + fisher_mode: str, + device: str, + input_mode: str, +) -> Tuple[ + List[Dict[str, object]], int, List[Dict[str, int]], Dict[str, object] +]: + """Collect diagonal Fisher with source MLP evaluated on anchor coordinates.""" + + if input_mode not in _VALID_MODES: + raise ValueError(f"Unsupported contextual Fisher input mode: {input_mode}") + if fisher_mode not in {"param", "tensor"}: + raise ValueError(f"Unsupported Fisher mode: {fisher_mode}") + + for param in model.parameters(): + param.requires_grad_(False) + for layer in (layer_a, layer_b): + for param in layer.parameters(): + param.requires_grad_(True) + + fisher_sums, param_numels = _allocate_fisher( + (layer_a, layer_b), fisher_mode + ) + anchor_mlp = _mlp(layer_a) + source_mlp = _mlp(layer_b) + anchor_norm = _post_attention_norm(layer_a) + cache: Dict[str, torch.Tensor] = {} + diagnostics = _InputDiagnostics() + + def capture_anchor_input(_module, args): + hidden = args[0] + if input_mode == "anchor_input": + # Detaching is intentional: only the source MLP Fisher should change. + # The anchor's native path still contributes its ordinary gradients. + with torch.no_grad(): + cache["context"] = anchor_norm(hidden.detach()).detach() + + def capture_anchor_mlp_input(_module, args): + if input_mode == "anchor_post_attention": + cache["context"] = args[0].detach() + + def replace_source_mlp_input(_module, args): + context = cache.get("context") + if context is None: + raise RuntimeError("Anchor context was not captured before source MLP") + native = args[0] + context = context.to(device=native.device, dtype=native.dtype) + diagnostics.add(native, context) + return (context,) + tuple(args[1:]) + + num_batches = 0 + model.eval() + iterator = dataloader + if tqdm is not None and _tqdm_enabled(): + iterator = tqdm(dataloader, desc=f"Fisher[{input_mode}]", unit="batch") + + with ExitStack() as stack: + handles = [ + layer_a.register_forward_pre_hook(capture_anchor_input), + anchor_mlp.register_forward_pre_hook(capture_anchor_mlp_input), + source_mlp.register_forward_pre_hook(replace_source_mlp_input), + ] + for handle in handles: + stack.callback(handle.remove) + + for batch in iterator: + cache.clear() + input_ids = batch[0].to(device) + outputs = model(input_ids=input_ids, labels=input_ids) + outputs.loss.backward() + for layer_idx, layer in enumerate((layer_a, layer_b)): + layer_sums = fisher_sums[layer_idx] + for name, param in layer.named_parameters(): + if not param.requires_grad or param.grad is None: + continue + grad_sq = param.grad.detach().float().pow(2) + if fisher_mode == "param": + layer_sums[name] += grad_sq.cpu() + else: + layer_sums[name] += float(grad_sq.sum().item()) + model.zero_grad(set_to_none=True) + num_batches += 1 + + if num_batches == 0: + raise RuntimeError("No batches processed; check dataset or text inputs") + return fisher_sums, num_batches, param_numels, diagnostics.summary(input_mode) + diff --git a/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/code/tests/test_abprune_contextual_fisher.py b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/code/tests/test_abprune_contextual_fisher.py new file mode 100644 index 0000000000000000000000000000000000000000..5bc9f5b1f9b848fa5ae203daf6d338b46dc2e4d6 --- /dev/null +++ b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/code/tests/test_abprune_contextual_fisher.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +import torch + +from fusegpt.abprune_contextual_fisher import compute_contextual_fisher + + +class ToyBlock(torch.nn.Module): + def __init__(self, width: int, attention_scale: float) -> None: + super().__init__() + self.input_layernorm = torch.nn.LayerNorm(width) + self.self_attn = torch.nn.Linear(width, width, bias=False) + self.post_attention_layernorm = torch.nn.LayerNorm(width) + self.mlp = torch.nn.Linear(width, width, bias=False) + with torch.no_grad(): + self.self_attn.weight.copy_(torch.eye(width) * attention_scale) + + def forward(self, hidden: torch.Tensor) -> torch.Tensor: + residual = hidden + hidden = residual + self.self_attn(self.input_layernorm(hidden)) + residual = hidden + hidden = residual + self.mlp(self.post_attention_layernorm(hidden)) + return hidden + + +class ToyLM(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.embed = torch.nn.Embedding(13, 4) + self.layers = torch.nn.ModuleList([ToyBlock(4, 0.25), ToyBlock(4, -0.1)]) + self.head = torch.nn.Linear(4, 13, bias=False) + + def forward(self, input_ids: torch.Tensor, labels: torch.Tensor): + hidden = self.embed(input_ids) + for layer in self.layers: + hidden = layer(hidden) + logits = self.head(hidden) + loss = torch.nn.functional.cross_entropy( + logits[:, :-1].reshape(-1, logits.shape[-1]), labels[:, 1:].reshape(-1) + ) + return type("ToyOutput", (), {"loss": loss}) + + +def _loader(): + return [(torch.tensor([[1, 2, 3, 4], [4, 3, 2, 1]]),)] + + +def test_contextual_fisher_modes_are_distinct_and_nonzero() -> None: + torch.manual_seed(7) + reports = {} + fishers = {} + for mode in ("anchor_input", "anchor_post_attention"): + model = ToyLM() + fisher, batches, numels, report = compute_contextual_fisher( + model, + model.layers[0], + model.layers[1], + _loader(), + fisher_mode="param", + device="cpu", + input_mode=mode, + ) + assert batches == 1 + assert numels == [{}, {}] + assert report["mode"] == mode + assert report["tokens"] == 8 + assert torch.count_nonzero(fisher[1]["mlp.weight"]) > 0 + reports[mode] = report + fishers[mode] = fisher[1]["mlp.weight"] + + assert reports["anchor_input"]["anchor_attention_included"] is False + assert reports["anchor_post_attention"]["anchor_attention_included"] is True + assert not torch.allclose( + fishers["anchor_input"], fishers["anchor_post_attention"] + ) + + +def test_hooks_are_removed_after_collection() -> None: + model = ToyLM() + compute_contextual_fisher( + model, + model.layers[0], + model.layers[1], + _loader(), + fisher_mode="tensor", + device="cpu", + input_mode="anchor_input", + ) + assert not model.layers[0]._forward_pre_hooks + assert not model.layers[0].mlp._forward_pre_hooks + assert not model.layers[1].mlp._forward_pre_hooks + diff --git a/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/manifest.json b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..7397a5cd7d3de661c281ca00eb403b3e3cb0a1bc --- /dev/null +++ b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/manifest.json @@ -0,0 +1,647 @@ +{ + "schema": "sha256-tree-v1", + "files": [ + { + "path": "arms/anchor_input_no_rho/COMPACT_COMPLETE", + "size": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "path": "arms/anchor_input_no_rho/COMPLETE", + "size": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "path": "arms/anchor_input_no_rho/TRAIN_COMPLETE", + "size": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "path": "arms/anchor_input_no_rho/compact/manifest.json", + "size": 3976, + "sha256": "4633135896192cde45c318dd88c4a233d8d6a14750ba8116737773034b6202fd" + }, + { + "path": "arms/anchor_input_no_rho/compact/stage_01/anchor_layer.pt", + "size": 201866630, + "sha256": "2ea21e63d8385b7c6b32cb47cb5a93198822649e15ac1ec7e1a401aba2f6ce2f" + }, + { + "path": "arms/anchor_input_no_rho/compact/stage_01/metadata.json", + "size": 598, + "sha256": "e2e7ba5b3349e2e7c3afb5bd16cbd6524618ba870e243c7f5417d4fedcfb603b" + }, + { + "path": "arms/anchor_input_no_rho/compact/stage_02/anchor_layer.pt", + "size": 201866630, + "sha256": "b4c0450e441f2366f09cc6b5b72fc17d1f845cb9e8be7add342281cc27b40a6c" + }, + { + "path": "arms/anchor_input_no_rho/compact/stage_02/metadata.json", + "size": 606, + "sha256": "ae4a3fb2b4b340474068f82d7dadfce2ebfad08223fe077eee187fd6094e678b" + }, + { + "path": "arms/anchor_input_no_rho/compact/stage_03/anchor_layer.pt", + "size": 201866630, + "sha256": "0c3aacef2687a4fdda11ff7fa75ceed7e55be1269fe40618b1e782f79d6745af" + }, + { + "path": "arms/anchor_input_no_rho/compact/stage_03/metadata.json", + "size": 614, + "sha256": "32283899328a89c129254fca4ec18138babbb3e78b7358d84c89a4a9a7ee10e0" + }, + { + "path": "arms/anchor_input_no_rho/compact/stage_04/anchor_layer.pt", + "size": 201866630, + "sha256": "36cdbc27d39227c16d250dd572efe3be8e4f38bacb89beac45d94102cf45ae11" + }, + { + "path": "arms/anchor_input_no_rho/compact/stage_04/metadata.json", + "size": 622, + "sha256": "6face005a3d03f6cb61065e357501a9d302500a06a6bbe192126a4fb6fa365ae" + }, + { + "path": "arms/anchor_input_no_rho/compact/stage_05/anchor_layer.pt", + "size": 201866630, + "sha256": "a7812d906d202c1bd843be4aff0faabad90d2da61812e26b0f164a62e7faaed8" + }, + { + "path": "arms/anchor_input_no_rho/compact/stage_05/metadata.json", + "size": 630, + "sha256": "2c56f2315e9c1128784f71857652e0ac32234bd7296fb7916d8cd8a1075ffab1" + }, + { + "path": "arms/anchor_input_no_rho/eval/stage_01_ppl.json", + "size": 869, + "sha256": "d995e49056d63940075c466ae3df812bac149db5cf8fb95f473eff99e141d23b" + }, + { + "path": "arms/anchor_input_no_rho/eval/stage_02_ppl.json", + "size": 871, + "sha256": "7f63a15b8be01b1066af3a1ee96960c2cfc503ab3d918af20ade9ef3602952f9" + }, + { + "path": "arms/anchor_input_no_rho/eval/stage_03_ppl.json", + "size": 871, + "sha256": "46749f97bcab7ca4650700a08b2b09831dcb86e63348e2eaec91116c85b8fc16" + }, + { + "path": "arms/anchor_input_no_rho/eval/stage_04_ppl.json", + "size": 871, + "sha256": "ce94d225cf96682e42b2d39cf39d5c9415c543044b435c1eb81c53a3e83aa4fe" + }, + { + "path": "arms/anchor_input_no_rho/eval/stage_05_ppl.json", + "size": 871, + "sha256": "f252c0c060de314a0408c7df8829c203ebd0e564719d1790612ee6bcf7ca4435" + }, + { + "path": "arms/anchor_input_no_rho/eval/stage_05_zs6.json", + "size": 3417, + "sha256": "76637c2d019c56361d9a3fba95e3c2d5a3cb1833521cb7cefd27c317b645352a" + }, + { + "path": "arms/anchor_input_no_rho/logs/eval.log", + "size": 133936, + "sha256": "6d05478772e6f87368ac3c9de962d71c0351bc1d25e860d91e9dafc99bd9ac12" + }, + { + "path": "arms/anchor_input_no_rho/logs/train.log", + "size": 19706, + "sha256": "aad02a14d588f838c51c62873bd6613541fa4db272d6c6953da5b9ec8d2d5605" + }, + { + "path": "arms/anchor_input_no_rho/metadata/cycle_01.json", + "size": 25108, + "sha256": "2d0a10e1081cb6eedefc5c59fb0da39e56f290756d2c6a186f36cb9e55ec2f81" + }, + { + "path": "arms/anchor_input_no_rho/metadata/cycle_02.json", + "size": 25138, + "sha256": "699aaf759335d848243a07b98400f912ef0d6d5e5d299b578b1b040b8e6a2abf" + }, + { + "path": "arms/anchor_input_no_rho/metadata/cycle_03.json", + "size": 25120, + "sha256": "df09a146a36b191754bea5a342c322e40ccec325b10c59e5c2373e61712b6bb6" + }, + { + "path": "arms/anchor_input_no_rho/metadata/cycle_04.json", + "size": 25122, + "sha256": "0d12460af4f5a034040042b9abac2da60a60fddce9b13b5e8c97965fedf19a29" + }, + { + "path": "arms/anchor_input_no_rho/metadata/cycle_05.json", + "size": 25113, + "sha256": "e0b6dcfad3b7e927de79a58d96a00223f6815c60e125f7c1f8db7ab09a16cc72" + }, + { + "path": "arms/anchor_input_no_rho/protocol.json", + "size": 516, + "sha256": "0395c367c16cc4a72c429491ecef3d2499b57c4d6cc44d24cd4905a5d5c0fb80" + }, + { + "path": "arms/anchor_input_rho/COMPACT_COMPLETE", + "size": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "path": "arms/anchor_input_rho/COMPLETE", + "size": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "path": "arms/anchor_input_rho/TRAIN_COMPLETE", + "size": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "path": "arms/anchor_input_rho/compact/manifest.json", + "size": 3958, + "sha256": "6c21a276cccaf0892d1839682a8deefdc95c17890021a8ac478bf20b0a37a765" + }, + { + "path": "arms/anchor_input_rho/compact/stage_01/anchor_layer.pt", + "size": 201866630, + "sha256": "b85fe2e31883e0c4c6c6881d5733ad1cb16cc828677ae95bfd9fc9dc6838a7f2" + }, + { + "path": "arms/anchor_input_rho/compact/stage_01/metadata.json", + "size": 595, + "sha256": "2d30689523b1e76d62a5e368afec8df4d0357073ad8949852446fb4359451908" + }, + { + "path": "arms/anchor_input_rho/compact/stage_02/anchor_layer.pt", + "size": 201866630, + "sha256": "62d25e32d3361f51f061100b532d978ae9e3c2ee68aef4b526d9768616416892" + }, + { + "path": "arms/anchor_input_rho/compact/stage_02/metadata.json", + "size": 603, + "sha256": "4b6a9aea455f1c41d32f1969be1547ba1b5a98f15c5cae56c9497fd7eec8a10b" + }, + { + "path": "arms/anchor_input_rho/compact/stage_03/anchor_layer.pt", + "size": 201866630, + "sha256": "589ee811db70a31db1a4e82a21eefe617b08f0c89200fd4f7c5d3aedf797479e" + }, + { + "path": "arms/anchor_input_rho/compact/stage_03/metadata.json", + "size": 611, + "sha256": "3831f721c32d0344715d9174673065f2940de70ba49669763f9a5cf939c70c13" + }, + { + "path": "arms/anchor_input_rho/compact/stage_04/anchor_layer.pt", + "size": 201866630, + "sha256": "824da6009a2af44c4fd1bacca61a2c74b8d582ce57c66889aee021a2efbacd3b" + }, + { + "path": "arms/anchor_input_rho/compact/stage_04/metadata.json", + "size": 619, + "sha256": "19abcd6cc051ac1965e66225fcf80f4021372a5fb293378e16925ee61e17bde9" + }, + { + "path": "arms/anchor_input_rho/compact/stage_05/anchor_layer.pt", + "size": 201866630, + "sha256": "310b58c5ed33748481809ecc404a0af8d41c67b3589e8a4cd2f627e58990b9a1" + }, + { + "path": "arms/anchor_input_rho/compact/stage_05/metadata.json", + "size": 627, + "sha256": "c6c86dcbbf0c597f5deb9407b1bd52f41ea8a99bfd9e5a3215e61b912925dca4" + }, + { + "path": "arms/anchor_input_rho/eval/stage_01_ppl.json", + "size": 867, + "sha256": "d82f98e131cc22cdd165b70359578f2dd41c1b858336f81201ced6d35acf2e04" + }, + { + "path": "arms/anchor_input_rho/eval/stage_02_ppl.json", + "size": 867, + "sha256": "384adf1d210cd384de79bd4156a257961bcf28e6957156531bc7fd8c5072df86" + }, + { + "path": "arms/anchor_input_rho/eval/stage_03_ppl.json", + "size": 868, + "sha256": "b1565af4cb1080ab6034c8a91431ceaa63c50cc1e58f48916b3f5dd5c1e5588a" + }, + { + "path": "arms/anchor_input_rho/eval/stage_04_ppl.json", + "size": 868, + "sha256": "5f6bcac32b325540b1bb5f52953c2d82c4bc61b2859ef3a810abe892f217e466" + }, + { + "path": "arms/anchor_input_rho/eval/stage_05_ppl.json", + "size": 868, + "sha256": "b49fc2223f7f411a311f649561db9b4af39cf756fe7c465b313292d61f7dc1db" + }, + { + "path": "arms/anchor_input_rho/eval/stage_05_zs6.json", + "size": 3412, + "sha256": "65971184072982a50034aeb95bc0e5b8c29b9413f238b93f4f8ba53508431ff8" + }, + { + "path": "arms/anchor_input_rho/logs/eval.log", + "size": 125569, + "sha256": "46216a0bc289c6f1f57410706df779bb60b3a296bc23df1211be10de4cf6ef71" + }, + { + "path": "arms/anchor_input_rho/logs/train.log", + "size": 33314, + "sha256": "03a1ef3d04b400a8d2357a34f27b63ac535da1a9f04a4868e31633e8e5be90b8" + }, + { + "path": "arms/anchor_input_rho/metadata/cycle_01.json", + "size": 27687, + "sha256": "736c3ed895da48de14384812a9d9d813076285b46303f37aaf1629af7aabede8" + }, + { + "path": "arms/anchor_input_rho/metadata/cycle_02.json", + "size": 27722, + "sha256": "03f2c7c911af31f60edae854cb8d7a58a36468b7b84fdcd50b12d1f1a8535fcb" + }, + { + "path": "arms/anchor_input_rho/metadata/cycle_03.json", + "size": 27707, + "sha256": "3d178d9d663c5ae360f9cce9068d8f07fa1a4e0e0e19d74cb839dc921f4b1087" + }, + { + "path": "arms/anchor_input_rho/metadata/cycle_04.json", + "size": 27664, + "sha256": "6d13f274469e3f4f9fb0853ec7b2852beaab881b55b96c7604d72d0d5f00ba17" + }, + { + "path": "arms/anchor_input_rho/metadata/cycle_05.json", + "size": 27686, + "sha256": "ff914d5587abbe3aeebd1c575be066a94317790b01dfe5bb9531a30a0f75d97c" + }, + { + "path": "arms/anchor_input_rho/protocol.json", + "size": 534, + "sha256": "da3c7a64f3d3108007269fbdcdc2697530d6fbbd91c89afd23ebc31236858e47" + }, + { + "path": "arms/anchor_post_attention_no_rho/COMPACT_COMPLETE", + "size": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "path": "arms/anchor_post_attention_no_rho/COMPLETE", + "size": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "path": "arms/anchor_post_attention_no_rho/TRAIN_COMPLETE", + "size": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "path": "arms/anchor_post_attention_no_rho/compact/manifest.json", + "size": 4030, + "sha256": "c1adbacdcc80df2629dfa271ea13b9e334e96e45cc3f6303760e4a56feae0ab5" + }, + { + "path": "arms/anchor_post_attention_no_rho/compact/stage_01/anchor_layer.pt", + "size": 201866630, + "sha256": "25b31a97c4035e8fd9636c98fceb78375fab7a5dd84a9830e2a724cf077d7179" + }, + { + "path": "arms/anchor_post_attention_no_rho/compact/stage_01/metadata.json", + "size": 607, + "sha256": "107eba6fa8d065765115ebc7246b250385771c6a971a5af6639c85621cec456d" + }, + { + "path": "arms/anchor_post_attention_no_rho/compact/stage_02/anchor_layer.pt", + "size": 201866630, + "sha256": "45e1b325a0a2611af07452976fec3028f20cbd18698299e66841bb86fab741ae" + }, + { + "path": "arms/anchor_post_attention_no_rho/compact/stage_02/metadata.json", + "size": 615, + "sha256": "98f3d3fea097af67e99491a4f7d30773fcb76e3219097b95431e1b796ee89666" + }, + { + "path": "arms/anchor_post_attention_no_rho/compact/stage_03/anchor_layer.pt", + "size": 201866630, + "sha256": "ea7194027d19b08674bea525a1cd5ecfc04f2c53487b9dc857287bddd373fd65" + }, + { + "path": "arms/anchor_post_attention_no_rho/compact/stage_03/metadata.json", + "size": 623, + "sha256": "331c989a29053426ea599513a7003bf4dfa6f20068c3b0007535f585e51cb787" + }, + { + "path": "arms/anchor_post_attention_no_rho/compact/stage_04/anchor_layer.pt", + "size": 201866630, + "sha256": "8467a7fd38b5cd6014e099cb792d26ef18e9c8db8278159a28da3fb40faf6f2a" + }, + { + "path": "arms/anchor_post_attention_no_rho/compact/stage_04/metadata.json", + "size": 631, + "sha256": "4b237ac027de3d7a0870ebf30b2dae321e248fb90b52d4b8471d4d798a323a7c" + }, + { + "path": "arms/anchor_post_attention_no_rho/compact/stage_05/anchor_layer.pt", + "size": 201866630, + "sha256": "c9a4dd0a93f2fc2111f0b04424231581e41a6a7a479f3fdc334bdc75cea15c65" + }, + { + "path": "arms/anchor_post_attention_no_rho/compact/stage_05/metadata.json", + "size": 639, + "sha256": "bd5779a88437e9b17dd2892ce16bb06a08463cfcae56f65ab37b34eae65db49f" + }, + { + "path": "arms/anchor_post_attention_no_rho/eval/stage_01_ppl.json", + "size": 879, + "sha256": "cc15e02a33e8f783f343aee53f8df3301693b48ea849208089efc25a5776c41c" + }, + { + "path": "arms/anchor_post_attention_no_rho/eval/stage_02_ppl.json", + "size": 879, + "sha256": "88f4579571a50b8fbe6378e0b87e0e7da36b9e03e2ab1189a12b57e9f5054b36" + }, + { + "path": "arms/anchor_post_attention_no_rho/eval/stage_03_ppl.json", + "size": 880, + "sha256": "9db0bfc4aefe863c9f3270a247acefb30a82e7c017a80722120a1dafed476002" + }, + { + "path": "arms/anchor_post_attention_no_rho/eval/stage_04_ppl.json", + "size": 881, + "sha256": "dc8ef2a402ca4264d1fd5ffc6d8f733ee2375c7d001d0ab57db402ad4b9b8b4e" + }, + { + "path": "arms/anchor_post_attention_no_rho/eval/stage_05_ppl.json", + "size": 880, + "sha256": "1a357e7484eea6542ac9f7e7135ac4891512518fb3eab5fa6373cee480521755" + }, + { + "path": "arms/anchor_post_attention_no_rho/eval/stage_05_zs6.json", + "size": 3430, + "sha256": "7d9095318e0c9ed35af107c2c2211da2eaa1e143ffb8cb37c42dff982e011c25" + }, + { + "path": "arms/anchor_post_attention_no_rho/logs/eval.log", + "size": 125609, + "sha256": "e7ca285988f5ee150d14dd3fa84f2c2ad0ca4925d4eb4477b4ce1463943a3f7b" + }, + { + "path": "arms/anchor_post_attention_no_rho/logs/train.log", + "size": 19689, + "sha256": "f2cc674cfec87ff80cc0518ad474932a53ce049e6f4584e67018cd66f9feb203" + }, + { + "path": "arms/anchor_post_attention_no_rho/metadata/cycle_01.json", + "size": 25106, + "sha256": "2526f0aa5ebe0a0b9d9cbc5b70b2f0b70f417348be25d776b35c65de968c6513" + }, + { + "path": "arms/anchor_post_attention_no_rho/metadata/cycle_02.json", + "size": 25157, + "sha256": "ef5bfddf70337938a55e05289de6ba1cfde44988167a0ae986384bbfd92aec35" + }, + { + "path": "arms/anchor_post_attention_no_rho/metadata/cycle_03.json", + "size": 25149, + "sha256": "6a228aaf04e22e991309b0bae6b5cc3964a1403ddffcfa250fa4b81dcfe16f85" + }, + { + "path": "arms/anchor_post_attention_no_rho/metadata/cycle_04.json", + "size": 25131, + "sha256": "eef515c4262c094dfad1a90be0eb295d20afbc1e124a5cba7a4106468a4a3cad" + }, + { + "path": "arms/anchor_post_attention_no_rho/metadata/cycle_05.json", + "size": 25138, + "sha256": "b73b004496c3d6721686708ba5462524fa0d885caf6c3dc6d7eb247fe30fdd69" + }, + { + "path": "arms/anchor_post_attention_no_rho/protocol.json", + "size": 525, + "sha256": "aebd7f72bb4a0306074db5b1c92e12fea90a772375f36b0f53824933ddebd6d5" + }, + { + "path": "arms/anchor_post_attention_rho/COMPACT_COMPLETE", + "size": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "path": "arms/anchor_post_attention_rho/COMPLETE", + "size": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "path": "arms/anchor_post_attention_rho/TRAIN_COMPLETE", + "size": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "path": "arms/anchor_post_attention_rho/compact/manifest.json", + "size": 4012, + "sha256": "d6bfabf0fc792452b2a7b8de4df87e61791e62e0df9df97e853d068410fb5762" + }, + { + "path": "arms/anchor_post_attention_rho/compact/stage_01/anchor_layer.pt", + "size": 201866630, + "sha256": "b583071fdd4cca58e17d62ab87e4838eadca620931031e3f1512e8ab252b7e49" + }, + { + "path": "arms/anchor_post_attention_rho/compact/stage_01/metadata.json", + "size": 604, + "sha256": "a3d0357dadefcf197329aaa1d54f328a09b261959ecf065f84fe250ced52e087" + }, + { + "path": "arms/anchor_post_attention_rho/compact/stage_02/anchor_layer.pt", + "size": 201866630, + "sha256": "bd7661105cfcc92ef55473a63c9579d16cabe43eb9496715223ca4d127d17002" + }, + { + "path": "arms/anchor_post_attention_rho/compact/stage_02/metadata.json", + "size": 612, + "sha256": "5dfac79f6e1116216b3f850b193ff1155896a64a664a71ed6187686b48319da5" + }, + { + "path": "arms/anchor_post_attention_rho/compact/stage_03/anchor_layer.pt", + "size": 201866630, + "sha256": "45be15ca59218c3f13e3b271a7d6815ef8e4d37000aed2185fd6d73f53b67d58" + }, + { + "path": "arms/anchor_post_attention_rho/compact/stage_03/metadata.json", + "size": 620, + "sha256": "2b8a53b74fe5e9a16c972eb4aac183e837d95be949e06e8d43ae4d41ad775d6e" + }, + { + "path": "arms/anchor_post_attention_rho/compact/stage_04/anchor_layer.pt", + "size": 201866630, + "sha256": "9f159f56702fe5980b3ed2a2b4ef4c2d7d6ba13ea0187254acf7e9b0f7db275f" + }, + { + "path": "arms/anchor_post_attention_rho/compact/stage_04/metadata.json", + "size": 628, + "sha256": "9abbf014c55443e1c527d5cd197cecfd08dfea050ed59194345eeb23c0507bd6" + }, + { + "path": "arms/anchor_post_attention_rho/compact/stage_05/anchor_layer.pt", + "size": 201866630, + "sha256": "501e21dde214c25e0650b217ef6561c9cde1b91290cb001cd5eeede1861c6538" + }, + { + "path": "arms/anchor_post_attention_rho/compact/stage_05/metadata.json", + "size": 636, + "sha256": "9ad1a5f4152c1a555a325fedaeb88e802cbc5eb938de13a53c9f492868d7fc41" + }, + { + "path": "arms/anchor_post_attention_rho/eval/stage_01_ppl.json", + "size": 877, + "sha256": "5493ee86159d683bfa224aa87ab8fc675e5c7ce2f303d4ac052fed1d69b959f8" + }, + { + "path": "arms/anchor_post_attention_rho/eval/stage_02_ppl.json", + "size": 875, + "sha256": "15c17f05a9364808582b9fb588c2638687c382f5ce5e7f35b619d1789aebadfd" + }, + { + "path": "arms/anchor_post_attention_rho/eval/stage_03_ppl.json", + "size": 876, + "sha256": "52128e55b7e3f3d53bc1ddabb816d9eac7d068d6c0341fad0322534ce979c3ea" + }, + { + "path": "arms/anchor_post_attention_rho/eval/stage_04_ppl.json", + "size": 878, + "sha256": "7358e0ecb07a1c66d0fe15263f263302bfcc253e21f654e7129296d84c1c7021" + }, + { + "path": "arms/anchor_post_attention_rho/eval/stage_05_ppl.json", + "size": 878, + "sha256": "f4b9084d33e052876522a9d8064720f85719be7bf19b7d981f2bdd960b377e8f" + }, + { + "path": "arms/anchor_post_attention_rho/eval/stage_05_zs6.json", + "size": 3426, + "sha256": "9f78048a64f87e564cd6ed88c2294dfbb4f8f955db1235e6a50e269e6ce65eaa" + }, + { + "path": "arms/anchor_post_attention_rho/logs/eval.log", + "size": 126004, + "sha256": "b158ad7bce557b58e9208855ad49e356273135bed74f98e19ac184c5a08028c2" + }, + { + "path": "arms/anchor_post_attention_rho/logs/train.log", + "size": 33405, + "sha256": "62f05e4a04181edd12ff8475854b08920f417cf16231eae52f252d77c66000fe" + }, + { + "path": "arms/anchor_post_attention_rho/metadata/cycle_01.json", + "size": 27713, + "sha256": "7e3603bc0ec7ec79faf8f7c9e2e770b477ccb905ca2310f4e8539c0a45410622" + }, + { + "path": "arms/anchor_post_attention_rho/metadata/cycle_02.json", + "size": 27741, + "sha256": "13e2782a336c5c7c344602540021a572319609412307f72091db7f72061accb4" + }, + { + "path": "arms/anchor_post_attention_rho/metadata/cycle_03.json", + "size": 27735, + "sha256": "d56fbec691c2502f5f3a9644bec615f5b09427bc5f15abc8a877bbc0773bb20d" + }, + { + "path": "arms/anchor_post_attention_rho/metadata/cycle_04.json", + "size": 27697, + "sha256": "3317cb0ef99d5e1fb8759bc5cbba0de2475356c06d6b4d8bf27536cbf3d522fb" + }, + { + "path": "arms/anchor_post_attention_rho/metadata/cycle_05.json", + "size": 27692, + "sha256": "2f4aa6363bdd08e991e35653a75c7cecd4f633c822933831f745d06372ac8f6b" + }, + { + "path": "arms/anchor_post_attention_rho/protocol.json", + "size": 543, + "sha256": "8b20540e038167425e3bc0e3bac5941d26824b0b72cd1269f39f60d6604756e7" + }, + { + "path": "code/patches/0003-Add-anchor-contextual-Fisher-input.patch", + "size": 3164, + "sha256": "590018723d2bd3418a438ced294937d028c61d5287e875a7900b07d6196b1e25" + }, + { + "path": "code/scripts/eval_abprune_compact_zeroshot.py", + "size": 6794, + "sha256": "45cde2b7e983ea793be175d8f504e84715bbd155802ab7ff821bb839411b6d41" + }, + { + "path": "code/scripts/evaluate_abprune_compact_ppl.py", + "size": 6229, + "sha256": "4a25e2e10cac70b3f72bfba8b7e91d322638021d8bfc02caafad9ffb495bf01a" + }, + { + "path": "code/scripts/package_contextual_fisher_host.py", + "size": 5699, + "sha256": "49a496a630e2c72ade528a070300af921412a97cfa105e6a69d1d6c30113eb6d" + }, + { + "path": "code/scripts/run_contextual_fisher_4b_queue.sh", + "size": 534, + "sha256": "99a388d604740f3dc12a38682515e6b0cf4f11c1ec7bdff37c8ce09196efe871" + }, + { + "path": "code/scripts/run_contextual_fisher_matrix_2gpu.sh", + "size": 2561, + "sha256": "174c14b63273137351f141dd0f5ea116df8c3e5691b4165b68d428df93c1243d" + }, + { + "path": "code/scripts/run_contextual_fisher_variant.sh", + "size": 5889, + "sha256": "07194ad645ca08d295f2347472ef3808b20dffbe43d56cd80e00044e606d9cb4" + }, + { + "path": "code/scripts/summarize_contextual_fisher_matrix.py", + "size": 3118, + "sha256": "5321509a57ae8afa3d4331319e24370df1fcf91d11d6cee490e7cbea53f0a23a" + }, + { + "path": "code/scripts/upload_verified_hf_tree.py", + "size": 3512, + "sha256": "d2e4321eb2dd8e90b2be598ea2f61ab861c4c768f57d9ea8f27c1dc1282331b8" + }, + { + "path": "code/scripts/write_tree_manifest.py", + "size": 1498, + "sha256": "240ecd37b6786bdb2de23bc43625dd4a85de0491a2ddb05203d99d3c8c072c98" + }, + { + "path": "code/src/fusegpt/abprune_compact.py", + "size": 11571, + "sha256": "5a453e543ed1748b833960a3da8c471454607c7c09ff636d88ece49b1b324ae4" + }, + { + "path": "code/src/fusegpt/abprune_contextual_fisher.py", + "size": 8269, + "sha256": "57330a397606ff5663293e82055d04aeda0d150ad6dc6c589fd97b8df75c426c" + }, + { + "path": "code/tests/test_abprune_contextual_fisher.py", + "size": 3118, + "sha256": "ac665dd41c9c281a292deea6d25cd5b4cd35028354e15360cc413e4bc03d9d68" + }, + { + "path": "provenance.json", + "size": 579, + "sha256": "2b75eb021e01e77c0f84a9dba9ca7f1d95a52b1f68f9fb247ef609f9e3d79c58" + }, + { + "path": "summary.json", + "size": 2356, + "sha256": "d00fb3a356ed0e072a9db2a094df21f1839c0ac87907d041bdcf897c99f11422" + }, + { + "path": "summary.md", + "size": 269, + "sha256": "8c9413befd913e56f45c6ace28ef7892620f70965ac5a2a782ba0f95f02af3d7" + } + ], + "total_files": 128, + "total_bytes": 4038604911 +} diff --git a/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/provenance.json b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..990a3ce846cb2822ad0437373c58e2889e6143e0 --- /dev/null +++ b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/provenance.json @@ -0,0 +1,19 @@ +{ + "experiment": "anchor_contextual_fisher_matrix_v1", + "model_key": "4b", + "session_alias": "vast-paper_revision-context-fisher-4b-20260920", + "vast_contract": 51712821, + "hardware": "1x-A100-SXM4-40GB", + "rate_usd_per_hour": 0.7315703704, + "git_commit": "c628fbf", + "external_abprune_commit": "dbf6582c9d38cda745ee0b4eb18c33e2f9aabcec", + "full_models_omitted": true, + "compact_stage_count": 20, + "compact_bytes": 4037360916, + "variants": [ + "anchor_input_no_rho", + "anchor_input_rho", + "anchor_post_attention_no_rho", + "anchor_post_attention_rho" + ] +} diff --git a/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/summary.json b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..28747d2eef373db171f8047d76fa051db37e152c --- /dev/null +++ b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/summary.json @@ -0,0 +1,87 @@ +{ + "experiment": "anchor_contextual_fisher_matrix_v1", + "model_key": "4b", + "cycles": 5, + "rows": [ + { + "variant": "anchor_input_no_rho", + "stage_ppl": [ + 8.54561942281943, + 9.899906192368073, + 11.03480077974187, + 13.289681558091948, + 17.268860422166572 + ], + "final_ppl": 17.268860422166572, + "final_zs6": 0.5427600462247986, + "task_scores": { + "hellaswag": 0.6079466241784505, + "piqa": 0.6931447225244831, + "winogrande": 0.6708760852407262, + "arc_challenge": 0.386518771331058, + "arc_easy": 0.5740740740740741, + "openbookqa": 0.324 + } + }, + { + "variant": "anchor_input_rho", + "stage_ppl": [ + 8.438081372774594, + 9.77480645692066, + 10.719274662057657, + 12.836020408000905, + 16.339577972080622 + ], + "final_ppl": 16.339577972080622, + "final_zs6": 0.5479890119171247, + "task_scores": { + "hellaswag": 0.617307309300936, + "piqa": 0.70620239390642, + "winogrande": 0.6677190213101816, + "arc_challenge": 0.3873720136518771, + "arc_easy": 0.5833333333333334, + "openbookqa": 0.326 + } + }, + { + "variant": "anchor_post_attention_no_rho", + "stage_ppl": [ + 8.548380241074604, + 9.904639344982495, + 11.03667434024624, + 13.292236778488792, + 17.278265712672614 + ], + "final_ppl": 17.278265712672614, + "final_zs6": 0.5419276678774586, + "task_scores": { + "hellaswag": 0.6070503883688508, + "piqa": 0.6969532100108814, + "winogrande": 0.6692975532754538, + "arc_challenge": 0.3856655290102389, + "arc_easy": 0.5765993265993266, + "openbookqa": 0.316 + } + }, + { + "variant": "anchor_post_attention_rho", + "stage_ppl": [ + 8.440246786377427, + 9.68061879965598, + 10.702639815217683, + 12.815089341144557, + 16.312433454579697 + ], + "final_ppl": 16.312433454579697, + "final_zs6": 0.5496731774620703, + "task_scores": { + "hellaswag": 0.6183031268671579, + "piqa": 0.7034820457018498, + "winogrande": 0.6677190213101816, + "arc_challenge": 0.3984641638225256, + "arc_easy": 0.5820707070707071, + "openbookqa": 0.328 + } + } + ] +} diff --git a/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/summary.md b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/summary.md new file mode 100644 index 0000000000000000000000000000000000000000..d570b67bf610b2b44009391ee42359fddb4eace7 --- /dev/null +++ b/runs/vast-paper_revision-context-fisher-4b-20260920/20260920_anchor-contextual-fisher-matrix-v1/4b/summary.md @@ -0,0 +1,8 @@ +# Anchor-contextual Fisher matrix: 4b + +| Variant | Final PPL | ZS6 | +|---|---:|---:| +| anchor_input_no_rho | 17.2689 | 54.28% | +| anchor_input_rho | 16.3396 | 54.80% | +| anchor_post_attention_no_rho | 17.2783 | 54.19% | +| anchor_post_attention_rho | 16.3124 | 54.97% |