twibiral commited on
Commit
a33acc4
·
verified ·
1 Parent(s): a3a31c1

Upload folder using huggingface_hub

Browse files
.gitattributes CHANGED
@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ batch-top-k/latent_metrics.csv filter=lfs diff=lfs merge=lfs -text
37
+ relative_decoder_norm_hist.png filter=lfs diff=lfs merge=lfs -text
batch-top-k/checkpoint_0.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:270a03f86a235b8fdc5393d1c0a4a6c37e61b8e891578bb5c8c0035cb070f166
3
+ size 4832309751
batch-top-k/config.json ADDED
@@ -0,0 +1,50 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "trainer": {
3
+ "trainer_class": "BatchTopKCrossCoderTrainer",
4
+ "dict_class": "BatchTopKCrossCoder",
5
+ "lr": 0.001,
6
+ "steps": 50000,
7
+ "auxk_alpha": 0.03125,
8
+ "warmup_steps": 1000,
9
+ "decay_start": null,
10
+ "threshold_beta": 0.999,
11
+ "threshold_start_step": 1000,
12
+ "top_k_aux": 512,
13
+ "seed": null,
14
+ "activation_dim": 3072,
15
+ "dict_size": 98304,
16
+ "k": 2560,
17
+ "k_target": 128,
18
+ "k_initial": 2560,
19
+ "k_annealing_steps": 5000,
20
+ "code_normalization": "CROSSCODER",
21
+ "code_normalization_alpha_sae": 1.0,
22
+ "code_normalization_alpha_cc": 0.1,
23
+ "device": "cuda",
24
+ "layer": -1,
25
+ "lm_name": "paired_capture",
26
+ "wandb_name": "crosscoder_batch-top-k",
27
+ "submodule_name": null,
28
+ "dict_class_kwargs": {},
29
+ "target_rms": 1.0
30
+ },
31
+ "buffer": {
32
+ "type": "Subset(PairedActivationCache)",
33
+ "n_total": 30172167,
34
+ "n_selected": 30172167,
35
+ "n_train": 27154950,
36
+ "n_val": 3017217,
37
+ "val_split": 0.1,
38
+ "batch_size": 2046,
39
+ "drop_last": true,
40
+ "shuffle": false,
41
+ "random_sampling": false,
42
+ "seed": 42,
43
+ "split": "train",
44
+ "cycle": true,
45
+ "batch_transform": {
46
+ "sanitize_activation_nans": true,
47
+ "normalize_per_model": false
48
+ }
49
+ }
50
+ }
batch-top-k/dictionary_model/README.md ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ tags:
3
+ - model_hub_mixin
4
+ - pytorch_model_hub_mixin
5
+ ---
6
+
7
+ This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration:
8
+ - Code: [More Information Needed]
9
+ - Paper: [More Information Needed]
10
+ - Docs: [More Information Needed]
batch-top-k/dictionary_model/config.json ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "activation_dim": 3072,
3
+ "activation_mean": null,
4
+ "activation_std": null,
5
+ "code_normalization": "CROSSCODER",
6
+ "dict_class": "BatchTopKCrossCoder",
7
+ "dict_size": 98304,
8
+ "k": 2560,
9
+ "norm_init_scale": 1.0,
10
+ "num_layers": 2,
11
+ "target_rms": 1.0
12
+ }
batch-top-k/dictionary_model/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:04a78ee8650ef5b297925f192e15a80c76b6c3370610b8b2356efd1f44d07096
3
+ size 4832306068
batch-top-k/feature_validation.csv ADDED
The diff for this file is too large to render. See raw diff
 
batch-top-k/last_eval_logs.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:13b2478c6d6c9a39c0e1a43c4048e71874b6cb96f4def7743def320f0429ee40
3
+ size 2336
batch-top-k/latent_metrics.csv ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d9d5e9978147ffd4136507491a951e496981bf1fa37c37e2ad6168340e033148
3
+ size 11096809
batch-top-k/loss_history.csv ADDED
The diff for this file is too large to render. See raw diff
 
batch-top-k/model_final.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9bbdec596e814b646a0a932760fa1358c3dccf19a581eef9b1cbcfafff3f9abb
3
+ size 4832309734
batch-top-k/results_summary.json ADDED
@@ -0,0 +1,102 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "capture_dir": "/data/twibiral/projects/sparseautoencoders/experiment_data/crosscoder_activations_base_vs_coding_l20_prompt_response_all_1m_subsampled",
3
+ "model_1_dir": "/data/twibiral/projects/sparseautoencoders/experiment_data/crosscoder_activations_base_vs_coding_l20_prompt_response_all_1m_subsampled/model_1__meta-llama__Llama-3.2-3B",
4
+ "model_2_dir": "/data/twibiral/projects/sparseautoencoders/experiment_data/crosscoder_activations_base_vs_coding_l20_prompt_response_all_1m_subsampled/model_2__MergeBench__Llama-3.2-3B_coding",
5
+ "submodule_name": "model_layers_idx20_out",
6
+ "seed": 42,
7
+ "split_seed": 42,
8
+ "n_tokens_available": 30172167,
9
+ "n_tokens_selected": 30172167,
10
+ "n_tokens_used": 27154950,
11
+ "n_tokens_val": 3017217,
12
+ "layer_index": null,
13
+ "diffing_toolkit_note": "Uses the same crosscoder variants as diffing-toolkit (relu and batch-top-k) via dictionary_learning trainers.",
14
+ "training": {
15
+ "variant": "batch-top-k",
16
+ "steps": 50000,
17
+ "batch_size": 2046,
18
+ "random_sampling": false,
19
+ "train_samples": 0,
20
+ "val_split": 0.1,
21
+ "activation_dim": 3072,
22
+ "dict_size": 98304,
23
+ "device": "cuda",
24
+ "lr": 0.001,
25
+ "warmup_steps": 1000,
26
+ "l1_penalty": null,
27
+ "topk": 128,
28
+ "topk_anneal_multiplier": 20.0,
29
+ "topk_anneal_steps": 5000,
30
+ "loss_track_every": 100,
31
+ "loss_history_path": "/data/twibiral/projects/sparseautoencoders/experiment_data/diffing_base_vs_coding_l20_prompt_response_all_1m/batch-top-k/loss_history.csv",
32
+ "auxk_alpha": 0.03125,
33
+ "auxk_topk": 512,
34
+ "cycle_train_loader": true,
35
+ "train_batches_per_epoch": 13272,
36
+ "effective_optimizer_steps": 50000
37
+ },
38
+ "preprocessing": {
39
+ "sanitize_activation_nans": true,
40
+ "normalize_per_model": false
41
+ },
42
+ "feature_thresholds": {
43
+ "shared_low": 0.4,
44
+ "shared_high": 0.6,
45
+ "exclusive_threshold": 0.1
46
+ },
47
+ "summary": {
48
+ "n_latents": 98304,
49
+ "relative_decoder_norm_mean": -0.0017445944249629974,
50
+ "relative_decoder_norm_std": 0.014968588016927242,
51
+ "relative_decoder_norm_q10": -0.011190600413829088,
52
+ "relative_decoder_norm_q50": -0.00128219707403332,
53
+ "relative_decoder_norm_q90": 0.006832759175449608,
54
+ "ratio_model2_over_model1_mean": 0.9969738125801086,
55
+ "ratio_model2_over_model1_median": 0.997438907623291,
56
+ "ratio_model2_over_model1_q90": 1.0137595295906068,
57
+ "decoder_cosine_mean": 0.8656144142150879,
58
+ "decoder_cosine_median": 0.8902400135993958,
59
+ "shared_decoder_cosine_mean": 0.8660304546356201,
60
+ "n_shared": 98242,
61
+ "n_model_1_specific": 0,
62
+ "n_model_2_specific": 0,
63
+ "n_mixed": 62
64
+ },
65
+ "feature_validation": {
66
+ "validation_tokens": 130944,
67
+ "mean_l0": 132.69653439638319,
68
+ "mean_activation_frequency": 0.0013498589517861245,
69
+ "mean_concentration_proxy": 0.0017056000798971137,
70
+ "class_summary": {
71
+ "shared": {
72
+ "n_latents": 98242,
73
+ "activation_frequency_mean": 0.0013429196839579592,
74
+ "activation_frequency_q90": 0.0,
75
+ "concentration_proxy_mean": 0.0017056824740553306,
76
+ "concentration_proxy_q90": 0.0
77
+ },
78
+ "model_1_specific": {
79
+ "n_latents": 0,
80
+ "activation_frequency_mean": NaN,
81
+ "activation_frequency_q90": NaN,
82
+ "concentration_proxy_mean": NaN,
83
+ "concentration_proxy_q90": NaN
84
+ },
85
+ "model_2_specific": {
86
+ "n_latents": 0,
87
+ "activation_frequency_mean": NaN,
88
+ "activation_frequency_q90": NaN,
89
+ "concentration_proxy_mean": NaN,
90
+ "concentration_proxy_q90": NaN
91
+ },
92
+ "mixed": {
93
+ "n_latents": 62,
94
+ "activation_frequency_mean": 0.012345464596537696,
95
+ "activation_frequency_q90": 0.02664726906158357,
96
+ "concentration_proxy_mean": 0.0015750425493886691,
97
+ "concentration_proxy_q90": 0.00204309951219652
98
+ }
99
+ }
100
+ },
101
+ "feature_validation_path": "/data/twibiral/projects/sparseautoencoders/experiment_data/diffing_base_vs_coding_l20_prompt_response_all_1m/batch-top-k/feature_validation.csv"
102
+ }
crosscoder_run_summary.csv ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ variant,n_latents,relative_decoder_norm_mean,relative_decoder_norm_std,relative_decoder_norm_q10,relative_decoder_norm_q50,relative_decoder_norm_q90,ratio_model2_over_model1_mean,ratio_model2_over_model1_median,ratio_model2_over_model1_q90,decoder_cosine_mean,decoder_cosine_median,shared_decoder_cosine_mean,n_shared,n_model_1_specific,n_model_2_specific,n_mixed,n_tokens_used,n_tokens_val,n_tokens_selected,dict_size,activation_dim
2
+ batch-top-k,98304,-0.0017445944249629974,0.014968588016927242,-0.011190600413829088,-0.00128219707403332,0.006832759175449608,0.9969738125801086,0.997438907623291,1.0137595295906068,0.8656144142150879,0.8902400135993958,0.8660304546356201,98242,0,0,62,27154950,3017217,30172167,98304,3072
crosscoder_run_summary.json ADDED
@@ -0,0 +1,149 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "capture_dir": "/data/twibiral/projects/sparseautoencoders/experiment_data/crosscoder_activations_base_vs_coding_l20_prompt_response_all_1m_subsampled",
3
+ "submodule_name": "model_layers_idx20_out",
4
+ "variants": [
5
+ "batch-top-k"
6
+ ],
7
+ "activation_dim": 3072,
8
+ "dict_size": 98304,
9
+ "seed": 42,
10
+ "split_seed": 42,
11
+ "val_split": 0.1,
12
+ "random_sampling": false,
13
+ "n_tokens_selected": 30172167,
14
+ "n_tokens_train": 27154950,
15
+ "n_tokens_val": 3017217,
16
+ "train_batches_per_epoch": 13272,
17
+ "cycle_train_loader": true,
18
+ "effective_optimizer_steps": 50000,
19
+ "preprocessing": {
20
+ "sanitize_activation_nans": true,
21
+ "normalize_per_model": false
22
+ },
23
+ "raw_run_config": {
24
+ "workflow": "capture_only_paired_models",
25
+ "pair": {
26
+ "name": "llama32_3b_base_vs_coding",
27
+ "model_1": "meta-llama/Llama-3.2-3B",
28
+ "model_2": "MergeBench/Llama-3.2-3B_coding",
29
+ "type": "generic",
30
+ "family": null,
31
+ "checkpoint_1": null,
32
+ "checkpoint_2": null
33
+ },
34
+ "dataset": {
35
+ "name": "allenai/Dolci-DPO-Model-Response-Pool",
36
+ "config": "gpt-4.1-2025-04-14",
37
+ "split": "train",
38
+ "response_model": "gpt-4.1-2025-04-14",
39
+ "text_mode": "prompt_response",
40
+ "max_samples": 1000000,
41
+ "use_full_dataset": false,
42
+ "selected_samples": 997147,
43
+ "selected_samples_before_chunk": 997147,
44
+ "shuffle_seed": 42,
45
+ "streaming": true,
46
+ "chunking": {
47
+ "enabled": false,
48
+ "chunk_size": null,
49
+ "chunk_nr": null,
50
+ "chunk_start": 0,
51
+ "chunk_end": 997147,
52
+ "total_chunks": 1,
53
+ "selected_samples_before_chunk": 997147,
54
+ "selected_samples_after_chunk": 997147
55
+ },
56
+ "coding_filter": {
57
+ "coding_only": false,
58
+ "annotations_csv": null,
59
+ "coding_domains": [],
60
+ "annotation_rows_total": 0,
61
+ "annotation_rows_matching_domain": 0,
62
+ "annotation_prompts_unique": 0,
63
+ "coding_lookup_checked": 0,
64
+ "coding_lookup_filtered_out": 0,
65
+ "coding_lookup_kept": 0,
66
+ "coding_lookup_enabled": false
67
+ }
68
+ },
69
+ "collection": {
70
+ "context_len": 512,
71
+ "batch_size": 512,
72
+ "shard_size": 200000,
73
+ "dtype": "bfloat16",
74
+ "token_selection": "all",
75
+ "output_dir": "experiment_data/crosscoder_activations_base_vs_coding_l20_prompt_response_all_1m_subsampled"
76
+ },
77
+ "models": [
78
+ {
79
+ "model_slot": "model_1",
80
+ "model_name_or_path": "meta-llama/Llama-3.2-3B",
81
+ "layer_path": "model.layers",
82
+ "layer_index": 20,
83
+ "hidden_size": 3072,
84
+ "cache_submodule_name": "model_layers_idx20",
85
+ "output_dir": "experiment_data/crosscoder_activations_base_vs_coding_l20_prompt_response_all_1m_subsampled/model_1__meta-llama__Llama-3.2-3B",
86
+ "cache_state": "explicit_overwrite",
87
+ "token_selection": {
88
+ "token_selection": "all",
89
+ "n_tokens_before": 400706696,
90
+ "n_tokens_after": 30172167,
91
+ "n_sequences": 997147
92
+ }
93
+ },
94
+ {
95
+ "model_slot": "model_2",
96
+ "model_name_or_path": "MergeBench/Llama-3.2-3B_coding",
97
+ "layer_path": "model.layers",
98
+ "layer_index": 20,
99
+ "hidden_size": 3072,
100
+ "cache_submodule_name": "model_layers_idx20",
101
+ "output_dir": "experiment_data/crosscoder_activations_base_vs_coding_l20_prompt_response_all_1m_subsampled/model_2__MergeBench__Llama-3.2-3B_coding",
102
+ "cache_state": "explicit_overwrite",
103
+ "token_selection": {
104
+ "token_selection": "all",
105
+ "n_tokens_before": 400706696,
106
+ "n_tokens_after": 30172167,
107
+ "n_sequences": 997147
108
+ }
109
+ }
110
+ ],
111
+ "subsampling": {
112
+ "source_run_dir": "/data/twibiral/projects/sparseautoencoders/experiment_data/crosscoder_activations_base_vs_coding_l20_prompt_response_all_1m",
113
+ "drop_first_n_tokens": 100,
114
+ "portion_tokens_to_keep": 0.1,
115
+ "seed": 42,
116
+ "target_dtype": "torch.bfloat16",
117
+ "n_tokens_before": 400706696,
118
+ "n_tokens_after": 30172167,
119
+ "sample_metadata_csv": "sample_subsample_metadata.csv",
120
+ "notes": "Use sequence_ranges.pt to map token rows to sample_id; sample_subsample_metadata.csv adds sample labels."
121
+ }
122
+ },
123
+ "results": [
124
+ {
125
+ "variant": "batch-top-k",
126
+ "n_latents": 98304,
127
+ "relative_decoder_norm_mean": -0.0017445944249629974,
128
+ "relative_decoder_norm_std": 0.014968588016927242,
129
+ "relative_decoder_norm_q10": -0.011190600413829088,
130
+ "relative_decoder_norm_q50": -0.00128219707403332,
131
+ "relative_decoder_norm_q90": 0.006832759175449608,
132
+ "ratio_model2_over_model1_mean": 0.9969738125801086,
133
+ "ratio_model2_over_model1_median": 0.997438907623291,
134
+ "ratio_model2_over_model1_q90": 1.0137595295906068,
135
+ "decoder_cosine_mean": 0.8656144142150879,
136
+ "decoder_cosine_median": 0.8902400135993958,
137
+ "shared_decoder_cosine_mean": 0.8660304546356201,
138
+ "n_shared": 98242,
139
+ "n_model_1_specific": 0,
140
+ "n_model_2_specific": 0,
141
+ "n_mixed": 62,
142
+ "n_tokens_used": 27154950,
143
+ "n_tokens_val": 3017217,
144
+ "n_tokens_selected": 30172167,
145
+ "dict_size": 98304,
146
+ "activation_dim": 3072
147
+ }
148
+ ]
149
+ }
relative_decoder_norm_hist.png ADDED

Git LFS Details

  • SHA256: e301f9d86f390c1c693db74f6b3e98896dac8077f5dd865ca9e1306bff19d4c9
  • Pointer size: 131 Bytes
  • Size of remote file: 122 kB