Upload folder using huggingface_hub
Browse files- .gitattributes +2 -0
- batch-top-k/checkpoint_0.pt +3 -0
- batch-top-k/config.json +50 -0
- batch-top-k/dictionary_model/README.md +10 -0
- batch-top-k/dictionary_model/config.json +12 -0
- batch-top-k/dictionary_model/model.safetensors +3 -0
- batch-top-k/feature_validation.csv +0 -0
- batch-top-k/last_eval_logs.pt +3 -0
- batch-top-k/latent_metrics.csv +3 -0
- batch-top-k/loss_history.csv +0 -0
- batch-top-k/model_final.pt +3 -0
- batch-top-k/results_summary.json +102 -0
- crosscoder_run_summary.csv +2 -0
- crosscoder_run_summary.json +149 -0
- relative_decoder_norm_hist.png +3 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
batch-top-k/latent_metrics.csv filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
relative_decoder_norm_hist.png filter=lfs diff=lfs merge=lfs -text
|
batch-top-k/checkpoint_0.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:270a03f86a235b8fdc5393d1c0a4a6c37e61b8e891578bb5c8c0035cb070f166
|
| 3 |
+
size 4832309751
|
batch-top-k/config.json
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"trainer": {
|
| 3 |
+
"trainer_class": "BatchTopKCrossCoderTrainer",
|
| 4 |
+
"dict_class": "BatchTopKCrossCoder",
|
| 5 |
+
"lr": 0.001,
|
| 6 |
+
"steps": 50000,
|
| 7 |
+
"auxk_alpha": 0.03125,
|
| 8 |
+
"warmup_steps": 1000,
|
| 9 |
+
"decay_start": null,
|
| 10 |
+
"threshold_beta": 0.999,
|
| 11 |
+
"threshold_start_step": 1000,
|
| 12 |
+
"top_k_aux": 512,
|
| 13 |
+
"seed": null,
|
| 14 |
+
"activation_dim": 3072,
|
| 15 |
+
"dict_size": 98304,
|
| 16 |
+
"k": 2560,
|
| 17 |
+
"k_target": 128,
|
| 18 |
+
"k_initial": 2560,
|
| 19 |
+
"k_annealing_steps": 5000,
|
| 20 |
+
"code_normalization": "CROSSCODER",
|
| 21 |
+
"code_normalization_alpha_sae": 1.0,
|
| 22 |
+
"code_normalization_alpha_cc": 0.1,
|
| 23 |
+
"device": "cuda",
|
| 24 |
+
"layer": -1,
|
| 25 |
+
"lm_name": "paired_capture",
|
| 26 |
+
"wandb_name": "crosscoder_batch-top-k",
|
| 27 |
+
"submodule_name": null,
|
| 28 |
+
"dict_class_kwargs": {},
|
| 29 |
+
"target_rms": 1.0
|
| 30 |
+
},
|
| 31 |
+
"buffer": {
|
| 32 |
+
"type": "Subset(PairedActivationCache)",
|
| 33 |
+
"n_total": 30172167,
|
| 34 |
+
"n_selected": 30172167,
|
| 35 |
+
"n_train": 27154950,
|
| 36 |
+
"n_val": 3017217,
|
| 37 |
+
"val_split": 0.1,
|
| 38 |
+
"batch_size": 2046,
|
| 39 |
+
"drop_last": true,
|
| 40 |
+
"shuffle": false,
|
| 41 |
+
"random_sampling": false,
|
| 42 |
+
"seed": 42,
|
| 43 |
+
"split": "train",
|
| 44 |
+
"cycle": true,
|
| 45 |
+
"batch_transform": {
|
| 46 |
+
"sanitize_activation_nans": true,
|
| 47 |
+
"normalize_per_model": false
|
| 48 |
+
}
|
| 49 |
+
}
|
| 50 |
+
}
|
batch-top-k/dictionary_model/README.md
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
tags:
|
| 3 |
+
- model_hub_mixin
|
| 4 |
+
- pytorch_model_hub_mixin
|
| 5 |
+
---
|
| 6 |
+
|
| 7 |
+
This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration:
|
| 8 |
+
- Code: [More Information Needed]
|
| 9 |
+
- Paper: [More Information Needed]
|
| 10 |
+
- Docs: [More Information Needed]
|
batch-top-k/dictionary_model/config.json
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"activation_dim": 3072,
|
| 3 |
+
"activation_mean": null,
|
| 4 |
+
"activation_std": null,
|
| 5 |
+
"code_normalization": "CROSSCODER",
|
| 6 |
+
"dict_class": "BatchTopKCrossCoder",
|
| 7 |
+
"dict_size": 98304,
|
| 8 |
+
"k": 2560,
|
| 9 |
+
"norm_init_scale": 1.0,
|
| 10 |
+
"num_layers": 2,
|
| 11 |
+
"target_rms": 1.0
|
| 12 |
+
}
|
batch-top-k/dictionary_model/model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:04a78ee8650ef5b297925f192e15a80c76b6c3370610b8b2356efd1f44d07096
|
| 3 |
+
size 4832306068
|
batch-top-k/feature_validation.csv
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
batch-top-k/last_eval_logs.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:13b2478c6d6c9a39c0e1a43c4048e71874b6cb96f4def7743def320f0429ee40
|
| 3 |
+
size 2336
|
batch-top-k/latent_metrics.csv
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d9d5e9978147ffd4136507491a951e496981bf1fa37c37e2ad6168340e033148
|
| 3 |
+
size 11096809
|
batch-top-k/loss_history.csv
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
batch-top-k/model_final.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9bbdec596e814b646a0a932760fa1358c3dccf19a581eef9b1cbcfafff3f9abb
|
| 3 |
+
size 4832309734
|
batch-top-k/results_summary.json
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"capture_dir": "/data/twibiral/projects/sparseautoencoders/experiment_data/crosscoder_activations_base_vs_coding_l20_prompt_response_all_1m_subsampled",
|
| 3 |
+
"model_1_dir": "/data/twibiral/projects/sparseautoencoders/experiment_data/crosscoder_activations_base_vs_coding_l20_prompt_response_all_1m_subsampled/model_1__meta-llama__Llama-3.2-3B",
|
| 4 |
+
"model_2_dir": "/data/twibiral/projects/sparseautoencoders/experiment_data/crosscoder_activations_base_vs_coding_l20_prompt_response_all_1m_subsampled/model_2__MergeBench__Llama-3.2-3B_coding",
|
| 5 |
+
"submodule_name": "model_layers_idx20_out",
|
| 6 |
+
"seed": 42,
|
| 7 |
+
"split_seed": 42,
|
| 8 |
+
"n_tokens_available": 30172167,
|
| 9 |
+
"n_tokens_selected": 30172167,
|
| 10 |
+
"n_tokens_used": 27154950,
|
| 11 |
+
"n_tokens_val": 3017217,
|
| 12 |
+
"layer_index": null,
|
| 13 |
+
"diffing_toolkit_note": "Uses the same crosscoder variants as diffing-toolkit (relu and batch-top-k) via dictionary_learning trainers.",
|
| 14 |
+
"training": {
|
| 15 |
+
"variant": "batch-top-k",
|
| 16 |
+
"steps": 50000,
|
| 17 |
+
"batch_size": 2046,
|
| 18 |
+
"random_sampling": false,
|
| 19 |
+
"train_samples": 0,
|
| 20 |
+
"val_split": 0.1,
|
| 21 |
+
"activation_dim": 3072,
|
| 22 |
+
"dict_size": 98304,
|
| 23 |
+
"device": "cuda",
|
| 24 |
+
"lr": 0.001,
|
| 25 |
+
"warmup_steps": 1000,
|
| 26 |
+
"l1_penalty": null,
|
| 27 |
+
"topk": 128,
|
| 28 |
+
"topk_anneal_multiplier": 20.0,
|
| 29 |
+
"topk_anneal_steps": 5000,
|
| 30 |
+
"loss_track_every": 100,
|
| 31 |
+
"loss_history_path": "/data/twibiral/projects/sparseautoencoders/experiment_data/diffing_base_vs_coding_l20_prompt_response_all_1m/batch-top-k/loss_history.csv",
|
| 32 |
+
"auxk_alpha": 0.03125,
|
| 33 |
+
"auxk_topk": 512,
|
| 34 |
+
"cycle_train_loader": true,
|
| 35 |
+
"train_batches_per_epoch": 13272,
|
| 36 |
+
"effective_optimizer_steps": 50000
|
| 37 |
+
},
|
| 38 |
+
"preprocessing": {
|
| 39 |
+
"sanitize_activation_nans": true,
|
| 40 |
+
"normalize_per_model": false
|
| 41 |
+
},
|
| 42 |
+
"feature_thresholds": {
|
| 43 |
+
"shared_low": 0.4,
|
| 44 |
+
"shared_high": 0.6,
|
| 45 |
+
"exclusive_threshold": 0.1
|
| 46 |
+
},
|
| 47 |
+
"summary": {
|
| 48 |
+
"n_latents": 98304,
|
| 49 |
+
"relative_decoder_norm_mean": -0.0017445944249629974,
|
| 50 |
+
"relative_decoder_norm_std": 0.014968588016927242,
|
| 51 |
+
"relative_decoder_norm_q10": -0.011190600413829088,
|
| 52 |
+
"relative_decoder_norm_q50": -0.00128219707403332,
|
| 53 |
+
"relative_decoder_norm_q90": 0.006832759175449608,
|
| 54 |
+
"ratio_model2_over_model1_mean": 0.9969738125801086,
|
| 55 |
+
"ratio_model2_over_model1_median": 0.997438907623291,
|
| 56 |
+
"ratio_model2_over_model1_q90": 1.0137595295906068,
|
| 57 |
+
"decoder_cosine_mean": 0.8656144142150879,
|
| 58 |
+
"decoder_cosine_median": 0.8902400135993958,
|
| 59 |
+
"shared_decoder_cosine_mean": 0.8660304546356201,
|
| 60 |
+
"n_shared": 98242,
|
| 61 |
+
"n_model_1_specific": 0,
|
| 62 |
+
"n_model_2_specific": 0,
|
| 63 |
+
"n_mixed": 62
|
| 64 |
+
},
|
| 65 |
+
"feature_validation": {
|
| 66 |
+
"validation_tokens": 130944,
|
| 67 |
+
"mean_l0": 132.69653439638319,
|
| 68 |
+
"mean_activation_frequency": 0.0013498589517861245,
|
| 69 |
+
"mean_concentration_proxy": 0.0017056000798971137,
|
| 70 |
+
"class_summary": {
|
| 71 |
+
"shared": {
|
| 72 |
+
"n_latents": 98242,
|
| 73 |
+
"activation_frequency_mean": 0.0013429196839579592,
|
| 74 |
+
"activation_frequency_q90": 0.0,
|
| 75 |
+
"concentration_proxy_mean": 0.0017056824740553306,
|
| 76 |
+
"concentration_proxy_q90": 0.0
|
| 77 |
+
},
|
| 78 |
+
"model_1_specific": {
|
| 79 |
+
"n_latents": 0,
|
| 80 |
+
"activation_frequency_mean": NaN,
|
| 81 |
+
"activation_frequency_q90": NaN,
|
| 82 |
+
"concentration_proxy_mean": NaN,
|
| 83 |
+
"concentration_proxy_q90": NaN
|
| 84 |
+
},
|
| 85 |
+
"model_2_specific": {
|
| 86 |
+
"n_latents": 0,
|
| 87 |
+
"activation_frequency_mean": NaN,
|
| 88 |
+
"activation_frequency_q90": NaN,
|
| 89 |
+
"concentration_proxy_mean": NaN,
|
| 90 |
+
"concentration_proxy_q90": NaN
|
| 91 |
+
},
|
| 92 |
+
"mixed": {
|
| 93 |
+
"n_latents": 62,
|
| 94 |
+
"activation_frequency_mean": 0.012345464596537696,
|
| 95 |
+
"activation_frequency_q90": 0.02664726906158357,
|
| 96 |
+
"concentration_proxy_mean": 0.0015750425493886691,
|
| 97 |
+
"concentration_proxy_q90": 0.00204309951219652
|
| 98 |
+
}
|
| 99 |
+
}
|
| 100 |
+
},
|
| 101 |
+
"feature_validation_path": "/data/twibiral/projects/sparseautoencoders/experiment_data/diffing_base_vs_coding_l20_prompt_response_all_1m/batch-top-k/feature_validation.csv"
|
| 102 |
+
}
|
crosscoder_run_summary.csv
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
variant,n_latents,relative_decoder_norm_mean,relative_decoder_norm_std,relative_decoder_norm_q10,relative_decoder_norm_q50,relative_decoder_norm_q90,ratio_model2_over_model1_mean,ratio_model2_over_model1_median,ratio_model2_over_model1_q90,decoder_cosine_mean,decoder_cosine_median,shared_decoder_cosine_mean,n_shared,n_model_1_specific,n_model_2_specific,n_mixed,n_tokens_used,n_tokens_val,n_tokens_selected,dict_size,activation_dim
|
| 2 |
+
batch-top-k,98304,-0.0017445944249629974,0.014968588016927242,-0.011190600413829088,-0.00128219707403332,0.006832759175449608,0.9969738125801086,0.997438907623291,1.0137595295906068,0.8656144142150879,0.8902400135993958,0.8660304546356201,98242,0,0,62,27154950,3017217,30172167,98304,3072
|
crosscoder_run_summary.json
ADDED
|
@@ -0,0 +1,149 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"capture_dir": "/data/twibiral/projects/sparseautoencoders/experiment_data/crosscoder_activations_base_vs_coding_l20_prompt_response_all_1m_subsampled",
|
| 3 |
+
"submodule_name": "model_layers_idx20_out",
|
| 4 |
+
"variants": [
|
| 5 |
+
"batch-top-k"
|
| 6 |
+
],
|
| 7 |
+
"activation_dim": 3072,
|
| 8 |
+
"dict_size": 98304,
|
| 9 |
+
"seed": 42,
|
| 10 |
+
"split_seed": 42,
|
| 11 |
+
"val_split": 0.1,
|
| 12 |
+
"random_sampling": false,
|
| 13 |
+
"n_tokens_selected": 30172167,
|
| 14 |
+
"n_tokens_train": 27154950,
|
| 15 |
+
"n_tokens_val": 3017217,
|
| 16 |
+
"train_batches_per_epoch": 13272,
|
| 17 |
+
"cycle_train_loader": true,
|
| 18 |
+
"effective_optimizer_steps": 50000,
|
| 19 |
+
"preprocessing": {
|
| 20 |
+
"sanitize_activation_nans": true,
|
| 21 |
+
"normalize_per_model": false
|
| 22 |
+
},
|
| 23 |
+
"raw_run_config": {
|
| 24 |
+
"workflow": "capture_only_paired_models",
|
| 25 |
+
"pair": {
|
| 26 |
+
"name": "llama32_3b_base_vs_coding",
|
| 27 |
+
"model_1": "meta-llama/Llama-3.2-3B",
|
| 28 |
+
"model_2": "MergeBench/Llama-3.2-3B_coding",
|
| 29 |
+
"type": "generic",
|
| 30 |
+
"family": null,
|
| 31 |
+
"checkpoint_1": null,
|
| 32 |
+
"checkpoint_2": null
|
| 33 |
+
},
|
| 34 |
+
"dataset": {
|
| 35 |
+
"name": "allenai/Dolci-DPO-Model-Response-Pool",
|
| 36 |
+
"config": "gpt-4.1-2025-04-14",
|
| 37 |
+
"split": "train",
|
| 38 |
+
"response_model": "gpt-4.1-2025-04-14",
|
| 39 |
+
"text_mode": "prompt_response",
|
| 40 |
+
"max_samples": 1000000,
|
| 41 |
+
"use_full_dataset": false,
|
| 42 |
+
"selected_samples": 997147,
|
| 43 |
+
"selected_samples_before_chunk": 997147,
|
| 44 |
+
"shuffle_seed": 42,
|
| 45 |
+
"streaming": true,
|
| 46 |
+
"chunking": {
|
| 47 |
+
"enabled": false,
|
| 48 |
+
"chunk_size": null,
|
| 49 |
+
"chunk_nr": null,
|
| 50 |
+
"chunk_start": 0,
|
| 51 |
+
"chunk_end": 997147,
|
| 52 |
+
"total_chunks": 1,
|
| 53 |
+
"selected_samples_before_chunk": 997147,
|
| 54 |
+
"selected_samples_after_chunk": 997147
|
| 55 |
+
},
|
| 56 |
+
"coding_filter": {
|
| 57 |
+
"coding_only": false,
|
| 58 |
+
"annotations_csv": null,
|
| 59 |
+
"coding_domains": [],
|
| 60 |
+
"annotation_rows_total": 0,
|
| 61 |
+
"annotation_rows_matching_domain": 0,
|
| 62 |
+
"annotation_prompts_unique": 0,
|
| 63 |
+
"coding_lookup_checked": 0,
|
| 64 |
+
"coding_lookup_filtered_out": 0,
|
| 65 |
+
"coding_lookup_kept": 0,
|
| 66 |
+
"coding_lookup_enabled": false
|
| 67 |
+
}
|
| 68 |
+
},
|
| 69 |
+
"collection": {
|
| 70 |
+
"context_len": 512,
|
| 71 |
+
"batch_size": 512,
|
| 72 |
+
"shard_size": 200000,
|
| 73 |
+
"dtype": "bfloat16",
|
| 74 |
+
"token_selection": "all",
|
| 75 |
+
"output_dir": "experiment_data/crosscoder_activations_base_vs_coding_l20_prompt_response_all_1m_subsampled"
|
| 76 |
+
},
|
| 77 |
+
"models": [
|
| 78 |
+
{
|
| 79 |
+
"model_slot": "model_1",
|
| 80 |
+
"model_name_or_path": "meta-llama/Llama-3.2-3B",
|
| 81 |
+
"layer_path": "model.layers",
|
| 82 |
+
"layer_index": 20,
|
| 83 |
+
"hidden_size": 3072,
|
| 84 |
+
"cache_submodule_name": "model_layers_idx20",
|
| 85 |
+
"output_dir": "experiment_data/crosscoder_activations_base_vs_coding_l20_prompt_response_all_1m_subsampled/model_1__meta-llama__Llama-3.2-3B",
|
| 86 |
+
"cache_state": "explicit_overwrite",
|
| 87 |
+
"token_selection": {
|
| 88 |
+
"token_selection": "all",
|
| 89 |
+
"n_tokens_before": 400706696,
|
| 90 |
+
"n_tokens_after": 30172167,
|
| 91 |
+
"n_sequences": 997147
|
| 92 |
+
}
|
| 93 |
+
},
|
| 94 |
+
{
|
| 95 |
+
"model_slot": "model_2",
|
| 96 |
+
"model_name_or_path": "MergeBench/Llama-3.2-3B_coding",
|
| 97 |
+
"layer_path": "model.layers",
|
| 98 |
+
"layer_index": 20,
|
| 99 |
+
"hidden_size": 3072,
|
| 100 |
+
"cache_submodule_name": "model_layers_idx20",
|
| 101 |
+
"output_dir": "experiment_data/crosscoder_activations_base_vs_coding_l20_prompt_response_all_1m_subsampled/model_2__MergeBench__Llama-3.2-3B_coding",
|
| 102 |
+
"cache_state": "explicit_overwrite",
|
| 103 |
+
"token_selection": {
|
| 104 |
+
"token_selection": "all",
|
| 105 |
+
"n_tokens_before": 400706696,
|
| 106 |
+
"n_tokens_after": 30172167,
|
| 107 |
+
"n_sequences": 997147
|
| 108 |
+
}
|
| 109 |
+
}
|
| 110 |
+
],
|
| 111 |
+
"subsampling": {
|
| 112 |
+
"source_run_dir": "/data/twibiral/projects/sparseautoencoders/experiment_data/crosscoder_activations_base_vs_coding_l20_prompt_response_all_1m",
|
| 113 |
+
"drop_first_n_tokens": 100,
|
| 114 |
+
"portion_tokens_to_keep": 0.1,
|
| 115 |
+
"seed": 42,
|
| 116 |
+
"target_dtype": "torch.bfloat16",
|
| 117 |
+
"n_tokens_before": 400706696,
|
| 118 |
+
"n_tokens_after": 30172167,
|
| 119 |
+
"sample_metadata_csv": "sample_subsample_metadata.csv",
|
| 120 |
+
"notes": "Use sequence_ranges.pt to map token rows to sample_id; sample_subsample_metadata.csv adds sample labels."
|
| 121 |
+
}
|
| 122 |
+
},
|
| 123 |
+
"results": [
|
| 124 |
+
{
|
| 125 |
+
"variant": "batch-top-k",
|
| 126 |
+
"n_latents": 98304,
|
| 127 |
+
"relative_decoder_norm_mean": -0.0017445944249629974,
|
| 128 |
+
"relative_decoder_norm_std": 0.014968588016927242,
|
| 129 |
+
"relative_decoder_norm_q10": -0.011190600413829088,
|
| 130 |
+
"relative_decoder_norm_q50": -0.00128219707403332,
|
| 131 |
+
"relative_decoder_norm_q90": 0.006832759175449608,
|
| 132 |
+
"ratio_model2_over_model1_mean": 0.9969738125801086,
|
| 133 |
+
"ratio_model2_over_model1_median": 0.997438907623291,
|
| 134 |
+
"ratio_model2_over_model1_q90": 1.0137595295906068,
|
| 135 |
+
"decoder_cosine_mean": 0.8656144142150879,
|
| 136 |
+
"decoder_cosine_median": 0.8902400135993958,
|
| 137 |
+
"shared_decoder_cosine_mean": 0.8660304546356201,
|
| 138 |
+
"n_shared": 98242,
|
| 139 |
+
"n_model_1_specific": 0,
|
| 140 |
+
"n_model_2_specific": 0,
|
| 141 |
+
"n_mixed": 62,
|
| 142 |
+
"n_tokens_used": 27154950,
|
| 143 |
+
"n_tokens_val": 3017217,
|
| 144 |
+
"n_tokens_selected": 30172167,
|
| 145 |
+
"dict_size": 98304,
|
| 146 |
+
"activation_dim": 3072
|
| 147 |
+
}
|
| 148 |
+
]
|
| 149 |
+
}
|
relative_decoder_norm_hist.png
ADDED
|
Git LFS Details
|