Create configs.apertus.matryoshka_k80.layer16.yaml

sae_matryoshka/layer16/apertus_mid16_k80/configs.apertus.matryoshka_k80.layer16.yaml

@@ -0,0 +1,34 @@
+collect_activations: false
+sae_train: true
+
+sae:
+  out_dir: # directory containing acts_*.npy
+  sae_save_dir: #where to save the sae 
+  sae_variant: matryoshka
+  sae_matryoshka_group_sizes: [6144,14336,30720]  # d_sae=51200 (~12.5x)
+  sae_steps: 300000
+  sae_lr: 0.001
+  sae_lr_warmup: 1000
+  sae_lr_decay_every: 20000
+  sae_lr_decay_factor: 0.9
+  sae_min_lr: 0.000001
+  sae_l1: 0.0
+  sae_lp: 1.0
+  sae_l1_warmup: 0
+  sae_batchtopk_k: 80              # Global batch top-K (same as experiment2)
+  sae_tie_weights: false
+  sae_aux_penalty: 0.03125          # 1/32, revives dead features
+  sae_top_k_aux: 128
+  sae_n_batches_to_dead: 50
+  sae_batch_size: 4096 # reduced batch size to fit into memory 
+  sae_num_workers: 4
+  sae_prefetch_factor: 2
+  sae_checkpoint_interval: 10000
+  sae_keep_checkpoints: 5
+  sae_append_timestamp: false
+  sae_resume_from: null
+  sae_auto_resume: true
+  sae_eval_interval: 2500
+  sae_eval_batches: 20
+  sae_eval_batch_size: 4096
+  sae_eval_num_workers: 2