Create configs.apertus.matching_pursuit.layer16.yaml

sae_mp/layer16/apertus_mid16_8x/configs.apertus.matching_pursuit.layer16.yaml

@@ -0,0 +1,28 @@
+collect_activations: false
+sae_train: true
+
+sae:
+  out_dir: # directory containing acts_*.npy
+  sae_save_dir: #where to save the sae 
+  sae_variant: mpsae
+  sae_expansion: 8.0            # d_sae = d_in * 8 (same as experiment2 for comparison)
+  sae_steps: 300000
+  sae_lr: 0.001
+  sae_lr_warmup: 1000
+  sae_lr_decay_every: 20000
+  sae_lr_decay_factor: 0.9
+  sae_min_lr: 0.000001
+  sae_mpsae_threshold: 5.0 #paper uses 0.05 for normalized activations, but our activations are not normalized and have higher L2 norms, so we use a higher threshold.. if sparsity to high, increase. if reconstruction to poor, decrease 
+  sae_mpsae_max_iter: 50        # safeguard: max greedy steps per forward pass
+  sae_batch_size: 4096
+  sae_num_workers: 4
+  sae_prefetch_factor: 2
+  sae_checkpoint_interval: 10000
+  sae_keep_checkpoints: 5
+  sae_append_timestamp: false
+  sae_resume_from: null
+  sae_auto_resume: true
+  sae_eval_interval: 2500
+  sae_eval_batches: 20
+  sae_eval_batch_size: 4096
+  sae_eval_num_workers: 2