| model_name: meta-llama/Llama-3.2-1B | |
| glp_kwargs: | |
| normalizer_config: | |
| rep_statistic: ./data/llama1b-layer07-fineweb-1M/rep_statistics.pt | |
| denoiser_config: | |
| d_input: 2048 | |
| d_model: 4096 | |
| d_mlp: 8192 | |
| n_layers: 3 | |
| multi_layer_n_layers: null | |
| tracedict_config: | |
| layer_prefix: model.layers | |
| layers: | |
| - 7 | |
| retain: output | |
| cold_diffusion_kwargs: | |
| degradation: vector_addition | |
| degradation_kwargs: | |
| scale: 5.2087 | |
| alpha_mode: random | |
| alpha_min: 0.2 | |
| alpha_max: 2.0 | |
| steering_vectors_path: ./data/steering_vectors/llama3.2-1b-layer07/steering_vectors.pt | |
| loss_type: l2 | |
| shuffle: true | |
| train_dataset: ./data/llama1b-layer07-fineweb-1M | |
| rep_statistic: ./data/llama1b-layer07-fineweb-1M/rep_statistics.pt | |
| dynamic_dataset: false | |
| use_bf16: true | |
| num_epochs: 1 | |
| epoch_size: null | |
| batch_size: 4096 | |
| learning_rate: 5.0e-05 | |
| lr_scheduler: | |
| scheduler_cls: cosine_scheduler_with_warmup | |
| warmup_ratio: 0.01 | |
| initial_factor: 0.01 | |
| final_factor: 0.1 | |
| gradient_accumulation_steps: 1 | |
| gradient_clipping_threshold: 1.0 | |
| log_every_n_steps: 10 | |
| save_every_n_steps: null | |
| save_epochs: | |
| - 1 | |
| save_opt_state: true | |
| output_path: !!python/object/apply:pathlib.PosixPath | |
| - runs | |
| - cold-vector-llama1b-d3-static-1M-random | |
| wandb_enabled: true | |
| wandb_entity: null | |
| wandb_project: glp | |
| wandb_run_name: cold-vector-llama1b-d3-static-1M-random | |
| save_root: . | |
| run_name: cold-vector-llama1b-d3-static-1M-random | |