File size: 1,399 Bytes
f827f04 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 | model_name: meta-llama/Llama-3.2-1B
glp_kwargs:
normalizer_config:
rep_statistic: ./data/llama1b-layer07-fineweb-1M/rep_statistics.pt
denoiser_config:
d_input: 2048
d_model: 4096
d_mlp: 8192
n_layers: 3
multi_layer_n_layers: null
tracedict_config:
layer_prefix: model.layers
layers:
- 7
retain: output
cold_diffusion_kwargs:
degradation: vector_addition
degradation_kwargs:
scale: 5.2087
alpha_mode: random
alpha_min: 0.2
alpha_max: 2.0
steering_vectors_path: ./data/steering_vectors/llama3.2-1b-layer07/steering_vectors.pt
loss_type: l2
shuffle: true
train_dataset: ./data/llama1b-layer07-fineweb-1M
rep_statistic: ./data/llama1b-layer07-fineweb-1M/rep_statistics.pt
dynamic_dataset: false
use_bf16: true
num_epochs: 1
epoch_size: null
batch_size: 4096
learning_rate: 5.0e-05
lr_scheduler:
scheduler_cls: cosine_scheduler_with_warmup
warmup_ratio: 0.01
initial_factor: 0.01
final_factor: 0.1
gradient_accumulation_steps: 1
gradient_clipping_threshold: 1.0
log_every_n_steps: 10
save_every_n_steps: null
save_epochs:
- 1
save_opt_state: true
output_path: !!python/object/apply:pathlib.PosixPath
- runs
- cold-vector-llama1b-d3-static-1M-random
wandb_enabled: true
wandb_entity: null
wandb_project: glp
wandb_run_name: cold-vector-llama1b-d3-static-1M-random
save_root: .
run_name: cold-vector-llama1b-d3-static-1M-random
|