Hugging Face's logo

asher577
/
ci_thresholding

PyTorch
Model card Files
xet
 Community
ci_thresholding
File size: 2,913 Bytes
8225c13
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
model:
  n_layer: 2
  d_model: 3072
  n_ctx: 512
  d_head: 16
  d_mlp: 12288
  vocab_size: 4096
  use_rms_norm: true
  tie_embeddings: false
  use_positional_embeddings: false
  use_bigram_table: false
  use_attention_sinks: true
  activation: gelu
  dropout: 0.0
  use_bias: true
  use_flash_attention: true
sparsity:
  enable_weight_sparsity: true
  target_l0_fraction: 0.05
  sparsity_anneal_start_fraction: 0.01
  sparsity_anneal_end_fraction: 0.5
  anneal_type: linear
  min_weights_per_neuron: 4
  enable_activation_sparsity: true
  activation_topk_fraction: 0.2
  activation_sparsity_locations: attn_q,attn_k,attn_v,attn_out,mlp_neuron,mlp_out
  activation_topk_mode: per_token
  mlp_neuron_pre_activation: true
  activation_topk_annealing: linear
  activation_topk_anneal_start_fraction: 0.01
  activation_topk_anneal_end_fraction: 0.5
optimizer:
  optimizer_type: adamw
  learning_rate: 0.001
  beta1: 0.9
  beta2: 0.95
  weight_decay: 0.001
  eps: 0.1
  enable_grad_clip: true
  grad_clip_rms: 1.0
  warmup_fraction: 0.01
  enable_lr_decay: true
  min_lr_fraction: 0.0
  lr_decay_end_fraction: 1.0
  use_sharkfin_schedule: true
training:
  dataset_name: jacobcd52/simplestories-tokenized
  dataset_split: train
  text_column: story
  tokenizer_name: SimpleStories/SimpleStories-1.25M
  total_tokens: 1000000000
  batch_size: 32
  gradient_accumulation_steps: 1
  mixed_precision: bf16
  checkpoint_dir: checkpoints/ablation_pretraining/pgd_d3072
  checkpoint_every_n_steps: 0
  keep_n_checkpoints: 0
  log_every_n_steps: 10
  log_gradients_every_n_steps: 100
  log_weights_every_n_steps: 100
  log_sparsity_every_n_steps: 100
  eval_every_n_steps: 100
  val_split: test
  val_holdout_fraction: 0.01
  val_max_batches: 20
  wandb_project: nodesparse_adv_pretraining
  wandb_run_name: ci_thresholding_l0_0.05
  wandb_entity: null
  use_wandb: true
  seed: 0
  hf_repo: asher577/ci_thresholding
  num_workers: 8
  resume_from_checkpoint: null
  use_torch_compile: false
  torch_compile_mode: default
  torch_compile_backend: inductor
ablation:
  ci_fn_type: global_reverse_residual
  d_resid_ci_fn: 1024
  reader_hidden_dims: 1024,1024
  use_attn_transitions: false
  attn_n_heads: 4
  attn_max_len: 2048
  ci_targets: attn_q,attn_k,attn_v,attn_out,mlp_pre,mlp_out
  ablation_mode: pgd
  stochastic_n_bins: 10
  pgd_init: random
  pgd_lr: 0.2
  pgd_n_steps: 2
  pgd_use_sign_grad: true
  pgd_outer_lr: 0.0
  minimality_coeff: 1.0e-05
  minimality_coeff_final_frac: 1.0
  minimality_coeff_annealing: true
  minimality_coeff_anneal_start_frac: 0.01
  minimality_coeff_anneal_end_frac: 0.7
  minimality_coeff_anneal_type: linear
  minimality_pnorm: 2.0
  minimality_eps: 5.0e-07
  minimality_beta: 0.5
  minimality_p_anneal_start_frac: 0.01
  minimality_p_anneal_final_p: 0.5
  minimality_p_anneal_end_frac: 0.5
  initial_ci: 1.0
  minimality_warmup_frac: 0.02
  ci_thresholding: 0.1
  ci_learning_rate: null