bobert / catch /training.yaml
token03's picture
Release v16
e4c567e verified
Raw History Blame Contribute Delete
1.23 kB
data:
mode: catch
features_dir: /home/jessiez/projects/bobert/data/features
strains_path: /home/jessiez/projects/bobert/data/strains.parquet
dataset_seed: 42
sample_size: null
max_seq_len: 4096
val_split: 0.1
min_sr: 2.0
max_sr: 14.0
dataloader:
num_workers: 2
pin_memory: true
persistent_workers: false
prefetch_factor: 2
validation_metadata_path: /home/jessiez/projects/bobert/runs/v15/catch/validation_groups.parquet
validation_group_column: validation_group_id
model:
d_model: 256
n_heads: 4
n_layers: 9
dim_feedforward: 688
dropout: 0.1
local_attention_window: 128
local_attention_block_size: 256
global_attention_layers:
- 2
- 5
- 8
feature_token_dim: 16
runtime:
compile_model: true
compile_mode: default
activation_checkpointing: false
training:
seed: 42
trainer:
epochs: 30
batch_size: 64
max_tokens: 32768
grad_accum: 2
grad_clip: 1.0
precision: bf16-mixed
save_checkpoints: true
optimizer:
adamw_lr: 0.0002
adamw_wd: 0.05
adamw_betas:
- 0.9
- 0.95
muon_lr: 0.01
muon_wd: 0.01
scheduler:
warmup_ratio: 0.01
min_lr_ratio: 0.005
masking:
ratio: 0.3
mean_span_length: 2