bobert / std /training.yaml
token03's picture
Release v16
e4c567e verified
Raw History Blame Contribute Delete
1.1 kB
data:
mode: std
features_dir: ./data/features
strains_path: ./data/strains.parquet
metadata_path: ./data/beatmaps.parquet
dataset_seed: 42
sample_size: 700000
max_seq_len: 4096
val_split: 0
min_sr: 2.0
max_sr: 14.0
dataloader:
num_workers: 2
pin_memory: true
persistent_workers: true
prefetch_factor: 2
model:
d_model: 512
n_heads: 8
n_layers: 9
dim_feedforward: 1408
dropout: 0.1
local_attention_window: 128
local_attention_block_size: 256
global_attention_layers:
- 2
- 5
- 8
feature_token_dim: 16
runtime:
compile_model: true
compile_mode: default
activation_checkpointing: false
training:
seed: 42
trainer:
epochs: 20
batch_size: 64
max_tokens: 32768
grad_accum: 2
grad_clip: 1.0
precision: bf16-mixed
save_checkpoints: true
optimizer:
adamw_lr: 0.0002
adamw_wd: 0.05
adamw_betas:
- 0.9
- 0.95
muon_lr: 0.01
muon_wd: 0.01
scheduler:
warmup_ratio: 0.01
min_lr_ratio: 0.005
section_contrast_weight: 0.02
masking:
ratio: 0.3
mean_span_length: 2