train_args.json

{
 "train": "data/mix_long/train.jsonl",
 "val": "data/mix_long/val.jsonl",
 "out": "checkpoints/jevlike-large-nf",
 "backbone": "answerdotai/ModernBERT-base",
 "max_len": 2048,
 "head_max_len": 192,
 "save_max_len": null,
 "device": null,
 "epochs": 2.0,
 "max_steps": 1000,
 "max_tokens": 2048,
 "max_batch": 64,
 "grad_accum": 8,
 "lr_backbone": 1.5e-05,
 "lr_head": 0.0001,
 "weight_decay": 0.01,
 "warmup": 0.05,
 "schedule": "linear",
 "clip": 1.0,
 "dropout": 0.1,
 "rps_weight": 1.0,
 "esc_weight": 0.2,
 "grad_checkpointing": true,
 "train_embeddings": false,
 "eval_every": 250,
 "log_every": 20,
 "select": "nll",
 "limit_train": 0,
 "limit_val": 3000,
 "seed": 0,
 "resume": false,
 "init_from": "checkpoints/jevlike-large",
 "optim": "adamw8bit",
 "freeze_layers": 10
}