| { |
| "step": 9552, |
| "epoch": 12, |
| "position": 0, |
| "tokens_seen": 1291124736, |
| "train_seconds_total": 54185.850963340024, |
| "peak_reserved_gb": 17.939453125, |
| "peak_allocated_gb": 17.494221687316895, |
| "last_loss": 1.523698394948786, |
| "last_val_loss": 1.9862932266429436, |
| "last_val_acc": 0.6291136684837473, |
| "invocations": 1, |
| "status": "complete", |
| "exit_code": 0, |
| "run_end_step": 9552, |
| "this_invocation": { |
| "steps": 9552, |
| "tokens": 1291124736, |
| "loop_seconds": 54190.0, |
| "mean_tokens_per_s": 23889, |
| "setup_seconds": 1.7 |
| }, |
| "mean_tokens_per_s": 23889, |
| "config": { |
| "model": { |
| "vocab_size": 32768, |
| "max_seq_len": 512, |
| "d_model": 1024, |
| "n_layers": 24, |
| "n_heads": 16, |
| "d_ff": 4096, |
| "dropout": 0.1, |
| "attention_dropout": 0.1, |
| "layer_norm_eps": 1e-05, |
| "pad_token_id": 0, |
| "tie_mlm_head": true, |
| "num_labels": 2 |
| }, |
| "grad_checkpointing": false, |
| "pretrain": { |
| "lr": 0.00015, |
| "betas": [ |
| 0.9, |
| 0.98 |
| ], |
| "eps": 1e-06, |
| "weight_decay": 0.01, |
| "grad_clip": 1.0, |
| "mlm_probability": 0.15, |
| "mask_replace_probs": [ |
| 0.8, |
| 0.1, |
| 0.1 |
| ], |
| "schedule": "cosine_to_10pct", |
| "effective_batch_sequences": 256 |
| }, |
| "seed": 42, |
| "micro_batch": 24, |
| "grad_accumulation": 11, |
| "effective_batch": 264, |
| "schedule": { |
| "kind": "cosine", |
| "peak_lr": 0.00015, |
| "warmup_steps": 573, |
| "total_steps": 9552, |
| "floor_ratio": 0.1 |
| }, |
| "steps_per_epoch": 796, |
| "epochs": 12, |
| "total_steps": 9552, |
| "data": { |
| "n_windows": 210368, |
| "n_val_windows": 1999, |
| "tokenizer_sha1": "9f50c5e775089d2ca3efc82e100cd6b3396ed7e7", |
| "n_docs": 226273, |
| "limit_docs": null, |
| "window": "[CLS] + 511 stream tokens" |
| } |
| }, |
| "params": { |
| "total": 337476608, |
| "trainable": 337476608, |
| "embeddings": 34080768, |
| "encoder_layers": 302309376, |
| "final_norm": 2048, |
| "head": 1084416 |
| }, |
| "finished": "2026-10-09T03:31:14", |
| "log": "/home/nathank/BITS/dev/Phishing Model/logs/pretrain-20261008-122802.log" |
| } |
|
|
