Add pretrain/step_076294 from udgamlabs/udgam3-base-ckpt

pretrain/step_076294/meta_076294.json

@@ -0,0 +1,148 @@
+{
+  "step": 76294,
+  "val_bpb": 0.7931417583583836,
+  "model_config": {
+    "sequence_len": 2048,
+    "vocab_size": 32768,
+    "n_layer": 28,
+    "n_head": 10,
+    "n_kv_head": 2,
+    "n_embd": 640,
+    "head_dim": 64,
+    "global_head_dim": 128,
+    "intermediate_size": 1792,
+    "ple_dim": 0,
+    "window_pattern": "SSSL",
+    "sliding_window": 512,
+    "rope_theta_sliding": 10000.0,
+    "rope_theta_full": 1000000.0,
+    "final_logit_softcap": 15.0,
+    "rms_norm_eps": 1e-06,
+    "loop_start": -1,
+    "loop_end": -1,
+    "n_loops": 1,
+    "mtp": false,
+    "loop_loss": false,
+    "loop_loss_stride": 4,
+    "loop_loss_weight": -1.0,
+    "loop_reinject": false,
+    "abacus": 0,
+    "abacus_offset": 0,
+    "digit_ids": [
+      291,
+      292,
+      293,
+      294,
+      295,
+      296,
+      297,
+      298,
+      299,
+      300
+    ],
+    "digit_space_ids": [
+      1150,
+      958,
+      994,
+      1106,
+      1189,
+      1182,
+      1348,
+      1506,
+      1500,
+      1564
+    ],
+    "tie_embeddings": true
+  },
+  "user_config": {
+    "run": "udgam3-base-v1",
+    "device_type": "",
+    "fp8": false,
+    "fp8_recipe": "tensorwise",
+    "depth": 28,
+    "aspect_ratio": 64,
+    "head_dim": 128,
+    "max_seq_len": 2048,
+    "window_pattern": "SSSL",
+    "arch": "gemma4",
+    "n_embd": 640,
+    "n_head": 10,
+    "n_kv_head": 2,
+    "g4_head_dim": 64,
+    "g4_global_head_dim": 128,
+    "intermediate_size": 1792,
+    "ple_dim": 0,
+    "sliding_window": 512,
+    "softcap": 15.0,
+    "loop_start": -1,
+    "loop_end": -1,
+    "n_loops": 1,
+    "mtp": false,
+    "mtp_weight": 0.3,
+    "tie_embeddings": true,
+    "loop_loss": false,
+    "loop_loss_weight": -1.0,
+    "loop_loss_stride": 4,
+    "loop_reinject": false,
+    "abacus": 0,
+    "abacus_offset": 0,
+    "digits_reversed": false,
+    "synth_arith_frac": 0.0,
+    "arith_probe": false,
+    "gsm_eval": "none",
+    "gsm_dev_file": "",
+    "gsm_dev_every": 0,
+    "gsm_eval_n": 0,
+    "results_json": "",
+    "optimizer": "muon",
+    "tst_s": 4,
+    "tst_frac": 0.25,
+    "extra_val": [
+      "web=/workspace/data/val_sets/val_web",
+      "task=/workspace/data/val_sets/val_task",
+      "reason=/workspace/data/val_sets/val_reason",
+      "fresh=/workspace/data/val_sets/val_fresh"
+    ],
+    "ema_start_frac": 0.9,
+    "ema_beta": 0.998,
+    "seed": 42,
+    "phase_b_data_dir": "/workspace/data/shards/phaseB",
+    "phase_b_start_frac": 0.8,
+    "init_from": "",
+    "num_iterations": 76294,
+    "target_flops": -1.0,
+    "target_param_data_ratio": 12,
+    "device_batch_size": 16,
+    "total_batch_size": 524288,
+    "embedding_lr": 0.3,
+    "unembedding_lr": 0.008,
+    "weight_decay": 0.28,
+    "matrix_lr": 0.02,
+    "scalar_lr": 0.5,
+    "warmup_steps": 40,
+    "warmdown_ratio": 0.65,
+    "final_lr_frac": 0.05,
+    "resume_from_step": -1,
+    "eval_every": 2000,
+    "eval_tokens": 4194304,
+    "core_metric_every": -1,
+    "core_metric_max_per_task": 500,
+    "sample_every": -1,
+    "save_every": 2000,
+    "model_tag": "udgam3-base-v1"
+  },
+  "device_batch_size": 16,
+  "max_seq_len": 2048,
+  "total_batch_size": 524288,
+  "dataloader_state_dict": {
+    "pq_idx": 63,
+    "rg_idx": 112,
+    "epoch": 1
+  },
+  "data_phase": "B",
+  "loop_state": {
+    "min_val_bpb": 0.7931417583583836,
+    "smooth_train_loss": 2.008527385027897,
+    "total_training_time": 45736.535438776016
+  }
+}