exp-a retrain era_a0_refinedweb

exp-a-core-retrain/era_a0_refinedweb/run_config.json

@@ -0,0 +1,68 @@
+{
+  "size": "small",
+  "arm": "",
+  "data_dir": "/workspace/datasets/refinedweb_gpt2",
+  "neutral_eval_dir": "/workspace/datasets/wiki_eval_union",
+  "token_budget": 8873228288,
+  "n_epochs": 1,
+  "out_dir": "/workspace/runs/era_a0_refinedweb",
+  "total_batch_tokens": 524288,
+  "device_batch_size": 64,
+  "block_size": 1024,
+  "lr": 0.0006,
+  "warmup_steps": 700,
+  "min_lr_ratio": 0.1,
+  "weight_decay": 0.1,
+  "grad_clip": 1.0,
+  "n_checkpoints": 25,
+  "first_ckpt_frac": 0.001,
+  "warmup_timed_steps": 10,
+  "max_steps": 0,
+  "eval_device_batch_size": 8,
+  "compile": false,
+  "seed": 1234,
+  "wandb": "",
+  "save_checkpoints": 1,
+  "keep_checkpoints": 3,
+  "world_size": 8,
+  "total_steps": 16925,
+  "tokens_per_step": 524288,
+  "grad_accum": 1,
+  "n_params": 123689472,
+  "device": "cuda:0",
+  "gpu_name": "NVIDIA H100 80GB HBM3",
+  "torch_version": "2.10.0+cu128",
+  "algorithm": "old_gpt2",
+  "ckpt_steps": [
+    17,
+    23,
+    30,
+    40,
+    54,
+    71,
+    95,
+    127,
+    169,
+    226,
+    301,
+    401,
+    535,
+    714,
+    952,
+    1269,
+    1692,
+    2257,
+    3010,
+    4014,
+    5352,
+    7137,
+    9518,
+    12692,
+    15232,
+    15571,
+    15910,
+    16248,
+    16586,
+    16925
+  ]
+}