File size: 4,278 Bytes
e5e74c7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
{
  "signature": {
    "plan": {
      "format": "warpstate-posttrain-plan-v1",
      "stage": "sft",
      "after_completion": "pause_for_evaluation",
      "base": {
        "step": 76294,
        "tokens_seen": 10000007168,
        "tokenizer_sha256": "ebae4b4310068ee102f7bf68ac68b79ee1425baa32de3ca8ff8138692529d84e"
      },
      "handoff": {
        "base_work_dir": "rtx6000_40b/work",
        "base_program": "warpstate-train",
        "supervisor_socket": "rtx6000_40b/work/control/supervisor.sock",
        "archive_dir": "rtx6000_40b/work/archives/step-00076294",
        "work_dir": "rtx6000_40b/work/posttrain/step-00076294",
        "data_dir": "rtx6000_40b/work/posttrain/data",
        "tokenizer": "rtx6000_40b/tokenizer.model",
        "processes": 2,
        "poll_seconds": 15
      },
      "training": {
        "sequence_length": 2048,
        "micro_batch": 4,
        "grad_accum": 4,
        "lr": 5e-06,
        "min_lr": 0.0,
        "warmup_ratio": 0.03,
        "epochs": 1,
        "max_steps": null,
        "weight_decay": 0.0,
        "seed": 42,
        "eval_every": 100,
        "save_every": 100,
        "compile_core": true,
        "loss_mode": "compiled-full",
        "bf16_residuals": true,
        "gradient_checkpointing": false,
        "workers": 8
      }
    },
    "settings": {
      "sequence_length": 2048,
      "micro_batch": 4,
      "grad_accum": 4,
      "lr": 5e-06,
      "min_lr": 0.0,
      "warmup_ratio": 0.03,
      "epochs": 1,
      "max_steps": null,
      "weight_decay": 0.0,
      "seed": 42,
      "eval_every": 100,
      "save_every": 100,
      "eval_batches": 32,
      "compile_core": true,
      "loss_mode": "compiled-full",
      "bf16_residuals": true,
      "gradient_checkpointing": false,
      "workers": 8,
      "deterministic": true,
      "clip_grad_norm": 1.0
    },
    "data_sha256": "70b144f08930fa2a6b4fde19bb26b8dee4c90bd6fe464cfb23282e7eb986801e",
    "tokenizer_sha256": "ebae4b4310068ee102f7bf68ac68b79ee1425baa32de3ca8ff8138692529d84e",
    "renderer": "warpstate-segmented-roles-v1",
    "base_sha256": "dc7605cecd6c807135e3d83e04496fa96079fef022bed1952c8cf21df5f573da",
    "world_size": 2,
    "device_type": "cuda",
    "torch_version": "2.14.0+cu130",
    "sources": {
      "posttrain.py": "bd10e97e61b3289e83b7e79c76042bf7f04ce58f4af9067db97ac1fe60d4a411",
      "ops/posttrain_data.py": "486d2e96e7e99ffd05caf0de7d92068ce47047d4641ede3515681cbb79a9ce10",
      "ops/loss_experiment.py": "12bf7822979692d5106e30544d0ed5d977fb06f556c068ef5f92ddf56ab8376e",
      "ops/residual_experiment.py": "e82aa68e36c76a228156016eb04a02ce1a632a63ac6eda567ec9c4d258aae8f1",
      "ops/checkpoint_writer.py": "3c7fe494e8782802b93a439c05ddd6f416711ca1b3f0f8a196a97a48e37f0216",
      "run_40b.py": "9ad6533d781e376b3b40c16e6c29ef15063d39f917a93800489f17115be15ba0",
      "warpstate/__init__.py": "f386c0880cb499d84159d8205667d250a72d0e8a102f0d7be6af0ba0ea33b22b",
      "warpstate/config.py": "1d716b6c582bab21e8d5fcbaa6d9f3be4f792baf9b6abe826c72422f8eced6ad",
      "warpstate/model.py": "025330ee58cca899695c7631e1a06d2efd006402c61e51106afe6fa2e1d35161"
    }
  },
  "origin": {
    "format": "warpstate-chat-v1",
    "inference_only": true,
    "config": {
      "vocab_size": 32768,
      "d_model": 2048,
      "n_heads": 32,
      "n_cores": 28,
      "logical_depth": 28,
      "ffn_hidden": 7168,
      "chunk_size": 128,
      "dropout": 0.0,
      "rms_eps": 1e-05,
      "gradient_checkpointing": false,
      "parallel_memory": true,
      "loss_chunk_size": 512
    },
    "step": 76294,
    "tokens_seen": 10000007168,
    "val_loss": 2.3496886491775513,
    "tokenizer_sha256": "ebae4b4310068ee102f7bf68ac68b79ee1425baa32de3ca8ff8138692529d84e",
    "captured_at_utc": "2026-09-14T14:27:16Z",
    "source": {
      "path": "/workspace/WarpState/rtx6000_40b/work/run/latest.pt",
      "device": 66,
      "inode": 8616307015,
      "size": 24061705357,
      "mtime_ns": 1789396019640599046
    },
    "weights_sha256": "dc7605cecd6c807135e3d83e04496fa96079fef022bed1952c8cf21df5f573da",
    "weights_path": "/workspace/WarpState/rtx6000_40b/work/posttrain/step-00076294/initialization/model.pt"
  },
  "total_steps": 10312,
  "train_rows": 329973,
  "val_rows": 1603
}