Upload 8 files
Browse files- model/PROVENANCE.json +96 -0
- model/SHA256SUMS +12 -0
- model/model.pt +3 -0
- model/settings.json +119 -0
- model/sft-execution.json +62 -0
- model/sft-handoff.json +114 -0
- model/snapshot.json +67 -0
- model/tokenizer.model +3 -0
model/PROVENANCE.json
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"data_sha256": "70b144f08930fa2a6b4fde19bb26b8dee4c90bd6fe464cfb23282e7eb986801e",
|
| 3 |
+
"format": "warpstate-sft-archive-v1",
|
| 4 |
+
"model_sha256": "ddf25c0260a189614613c4ceb3b778983aec601c55ff4a175aff350d85ce1f44",
|
| 5 |
+
"raw_base_sha256": "dc7605cecd6c807135e3d83e04496fa96079fef022bed1952c8cf21df5f573da",
|
| 6 |
+
"raw_base_step": 76294,
|
| 7 |
+
"raw_base_tokens": 10000007168,
|
| 8 |
+
"repository_revision": "774890d7f88c31fbe3e417aebab45ab08a177208",
|
| 9 |
+
"sft_assistant_tokens": 143695039,
|
| 10 |
+
"sft_input_tokens": 212321372,
|
| 11 |
+
"sft_signature": {
|
| 12 |
+
"base_sha256": "dc7605cecd6c807135e3d83e04496fa96079fef022bed1952c8cf21df5f573da",
|
| 13 |
+
"data_sha256": "70b144f08930fa2a6b4fde19bb26b8dee4c90bd6fe464cfb23282e7eb986801e",
|
| 14 |
+
"device_type": "cuda",
|
| 15 |
+
"plan": {
|
| 16 |
+
"after_completion": "pause_for_evaluation",
|
| 17 |
+
"base": {
|
| 18 |
+
"step": 76294,
|
| 19 |
+
"tokenizer_sha256": "ebae4b4310068ee102f7bf68ac68b79ee1425baa32de3ca8ff8138692529d84e",
|
| 20 |
+
"tokens_seen": 10000007168
|
| 21 |
+
},
|
| 22 |
+
"format": "warpstate-posttrain-plan-v1",
|
| 23 |
+
"handoff": {
|
| 24 |
+
"archive_dir": "rtx6000_40b/work/archives/step-00076294",
|
| 25 |
+
"base_program": "warpstate-train",
|
| 26 |
+
"base_work_dir": "rtx6000_40b/work",
|
| 27 |
+
"data_dir": "rtx6000_40b/work/posttrain/data",
|
| 28 |
+
"poll_seconds": 15,
|
| 29 |
+
"processes": 2,
|
| 30 |
+
"supervisor_socket": "rtx6000_40b/work/control/supervisor.sock",
|
| 31 |
+
"tokenizer": "rtx6000_40b/tokenizer.model",
|
| 32 |
+
"work_dir": "rtx6000_40b/work/posttrain/step-00076294"
|
| 33 |
+
},
|
| 34 |
+
"stage": "sft",
|
| 35 |
+
"training": {
|
| 36 |
+
"bf16_residuals": true,
|
| 37 |
+
"compile_core": true,
|
| 38 |
+
"epochs": 1,
|
| 39 |
+
"eval_every": 100,
|
| 40 |
+
"grad_accum": 4,
|
| 41 |
+
"gradient_checkpointing": false,
|
| 42 |
+
"loss_mode": "compiled-full",
|
| 43 |
+
"lr": 5e-06,
|
| 44 |
+
"max_steps": null,
|
| 45 |
+
"micro_batch": 4,
|
| 46 |
+
"min_lr": 0.0,
|
| 47 |
+
"save_every": 100,
|
| 48 |
+
"seed": 42,
|
| 49 |
+
"sequence_length": 2048,
|
| 50 |
+
"warmup_ratio": 0.03,
|
| 51 |
+
"weight_decay": 0.0,
|
| 52 |
+
"workers": 8
|
| 53 |
+
}
|
| 54 |
+
},
|
| 55 |
+
"renderer": "warpstate-segmented-roles-v1",
|
| 56 |
+
"settings": {
|
| 57 |
+
"bf16_residuals": true,
|
| 58 |
+
"clip_grad_norm": 1.0,
|
| 59 |
+
"compile_core": true,
|
| 60 |
+
"deterministic": true,
|
| 61 |
+
"epochs": 1,
|
| 62 |
+
"eval_batches": 32,
|
| 63 |
+
"eval_every": 100,
|
| 64 |
+
"grad_accum": 4,
|
| 65 |
+
"gradient_checkpointing": false,
|
| 66 |
+
"loss_mode": "compiled-full",
|
| 67 |
+
"lr": 5e-06,
|
| 68 |
+
"max_steps": null,
|
| 69 |
+
"micro_batch": 4,
|
| 70 |
+
"min_lr": 0.0,
|
| 71 |
+
"save_every": 100,
|
| 72 |
+
"seed": 42,
|
| 73 |
+
"sequence_length": 2048,
|
| 74 |
+
"warmup_ratio": 0.03,
|
| 75 |
+
"weight_decay": 0.0,
|
| 76 |
+
"workers": 8
|
| 77 |
+
},
|
| 78 |
+
"sources": {
|
| 79 |
+
"ops/checkpoint_writer.py": "3c7fe494e8782802b93a439c05ddd6f416711ca1b3f0f8a196a97a48e37f0216",
|
| 80 |
+
"ops/loss_experiment.py": "12bf7822979692d5106e30544d0ed5d977fb06f556c068ef5f92ddf56ab8376e",
|
| 81 |
+
"ops/posttrain_data.py": "486d2e96e7e99ffd05caf0de7d92068ce47047d4641ede3515681cbb79a9ce10",
|
| 82 |
+
"ops/residual_experiment.py": "e82aa68e36c76a228156016eb04a02ce1a632a63ac6eda567ec9c4d258aae8f1",
|
| 83 |
+
"posttrain.py": "bd10e97e61b3289e83b7e79c76042bf7f04ce58f4af9067db97ac1fe60d4a411",
|
| 84 |
+
"run_40b.py": "9ad6533d781e376b3b40c16e6c29ef15063d39f917a93800489f17115be15ba0",
|
| 85 |
+
"warpstate/__init__.py": "f386c0880cb499d84159d8205667d250a72d0e8a102f0d7be6af0ba0ea33b22b",
|
| 86 |
+
"warpstate/config.py": "1d716b6c582bab21e8d5fcbaa6d9f3be4f792baf9b6abe826c72422f8eced6ad",
|
| 87 |
+
"warpstate/model.py": "025330ee58cca899695c7631e1a06d2efd006402c61e51106afe6fa2e1d35161"
|
| 88 |
+
},
|
| 89 |
+
"tokenizer_sha256": "ebae4b4310068ee102f7bf68ac68b79ee1425baa32de3ca8ff8138692529d84e",
|
| 90 |
+
"torch_version": "2.14.0+cu130",
|
| 91 |
+
"world_size": 2
|
| 92 |
+
},
|
| 93 |
+
"sft_step": 10312,
|
| 94 |
+
"source_checkpoint_sha256": "b94426834e5c56895ed34529479fe540ff14c2b6cef9c6804b9cf16a34ec27c8",
|
| 95 |
+
"tokenizer_sha256": "ebae4b4310068ee102f7bf68ac68b79ee1425baa32de3ca8ff8138692529d84e"
|
| 96 |
+
}
|
model/SHA256SUMS
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
b94426834e5c56895ed34529479fe540ff14c2b6cef9c6804b9cf16a34ec27c8 checkpoint.pt
|
| 2 |
+
ddf25c0260a189614613c4ceb3b778983aec601c55ff4a175aff350d85ce1f44 model.pt
|
| 3 |
+
93a95e30082b1f9c833fbb230a5a866a562fc5cc4b7d794208ab88c1ba20c07f snapshot.json
|
| 4 |
+
ebae4b4310068ee102f7bf68ac68b79ee1425baa32de3ca8ff8138692529d84e tokenizer.model
|
| 5 |
+
dc4d1e696a61c86bcc9a42bfaae8bc3251de582b84d8ad2cd89a0776b9f4b909 settings.json
|
| 6 |
+
2344f247a52ce1d7071e9048e0eba8a12763951af0fc4b2d0dd786b57cdd11ee evaluation.json
|
| 7 |
+
50e49a76703db6799b029ba7fb09581165094693178d29010d069fc2026e4682 completion.json
|
| 8 |
+
2e238b21e0935edeadea9a2e172931c9678943de610c6fe6099cbbd336ea3c72 metrics.jsonl
|
| 9 |
+
d83a09bc5ff9b3a7811f3ae5e4257181ab6adf94122daebd902a7297b177e7c4 sft-execution.json
|
| 10 |
+
0c29ef1760ed3497d35f9ccc7d75a0f50d32af66b4e2f492a870142cf50c0188 sft-handoff.json
|
| 11 |
+
963c1ffdfd55df0689d898986247f4c326c7b0b89e7a46defbc3eb0f98012882 PROVENANCE.json
|
| 12 |
+
e4f114245d0789a3ce5270e2dfa10292fd1e193f7be3344143dd4b819fd40b19 ARCHIVE.md
|
model/model.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ddf25c0260a189614613c4ceb3b778983aec601c55ff4a175aff350d85ce1f44
|
| 3 |
+
size 8020534643
|
model/settings.json
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"signature": {
|
| 3 |
+
"plan": {
|
| 4 |
+
"format": "warpstate-posttrain-plan-v1",
|
| 5 |
+
"stage": "sft",
|
| 6 |
+
"after_completion": "pause_for_evaluation",
|
| 7 |
+
"base": {
|
| 8 |
+
"step": 76294,
|
| 9 |
+
"tokens_seen": 10000007168,
|
| 10 |
+
"tokenizer_sha256": "ebae4b4310068ee102f7bf68ac68b79ee1425baa32de3ca8ff8138692529d84e"
|
| 11 |
+
},
|
| 12 |
+
"handoff": {
|
| 13 |
+
"base_work_dir": "rtx6000_40b/work",
|
| 14 |
+
"base_program": "warpstate-train",
|
| 15 |
+
"supervisor_socket": "rtx6000_40b/work/control/supervisor.sock",
|
| 16 |
+
"archive_dir": "rtx6000_40b/work/archives/step-00076294",
|
| 17 |
+
"work_dir": "rtx6000_40b/work/posttrain/step-00076294",
|
| 18 |
+
"data_dir": "rtx6000_40b/work/posttrain/data",
|
| 19 |
+
"tokenizer": "rtx6000_40b/tokenizer.model",
|
| 20 |
+
"processes": 2,
|
| 21 |
+
"poll_seconds": 15
|
| 22 |
+
},
|
| 23 |
+
"training": {
|
| 24 |
+
"sequence_length": 2048,
|
| 25 |
+
"micro_batch": 4,
|
| 26 |
+
"grad_accum": 4,
|
| 27 |
+
"lr": 5e-06,
|
| 28 |
+
"min_lr": 0.0,
|
| 29 |
+
"warmup_ratio": 0.03,
|
| 30 |
+
"epochs": 1,
|
| 31 |
+
"max_steps": null,
|
| 32 |
+
"weight_decay": 0.0,
|
| 33 |
+
"seed": 42,
|
| 34 |
+
"eval_every": 100,
|
| 35 |
+
"save_every": 100,
|
| 36 |
+
"compile_core": true,
|
| 37 |
+
"loss_mode": "compiled-full",
|
| 38 |
+
"bf16_residuals": true,
|
| 39 |
+
"gradient_checkpointing": false,
|
| 40 |
+
"workers": 8
|
| 41 |
+
}
|
| 42 |
+
},
|
| 43 |
+
"settings": {
|
| 44 |
+
"sequence_length": 2048,
|
| 45 |
+
"micro_batch": 4,
|
| 46 |
+
"grad_accum": 4,
|
| 47 |
+
"lr": 5e-06,
|
| 48 |
+
"min_lr": 0.0,
|
| 49 |
+
"warmup_ratio": 0.03,
|
| 50 |
+
"epochs": 1,
|
| 51 |
+
"max_steps": null,
|
| 52 |
+
"weight_decay": 0.0,
|
| 53 |
+
"seed": 42,
|
| 54 |
+
"eval_every": 100,
|
| 55 |
+
"save_every": 100,
|
| 56 |
+
"eval_batches": 32,
|
| 57 |
+
"compile_core": true,
|
| 58 |
+
"loss_mode": "compiled-full",
|
| 59 |
+
"bf16_residuals": true,
|
| 60 |
+
"gradient_checkpointing": false,
|
| 61 |
+
"workers": 8,
|
| 62 |
+
"deterministic": true,
|
| 63 |
+
"clip_grad_norm": 1.0
|
| 64 |
+
},
|
| 65 |
+
"data_sha256": "70b144f08930fa2a6b4fde19bb26b8dee4c90bd6fe464cfb23282e7eb986801e",
|
| 66 |
+
"tokenizer_sha256": "ebae4b4310068ee102f7bf68ac68b79ee1425baa32de3ca8ff8138692529d84e",
|
| 67 |
+
"renderer": "warpstate-segmented-roles-v1",
|
| 68 |
+
"base_sha256": "dc7605cecd6c807135e3d83e04496fa96079fef022bed1952c8cf21df5f573da",
|
| 69 |
+
"world_size": 2,
|
| 70 |
+
"device_type": "cuda",
|
| 71 |
+
"torch_version": "2.14.0+cu130",
|
| 72 |
+
"sources": {
|
| 73 |
+
"posttrain.py": "bd10e97e61b3289e83b7e79c76042bf7f04ce58f4af9067db97ac1fe60d4a411",
|
| 74 |
+
"ops/posttrain_data.py": "486d2e96e7e99ffd05caf0de7d92068ce47047d4641ede3515681cbb79a9ce10",
|
| 75 |
+
"ops/loss_experiment.py": "12bf7822979692d5106e30544d0ed5d977fb06f556c068ef5f92ddf56ab8376e",
|
| 76 |
+
"ops/residual_experiment.py": "e82aa68e36c76a228156016eb04a02ce1a632a63ac6eda567ec9c4d258aae8f1",
|
| 77 |
+
"ops/checkpoint_writer.py": "3c7fe494e8782802b93a439c05ddd6f416711ca1b3f0f8a196a97a48e37f0216",
|
| 78 |
+
"run_40b.py": "9ad6533d781e376b3b40c16e6c29ef15063d39f917a93800489f17115be15ba0",
|
| 79 |
+
"warpstate/__init__.py": "f386c0880cb499d84159d8205667d250a72d0e8a102f0d7be6af0ba0ea33b22b",
|
| 80 |
+
"warpstate/config.py": "1d716b6c582bab21e8d5fcbaa6d9f3be4f792baf9b6abe826c72422f8eced6ad",
|
| 81 |
+
"warpstate/model.py": "025330ee58cca899695c7631e1a06d2efd006402c61e51106afe6fa2e1d35161"
|
| 82 |
+
}
|
| 83 |
+
},
|
| 84 |
+
"origin": {
|
| 85 |
+
"format": "warpstate-chat-v1",
|
| 86 |
+
"inference_only": true,
|
| 87 |
+
"config": {
|
| 88 |
+
"vocab_size": 32768,
|
| 89 |
+
"d_model": 2048,
|
| 90 |
+
"n_heads": 32,
|
| 91 |
+
"n_cores": 28,
|
| 92 |
+
"logical_depth": 28,
|
| 93 |
+
"ffn_hidden": 7168,
|
| 94 |
+
"chunk_size": 128,
|
| 95 |
+
"dropout": 0.0,
|
| 96 |
+
"rms_eps": 1e-05,
|
| 97 |
+
"gradient_checkpointing": false,
|
| 98 |
+
"parallel_memory": true,
|
| 99 |
+
"loss_chunk_size": 512
|
| 100 |
+
},
|
| 101 |
+
"step": 76294,
|
| 102 |
+
"tokens_seen": 10000007168,
|
| 103 |
+
"val_loss": 2.3496886491775513,
|
| 104 |
+
"tokenizer_sha256": "ebae4b4310068ee102f7bf68ac68b79ee1425baa32de3ca8ff8138692529d84e",
|
| 105 |
+
"captured_at_utc": "2026-09-14T14:27:16Z",
|
| 106 |
+
"source": {
|
| 107 |
+
"path": "/workspace/WarpState/rtx6000_40b/work/run/latest.pt",
|
| 108 |
+
"device": 66,
|
| 109 |
+
"inode": 8616307015,
|
| 110 |
+
"size": 24061705357,
|
| 111 |
+
"mtime_ns": 1789396019640599046
|
| 112 |
+
},
|
| 113 |
+
"weights_sha256": "dc7605cecd6c807135e3d83e04496fa96079fef022bed1952c8cf21df5f573da",
|
| 114 |
+
"weights_path": "/workspace/WarpState/rtx6000_40b/work/posttrain/step-00076294/initialization/model.pt"
|
| 115 |
+
},
|
| 116 |
+
"total_steps": 10312,
|
| 117 |
+
"train_rows": 329973,
|
| 118 |
+
"val_rows": 1603
|
| 119 |
+
}
|
model/sft-execution.json
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"archive_sha256": "dc7605cecd6c807135e3d83e04496fa96079fef022bed1952c8cf21df5f573da",
|
| 3 |
+
"data_manifest_sha256": "70b144f08930fa2a6b4fde19bb26b8dee4c90bd6fe464cfb23282e7eb986801e",
|
| 4 |
+
"git_revision": "ea63c41cc541fdef8c36da8a85f7b625a2bcec40",
|
| 5 |
+
"plan": {
|
| 6 |
+
"after_completion": "pause_for_evaluation",
|
| 7 |
+
"base": {
|
| 8 |
+
"step": 76294,
|
| 9 |
+
"tokenizer_sha256": "ebae4b4310068ee102f7bf68ac68b79ee1425baa32de3ca8ff8138692529d84e",
|
| 10 |
+
"tokens_seen": 10000007168
|
| 11 |
+
},
|
| 12 |
+
"format": "warpstate-posttrain-plan-v1",
|
| 13 |
+
"handoff": {
|
| 14 |
+
"archive_dir": "rtx6000_40b/work/archives/step-00076294",
|
| 15 |
+
"base_program": "warpstate-train",
|
| 16 |
+
"base_work_dir": "rtx6000_40b/work",
|
| 17 |
+
"data_dir": "rtx6000_40b/work/posttrain/data",
|
| 18 |
+
"poll_seconds": 15,
|
| 19 |
+
"processes": 2,
|
| 20 |
+
"supervisor_socket": "rtx6000_40b/work/control/supervisor.sock",
|
| 21 |
+
"tokenizer": "rtx6000_40b/tokenizer.model",
|
| 22 |
+
"work_dir": "rtx6000_40b/work/posttrain/step-00076294"
|
| 23 |
+
},
|
| 24 |
+
"stage": "sft",
|
| 25 |
+
"training": {
|
| 26 |
+
"bf16_residuals": true,
|
| 27 |
+
"compile_core": true,
|
| 28 |
+
"epochs": 1,
|
| 29 |
+
"eval_every": 100,
|
| 30 |
+
"grad_accum": 4,
|
| 31 |
+
"gradient_checkpointing": false,
|
| 32 |
+
"loss_mode": "compiled-full",
|
| 33 |
+
"lr": 5e-06,
|
| 34 |
+
"max_steps": null,
|
| 35 |
+
"micro_batch": 4,
|
| 36 |
+
"min_lr": 0.0,
|
| 37 |
+
"save_every": 100,
|
| 38 |
+
"seed": 42,
|
| 39 |
+
"sequence_length": 2048,
|
| 40 |
+
"warmup_ratio": 0.03,
|
| 41 |
+
"weight_decay": 0.0,
|
| 42 |
+
"workers": 8
|
| 43 |
+
}
|
| 44 |
+
},
|
| 45 |
+
"source_sha256": {
|
| 46 |
+
"rtx6000_40b/chat.py": "148fd4f20d445ba01723d4c2625372132270884d5c44104f31a3cc13f7c02774",
|
| 47 |
+
"rtx6000_40b/configs/posttrain_10b.json": "a0f8d06e38f72245e046a6a888053c92aa23627256599348f5889886d61df179",
|
| 48 |
+
"rtx6000_40b/ops/checkpoint_writer.py": "3c7fe494e8782802b93a439c05ddd6f416711ca1b3f0f8a196a97a48e37f0216",
|
| 49 |
+
"rtx6000_40b/ops/launch_training.py": "fe9a204023300632766aabe7ff4b0cd22dba5ca7a306040da7ce3b21c9305e08",
|
| 50 |
+
"rtx6000_40b/ops/loss_experiment.py": "12bf7822979692d5106e30544d0ed5d977fb06f556c068ef5f92ddf56ab8376e",
|
| 51 |
+
"rtx6000_40b/ops/posttrain_data.py": "486d2e96e7e99ffd05caf0de7d92068ce47047d4641ede3515681cbb79a9ce10",
|
| 52 |
+
"rtx6000_40b/ops/posttrain_handoff.py": "8f7bb104183bb59802774c5500965d3ac2d1ffd1cb3de18256d075d5414f7c08",
|
| 53 |
+
"rtx6000_40b/ops/posttrain_preflight.py": "4489ae1f37c13e3226d6687147423a9f6a95a0509c135357291ea0a0a9d9664a",
|
| 54 |
+
"rtx6000_40b/ops/residual_experiment.py": "e82aa68e36c76a228156016eb04a02ce1a632a63ac6eda567ec9c4d258aae8f1",
|
| 55 |
+
"rtx6000_40b/ops/training-env.sh": "f5c9d6557d1dd9721e2e4c7bdeef001d994481c863c41bdd4080fd88484aa6f1",
|
| 56 |
+
"rtx6000_40b/posttrain.py": "bd10e97e61b3289e83b7e79c76042bf7f04ce58f4af9067db97ac1fe60d4a411",
|
| 57 |
+
"rtx6000_40b/run_40b.py": "9ad6533d781e376b3b40c16e6c29ef15063d39f917a93800489f17115be15ba0",
|
| 58 |
+
"rtx6000_40b/warpstate/__init__.py": "f386c0880cb499d84159d8205667d250a72d0e8a102f0d7be6af0ba0ea33b22b",
|
| 59 |
+
"rtx6000_40b/warpstate/config.py": "1d716b6c582bab21e8d5fcbaa6d9f3be4f792baf9b6abe826c72422f8eced6ad",
|
| 60 |
+
"rtx6000_40b/warpstate/model.py": "025330ee58cca899695c7631e1a06d2efd006402c61e51106afe6fa2e1d35161"
|
| 61 |
+
}
|
| 62 |
+
}
|
model/sft-handoff.json
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"archive_dir": "/workspace/WarpState/rtx6000_40b/work/archives/step-00076294",
|
| 3 |
+
"archive_sha256": "dc7605cecd6c807135e3d83e04496fa96079fef022bed1952c8cf21df5f573da",
|
| 4 |
+
"armed_at": "2026-09-14T09:43:24Z",
|
| 5 |
+
"command": [
|
| 6 |
+
"/workspace/WarpState/rtx6000_40b/.server-venv/bin/python",
|
| 7 |
+
"-B",
|
| 8 |
+
"-m",
|
| 9 |
+
"torch.distributed.run",
|
| 10 |
+
"--standalone",
|
| 11 |
+
"--nnodes",
|
| 12 |
+
"1",
|
| 13 |
+
"--nproc-per-node",
|
| 14 |
+
"2",
|
| 15 |
+
"--log-dir",
|
| 16 |
+
"/workspace/WarpState/rtx6000_40b/work/posttrain/step-00076294/logs/torchrun",
|
| 17 |
+
"/workspace/WarpState/rtx6000_40b/posttrain.py",
|
| 18 |
+
"--config",
|
| 19 |
+
"/workspace/WarpState/rtx6000_40b/configs/posttrain_10b.json",
|
| 20 |
+
"--work-dir",
|
| 21 |
+
"/workspace/WarpState/rtx6000_40b/work/posttrain/step-00076294/run",
|
| 22 |
+
"--data-dir",
|
| 23 |
+
"/workspace/WarpState/rtx6000_40b/work/posttrain/data",
|
| 24 |
+
"--weights",
|
| 25 |
+
"/workspace/WarpState/rtx6000_40b/work/posttrain/step-00076294/initialization/model.pt",
|
| 26 |
+
"--tokenizer",
|
| 27 |
+
"/workspace/WarpState/rtx6000_40b/tokenizer.model",
|
| 28 |
+
"--stop-file",
|
| 29 |
+
"/workspace/WarpState/rtx6000_40b/work/posttrain/step-00076294/control/training-stop-request"
|
| 30 |
+
],
|
| 31 |
+
"completion": {
|
| 32 |
+
"assistant_tokens_seen": 143695039,
|
| 33 |
+
"base_step": 76294,
|
| 34 |
+
"checkpoint": "/workspace/WarpState/rtx6000_40b/work/posttrain/step-00076294/run/latest.pt",
|
| 35 |
+
"completed": true,
|
| 36 |
+
"evaluation_file": "/workspace/WarpState/rtx6000_40b/work/posttrain/step-00076294/run/evaluation.json",
|
| 37 |
+
"format": "warpstate-posttrain-completion-v1",
|
| 38 |
+
"paused_for_evaluation": true,
|
| 39 |
+
"stage": "sft",
|
| 40 |
+
"step": 10312,
|
| 41 |
+
"timestamp": 1789419903.0140994,
|
| 42 |
+
"tokens_seen": 212321372,
|
| 43 |
+
"total_steps": 10312,
|
| 44 |
+
"val_loss": 1.1157395899572582
|
| 45 |
+
},
|
| 46 |
+
"data_ready": true,
|
| 47 |
+
"format": "warpstate-posttrain-handoff-v1",
|
| 48 |
+
"git_revision": "ea63c41cc541fdef8c36da8a85f7b625a2bcec40",
|
| 49 |
+
"message": "SFT finished; paused for evaluation. DPO has not been started.",
|
| 50 |
+
"phase": "complete",
|
| 51 |
+
"preflight": {
|
| 52 |
+
"accepted": true,
|
| 53 |
+
"bf16_compute": true,
|
| 54 |
+
"exact_resume_bookkeeping": true,
|
| 55 |
+
"final_validation_loss": 10.38550066947937,
|
| 56 |
+
"fixture_sha256": "bb6a842bbdc14a0a0b0537ba58e7630e16bbd09fc0613520303cff674a900dd3",
|
| 57 |
+
"format": "warpstate-posttrain-preflight-v1",
|
| 58 |
+
"fp32_parameters_and_adam": true,
|
| 59 |
+
"generator_sha256": "4489ae1f37c13e3226d6687147423a9f6a95a0509c135357291ea0a0a9d9664a",
|
| 60 |
+
"masked_tail": true,
|
| 61 |
+
"numerical_atol": 1e-07,
|
| 62 |
+
"numerical_comparison": {
|
| 63 |
+
"model": {
|
| 64 |
+
"bitwise_equal": true,
|
| 65 |
+
"max_absolute_difference": 0.0,
|
| 66 |
+
"tensor_count": 13
|
| 67 |
+
},
|
| 68 |
+
"optimizer": {
|
| 69 |
+
"bitwise_equal": true,
|
| 70 |
+
"max_absolute_difference": 0.0,
|
| 71 |
+
"tensor_count": 39
|
| 72 |
+
}
|
| 73 |
+
},
|
| 74 |
+
"numerical_rtol": 1e-05,
|
| 75 |
+
"optimizer_steps": 2,
|
| 76 |
+
"processes": 2,
|
| 77 |
+
"source_sha256": {
|
| 78 |
+
"ops/checkpoint_writer.py": "3c7fe494e8782802b93a439c05ddd6f416711ca1b3f0f8a196a97a48e37f0216",
|
| 79 |
+
"ops/loss_experiment.py": "12bf7822979692d5106e30544d0ed5d977fb06f556c068ef5f92ddf56ab8376e",
|
| 80 |
+
"ops/posttrain_data.py": "486d2e96e7e99ffd05caf0de7d92068ce47047d4641ede3515681cbb79a9ce10",
|
| 81 |
+
"ops/residual_experiment.py": "e82aa68e36c76a228156016eb04a02ce1a632a63ac6eda567ec9c4d258aae8f1",
|
| 82 |
+
"posttrain.py": "bd10e97e61b3289e83b7e79c76042bf7f04ce58f4af9067db97ac1fe60d4a411",
|
| 83 |
+
"run_40b.py": "9ad6533d781e376b3b40c16e6c29ef15063d39f917a93800489f17115be15ba0",
|
| 84 |
+
"warpstate/__init__.py": "f386c0880cb499d84159d8205667d250a72d0e8a102f0d7be6af0ba0ea33b22b",
|
| 85 |
+
"warpstate/config.py": "1d716b6c582bab21e8d5fcbaa6d9f3be4f792baf9b6abe826c72422f8eced6ad",
|
| 86 |
+
"warpstate/model.py": "025330ee58cca899695c7631e1a06d2efd006402c61e51106afe6fa2e1d35161"
|
| 87 |
+
},
|
| 88 |
+
"total_samples": 5
|
| 89 |
+
},
|
| 90 |
+
"source_sha256": {
|
| 91 |
+
"rtx6000_40b/chat.py": "148fd4f20d445ba01723d4c2625372132270884d5c44104f31a3cc13f7c02774",
|
| 92 |
+
"rtx6000_40b/configs/posttrain_10b.json": "a0f8d06e38f72245e046a6a888053c92aa23627256599348f5889886d61df179",
|
| 93 |
+
"rtx6000_40b/ops/checkpoint_writer.py": "3c7fe494e8782802b93a439c05ddd6f416711ca1b3f0f8a196a97a48e37f0216",
|
| 94 |
+
"rtx6000_40b/ops/launch_training.py": "fe9a204023300632766aabe7ff4b0cd22dba5ca7a306040da7ce3b21c9305e08",
|
| 95 |
+
"rtx6000_40b/ops/loss_experiment.py": "12bf7822979692d5106e30544d0ed5d977fb06f556c068ef5f92ddf56ab8376e",
|
| 96 |
+
"rtx6000_40b/ops/posttrain_data.py": "486d2e96e7e99ffd05caf0de7d92068ce47047d4641ede3515681cbb79a9ce10",
|
| 97 |
+
"rtx6000_40b/ops/posttrain_handoff.py": "8f7bb104183bb59802774c5500965d3ac2d1ffd1cb3de18256d075d5414f7c08",
|
| 98 |
+
"rtx6000_40b/ops/posttrain_preflight.py": "4489ae1f37c13e3226d6687147423a9f6a95a0509c135357291ea0a0a9d9664a",
|
| 99 |
+
"rtx6000_40b/ops/residual_experiment.py": "e82aa68e36c76a228156016eb04a02ce1a632a63ac6eda567ec9c4d258aae8f1",
|
| 100 |
+
"rtx6000_40b/ops/training-env.sh": "f5c9d6557d1dd9721e2e4c7bdeef001d994481c863c41bdd4080fd88484aa6f1",
|
| 101 |
+
"rtx6000_40b/posttrain.py": "bd10e97e61b3289e83b7e79c76042bf7f04ce58f4af9067db97ac1fe60d4a411",
|
| 102 |
+
"rtx6000_40b/run_40b.py": "9ad6533d781e376b3b40c16e6c29ef15063d39f917a93800489f17115be15ba0",
|
| 103 |
+
"rtx6000_40b/warpstate/__init__.py": "f386c0880cb499d84159d8205667d250a72d0e8a102f0d7be6af0ba0ea33b22b",
|
| 104 |
+
"rtx6000_40b/warpstate/config.py": "1d716b6c582bab21e8d5fcbaa6d9f3be4f792baf9b6abe826c72422f8eced6ad",
|
| 105 |
+
"rtx6000_40b/warpstate/model.py": "025330ee58cca899695c7631e1a06d2efd006402c61e51106afe6fa2e1d35161"
|
| 106 |
+
},
|
| 107 |
+
"target": {
|
| 108 |
+
"step": 76294,
|
| 109 |
+
"tokenizer_sha256": "ebae4b4310068ee102f7bf68ac68b79ee1425baa32de3ca8ff8138692529d84e",
|
| 110 |
+
"tokens_seen": 10000007168
|
| 111 |
+
},
|
| 112 |
+
"updated_at": "2026-09-14T21:05:23Z",
|
| 113 |
+
"work_dir": "/workspace/WarpState/rtx6000_40b/work/posttrain/step-00076294"
|
| 114 |
+
}
|
model/snapshot.json
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"assistant_tokens_seen": 143695039,
|
| 3 |
+
"captured_at_utc": "2026-09-14T21:06:02Z",
|
| 4 |
+
"config": {
|
| 5 |
+
"chunk_size": 128,
|
| 6 |
+
"d_model": 2048,
|
| 7 |
+
"dropout": 0.0,
|
| 8 |
+
"ffn_hidden": 7168,
|
| 9 |
+
"gradient_checkpointing": false,
|
| 10 |
+
"logical_depth": 28,
|
| 11 |
+
"loss_chunk_size": 512,
|
| 12 |
+
"n_cores": 28,
|
| 13 |
+
"n_heads": 32,
|
| 14 |
+
"parallel_memory": true,
|
| 15 |
+
"rms_eps": 1e-05,
|
| 16 |
+
"vocab_size": 32768
|
| 17 |
+
},
|
| 18 |
+
"format": "warpstate-chat-v1",
|
| 19 |
+
"inference_only": true,
|
| 20 |
+
"origin": {
|
| 21 |
+
"captured_at_utc": "2026-09-14T14:27:16Z",
|
| 22 |
+
"config": {
|
| 23 |
+
"chunk_size": 128,
|
| 24 |
+
"d_model": 2048,
|
| 25 |
+
"dropout": 0.0,
|
| 26 |
+
"ffn_hidden": 7168,
|
| 27 |
+
"gradient_checkpointing": false,
|
| 28 |
+
"logical_depth": 28,
|
| 29 |
+
"loss_chunk_size": 512,
|
| 30 |
+
"n_cores": 28,
|
| 31 |
+
"n_heads": 32,
|
| 32 |
+
"parallel_memory": true,
|
| 33 |
+
"rms_eps": 1e-05,
|
| 34 |
+
"vocab_size": 32768
|
| 35 |
+
},
|
| 36 |
+
"format": "warpstate-chat-v1",
|
| 37 |
+
"inference_only": true,
|
| 38 |
+
"source": {
|
| 39 |
+
"device": 66,
|
| 40 |
+
"inode": 8616307015,
|
| 41 |
+
"mtime_ns": 1789396019640599046,
|
| 42 |
+
"path": "/workspace/WarpState/rtx6000_40b/work/run/latest.pt",
|
| 43 |
+
"size": 24061705357
|
| 44 |
+
},
|
| 45 |
+
"step": 76294,
|
| 46 |
+
"tokenizer_sha256": "ebae4b4310068ee102f7bf68ac68b79ee1425baa32de3ca8ff8138692529d84e",
|
| 47 |
+
"tokens_seen": 10000007168,
|
| 48 |
+
"val_loss": 2.3496886491775513,
|
| 49 |
+
"weights_path": "/workspace/WarpState/rtx6000_40b/work/posttrain/step-00076294/initialization/model.pt",
|
| 50 |
+
"weights_sha256": "dc7605cecd6c807135e3d83e04496fa96079fef022bed1952c8cf21df5f573da"
|
| 51 |
+
},
|
| 52 |
+
"snapshot_bytes": 8020534643,
|
| 53 |
+
"snapshot_sha256": "ddf25c0260a189614613c4ceb3b778983aec601c55ff4a175aff350d85ce1f44",
|
| 54 |
+
"source": {
|
| 55 |
+
"device": 66,
|
| 56 |
+
"inode": 4305102787,
|
| 57 |
+
"mtime_ns": 1789419900073676679,
|
| 58 |
+
"path": "/workspace/WarpState/rtx6000_40b/work/posttrain/step-00076294/run/latest.pt",
|
| 59 |
+
"sha256": "b94426834e5c56895ed34529479fe540ff14c2b6cef9c6804b9cf16a34ec27c8",
|
| 60 |
+
"size": 24061672375
|
| 61 |
+
},
|
| 62 |
+
"stage": "sft",
|
| 63 |
+
"step": 10312,
|
| 64 |
+
"tokenizer_sha256": "ebae4b4310068ee102f7bf68ac68b79ee1425baa32de3ca8ff8138692529d84e",
|
| 65 |
+
"tokens_seen": 212321372,
|
| 66 |
+
"val_loss": 1.1157395899572582
|
| 67 |
+
}
|
model/tokenizer.model
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ebae4b4310068ee102f7bf68ac68b79ee1425baa32de3ca8ff8138692529d84e
|
| 3 |
+
size 545080
|