{ "archive_dir": "/workspace/WarpState/rtx6000_40b/work/archives/step-00076294", "archive_sha256": "dc7605cecd6c807135e3d83e04496fa96079fef022bed1952c8cf21df5f573da", "armed_at": "2026-09-14T09:43:24Z", "command": [ "/workspace/WarpState/rtx6000_40b/.server-venv/bin/python", "-B", "-m", "torch.distributed.run", "--standalone", "--nnodes", "1", "--nproc-per-node", "2", "--log-dir", "/workspace/WarpState/rtx6000_40b/work/posttrain/step-00076294/logs/torchrun", "/workspace/WarpState/rtx6000_40b/posttrain.py", "--config", "/workspace/WarpState/rtx6000_40b/configs/posttrain_10b.json", "--work-dir", "/workspace/WarpState/rtx6000_40b/work/posttrain/step-00076294/run", "--data-dir", "/workspace/WarpState/rtx6000_40b/work/posttrain/data", "--weights", "/workspace/WarpState/rtx6000_40b/work/posttrain/step-00076294/initialization/model.pt", "--tokenizer", "/workspace/WarpState/rtx6000_40b/tokenizer.model", "--stop-file", "/workspace/WarpState/rtx6000_40b/work/posttrain/step-00076294/control/training-stop-request" ], "completion": { "assistant_tokens_seen": 143695039, "base_step": 76294, "checkpoint": "/workspace/WarpState/rtx6000_40b/work/posttrain/step-00076294/run/latest.pt", "completed": true, "evaluation_file": "/workspace/WarpState/rtx6000_40b/work/posttrain/step-00076294/run/evaluation.json", "format": "warpstate-posttrain-completion-v1", "paused_for_evaluation": true, "stage": "sft", "step": 10312, "timestamp": 1789419903.0140994, "tokens_seen": 212321372, "total_steps": 10312, "val_loss": 1.1157395899572582 }, "data_ready": true, "format": "warpstate-posttrain-handoff-v1", "git_revision": "ea63c41cc541fdef8c36da8a85f7b625a2bcec40", "message": "SFT finished; paused for evaluation. DPO has not been started.", "phase": "complete", "preflight": { "accepted": true, "bf16_compute": true, "exact_resume_bookkeeping": true, "final_validation_loss": 10.38550066947937, "fixture_sha256": "bb6a842bbdc14a0a0b0537ba58e7630e16bbd09fc0613520303cff674a900dd3", "format": "warpstate-posttrain-preflight-v1", "fp32_parameters_and_adam": true, "generator_sha256": "4489ae1f37c13e3226d6687147423a9f6a95a0509c135357291ea0a0a9d9664a", "masked_tail": true, "numerical_atol": 1e-07, "numerical_comparison": { "model": { "bitwise_equal": true, "max_absolute_difference": 0.0, "tensor_count": 13 }, "optimizer": { "bitwise_equal": true, "max_absolute_difference": 0.0, "tensor_count": 39 } }, "numerical_rtol": 1e-05, "optimizer_steps": 2, "processes": 2, "source_sha256": { "ops/checkpoint_writer.py": "3c7fe494e8782802b93a439c05ddd6f416711ca1b3f0f8a196a97a48e37f0216", "ops/loss_experiment.py": "12bf7822979692d5106e30544d0ed5d977fb06f556c068ef5f92ddf56ab8376e", "ops/posttrain_data.py": "486d2e96e7e99ffd05caf0de7d92068ce47047d4641ede3515681cbb79a9ce10", "ops/residual_experiment.py": "e82aa68e36c76a228156016eb04a02ce1a632a63ac6eda567ec9c4d258aae8f1", "posttrain.py": "bd10e97e61b3289e83b7e79c76042bf7f04ce58f4af9067db97ac1fe60d4a411", "run_40b.py": "9ad6533d781e376b3b40c16e6c29ef15063d39f917a93800489f17115be15ba0", "warpstate/__init__.py": "f386c0880cb499d84159d8205667d250a72d0e8a102f0d7be6af0ba0ea33b22b", "warpstate/config.py": "1d716b6c582bab21e8d5fcbaa6d9f3be4f792baf9b6abe826c72422f8eced6ad", "warpstate/model.py": "025330ee58cca899695c7631e1a06d2efd006402c61e51106afe6fa2e1d35161" }, "total_samples": 5 }, "source_sha256": { "rtx6000_40b/chat.py": "148fd4f20d445ba01723d4c2625372132270884d5c44104f31a3cc13f7c02774", "rtx6000_40b/configs/posttrain_10b.json": "a0f8d06e38f72245e046a6a888053c92aa23627256599348f5889886d61df179", "rtx6000_40b/ops/checkpoint_writer.py": "3c7fe494e8782802b93a439c05ddd6f416711ca1b3f0f8a196a97a48e37f0216", "rtx6000_40b/ops/launch_training.py": "fe9a204023300632766aabe7ff4b0cd22dba5ca7a306040da7ce3b21c9305e08", "rtx6000_40b/ops/loss_experiment.py": "12bf7822979692d5106e30544d0ed5d977fb06f556c068ef5f92ddf56ab8376e", "rtx6000_40b/ops/posttrain_data.py": "486d2e96e7e99ffd05caf0de7d92068ce47047d4641ede3515681cbb79a9ce10", "rtx6000_40b/ops/posttrain_handoff.py": "8f7bb104183bb59802774c5500965d3ac2d1ffd1cb3de18256d075d5414f7c08", "rtx6000_40b/ops/posttrain_preflight.py": "4489ae1f37c13e3226d6687147423a9f6a95a0509c135357291ea0a0a9d9664a", "rtx6000_40b/ops/residual_experiment.py": "e82aa68e36c76a228156016eb04a02ce1a632a63ac6eda567ec9c4d258aae8f1", "rtx6000_40b/ops/training-env.sh": "f5c9d6557d1dd9721e2e4c7bdeef001d994481c863c41bdd4080fd88484aa6f1", "rtx6000_40b/posttrain.py": "bd10e97e61b3289e83b7e79c76042bf7f04ce58f4af9067db97ac1fe60d4a411", "rtx6000_40b/run_40b.py": "9ad6533d781e376b3b40c16e6c29ef15063d39f917a93800489f17115be15ba0", "rtx6000_40b/warpstate/__init__.py": "f386c0880cb499d84159d8205667d250a72d0e8a102f0d7be6af0ba0ea33b22b", "rtx6000_40b/warpstate/config.py": "1d716b6c582bab21e8d5fcbaa6d9f3be4f792baf9b6abe826c72422f8eced6ad", "rtx6000_40b/warpstate/model.py": "025330ee58cca899695c7631e1a06d2efd006402c61e51106afe6fa2e1d35161" }, "target": { "step": 76294, "tokenizer_sha256": "ebae4b4310068ee102f7bf68ac68b79ee1425baa32de3ca8ff8138692529d84e", "tokens_seen": 10000007168 }, "updated_at": "2026-09-14T21:05:23Z", "work_dir": "/workspace/WarpState/rtx6000_40b/work/posttrain/step-00076294" }