| { |
| "run": { |
| "git_commit": null, |
| "config": { |
| "path": "/home/nates/musicxml/configs/train_200m_sft_tier1_vast.yaml", |
| "sha256": "02851fe6647c34f735ad7852f1226516da6ebd56e482d920da5de5a67643d770", |
| "values": { |
| "model_config": "configs/model_200m_tier1.yaml", |
| "shard_root": "data/shards/sft_tier1", |
| "tokenizer_root": "artifacts/tokenizer_v2_tier1", |
| "seq_len": 2048, |
| "init_from": "artifacts/model_200m_tier1/ckpt_latest.pt", |
| "micro_batch": 32, |
| "grad_accum": 8, |
| "gradient_checkpointing": false, |
| "fp8": true, |
| "fp8_recipe": "tensorwise", |
| "compile": true, |
| "dtype": "bfloat16", |
| "max_steps": 760, |
| "warmup_steps": 38, |
| "lr": 3e-05, |
| "min_lr": 3e-06, |
| "weight_decay": 0.1, |
| "beta1": 0.9, |
| "beta2": 0.95, |
| "grad_clip": 1.0, |
| "seed": 20260910, |
| "shuffle_data": true, |
| "deterministic_data_order": true, |
| "log_interval": 20, |
| "eval_interval": 100, |
| "eval_max_batches": 20, |
| "checkpoint_interval": 100, |
| "ckpt_path": "artifacts/model_200m_sft_tier1/ckpt_best.pt", |
| "latest_ckpt_path": "artifacts/model_200m_sft_tier1/ckpt_latest.pt", |
| "summary_path": "artifacts/model_200m_sft_tier1/run_summary.json" |
| } |
| }, |
| "model_config": { |
| "path": "/home/nates/musicxml/configs/model_200m_tier1.yaml", |
| "sha256": "715bfb27addb075a7d1fb0e43fabb73886df1923d0c77a20b901d178d42fd8b5" |
| }, |
| "tokenizer": { |
| "root": "/home/nates/musicxml/artifacts/tokenizer_v2_tier1", |
| "files": [ |
| "bpe.json", |
| "vocab.json" |
| ], |
| "sha256": "e354a24a066e5060f0c21b9f516c8f49d32f190852563ffb354c450bdac40885" |
| }, |
| "dataset": { |
| "root": "/home/nates/musicxml/data/shards/sft_tier1", |
| "manifest": "/home/nates/musicxml/data/shards/sft_tier1/manifest.json", |
| "manifest_sha256": "9de77cf72070b8ca22b75243bed59157404deb04449ff2aaffe528b9954ae8e8", |
| "split_descriptor_sha256": "7e73c42cc3f52eef9e8f4ccaf6a81ce3a387f125ad487f6fa5c891eaa5d4b32b", |
| "splits": { |
| "train": { |
| "rows": 97157, |
| "seq_len": 2048, |
| "shards": [ |
| { |
| "tokens": "train/tokens_0000.npy", |
| "mask": "train/mask_0000.npy", |
| "rows": 51200 |
| }, |
| { |
| "tokens": "train/tokens_0001.npy", |
| "mask": "train/mask_0001.npy", |
| "rows": 45957 |
| } |
| ] |
| }, |
| "validation": { |
| "rows": 1258, |
| "seq_len": 2048, |
| "shards": [ |
| { |
| "tokens": "validation/tokens_0000.npy", |
| "mask": "validation/mask_0000.npy", |
| "rows": 1258 |
| } |
| ] |
| } |
| } |
| }, |
| "environment": { |
| "python": "3.10.12 (main, Aug 31 2026, 10:18:17) [GCC 11.4.0]", |
| "platform": "Linux-6.8.0-1066-gcp-x86_64-with-glibc2.35", |
| "pytorch": "2.9.1+cu129", |
| "numpy": "2.2.6", |
| "cuda_runtime": "12.9", |
| "cudnn": 91002 |
| }, |
| "gpu": { |
| "name": "NVIDIA RTX PRO 6000 Blackwell Server Edition", |
| "total_memory_bytes": 101973819392, |
| "capability": [ |
| 12, |
| 0 |
| ], |
| "driver": "580.178.04" |
| }, |
| "seed": 20260910 |
| }, |
| "params_total": 201398272, |
| "params_trainable": 201398272, |
| "start_step": 0, |
| "steps_completed": 760, |
| "elapsed_seconds_this_invocation": 2683.6809680461884, |
| "peak_vram_gb": 45.785115648, |
| "loss_curve": [ |
| [ |
| 0, |
| 0.2846265509724617 |
| ], |
| [ |
| 20, |
| 0.2832777798175812 |
| ], |
| [ |
| 40, |
| 0.2576543968170881 |
| ], |
| [ |
| 60, |
| 0.2751853410154581 |
| ], |
| [ |
| 80, |
| 0.27040398120880127 |
| ], |
| [ |
| 100, |
| 0.23810604959726334 |
| ], |
| [ |
| 120, |
| 0.2701740376651287 |
| ], |
| [ |
| 140, |
| 0.264717735350132 |
| ], |
| [ |
| 160, |
| 0.23320921510457993 |
| ], |
| [ |
| 180, |
| 0.2678709514439106 |
| ], |
| [ |
| 200, |
| 0.24964266642928123 |
| ], |
| [ |
| 220, |
| 0.26365517638623714 |
| ], |
| [ |
| 240, |
| 0.24836344085633755 |
| ], |
| [ |
| 260, |
| 0.23373913019895554 |
| ], |
| [ |
| 280, |
| 0.2419311013072729 |
| ], |
| [ |
| 300, |
| 0.2591604311019182 |
| ], |
| [ |
| 320, |
| 0.25108602456748486 |
| ], |
| [ |
| 340, |
| 0.22293414361774921 |
| ], |
| [ |
| 360, |
| 0.23284843936562538 |
| ], |
| [ |
| 380, |
| 0.2620360627770424 |
| ], |
| [ |
| 400, |
| 0.23341848328709602 |
| ], |
| [ |
| 420, |
| 0.2336137406527996 |
| ], |
| [ |
| 440, |
| 0.22174756787717342 |
| ], |
| [ |
| 460, |
| 0.20786287263035774 |
| ], |
| [ |
| 480, |
| 0.23133441992104053 |
| ], |
| [ |
| 500, |
| 0.24537247605621815 |
| ], |
| [ |
| 520, |
| 0.22828913666307926 |
| ], |
| [ |
| 540, |
| 0.24769587628543377 |
| ], |
| [ |
| 560, |
| 0.24742664769291878 |
| ], |
| [ |
| 580, |
| 0.2207590974867344 |
| ], |
| [ |
| 600, |
| 0.2361213807016611 |
| ], |
| [ |
| 620, |
| 0.22333898954093456 |
| ], |
| [ |
| 640, |
| 0.2485098633915186 |
| ], |
| [ |
| 660, |
| 0.24824612773954868 |
| ], |
| [ |
| 680, |
| 0.24885406717658043 |
| ], |
| [ |
| 700, |
| 0.2211875542998314 |
| ], |
| [ |
| 720, |
| 0.20957332104444504 |
| ], |
| [ |
| 740, |
| 0.2173500619828701 |
| ], |
| [ |
| 759, |
| 0.23597008362412453 |
| ] |
| ], |
| "val_losses": [ |
| [ |
| 99, |
| 0.31183503886185754 |
| ], |
| [ |
| 199, |
| 0.31025536271044013 |
| ], |
| [ |
| 299, |
| 0.3087775707864169 |
| ], |
| [ |
| 399, |
| 0.3122648150469459 |
| ], |
| [ |
| 499, |
| 0.31319335526241254 |
| ], |
| [ |
| 599, |
| 0.31266183949211407 |
| ], |
| [ |
| 699, |
| 0.31301069269288756 |
| ], |
| [ |
| 759, |
| 0.3136875436344451 |
| ] |
| ], |
| "best_val_loss": 0.3087775707864169, |
| "best_step": 299, |
| "micro_batch": 32, |
| "grad_accum": 8, |
| "seq_len": 2048, |
| "tokens_per_step_max": 524032, |
| "tokens_seen": 199562135, |
| "resume_from": null, |
| "best_checkpoint": "/home/nates/musicxml/artifacts/model_200m_sft_tier1/ckpt_best.pt", |
| "latest_checkpoint": "/home/nates/musicxml/artifacts/model_200m_sft_tier1/ckpt_latest.pt", |
| "precision": "torchao_fp8", |
| "te_recipe": null, |
| "throughput_toks_per_s": [ |
| [ |
| 0, |
| 2934.2845468938017 |
| ], |
| [ |
| 20, |
| 79685.4051552035 |
| ], |
| [ |
| 40, |
| 80436.13652294016 |
| ], |
| [ |
| 60, |
| 81103.02826918027 |
| ], |
| [ |
| 80, |
| 80978.39934958126 |
| ], |
| [ |
| 100, |
| 61124.23692212198 |
| ], |
| [ |
| 120, |
| 80949.35429477402 |
| ], |
| [ |
| 140, |
| 81100.49637965474 |
| ], |
| [ |
| 160, |
| 81782.3154442603 |
| ], |
| [ |
| 180, |
| 81352.39963476159 |
| ], |
| [ |
| 200, |
| 61476.625790909646 |
| ], |
| [ |
| 220, |
| 79335.2938181554 |
| ], |
| [ |
| 240, |
| 81202.6368181386 |
| ], |
| [ |
| 260, |
| 81061.85491368544 |
| ], |
| [ |
| 280, |
| 81568.15493084869 |
| ], |
| [ |
| 300, |
| 61402.92680827696 |
| ], |
| [ |
| 320, |
| 81593.74368636096 |
| ], |
| [ |
| 340, |
| 79963.94063424174 |
| ], |
| [ |
| 360, |
| 80044.8911001716 |
| ], |
| [ |
| 380, |
| 79824.95694230743 |
| ], |
| [ |
| 400, |
| 68050.27872487987 |
| ], |
| [ |
| 420, |
| 79631.7820692823 |
| ], |
| [ |
| 440, |
| 80400.248119905 |
| ], |
| [ |
| 460, |
| 79856.98116091211 |
| ], |
| [ |
| 480, |
| 81333.78704798948 |
| ], |
| [ |
| 500, |
| 68311.21101458241 |
| ], |
| [ |
| 520, |
| 79491.89788235497 |
| ], |
| [ |
| 540, |
| 80735.1329593898 |
| ], |
| [ |
| 560, |
| 83276.14449517097 |
| ], |
| [ |
| 580, |
| 81067.41114257302 |
| ], |
| [ |
| 600, |
| 69974.22693335732 |
| ], |
| [ |
| 620, |
| 82367.9521560698 |
| ], |
| [ |
| 640, |
| 80877.23511417964 |
| ], |
| [ |
| 660, |
| 81774.84449776652 |
| ], |
| [ |
| 680, |
| 80270.33452565636 |
| ], |
| [ |
| 700, |
| 69265.66820467077 |
| ], |
| [ |
| 720, |
| 78387.42052580223 |
| ], |
| [ |
| 740, |
| 80889.83415720014 |
| ], |
| [ |
| 759, |
| 81170.87283314017 |
| ] |
| ], |
| "ppl_curve": [ |
| [ |
| 99, |
| 1.3659293491834388 |
| ], |
| [ |
| 199, |
| 1.3637733265231424 |
| ], |
| [ |
| 299, |
| 1.3617594417328527 |
| ], |
| [ |
| 399, |
| 1.36651651925505 |
| ], |
| [ |
| 499, |
| 1.3677859740769955 |
| ], |
| [ |
| 599, |
| 1.367059167433088 |
| ], |
| [ |
| 699, |
| 1.3675361535935309 |
| ], |
| [ |
| 759, |
| 1.368462085050144 |
| ] |
| ] |
| } |
