Buckets:
| { | |
| "success": true, | |
| "image": "sha256:63e5cf4304efec771bdf3ad56291db7ad0168d86ea8d34a7b4629e1ce43d905b", | |
| "gpu_uuids": [ | |
| "GPU-2193d587-5242-e4af-f9ff-ce9da68bd83c", | |
| "GPU-e6205be5-b1a7-bc81-55de-8f79262cf524" | |
| ], | |
| "runs": [ | |
| { | |
| "program": "original", | |
| "status": "valid", | |
| "correctness": 1.0, | |
| "reward": 0.27382720778055364, | |
| "raw_score": 5477.907079277918, | |
| "score_us": 5477.907079277918, | |
| "wall_seconds": 27.8378162458539, | |
| "official_commit": "6c40e82dab9d5de7416ac873ad5cd3106084aaed", | |
| "manifest_sha256": "3a909e8f95197114714ca34d28c929ca186fece71fc4b5932ff3be67a10815c9", | |
| "result": { | |
| "success": true, | |
| "error": "", | |
| "system": { | |
| "gpu": "NVIDIA H100 80GB HBM3", | |
| "device_count": 1, | |
| "cpu": "Intel(R) Xeon(R) Platinum 8480C", | |
| "runtime": "CUDA", | |
| "platform": "Linux-5.15.0-1098-azure-x86_64-with-glibc2.39", | |
| "torch": "2.7.1+cu128", | |
| "hostname": "b296da5d37ab", | |
| "requeues": 0 | |
| }, | |
| "runs": { | |
| "test": { | |
| "start": "2026-11-05 10:59:11.901818", | |
| "end": "2026-11-05 10:59:23.743949", | |
| "compilation": null, | |
| "run": { | |
| "success": true, | |
| "passed": true, | |
| "command": "python3 eval.py test /tmp/tmpwopv3dv4", | |
| "stdout": "", | |
| "stderr": "", | |
| "exit_code": 0, | |
| "duration": 8.6984375230968, | |
| "result": { | |
| "test-count": "18", | |
| "test.0.spec": "seqlen: 32; bs: 1; dim: 128; hiddendim: 128; seed: 9371; nomask: True; distribution: normal", | |
| "test.0.status": "pass", | |
| "test.0.message": "Maximum error: 0.003163546323776245", | |
| "test.1.spec": "seqlen: 32; bs: 1; dim: 128; hiddendim: 128; seed: 1092; nomask: False; distribution: normal", | |
| "test.1.status": "pass", | |
| "test.1.message": "Maximum error: 0.006548404693603516", | |
| "test.2.spec": "seqlen: 64; bs: 2; dim: 256; hiddendim: 128; seed: 2291; nomask: True; distribution: normal", | |
| "test.2.status": "pass", | |
| "test.2.message": "Maximum error: 0.0018312931060791016", | |
| "test.3.spec": "seqlen: 64; bs: 2; dim: 256; hiddendim: 128; seed: 210284; nomask: False; distribution: normal", | |
| "test.3.status": "pass", | |
| "test.3.message": "Maximum error: 0.0023785829544067383", | |
| "test.4.spec": "seqlen: 128; bs: 1; dim: 768; hiddendim: 128; seed: 81934; nomask: True; distribution: normal", | |
| "test.4.status": "pass", | |
| "test.4.message": "Maximum error: 0.002499222755432129", | |
| "test.5.spec": "seqlen: 256; bs: 1; dim: 128; hiddendim: 128; seed: 1932; nomask: True; distribution: normal", | |
| "test.5.status": "pass", | |
| "test.5.message": "Maximum error: 0.00362396240234375", | |
| "test.6.spec": "seqlen: 256; bs: 1; dim: 128; hiddendim: 128; seed: 10432; nomask: False; distribution: normal", | |
| "test.6.status": "pass", | |
| "test.6.message": "Maximum error: 0.004271507263183594", | |
| "test.7.spec": "seqlen: 768; bs: 2; dim: 128; hiddendim: 128; seed: 731; nomask: True; distribution: normal", | |
| "test.7.status": "pass", | |
| "test.7.message": "Maximum error: 0.004614591598510742", | |
| "test.8.spec": "seqlen: 1024; bs: 1; dim: 384; hiddendim: 128; seed: 53121; nomask: False; distribution: normal", | |
| "test.8.status": "pass", | |
| "test.8.message": "Maximum error: 0.0018085241317749023", | |
| "test.9.spec": "seqlen: 1024; bs: 1; dim: 768; hiddendim: 128; seed: 31; nomask: True; distribution: normal", | |
| "test.9.status": "pass", | |
| "test.9.message": "Maximum error: 0.0017676353454589844", | |
| "test.10.spec": "seqlen: 1024; bs: 1; dim: 768; hiddendim: 128; seed: 4921; nomask: False; distribution: normal", | |
| "test.10.status": "pass", | |
| "test.10.message": "Maximum error: 0.002179384231567383", | |
| "test.11.spec": "seqlen: 32; bs: 1; dim: 128; hiddendim: 128; seed: 937321; nomask: True; distribution: cauchy", | |
| "test.11.status": "pass", | |
| "test.11.message": "Maximum error: 0.0033462047576904297", | |
| "test.12.spec": "seqlen: 64; bs: 2; dim: 256; hiddendim: 128; seed: 2291; nomask: True; distribution: cauchy", | |
| "test.12.status": "pass", | |
| "test.12.message": "Maximum error: 0.0016717910766601562", | |
| "test.13.spec": "seqlen: 128; bs: 1; dim: 768; hiddendim: 128; seed: 8134; nomask: True; distribution: cauchy", | |
| "test.13.status": "pass", | |
| "test.13.message": "Maximum error: 0.001565694808959961", | |
| "test.14.spec": "seqlen: 256; bs: 1; dim: 128; hiddendim: 128; seed: 932; nomask: True; distribution: cauchy", | |
| "test.14.status": "pass", | |
| "test.14.message": "Maximum error: 0.0031557083129882812", | |
| "test.15.spec": "seqlen: 768; bs: 2; dim: 128; hiddendim: 128; seed: 31; nomask: True; distribution: cauchy", | |
| "test.15.status": "pass", | |
| "test.15.message": "Maximum error: 0.0031419992446899414", | |
| "test.16.spec": "seqlen: 1024; bs: 1; dim: 384; hiddendim: 128; seed: 5321; nomask: False; distribution: cauchy", | |
| "test.16.status": "pass", | |
| "test.16.message": "Maximum error: 0.0022017955780029297", | |
| "test.17.spec": "seqlen: 1024; bs: 1; dim: 768; hiddendim: 128; seed: 491; nomask: False; distribution: cauchy", | |
| "test.17.status": "pass", | |
| "test.17.message": "Maximum error: 0.0017014741897583008", | |
| "check": "pass" | |
| } | |
| }, | |
| "profile": null | |
| }, | |
| "leaderboard": { | |
| "start": "2026-11-05 10:59:23.744062", | |
| "end": "2026-11-05 10:59:35.856388", | |
| "compilation": null, | |
| "run": { | |
| "success": true, | |
| "passed": true, | |
| "command": "python3 eval.py leaderboard /tmp/tmpl2y_2wbg", | |
| "stdout": "", | |
| "stderr": "", | |
| "exit_code": 0, | |
| "duration": 8.951863212510943, | |
| "result": { | |
| "benchmark-count": "7", | |
| "benchmark.0.spec": "seqlen: 256; bs: 2; dim: 128; hiddendim: 128; seed: 9371; nomask: True; distribution: normal", | |
| "benchmark.0.runs": "53", | |
| "benchmark.0.mean": "1251079.242184477", | |
| "benchmark.0.std": "9036.581630376093", | |
| "benchmark.0.err": "1241.269948916824", | |
| "benchmark.0.best": "1239871.9787597656", | |
| "benchmark.0.worst": "1286656.0220718384", | |
| "benchmark.1.spec": "seqlen: 768; bs: 1; dim: 128; hiddendim: 128; seed: 381; nomask: True; distribution: cauchy", | |
| "benchmark.1.runs": "3", | |
| "benchmark.1.mean": "8538026.491800943", | |
| "benchmark.1.std": "8666.433498375583", | |
| "benchmark.1.err": "5003.567713201133", | |
| "benchmark.1.best": "8528096.199035645", | |
| "benchmark.1.worst": "8544063.568115234", | |
| "benchmark.2.spec": "seqlen: 256; bs: 2; dim: 384; hiddendim: 128; seed: 2301; nomask: False; distribution: normal", | |
| "benchmark.2.runs": "8", | |
| "benchmark.2.mean": "1644200.012087822", | |
| "benchmark.2.std": "4578.963810479211", | |
| "benchmark.2.err": "1618.9081805988217", | |
| "benchmark.2.best": "1635040.044784546", | |
| "benchmark.2.worst": "1649984.0021133423", | |
| "benchmark.3.spec": "seqlen: 512; bs: 1; dim: 128; hiddendim: 128; seed: 12819; nomask: True; distribution: normal", | |
| "benchmark.3.runs": "15", | |
| "benchmark.3.mean": "2632354.132334391", | |
| "benchmark.3.std": "10145.229093277756", | |
| "benchmark.3.err": "2619.4868881149146", | |
| "benchmark.3.best": "2620383.9778900146", | |
| "benchmark.3.worst": "2656224.012374878", | |
| "benchmark.4.spec": "seqlen: 1024; bs: 1; dim: 128; hiddendim: 128; seed: 381; nomask: True; distribution: cauchy", | |
| "benchmark.4.runs": "5", | |
| "benchmark.4.mean": "16315340.423583984", | |
| "benchmark.4.std": "32547.727307042802", | |
| "benchmark.4.err": "14555.786154334774", | |
| "benchmark.4.best": "16280191.421508789", | |
| "benchmark.4.worst": "16355615.615844727", | |
| "benchmark.5.spec": "seqlen: 768; bs: 1; dim: 384; hiddendim: 128; seed: 481; nomask: False; distribution: normal", | |
| "benchmark.5.runs": "3", | |
| "benchmark.5.mean": "10185013.45316569", | |
| "benchmark.5.std": "10433.306742851937", | |
| "benchmark.5.err": "6023.672456523504", | |
| "benchmark.5.best": "10174943.923950195", | |
| "benchmark.5.worst": "10195775.985717773", | |
| "benchmark.6.spec": "seqlen: 1024; bs: 1; dim: 384; hiddendim: 128; seed: 23291; nomask: True; distribution: normal", | |
| "benchmark.6.runs": "5", | |
| "benchmark.6.mean": "19266534.423828125", | |
| "benchmark.6.std": "40055.48160779685", | |
| "benchmark.6.err": "17913.355949305263", | |
| "benchmark.6.best": "19244895.935058594", | |
| "benchmark.6.worst": "19338016.510009766", | |
| "check": "pass" | |
| } | |
| }, | |
| "profile": null | |
| } | |
| } | |
| }, | |
| "request_id": "b8d7f5578034367847aa87396050e287", | |
| "gpu_index": 6, | |
| "gpu_uuid": "GPU-2193d587-5242-e4af-f9ff-ce9da68bd83c", | |
| "image": "sha256:63e5cf4304efec771bdf3ad56291db7ad0168d86ea8d34a7b4629e1ce43d905b", | |
| "canonical_image_config_sha256": "sha256:63e5cf4304efec771bdf3ad56291db7ad0168d86ea8d34a7b4629e1ce43d905b", | |
| "image_manifest_sha256": "sha256:10c2bd0309edcea7cdc78723f9b8a8e893995c1b014d8305c33cf7b38876a075", | |
| "queue_seconds": 0.4665392003953457, | |
| "host_execution_seconds": 30.544493351131678, | |
| "execution_timeout_seconds": 530, | |
| "program_sha256": "d4fe30c2270c484dace78e1462a1d1d3a2e172fe8aefd0212f682e33bdc18174", | |
| "node": "datava270000004", | |
| "monitor_revision": "stale-pid-race-fix-v1", | |
| "pool_queue_seconds": 0.4722871035337448, | |
| "evaluator_gpus": [ | |
| 6, | |
| 7 | |
| ] | |
| }, | |
| { | |
| "program": "split", | |
| "status": "valid", | |
| "correctness": 1.0, | |
| "reward": 0.27418048936086675, | |
| "raw_score": 5470.848795611246, | |
| "score_us": 5470.848795611246, | |
| "wall_seconds": 27.317062279209495, | |
| "official_commit": "6c40e82dab9d5de7416ac873ad5cd3106084aaed", | |
| "manifest_sha256": "3a909e8f95197114714ca34d28c929ca186fece71fc4b5932ff3be67a10815c9", | |
| "result": { | |
| "success": true, | |
| "error": "", | |
| "system": { | |
| "gpu": "NVIDIA H100 80GB HBM3", | |
| "device_count": 1, | |
| "cpu": "Intel(R) Xeon(R) Platinum 8480C", | |
| "runtime": "CUDA", | |
| "platform": "Linux-5.15.0-1098-azure-x86_64-with-glibc2.39", | |
| "torch": "2.7.1+cu128", | |
| "hostname": "63933b9fa20f", | |
| "requeues": 0 | |
| }, | |
| "runs": { | |
| "test": { | |
| "start": "2026-11-05 10:59:11.585405", | |
| "end": "2026-11-05 10:59:23.933745", | |
| "compilation": null, | |
| "run": { | |
| "success": true, | |
| "passed": true, | |
| "command": "python3 eval.py test /tmp/tmp28lwe2v6", | |
| "stdout": "", | |
| "stderr": "", | |
| "exit_code": 0, | |
| "duration": 9.23547207377851, | |
| "result": { | |
| "test-count": "18", | |
| "test.0.spec": "seqlen: 32; bs: 1; dim: 128; hiddendim: 128; seed: 9371; nomask: True; distribution: normal", | |
| "test.0.status": "pass", | |
| "test.0.message": "Maximum error: 0.0025129318237304688", | |
| "test.1.spec": "seqlen: 32; bs: 1; dim: 128; hiddendim: 128; seed: 1092; nomask: False; distribution: normal", | |
| "test.1.status": "pass", | |
| "test.1.message": "Maximum error: 0.0027701854705810547", | |
| "test.2.spec": "seqlen: 64; bs: 2; dim: 256; hiddendim: 128; seed: 2291; nomask: True; distribution: normal", | |
| "test.2.status": "pass", | |
| "test.2.message": "Maximum error: 0.0022954940795898438", | |
| "test.3.spec": "seqlen: 64; bs: 2; dim: 256; hiddendim: 128; seed: 210284; nomask: False; distribution: normal", | |
| "test.3.status": "pass", | |
| "test.3.message": "Maximum error: 0.0018486976623535156", | |
| "test.4.spec": "seqlen: 128; bs: 1; dim: 768; hiddendim: 128; seed: 81934; nomask: True; distribution: normal", | |
| "test.4.status": "pass", | |
| "test.4.message": "Maximum error: 0.0013532638549804688", | |
| "test.5.spec": "seqlen: 256; bs: 1; dim: 128; hiddendim: 128; seed: 1932; nomask: True; distribution: normal", | |
| "test.5.status": "pass", | |
| "test.5.message": "Maximum error: 0.003708362579345703", | |
| "test.6.spec": "seqlen: 256; bs: 1; dim: 128; hiddendim: 128; seed: 10432; nomask: False; distribution: normal", | |
| "test.6.status": "pass", | |
| "test.6.message": "Maximum error: 0.0034810304641723633", | |
| "test.7.spec": "seqlen: 768; bs: 2; dim: 128; hiddendim: 128; seed: 731; nomask: True; distribution: normal", | |
| "test.7.status": "pass", | |
| "test.7.message": "Maximum error: 0.004608571529388428", | |
| "test.8.spec": "seqlen: 1024; bs: 1; dim: 384; hiddendim: 128; seed: 53121; nomask: False; distribution: normal", | |
| "test.8.status": "pass", | |
| "test.8.message": "Maximum error: 0.0032002925872802734", | |
| "test.9.spec": "seqlen: 1024; bs: 1; dim: 768; hiddendim: 128; seed: 31; nomask: True; distribution: normal", | |
| "test.9.status": "pass", | |
| "test.9.message": "Maximum error: 0.0025271177291870117", | |
| "test.10.spec": "seqlen: 1024; bs: 1; dim: 768; hiddendim: 128; seed: 4921; nomask: False; distribution: normal", | |
| "test.10.status": "pass", | |
| "test.10.message": "Maximum error: 0.0026807785034179688", | |
| "test.11.spec": "seqlen: 32; bs: 1; dim: 128; hiddendim: 128; seed: 937321; nomask: True; distribution: cauchy", | |
| "test.11.status": "pass", | |
| "test.11.message": "Maximum error: 0.003023982048034668", | |
| "test.12.spec": "seqlen: 64; bs: 2; dim: 256; hiddendim: 128; seed: 2291; nomask: True; distribution: cauchy", | |
| "test.12.status": "pass", | |
| "test.12.message": "Maximum error: 0.002574920654296875", | |
| "test.13.spec": "seqlen: 128; bs: 1; dim: 768; hiddendim: 128; seed: 8134; nomask: True; distribution: cauchy", | |
| "test.13.status": "pass", | |
| "test.13.message": "Maximum error: 0.002186298370361328", | |
| "test.14.spec": "seqlen: 256; bs: 1; dim: 128; hiddendim: 128; seed: 932; nomask: True; distribution: cauchy", | |
| "test.14.status": "pass", | |
| "test.14.message": "Maximum error: 0.004027128219604492", | |
| "test.15.spec": "seqlen: 768; bs: 2; dim: 128; hiddendim: 128; seed: 31; nomask: True; distribution: cauchy", | |
| "test.15.status": "pass", | |
| "test.15.message": "Maximum error: 0.005215167999267578", | |
| "test.16.spec": "seqlen: 1024; bs: 1; dim: 384; hiddendim: 128; seed: 5321; nomask: False; distribution: cauchy", | |
| "test.16.status": "pass", | |
| "test.16.message": "Maximum error: 0.002176046371459961", | |
| "test.17.spec": "seqlen: 1024; bs: 1; dim: 768; hiddendim: 128; seed: 491; nomask: False; distribution: cauchy", | |
| "test.17.status": "pass", | |
| "test.17.message": "Maximum error: 0.0012012720108032227", | |
| "check": "pass" | |
| } | |
| }, | |
| "profile": null | |
| }, | |
| "leaderboard": { | |
| "start": "2026-11-05 10:59:23.933848", | |
| "end": "2026-11-05 10:59:35.317517", | |
| "compilation": null, | |
| "run": { | |
| "success": true, | |
| "passed": true, | |
| "command": "python3 eval.py leaderboard /tmp/tmpd5dodqkt", | |
| "stdout": "", | |
| "stderr": "", | |
| "exit_code": 0, | |
| "duration": 8.330479983240366, | |
| "result": { | |
| "benchmark-count": "7", | |
| "benchmark.0.spec": "seqlen: 256; bs: 2; dim: 128; hiddendim: 128; seed: 9371; nomask: True; distribution: normal", | |
| "benchmark.0.runs": "10", | |
| "benchmark.0.mean": "1251363.1939888", | |
| "benchmark.0.std": "3954.0719666795744", | |
| "benchmark.0.err": "1250.3873446928865", | |
| "benchmark.0.best": "1247807.9795837402", | |
| "benchmark.0.worst": "1260607.9578399658", | |
| "benchmark.1.spec": "seqlen: 768; bs: 1; dim: 128; hiddendim: 128; seed: 381; nomask: True; distribution: cauchy", | |
| "benchmark.1.runs": "3", | |
| "benchmark.1.mean": "8552757.263183594", | |
| "benchmark.1.std": "5232.061729094771", | |
| "benchmark.1.err": "3020.732247709605", | |
| "benchmark.1.best": "8547552.108764648", | |
| "benchmark.1.worst": "8558015.823364258", | |
| "benchmark.2.spec": "seqlen: 256; bs: 2; dim: 384; hiddendim: 128; seed: 2301; nomask: False; distribution: normal", | |
| "benchmark.2.runs": "7", | |
| "benchmark.2.mean": "1633097.1377236503", | |
| "benchmark.2.std": "4293.5494292504645", | |
| "benchmark.2.err": "1622.80914736572", | |
| "benchmark.2.best": "1627807.9748153687", | |
| "benchmark.2.worst": "1640735.9838485718", | |
| "benchmark.3.spec": "seqlen: 512; bs: 1; dim: 128; hiddendim: 128; seed: 12819; nomask: True; distribution: normal", | |
| "benchmark.3.runs": "3", | |
| "benchmark.3.mean": "2640074.7299194336", | |
| "benchmark.3.std": "2416.0325376541073", | |
| "benchmark.3.err": "1394.8970359854936", | |
| "benchmark.3.best": "2637664.0796661377", | |
| "benchmark.3.worst": "2642496.109008789", | |
| "benchmark.4.spec": "seqlen: 1024; bs: 1; dim: 128; hiddendim: 128; seed: 381; nomask: True; distribution: cauchy", | |
| "benchmark.4.runs": "3", | |
| "benchmark.4.mean": "16270399.729410807", | |
| "benchmark.4.std": "4755.303723422112", | |
| "benchmark.4.err": "2745.475884796186", | |
| "benchmark.4.best": "16266239.166259766", | |
| "benchmark.4.worst": "16275583.267211914", | |
| "benchmark.5.spec": "seqlen: 768; bs: 1; dim: 384; hiddendim: 128; seed: 481; nomask: False; distribution: normal", | |
| "benchmark.5.runs": "3", | |
| "benchmark.5.mean": "10191786.766052246", | |
| "benchmark.5.std": "5355.786309062029", | |
| "benchmark.5.err": "3092.164667259075", | |
| "benchmark.5.best": "10188032.150268555", | |
| "benchmark.5.worst": "10197919.845581055", | |
| "benchmark.6.spec": "seqlen: 1024; bs: 1; dim: 384; hiddendim: 128; seed: 23291; nomask: True; distribution: normal", | |
| "benchmark.6.runs": "3", | |
| "benchmark.6.mean": "19169759.75036621", | |
| "benchmark.6.std": "6634.813864212958", | |
| "benchmark.6.err": "3830.6115705264124", | |
| "benchmark.6.best": "19162624.35913086", | |
| "benchmark.6.worst": "19175743.103027344", | |
| "check": "pass" | |
| } | |
| }, | |
| "profile": null | |
| } | |
| } | |
| }, | |
| "request_id": "611df10a485536588c0e238519bbc5cc", | |
| "gpu_index": 7, | |
| "gpu_uuid": "GPU-e6205be5-b1a7-bc81-55de-8f79262cf524", | |
| "image": "sha256:63e5cf4304efec771bdf3ad56291db7ad0168d86ea8d34a7b4629e1ce43d905b", | |
| "canonical_image_config_sha256": "sha256:63e5cf4304efec771bdf3ad56291db7ad0168d86ea8d34a7b4629e1ce43d905b", | |
| "image_manifest_sha256": "sha256:10c2bd0309edcea7cdc78723f9b8a8e893995c1b014d8305c33cf7b38876a075", | |
| "queue_seconds": 0.4641108028590679, | |
| "host_execution_seconds": 29.862283227965236, | |
| "execution_timeout_seconds": 530, | |
| "program_sha256": "edc2fba210584be16adab1883fd54586fd1a25b976f6ceb9c4969cf7a3473b7c", | |
| "node": "datava270000004", | |
| "monitor_revision": "stale-pid-race-fix-v1", | |
| "pool_queue_seconds": 0.46920238621532917, | |
| "evaluator_gpus": [ | |
| 6, | |
| 7 | |
| ] | |
| }, | |
| { | |
| "program": "split", | |
| "status": "valid", | |
| "correctness": 1.0, | |
| "reward": 0.27373407491630786, | |
| "raw_score": 5479.77083400601, | |
| "score_us": 5479.77083400601, | |
| "wall_seconds": 25.96976548433304, | |
| "official_commit": "6c40e82dab9d5de7416ac873ad5cd3106084aaed", | |
| "manifest_sha256": "3a909e8f95197114714ca34d28c929ca186fece71fc4b5932ff3be67a10815c9", | |
| "result": { | |
| "success": true, | |
| "error": "", | |
| "system": { | |
| "gpu": "NVIDIA H100 80GB HBM3", | |
| "device_count": 1, | |
| "cpu": "Intel(R) Xeon(R) Platinum 8480C", | |
| "runtime": "CUDA", | |
| "platform": "Linux-5.15.0-1098-azure-x86_64-with-glibc2.39", | |
| "torch": "2.7.1+cu128", | |
| "hostname": "c7ee82049ec9", | |
| "requeues": 0 | |
| }, | |
| "runs": { | |
| "test": { | |
| "start": "2026-11-05 10:59:41.068966", | |
| "end": "2026-11-05 10:59:52.533032", | |
| "compilation": null, | |
| "run": { | |
| "success": true, | |
| "passed": true, | |
| "command": "python3 eval.py test /tmp/tmp61c8rn39", | |
| "stdout": "", | |
| "stderr": "", | |
| "exit_code": 0, | |
| "duration": 8.373281108215451, | |
| "result": { | |
| "test-count": "18", | |
| "test.0.spec": "seqlen: 32; bs: 1; dim: 128; hiddendim: 128; seed: 9371; nomask: True; distribution: normal", | |
| "test.0.status": "pass", | |
| "test.0.message": "Maximum error: 0.002804994583129883", | |
| "test.1.spec": "seqlen: 32; bs: 1; dim: 128; hiddendim: 128; seed: 1092; nomask: False; distribution: normal", | |
| "test.1.status": "pass", | |
| "test.1.message": "Maximum error: 0.0036276578903198242", | |
| "test.2.spec": "seqlen: 64; bs: 2; dim: 256; hiddendim: 128; seed: 2291; nomask: True; distribution: normal", | |
| "test.2.status": "pass", | |
| "test.2.message": "Maximum error: 0.0024330615997314453", | |
| "test.3.spec": "seqlen: 64; bs: 2; dim: 256; hiddendim: 128; seed: 210284; nomask: False; distribution: normal", | |
| "test.3.status": "pass", | |
| "test.3.message": "Maximum error: 0.0027649402618408203", | |
| "test.4.spec": "seqlen: 128; bs: 1; dim: 768; hiddendim: 128; seed: 81934; nomask: True; distribution: normal", | |
| "test.4.status": "pass", | |
| "test.4.message": "Maximum error: 0.002653360366821289", | |
| "test.5.spec": "seqlen: 256; bs: 1; dim: 128; hiddendim: 128; seed: 1932; nomask: True; distribution: normal", | |
| "test.5.status": "pass", | |
| "test.5.message": "Maximum error: 0.0040130615234375", | |
| "test.6.spec": "seqlen: 256; bs: 1; dim: 128; hiddendim: 128; seed: 10432; nomask: False; distribution: normal", | |
| "test.6.status": "pass", | |
| "test.6.message": "Maximum error: 0.0029811859130859375", | |
| "test.7.spec": "seqlen: 768; bs: 2; dim: 128; hiddendim: 128; seed: 731; nomask: True; distribution: normal", | |
| "test.7.status": "pass", | |
| "test.7.message": "Maximum error: 0.0035283565521240234", | |
| "test.8.spec": "seqlen: 1024; bs: 1; dim: 384; hiddendim: 128; seed: 53121; nomask: False; distribution: normal", | |
| "test.8.status": "pass", | |
| "test.8.message": "Maximum error: 0.0024567842483520508", | |
| "test.9.spec": "seqlen: 1024; bs: 1; dim: 768; hiddendim: 128; seed: 31; nomask: True; distribution: normal", | |
| "test.9.status": "pass", | |
| "test.9.message": "Maximum error: 0.0017796754837036133", | |
| "test.10.spec": "seqlen: 1024; bs: 1; dim: 768; hiddendim: 128; seed: 4921; nomask: False; distribution: normal", | |
| "test.10.status": "pass", | |
| "test.10.message": "Maximum error: 0.002239823341369629", | |
| "test.11.spec": "seqlen: 32; bs: 1; dim: 128; hiddendim: 128; seed: 937321; nomask: True; distribution: cauchy", | |
| "test.11.status": "pass", | |
| "test.11.message": "Maximum error: 0.0020360946655273438", | |
| "test.12.spec": "seqlen: 64; bs: 2; dim: 256; hiddendim: 128; seed: 2291; nomask: True; distribution: cauchy", | |
| "test.12.status": "pass", | |
| "test.12.message": "Maximum error: 0.0020117759704589844", | |
| "test.13.spec": "seqlen: 128; bs: 1; dim: 768; hiddendim: 128; seed: 8134; nomask: True; distribution: cauchy", | |
| "test.13.status": "pass", | |
| "test.13.message": "Maximum error: 0.0011234283447265625", | |
| "test.14.spec": "seqlen: 256; bs: 1; dim: 128; hiddendim: 128; seed: 932; nomask: True; distribution: cauchy", | |
| "test.14.status": "pass", | |
| "test.14.message": "Maximum error: 0.0039560794830322266", | |
| "test.15.spec": "seqlen: 768; bs: 2; dim: 128; hiddendim: 128; seed: 31; nomask: True; distribution: cauchy", | |
| "test.15.status": "pass", | |
| "test.15.message": "Maximum error: 0.003553420305252075", | |
| "test.16.spec": "seqlen: 1024; bs: 1; dim: 384; hiddendim: 128; seed: 5321; nomask: False; distribution: cauchy", | |
| "test.16.status": "pass", | |
| "test.16.message": "Maximum error: 0.0020943880081176758", | |
| "test.17.spec": "seqlen: 1024; bs: 1; dim: 768; hiddendim: 128; seed: 491; nomask: False; distribution: cauchy", | |
| "test.17.status": "pass", | |
| "test.17.message": "Maximum error: 0.001437366008758545", | |
| "check": "pass" | |
| } | |
| }, | |
| "profile": null | |
| }, | |
| "leaderboard": { | |
| "start": "2026-11-05 10:59:52.533135", | |
| "end": "2026-11-05 11:00:03.686580", | |
| "compilation": null, | |
| "run": { | |
| "success": true, | |
| "passed": true, | |
| "command": "python3 eval.py leaderboard /tmp/tmpccprzwd9", | |
| "stdout": "", | |
| "stderr": "", | |
| "exit_code": 0, | |
| "duration": 7.935054844245315, | |
| "result": { | |
| "benchmark-count": "7", | |
| "benchmark.0.spec": "seqlen: 256; bs: 2; dim: 128; hiddendim: 128; seed: 9371; nomask: True; distribution: normal", | |
| "benchmark.0.runs": "23", | |
| "benchmark.0.mean": "1253131.135650303", | |
| "benchmark.0.std": "5940.0892705925835", | |
| "benchmark.0.err": "1238.5942337043293", | |
| "benchmark.0.best": "1243520.0214385986", | |
| "benchmark.0.worst": "1269024.0144729614", | |
| "benchmark.1.spec": "seqlen: 768; bs: 1; dim: 128; hiddendim: 128; seed: 381; nomask: True; distribution: cauchy", | |
| "benchmark.1.runs": "3", | |
| "benchmark.1.mean": "8566016.19720459", | |
| "benchmark.1.std": "4873.838130953106", | |
| "benchmark.1.err": "2813.9117568924385", | |
| "benchmark.1.best": "8561632.15637207", | |
| "benchmark.1.worst": "8571264.266967773", | |
| "benchmark.2.spec": "seqlen: 256; bs: 2; dim: 384; hiddendim: 128; seed: 2301; nomask: False; distribution: normal", | |
| "benchmark.2.runs": "3", | |
| "benchmark.2.mean": "1637909.2931747437", | |
| "benchmark.2.std": "2685.2197782959065", | |
| "benchmark.2.err": "1550.312361832449", | |
| "benchmark.2.best": "1635903.9545059204", | |
| "benchmark.2.worst": "1640959.9781036377", | |
| "benchmark.3.spec": "seqlen: 512; bs: 1; dim: 128; hiddendim: 128; seed: 12819; nomask: True; distribution: normal", | |
| "benchmark.3.runs": "3", | |
| "benchmark.3.mean": "2640469.3126678467", | |
| "benchmark.3.std": "2689.634291960447", | |
| "benchmark.3.err": "1552.8610824850127", | |
| "benchmark.3.best": "2637727.975845337", | |
| "benchmark.3.worst": "2643104.076385498", | |
| "benchmark.4.spec": "seqlen: 1024; bs: 1; dim: 128; hiddendim: 128; seed: 381; nomask: True; distribution: cauchy", | |
| "benchmark.4.runs": "3", | |
| "benchmark.4.mean": "16308778.762817383", | |
| "benchmark.4.std": "8586.715771168558", | |
| "benchmark.4.err": "4957.542661938972", | |
| "benchmark.4.best": "16299615.859985352", | |
| "benchmark.4.worst": "16316640.853881836", | |
| "benchmark.5.spec": "seqlen: 768; bs: 1; dim: 384; hiddendim: 128; seed: 481; nomask: False; distribution: normal", | |
| "benchmark.5.runs": "3", | |
| "benchmark.5.mean": "10205248.19691976", | |
| "benchmark.5.std": "14750.092115041212", | |
| "benchmark.5.err": "8515.969653190821", | |
| "benchmark.5.best": "10188511.848449707", | |
| "benchmark.5.worst": "10216352.462768555", | |
| "benchmark.6.spec": "seqlen: 1024; bs: 1; dim: 384; hiddendim: 128; seed: 23291; nomask: True; distribution: normal", | |
| "benchmark.6.runs": "3", | |
| "benchmark.6.mean": "19201951.98059082", | |
| "benchmark.6.std": "6541.989770553007", | |
| "benchmark.6.err": "3777.0195550645567", | |
| "benchmark.6.best": "19194591.522216797", | |
| "benchmark.6.worst": "19207103.729248047", | |
| "check": "pass" | |
| } | |
| }, | |
| "profile": null | |
| } | |
| } | |
| }, | |
| "request_id": "f1d536f6276fdece60f3419c7088f29e", | |
| "gpu_index": 6, | |
| "gpu_uuid": "GPU-2193d587-5242-e4af-f9ff-ce9da68bd83c", | |
| "image": "sha256:63e5cf4304efec771bdf3ad56291db7ad0168d86ea8d34a7b4629e1ce43d905b", | |
| "canonical_image_config_sha256": "sha256:63e5cf4304efec771bdf3ad56291db7ad0168d86ea8d34a7b4629e1ce43d905b", | |
| "image_manifest_sha256": "sha256:10c2bd0309edcea7cdc78723f9b8a8e893995c1b014d8305c33cf7b38876a075", | |
| "queue_seconds": 0.4837036971002817, | |
| "host_execution_seconds": 27.311977170407772, | |
| "execution_timeout_seconds": 530, | |
| "program_sha256": "edc2fba210584be16adab1883fd54586fd1a25b976f6ceb9c4969cf7a3473b7c", | |
| "node": "datava270000004", | |
| "monitor_revision": "stale-pid-race-fix-v1", | |
| "pool_queue_seconds": 0.4877773430198431, | |
| "evaluator_gpus": [ | |
| 6, | |
| 7 | |
| ] | |
| }, | |
| { | |
| "program": "original", | |
| "status": "valid", | |
| "correctness": 1.0, | |
| "reward": 0.2735808927990231, | |
| "raw_score": 5482.839041328533, | |
| "score_us": 5482.839041328533, | |
| "wall_seconds": 27.579238647595048, | |
| "official_commit": "6c40e82dab9d5de7416ac873ad5cd3106084aaed", | |
| "manifest_sha256": "3a909e8f95197114714ca34d28c929ca186fece71fc4b5932ff3be67a10815c9", | |
| "result": { | |
| "success": true, | |
| "error": "", | |
| "system": { | |
| "gpu": "NVIDIA H100 80GB HBM3", | |
| "device_count": 1, | |
| "cpu": "Intel(R) Xeon(R) Platinum 8480C", | |
| "runtime": "CUDA", | |
| "platform": "Linux-5.15.0-1098-azure-x86_64-with-glibc2.39", | |
| "torch": "2.7.1+cu128", | |
| "hostname": "fffdd913906a", | |
| "requeues": 0 | |
| }, | |
| "runs": { | |
| "test": { | |
| "start": "2026-11-05 10:59:41.513643", | |
| "end": "2026-11-05 10:59:53.532433", | |
| "compilation": null, | |
| "run": { | |
| "success": true, | |
| "passed": true, | |
| "command": "python3 eval.py test /tmp/tmpuzlkfokp", | |
| "stdout": "", | |
| "stderr": "", | |
| "exit_code": 0, | |
| "duration": 8.913430538028479, | |
| "result": { | |
| "test-count": "18", | |
| "test.0.spec": "seqlen: 32; bs: 1; dim: 128; hiddendim: 128; seed: 9371; nomask: True; distribution: normal", | |
| "test.0.status": "pass", | |
| "test.0.message": "Maximum error: 0.0021506547927856445", | |
| "test.1.spec": "seqlen: 32; bs: 1; dim: 128; hiddendim: 128; seed: 1092; nomask: False; distribution: normal", | |
| "test.1.status": "pass", | |
| "test.1.message": "Maximum error: 0.002777099609375", | |
| "test.2.spec": "seqlen: 64; bs: 2; dim: 256; hiddendim: 128; seed: 2291; nomask: True; distribution: normal", | |
| "test.2.status": "pass", | |
| "test.2.message": "Maximum error: 0.0018999576568603516", | |
| "test.3.spec": "seqlen: 64; bs: 2; dim: 256; hiddendim: 128; seed: 210284; nomask: False; distribution: normal", | |
| "test.3.status": "pass", | |
| "test.3.message": "Maximum error: 0.002408742904663086", | |
| "test.4.spec": "seqlen: 128; bs: 1; dim: 768; hiddendim: 128; seed: 81934; nomask: True; distribution: normal", | |
| "test.4.status": "pass", | |
| "test.4.message": "Maximum error: 0.0017777681350708008", | |
| "test.5.spec": "seqlen: 256; bs: 1; dim: 128; hiddendim: 128; seed: 1932; nomask: True; distribution: normal", | |
| "test.5.status": "pass", | |
| "test.5.message": "Maximum error: 0.0038526058197021484", | |
| "test.6.spec": "seqlen: 256; bs: 1; dim: 128; hiddendim: 128; seed: 10432; nomask: False; distribution: normal", | |
| "test.6.status": "pass", | |
| "test.6.message": "Maximum error: 0.002830982208251953", | |
| "test.7.spec": "seqlen: 768; bs: 2; dim: 128; hiddendim: 128; seed: 731; nomask: True; distribution: normal", | |
| "test.7.status": "pass", | |
| "test.7.message": "Maximum error: 0.004423856735229492", | |
| "test.8.spec": "seqlen: 1024; bs: 1; dim: 384; hiddendim: 128; seed: 53121; nomask: False; distribution: normal", | |
| "test.8.status": "pass", | |
| "test.8.message": "Maximum error: 0.00204622745513916", | |
| "test.9.spec": "seqlen: 1024; bs: 1; dim: 768; hiddendim: 128; seed: 31; nomask: True; distribution: normal", | |
| "test.9.status": "pass", | |
| "test.9.message": "Maximum error: 0.001607358455657959", | |
| "test.10.spec": "seqlen: 1024; bs: 1; dim: 768; hiddendim: 128; seed: 4921; nomask: False; distribution: normal", | |
| "test.10.status": "pass", | |
| "test.10.message": "Maximum error: 0.002584695816040039", | |
| "test.11.spec": "seqlen: 32; bs: 1; dim: 128; hiddendim: 128; seed: 937321; nomask: True; distribution: cauchy", | |
| "test.11.status": "pass", | |
| "test.11.message": "Maximum error: 0.002385735511779785", | |
| "test.12.spec": "seqlen: 64; bs: 2; dim: 256; hiddendim: 128; seed: 2291; nomask: True; distribution: cauchy", | |
| "test.12.status": "pass", | |
| "test.12.message": "Maximum error: 0.002347707748413086", | |
| "test.13.spec": "seqlen: 128; bs: 1; dim: 768; hiddendim: 128; seed: 8134; nomask: True; distribution: cauchy", | |
| "test.13.status": "pass", | |
| "test.13.message": "Maximum error: 0.0014344453811645508", | |
| "test.14.spec": "seqlen: 256; bs: 1; dim: 128; hiddendim: 128; seed: 932; nomask: True; distribution: cauchy", | |
| "test.14.status": "pass", | |
| "test.14.message": "Maximum error: 0.002925872802734375", | |
| "test.15.spec": "seqlen: 768; bs: 2; dim: 128; hiddendim: 128; seed: 31; nomask: True; distribution: cauchy", | |
| "test.15.status": "pass", | |
| "test.15.message": "Maximum error: 0.003274202346801758", | |
| "test.16.spec": "seqlen: 1024; bs: 1; dim: 384; hiddendim: 128; seed: 5321; nomask: False; distribution: cauchy", | |
| "test.16.status": "pass", | |
| "test.16.message": "Maximum error: 0.001819014549255371", | |
| "test.17.spec": "seqlen: 1024; bs: 1; dim: 768; hiddendim: 128; seed: 491; nomask: False; distribution: cauchy", | |
| "test.17.status": "pass", | |
| "test.17.message": "Maximum error: 0.0016510486602783203", | |
| "check": "pass" | |
| } | |
| }, | |
| "profile": null | |
| }, | |
| "leaderboard": { | |
| "start": "2026-11-05 10:59:53.532541", | |
| "end": "2026-11-05 11:00:05.264538", | |
| "compilation": null, | |
| "run": { | |
| "success": true, | |
| "passed": true, | |
| "command": "python3 eval.py leaderboard /tmp/tmpe22e2kvp", | |
| "stdout": "", | |
| "stderr": "", | |
| "exit_code": 0, | |
| "duration": 8.489543667063117, | |
| "result": { | |
| "benchmark-count": "7", | |
| "benchmark.0.spec": "seqlen: 256; bs: 2; dim: 128; hiddendim: 128; seed: 9371; nomask: True; distribution: normal", | |
| "benchmark.0.runs": "24", | |
| "benchmark.0.mean": "1254465.3366009395", | |
| "benchmark.0.std": "6046.224580521765", | |
| "benchmark.0.err": "1234.1804243792992", | |
| "benchmark.0.best": "1247295.9756851196", | |
| "benchmark.0.worst": "1268704.0567398071", | |
| "benchmark.1.spec": "seqlen: 768; bs: 1; dim: 128; hiddendim: 128; seed: 381; nomask: True; distribution: cauchy", | |
| "benchmark.1.runs": "3", | |
| "benchmark.1.mean": "8542709.032694498", | |
| "benchmark.1.std": "8969.233497536574", | |
| "benchmark.1.err": "5178.389374227349", | |
| "benchmark.1.best": "8532511.711120605", | |
| "benchmark.1.worst": "8549375.534057617", | |
| "benchmark.2.spec": "seqlen: 256; bs: 2; dim: 384; hiddendim: 128; seed: 2301; nomask: False; distribution: normal", | |
| "benchmark.2.runs": "10", | |
| "benchmark.2.mean": "1640451.192855835", | |
| "benchmark.2.std": "4782.600514541117", | |
| "benchmark.2.err": "1512.391076464317", | |
| "benchmark.2.best": "1636479.9737930298", | |
| "benchmark.2.worst": "1652863.9793395996", | |
| "benchmark.3.spec": "seqlen: 512; bs: 1; dim: 128; hiddendim: 128; seed: 12819; nomask: True; distribution: normal", | |
| "benchmark.3.runs": "8", | |
| "benchmark.3.mean": "2636548.012495041", | |
| "benchmark.3.std": "7089.02954310995", | |
| "benchmark.3.err": "2506.3504309824093", | |
| "benchmark.3.best": "2628159.999847412", | |
| "benchmark.3.worst": "2647968.053817749", | |
| "benchmark.4.spec": "seqlen: 1024; bs: 1; dim: 128; hiddendim: 128; seed: 381; nomask: True; distribution: cauchy", | |
| "benchmark.4.runs": "3", | |
| "benchmark.4.mean": "16387200.037638346", | |
| "benchmark.4.std": "24449.40670968853", | |
| "benchmark.4.err": "14115.871545365317", | |
| "benchmark.4.best": "16362623.21472168", | |
| "benchmark.4.worst": "16411520.004272461", | |
| "benchmark.5.spec": "seqlen: 768; bs: 1; dim: 384; hiddendim: 128; seed: 481; nomask: False; distribution: normal", | |
| "benchmark.5.runs": "3", | |
| "benchmark.5.mean": "10195850.690205893", | |
| "benchmark.5.std": "8691.630150317755", | |
| "benchmark.5.err": "5018.1150069826235", | |
| "benchmark.5.best": "10188159.942626953", | |
| "benchmark.5.worst": "10205280.303955078", | |
| "benchmark.6.spec": "seqlen: 1024; bs: 1; dim: 384; hiddendim: 128; seed: 23291; nomask: True; distribution: normal", | |
| "benchmark.6.runs": "3", | |
| "benchmark.6.mean": "19233440.399169922", | |
| "benchmark.6.std": "27268.61009893384", | |
| "benchmark.6.err": "15743.539381046401", | |
| "benchmark.6.best": "19208160.400390625", | |
| "benchmark.6.worst": "19262336.73095703", | |
| "check": "pass" | |
| } | |
| }, | |
| "profile": null | |
| } | |
| } | |
| }, | |
| "request_id": "1d3884b339f77cdd313e4c2931f486df", | |
| "gpu_index": 7, | |
| "gpu_uuid": "GPU-e6205be5-b1a7-bc81-55de-8f79262cf524", | |
| "image": "sha256:63e5cf4304efec771bdf3ad56291db7ad0168d86ea8d34a7b4629e1ce43d905b", | |
| "canonical_image_config_sha256": "sha256:63e5cf4304efec771bdf3ad56291db7ad0168d86ea8d34a7b4629e1ce43d905b", | |
| "image_manifest_sha256": "sha256:10c2bd0309edcea7cdc78723f9b8a8e893995c1b014d8305c33cf7b38876a075", | |
| "queue_seconds": 0.48289336264133453, | |
| "host_execution_seconds": 28.84198692254722, | |
| "execution_timeout_seconds": 530, | |
| "program_sha256": "d4fe30c2270c484dace78e1462a1d1d3a2e172fe8aefd0212f682e33bdc18174", | |
| "node": "datava270000004", | |
| "monitor_revision": "stale-pid-race-fix-v1", | |
| "pool_queue_seconds": 0.4879708234220743, | |
| "evaluator_gpus": [ | |
| 6, | |
| 7 | |
| ] | |
| } | |
| ], | |
| "new_root_us": 5479.77083400601, | |
| "new_root_source": "actual split Step0 on primary GPU6", | |
| "official_correctness_cases": 18, | |
| "official_timing_cases": 7, | |
| "paid_requests": 0, | |
| "time": 1793876405.8683152 | |
| } |
Xet Storage Details
- Size:
- 42.6 kB
- Xet hash:
- 216a269a76b75d2ed8376b3310b951a05393ea312db63c36110d691a6135d2fe
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.