zefancainlp's picture
download
raw
42.6 kB
{
"success": true,
"image": "sha256:63e5cf4304efec771bdf3ad56291db7ad0168d86ea8d34a7b4629e1ce43d905b",
"gpu_uuids": [
"GPU-2193d587-5242-e4af-f9ff-ce9da68bd83c",
"GPU-e6205be5-b1a7-bc81-55de-8f79262cf524"
],
"runs": [
{
"program": "original",
"status": "valid",
"correctness": 1.0,
"reward": 0.27382720778055364,
"raw_score": 5477.907079277918,
"score_us": 5477.907079277918,
"wall_seconds": 27.8378162458539,
"official_commit": "6c40e82dab9d5de7416ac873ad5cd3106084aaed",
"manifest_sha256": "3a909e8f95197114714ca34d28c929ca186fece71fc4b5932ff3be67a10815c9",
"result": {
"success": true,
"error": "",
"system": {
"gpu": "NVIDIA H100 80GB HBM3",
"device_count": 1,
"cpu": "Intel(R) Xeon(R) Platinum 8480C",
"runtime": "CUDA",
"platform": "Linux-5.15.0-1098-azure-x86_64-with-glibc2.39",
"torch": "2.7.1+cu128",
"hostname": "b296da5d37ab",
"requeues": 0
},
"runs": {
"test": {
"start": "2026-11-05 10:59:11.901818",
"end": "2026-11-05 10:59:23.743949",
"compilation": null,
"run": {
"success": true,
"passed": true,
"command": "python3 eval.py test /tmp/tmpwopv3dv4",
"stdout": "",
"stderr": "",
"exit_code": 0,
"duration": 8.6984375230968,
"result": {
"test-count": "18",
"test.0.spec": "seqlen: 32; bs: 1; dim: 128; hiddendim: 128; seed: 9371; nomask: True; distribution: normal",
"test.0.status": "pass",
"test.0.message": "Maximum error: 0.003163546323776245",
"test.1.spec": "seqlen: 32; bs: 1; dim: 128; hiddendim: 128; seed: 1092; nomask: False; distribution: normal",
"test.1.status": "pass",
"test.1.message": "Maximum error: 0.006548404693603516",
"test.2.spec": "seqlen: 64; bs: 2; dim: 256; hiddendim: 128; seed: 2291; nomask: True; distribution: normal",
"test.2.status": "pass",
"test.2.message": "Maximum error: 0.0018312931060791016",
"test.3.spec": "seqlen: 64; bs: 2; dim: 256; hiddendim: 128; seed: 210284; nomask: False; distribution: normal",
"test.3.status": "pass",
"test.3.message": "Maximum error: 0.0023785829544067383",
"test.4.spec": "seqlen: 128; bs: 1; dim: 768; hiddendim: 128; seed: 81934; nomask: True; distribution: normal",
"test.4.status": "pass",
"test.4.message": "Maximum error: 0.002499222755432129",
"test.5.spec": "seqlen: 256; bs: 1; dim: 128; hiddendim: 128; seed: 1932; nomask: True; distribution: normal",
"test.5.status": "pass",
"test.5.message": "Maximum error: 0.00362396240234375",
"test.6.spec": "seqlen: 256; bs: 1; dim: 128; hiddendim: 128; seed: 10432; nomask: False; distribution: normal",
"test.6.status": "pass",
"test.6.message": "Maximum error: 0.004271507263183594",
"test.7.spec": "seqlen: 768; bs: 2; dim: 128; hiddendim: 128; seed: 731; nomask: True; distribution: normal",
"test.7.status": "pass",
"test.7.message": "Maximum error: 0.004614591598510742",
"test.8.spec": "seqlen: 1024; bs: 1; dim: 384; hiddendim: 128; seed: 53121; nomask: False; distribution: normal",
"test.8.status": "pass",
"test.8.message": "Maximum error: 0.0018085241317749023",
"test.9.spec": "seqlen: 1024; bs: 1; dim: 768; hiddendim: 128; seed: 31; nomask: True; distribution: normal",
"test.9.status": "pass",
"test.9.message": "Maximum error: 0.0017676353454589844",
"test.10.spec": "seqlen: 1024; bs: 1; dim: 768; hiddendim: 128; seed: 4921; nomask: False; distribution: normal",
"test.10.status": "pass",
"test.10.message": "Maximum error: 0.002179384231567383",
"test.11.spec": "seqlen: 32; bs: 1; dim: 128; hiddendim: 128; seed: 937321; nomask: True; distribution: cauchy",
"test.11.status": "pass",
"test.11.message": "Maximum error: 0.0033462047576904297",
"test.12.spec": "seqlen: 64; bs: 2; dim: 256; hiddendim: 128; seed: 2291; nomask: True; distribution: cauchy",
"test.12.status": "pass",
"test.12.message": "Maximum error: 0.0016717910766601562",
"test.13.spec": "seqlen: 128; bs: 1; dim: 768; hiddendim: 128; seed: 8134; nomask: True; distribution: cauchy",
"test.13.status": "pass",
"test.13.message": "Maximum error: 0.001565694808959961",
"test.14.spec": "seqlen: 256; bs: 1; dim: 128; hiddendim: 128; seed: 932; nomask: True; distribution: cauchy",
"test.14.status": "pass",
"test.14.message": "Maximum error: 0.0031557083129882812",
"test.15.spec": "seqlen: 768; bs: 2; dim: 128; hiddendim: 128; seed: 31; nomask: True; distribution: cauchy",
"test.15.status": "pass",
"test.15.message": "Maximum error: 0.0031419992446899414",
"test.16.spec": "seqlen: 1024; bs: 1; dim: 384; hiddendim: 128; seed: 5321; nomask: False; distribution: cauchy",
"test.16.status": "pass",
"test.16.message": "Maximum error: 0.0022017955780029297",
"test.17.spec": "seqlen: 1024; bs: 1; dim: 768; hiddendim: 128; seed: 491; nomask: False; distribution: cauchy",
"test.17.status": "pass",
"test.17.message": "Maximum error: 0.0017014741897583008",
"check": "pass"
}
},
"profile": null
},
"leaderboard": {
"start": "2026-11-05 10:59:23.744062",
"end": "2026-11-05 10:59:35.856388",
"compilation": null,
"run": {
"success": true,
"passed": true,
"command": "python3 eval.py leaderboard /tmp/tmpl2y_2wbg",
"stdout": "",
"stderr": "",
"exit_code": 0,
"duration": 8.951863212510943,
"result": {
"benchmark-count": "7",
"benchmark.0.spec": "seqlen: 256; bs: 2; dim: 128; hiddendim: 128; seed: 9371; nomask: True; distribution: normal",
"benchmark.0.runs": "53",
"benchmark.0.mean": "1251079.242184477",
"benchmark.0.std": "9036.581630376093",
"benchmark.0.err": "1241.269948916824",
"benchmark.0.best": "1239871.9787597656",
"benchmark.0.worst": "1286656.0220718384",
"benchmark.1.spec": "seqlen: 768; bs: 1; dim: 128; hiddendim: 128; seed: 381; nomask: True; distribution: cauchy",
"benchmark.1.runs": "3",
"benchmark.1.mean": "8538026.491800943",
"benchmark.1.std": "8666.433498375583",
"benchmark.1.err": "5003.567713201133",
"benchmark.1.best": "8528096.199035645",
"benchmark.1.worst": "8544063.568115234",
"benchmark.2.spec": "seqlen: 256; bs: 2; dim: 384; hiddendim: 128; seed: 2301; nomask: False; distribution: normal",
"benchmark.2.runs": "8",
"benchmark.2.mean": "1644200.012087822",
"benchmark.2.std": "4578.963810479211",
"benchmark.2.err": "1618.9081805988217",
"benchmark.2.best": "1635040.044784546",
"benchmark.2.worst": "1649984.0021133423",
"benchmark.3.spec": "seqlen: 512; bs: 1; dim: 128; hiddendim: 128; seed: 12819; nomask: True; distribution: normal",
"benchmark.3.runs": "15",
"benchmark.3.mean": "2632354.132334391",
"benchmark.3.std": "10145.229093277756",
"benchmark.3.err": "2619.4868881149146",
"benchmark.3.best": "2620383.9778900146",
"benchmark.3.worst": "2656224.012374878",
"benchmark.4.spec": "seqlen: 1024; bs: 1; dim: 128; hiddendim: 128; seed: 381; nomask: True; distribution: cauchy",
"benchmark.4.runs": "5",
"benchmark.4.mean": "16315340.423583984",
"benchmark.4.std": "32547.727307042802",
"benchmark.4.err": "14555.786154334774",
"benchmark.4.best": "16280191.421508789",
"benchmark.4.worst": "16355615.615844727",
"benchmark.5.spec": "seqlen: 768; bs: 1; dim: 384; hiddendim: 128; seed: 481; nomask: False; distribution: normal",
"benchmark.5.runs": "3",
"benchmark.5.mean": "10185013.45316569",
"benchmark.5.std": "10433.306742851937",
"benchmark.5.err": "6023.672456523504",
"benchmark.5.best": "10174943.923950195",
"benchmark.5.worst": "10195775.985717773",
"benchmark.6.spec": "seqlen: 1024; bs: 1; dim: 384; hiddendim: 128; seed: 23291; nomask: True; distribution: normal",
"benchmark.6.runs": "5",
"benchmark.6.mean": "19266534.423828125",
"benchmark.6.std": "40055.48160779685",
"benchmark.6.err": "17913.355949305263",
"benchmark.6.best": "19244895.935058594",
"benchmark.6.worst": "19338016.510009766",
"check": "pass"
}
},
"profile": null
}
}
},
"request_id": "b8d7f5578034367847aa87396050e287",
"gpu_index": 6,
"gpu_uuid": "GPU-2193d587-5242-e4af-f9ff-ce9da68bd83c",
"image": "sha256:63e5cf4304efec771bdf3ad56291db7ad0168d86ea8d34a7b4629e1ce43d905b",
"canonical_image_config_sha256": "sha256:63e5cf4304efec771bdf3ad56291db7ad0168d86ea8d34a7b4629e1ce43d905b",
"image_manifest_sha256": "sha256:10c2bd0309edcea7cdc78723f9b8a8e893995c1b014d8305c33cf7b38876a075",
"queue_seconds": 0.4665392003953457,
"host_execution_seconds": 30.544493351131678,
"execution_timeout_seconds": 530,
"program_sha256": "d4fe30c2270c484dace78e1462a1d1d3a2e172fe8aefd0212f682e33bdc18174",
"node": "datava270000004",
"monitor_revision": "stale-pid-race-fix-v1",
"pool_queue_seconds": 0.4722871035337448,
"evaluator_gpus": [
6,
7
]
},
{
"program": "split",
"status": "valid",
"correctness": 1.0,
"reward": 0.27418048936086675,
"raw_score": 5470.848795611246,
"score_us": 5470.848795611246,
"wall_seconds": 27.317062279209495,
"official_commit": "6c40e82dab9d5de7416ac873ad5cd3106084aaed",
"manifest_sha256": "3a909e8f95197114714ca34d28c929ca186fece71fc4b5932ff3be67a10815c9",
"result": {
"success": true,
"error": "",
"system": {
"gpu": "NVIDIA H100 80GB HBM3",
"device_count": 1,
"cpu": "Intel(R) Xeon(R) Platinum 8480C",
"runtime": "CUDA",
"platform": "Linux-5.15.0-1098-azure-x86_64-with-glibc2.39",
"torch": "2.7.1+cu128",
"hostname": "63933b9fa20f",
"requeues": 0
},
"runs": {
"test": {
"start": "2026-11-05 10:59:11.585405",
"end": "2026-11-05 10:59:23.933745",
"compilation": null,
"run": {
"success": true,
"passed": true,
"command": "python3 eval.py test /tmp/tmp28lwe2v6",
"stdout": "",
"stderr": "",
"exit_code": 0,
"duration": 9.23547207377851,
"result": {
"test-count": "18",
"test.0.spec": "seqlen: 32; bs: 1; dim: 128; hiddendim: 128; seed: 9371; nomask: True; distribution: normal",
"test.0.status": "pass",
"test.0.message": "Maximum error: 0.0025129318237304688",
"test.1.spec": "seqlen: 32; bs: 1; dim: 128; hiddendim: 128; seed: 1092; nomask: False; distribution: normal",
"test.1.status": "pass",
"test.1.message": "Maximum error: 0.0027701854705810547",
"test.2.spec": "seqlen: 64; bs: 2; dim: 256; hiddendim: 128; seed: 2291; nomask: True; distribution: normal",
"test.2.status": "pass",
"test.2.message": "Maximum error: 0.0022954940795898438",
"test.3.spec": "seqlen: 64; bs: 2; dim: 256; hiddendim: 128; seed: 210284; nomask: False; distribution: normal",
"test.3.status": "pass",
"test.3.message": "Maximum error: 0.0018486976623535156",
"test.4.spec": "seqlen: 128; bs: 1; dim: 768; hiddendim: 128; seed: 81934; nomask: True; distribution: normal",
"test.4.status": "pass",
"test.4.message": "Maximum error: 0.0013532638549804688",
"test.5.spec": "seqlen: 256; bs: 1; dim: 128; hiddendim: 128; seed: 1932; nomask: True; distribution: normal",
"test.5.status": "pass",
"test.5.message": "Maximum error: 0.003708362579345703",
"test.6.spec": "seqlen: 256; bs: 1; dim: 128; hiddendim: 128; seed: 10432; nomask: False; distribution: normal",
"test.6.status": "pass",
"test.6.message": "Maximum error: 0.0034810304641723633",
"test.7.spec": "seqlen: 768; bs: 2; dim: 128; hiddendim: 128; seed: 731; nomask: True; distribution: normal",
"test.7.status": "pass",
"test.7.message": "Maximum error: 0.004608571529388428",
"test.8.spec": "seqlen: 1024; bs: 1; dim: 384; hiddendim: 128; seed: 53121; nomask: False; distribution: normal",
"test.8.status": "pass",
"test.8.message": "Maximum error: 0.0032002925872802734",
"test.9.spec": "seqlen: 1024; bs: 1; dim: 768; hiddendim: 128; seed: 31; nomask: True; distribution: normal",
"test.9.status": "pass",
"test.9.message": "Maximum error: 0.0025271177291870117",
"test.10.spec": "seqlen: 1024; bs: 1; dim: 768; hiddendim: 128; seed: 4921; nomask: False; distribution: normal",
"test.10.status": "pass",
"test.10.message": "Maximum error: 0.0026807785034179688",
"test.11.spec": "seqlen: 32; bs: 1; dim: 128; hiddendim: 128; seed: 937321; nomask: True; distribution: cauchy",
"test.11.status": "pass",
"test.11.message": "Maximum error: 0.003023982048034668",
"test.12.spec": "seqlen: 64; bs: 2; dim: 256; hiddendim: 128; seed: 2291; nomask: True; distribution: cauchy",
"test.12.status": "pass",
"test.12.message": "Maximum error: 0.002574920654296875",
"test.13.spec": "seqlen: 128; bs: 1; dim: 768; hiddendim: 128; seed: 8134; nomask: True; distribution: cauchy",
"test.13.status": "pass",
"test.13.message": "Maximum error: 0.002186298370361328",
"test.14.spec": "seqlen: 256; bs: 1; dim: 128; hiddendim: 128; seed: 932; nomask: True; distribution: cauchy",
"test.14.status": "pass",
"test.14.message": "Maximum error: 0.004027128219604492",
"test.15.spec": "seqlen: 768; bs: 2; dim: 128; hiddendim: 128; seed: 31; nomask: True; distribution: cauchy",
"test.15.status": "pass",
"test.15.message": "Maximum error: 0.005215167999267578",
"test.16.spec": "seqlen: 1024; bs: 1; dim: 384; hiddendim: 128; seed: 5321; nomask: False; distribution: cauchy",
"test.16.status": "pass",
"test.16.message": "Maximum error: 0.002176046371459961",
"test.17.spec": "seqlen: 1024; bs: 1; dim: 768; hiddendim: 128; seed: 491; nomask: False; distribution: cauchy",
"test.17.status": "pass",
"test.17.message": "Maximum error: 0.0012012720108032227",
"check": "pass"
}
},
"profile": null
},
"leaderboard": {
"start": "2026-11-05 10:59:23.933848",
"end": "2026-11-05 10:59:35.317517",
"compilation": null,
"run": {
"success": true,
"passed": true,
"command": "python3 eval.py leaderboard /tmp/tmpd5dodqkt",
"stdout": "",
"stderr": "",
"exit_code": 0,
"duration": 8.330479983240366,
"result": {
"benchmark-count": "7",
"benchmark.0.spec": "seqlen: 256; bs: 2; dim: 128; hiddendim: 128; seed: 9371; nomask: True; distribution: normal",
"benchmark.0.runs": "10",
"benchmark.0.mean": "1251363.1939888",
"benchmark.0.std": "3954.0719666795744",
"benchmark.0.err": "1250.3873446928865",
"benchmark.0.best": "1247807.9795837402",
"benchmark.0.worst": "1260607.9578399658",
"benchmark.1.spec": "seqlen: 768; bs: 1; dim: 128; hiddendim: 128; seed: 381; nomask: True; distribution: cauchy",
"benchmark.1.runs": "3",
"benchmark.1.mean": "8552757.263183594",
"benchmark.1.std": "5232.061729094771",
"benchmark.1.err": "3020.732247709605",
"benchmark.1.best": "8547552.108764648",
"benchmark.1.worst": "8558015.823364258",
"benchmark.2.spec": "seqlen: 256; bs: 2; dim: 384; hiddendim: 128; seed: 2301; nomask: False; distribution: normal",
"benchmark.2.runs": "7",
"benchmark.2.mean": "1633097.1377236503",
"benchmark.2.std": "4293.5494292504645",
"benchmark.2.err": "1622.80914736572",
"benchmark.2.best": "1627807.9748153687",
"benchmark.2.worst": "1640735.9838485718",
"benchmark.3.spec": "seqlen: 512; bs: 1; dim: 128; hiddendim: 128; seed: 12819; nomask: True; distribution: normal",
"benchmark.3.runs": "3",
"benchmark.3.mean": "2640074.7299194336",
"benchmark.3.std": "2416.0325376541073",
"benchmark.3.err": "1394.8970359854936",
"benchmark.3.best": "2637664.0796661377",
"benchmark.3.worst": "2642496.109008789",
"benchmark.4.spec": "seqlen: 1024; bs: 1; dim: 128; hiddendim: 128; seed: 381; nomask: True; distribution: cauchy",
"benchmark.4.runs": "3",
"benchmark.4.mean": "16270399.729410807",
"benchmark.4.std": "4755.303723422112",
"benchmark.4.err": "2745.475884796186",
"benchmark.4.best": "16266239.166259766",
"benchmark.4.worst": "16275583.267211914",
"benchmark.5.spec": "seqlen: 768; bs: 1; dim: 384; hiddendim: 128; seed: 481; nomask: False; distribution: normal",
"benchmark.5.runs": "3",
"benchmark.5.mean": "10191786.766052246",
"benchmark.5.std": "5355.786309062029",
"benchmark.5.err": "3092.164667259075",
"benchmark.5.best": "10188032.150268555",
"benchmark.5.worst": "10197919.845581055",
"benchmark.6.spec": "seqlen: 1024; bs: 1; dim: 384; hiddendim: 128; seed: 23291; nomask: True; distribution: normal",
"benchmark.6.runs": "3",
"benchmark.6.mean": "19169759.75036621",
"benchmark.6.std": "6634.813864212958",
"benchmark.6.err": "3830.6115705264124",
"benchmark.6.best": "19162624.35913086",
"benchmark.6.worst": "19175743.103027344",
"check": "pass"
}
},
"profile": null
}
}
},
"request_id": "611df10a485536588c0e238519bbc5cc",
"gpu_index": 7,
"gpu_uuid": "GPU-e6205be5-b1a7-bc81-55de-8f79262cf524",
"image": "sha256:63e5cf4304efec771bdf3ad56291db7ad0168d86ea8d34a7b4629e1ce43d905b",
"canonical_image_config_sha256": "sha256:63e5cf4304efec771bdf3ad56291db7ad0168d86ea8d34a7b4629e1ce43d905b",
"image_manifest_sha256": "sha256:10c2bd0309edcea7cdc78723f9b8a8e893995c1b014d8305c33cf7b38876a075",
"queue_seconds": 0.4641108028590679,
"host_execution_seconds": 29.862283227965236,
"execution_timeout_seconds": 530,
"program_sha256": "edc2fba210584be16adab1883fd54586fd1a25b976f6ceb9c4969cf7a3473b7c",
"node": "datava270000004",
"monitor_revision": "stale-pid-race-fix-v1",
"pool_queue_seconds": 0.46920238621532917,
"evaluator_gpus": [
6,
7
]
},
{
"program": "split",
"status": "valid",
"correctness": 1.0,
"reward": 0.27373407491630786,
"raw_score": 5479.77083400601,
"score_us": 5479.77083400601,
"wall_seconds": 25.96976548433304,
"official_commit": "6c40e82dab9d5de7416ac873ad5cd3106084aaed",
"manifest_sha256": "3a909e8f95197114714ca34d28c929ca186fece71fc4b5932ff3be67a10815c9",
"result": {
"success": true,
"error": "",
"system": {
"gpu": "NVIDIA H100 80GB HBM3",
"device_count": 1,
"cpu": "Intel(R) Xeon(R) Platinum 8480C",
"runtime": "CUDA",
"platform": "Linux-5.15.0-1098-azure-x86_64-with-glibc2.39",
"torch": "2.7.1+cu128",
"hostname": "c7ee82049ec9",
"requeues": 0
},
"runs": {
"test": {
"start": "2026-11-05 10:59:41.068966",
"end": "2026-11-05 10:59:52.533032",
"compilation": null,
"run": {
"success": true,
"passed": true,
"command": "python3 eval.py test /tmp/tmp61c8rn39",
"stdout": "",
"stderr": "",
"exit_code": 0,
"duration": 8.373281108215451,
"result": {
"test-count": "18",
"test.0.spec": "seqlen: 32; bs: 1; dim: 128; hiddendim: 128; seed: 9371; nomask: True; distribution: normal",
"test.0.status": "pass",
"test.0.message": "Maximum error: 0.002804994583129883",
"test.1.spec": "seqlen: 32; bs: 1; dim: 128; hiddendim: 128; seed: 1092; nomask: False; distribution: normal",
"test.1.status": "pass",
"test.1.message": "Maximum error: 0.0036276578903198242",
"test.2.spec": "seqlen: 64; bs: 2; dim: 256; hiddendim: 128; seed: 2291; nomask: True; distribution: normal",
"test.2.status": "pass",
"test.2.message": "Maximum error: 0.0024330615997314453",
"test.3.spec": "seqlen: 64; bs: 2; dim: 256; hiddendim: 128; seed: 210284; nomask: False; distribution: normal",
"test.3.status": "pass",
"test.3.message": "Maximum error: 0.0027649402618408203",
"test.4.spec": "seqlen: 128; bs: 1; dim: 768; hiddendim: 128; seed: 81934; nomask: True; distribution: normal",
"test.4.status": "pass",
"test.4.message": "Maximum error: 0.002653360366821289",
"test.5.spec": "seqlen: 256; bs: 1; dim: 128; hiddendim: 128; seed: 1932; nomask: True; distribution: normal",
"test.5.status": "pass",
"test.5.message": "Maximum error: 0.0040130615234375",
"test.6.spec": "seqlen: 256; bs: 1; dim: 128; hiddendim: 128; seed: 10432; nomask: False; distribution: normal",
"test.6.status": "pass",
"test.6.message": "Maximum error: 0.0029811859130859375",
"test.7.spec": "seqlen: 768; bs: 2; dim: 128; hiddendim: 128; seed: 731; nomask: True; distribution: normal",
"test.7.status": "pass",
"test.7.message": "Maximum error: 0.0035283565521240234",
"test.8.spec": "seqlen: 1024; bs: 1; dim: 384; hiddendim: 128; seed: 53121; nomask: False; distribution: normal",
"test.8.status": "pass",
"test.8.message": "Maximum error: 0.0024567842483520508",
"test.9.spec": "seqlen: 1024; bs: 1; dim: 768; hiddendim: 128; seed: 31; nomask: True; distribution: normal",
"test.9.status": "pass",
"test.9.message": "Maximum error: 0.0017796754837036133",
"test.10.spec": "seqlen: 1024; bs: 1; dim: 768; hiddendim: 128; seed: 4921; nomask: False; distribution: normal",
"test.10.status": "pass",
"test.10.message": "Maximum error: 0.002239823341369629",
"test.11.spec": "seqlen: 32; bs: 1; dim: 128; hiddendim: 128; seed: 937321; nomask: True; distribution: cauchy",
"test.11.status": "pass",
"test.11.message": "Maximum error: 0.0020360946655273438",
"test.12.spec": "seqlen: 64; bs: 2; dim: 256; hiddendim: 128; seed: 2291; nomask: True; distribution: cauchy",
"test.12.status": "pass",
"test.12.message": "Maximum error: 0.0020117759704589844",
"test.13.spec": "seqlen: 128; bs: 1; dim: 768; hiddendim: 128; seed: 8134; nomask: True; distribution: cauchy",
"test.13.status": "pass",
"test.13.message": "Maximum error: 0.0011234283447265625",
"test.14.spec": "seqlen: 256; bs: 1; dim: 128; hiddendim: 128; seed: 932; nomask: True; distribution: cauchy",
"test.14.status": "pass",
"test.14.message": "Maximum error: 0.0039560794830322266",
"test.15.spec": "seqlen: 768; bs: 2; dim: 128; hiddendim: 128; seed: 31; nomask: True; distribution: cauchy",
"test.15.status": "pass",
"test.15.message": "Maximum error: 0.003553420305252075",
"test.16.spec": "seqlen: 1024; bs: 1; dim: 384; hiddendim: 128; seed: 5321; nomask: False; distribution: cauchy",
"test.16.status": "pass",
"test.16.message": "Maximum error: 0.0020943880081176758",
"test.17.spec": "seqlen: 1024; bs: 1; dim: 768; hiddendim: 128; seed: 491; nomask: False; distribution: cauchy",
"test.17.status": "pass",
"test.17.message": "Maximum error: 0.001437366008758545",
"check": "pass"
}
},
"profile": null
},
"leaderboard": {
"start": "2026-11-05 10:59:52.533135",
"end": "2026-11-05 11:00:03.686580",
"compilation": null,
"run": {
"success": true,
"passed": true,
"command": "python3 eval.py leaderboard /tmp/tmpccprzwd9",
"stdout": "",
"stderr": "",
"exit_code": 0,
"duration": 7.935054844245315,
"result": {
"benchmark-count": "7",
"benchmark.0.spec": "seqlen: 256; bs: 2; dim: 128; hiddendim: 128; seed: 9371; nomask: True; distribution: normal",
"benchmark.0.runs": "23",
"benchmark.0.mean": "1253131.135650303",
"benchmark.0.std": "5940.0892705925835",
"benchmark.0.err": "1238.5942337043293",
"benchmark.0.best": "1243520.0214385986",
"benchmark.0.worst": "1269024.0144729614",
"benchmark.1.spec": "seqlen: 768; bs: 1; dim: 128; hiddendim: 128; seed: 381; nomask: True; distribution: cauchy",
"benchmark.1.runs": "3",
"benchmark.1.mean": "8566016.19720459",
"benchmark.1.std": "4873.838130953106",
"benchmark.1.err": "2813.9117568924385",
"benchmark.1.best": "8561632.15637207",
"benchmark.1.worst": "8571264.266967773",
"benchmark.2.spec": "seqlen: 256; bs: 2; dim: 384; hiddendim: 128; seed: 2301; nomask: False; distribution: normal",
"benchmark.2.runs": "3",
"benchmark.2.mean": "1637909.2931747437",
"benchmark.2.std": "2685.2197782959065",
"benchmark.2.err": "1550.312361832449",
"benchmark.2.best": "1635903.9545059204",
"benchmark.2.worst": "1640959.9781036377",
"benchmark.3.spec": "seqlen: 512; bs: 1; dim: 128; hiddendim: 128; seed: 12819; nomask: True; distribution: normal",
"benchmark.3.runs": "3",
"benchmark.3.mean": "2640469.3126678467",
"benchmark.3.std": "2689.634291960447",
"benchmark.3.err": "1552.8610824850127",
"benchmark.3.best": "2637727.975845337",
"benchmark.3.worst": "2643104.076385498",
"benchmark.4.spec": "seqlen: 1024; bs: 1; dim: 128; hiddendim: 128; seed: 381; nomask: True; distribution: cauchy",
"benchmark.4.runs": "3",
"benchmark.4.mean": "16308778.762817383",
"benchmark.4.std": "8586.715771168558",
"benchmark.4.err": "4957.542661938972",
"benchmark.4.best": "16299615.859985352",
"benchmark.4.worst": "16316640.853881836",
"benchmark.5.spec": "seqlen: 768; bs: 1; dim: 384; hiddendim: 128; seed: 481; nomask: False; distribution: normal",
"benchmark.5.runs": "3",
"benchmark.5.mean": "10205248.19691976",
"benchmark.5.std": "14750.092115041212",
"benchmark.5.err": "8515.969653190821",
"benchmark.5.best": "10188511.848449707",
"benchmark.5.worst": "10216352.462768555",
"benchmark.6.spec": "seqlen: 1024; bs: 1; dim: 384; hiddendim: 128; seed: 23291; nomask: True; distribution: normal",
"benchmark.6.runs": "3",
"benchmark.6.mean": "19201951.98059082",
"benchmark.6.std": "6541.989770553007",
"benchmark.6.err": "3777.0195550645567",
"benchmark.6.best": "19194591.522216797",
"benchmark.6.worst": "19207103.729248047",
"check": "pass"
}
},
"profile": null
}
}
},
"request_id": "f1d536f6276fdece60f3419c7088f29e",
"gpu_index": 6,
"gpu_uuid": "GPU-2193d587-5242-e4af-f9ff-ce9da68bd83c",
"image": "sha256:63e5cf4304efec771bdf3ad56291db7ad0168d86ea8d34a7b4629e1ce43d905b",
"canonical_image_config_sha256": "sha256:63e5cf4304efec771bdf3ad56291db7ad0168d86ea8d34a7b4629e1ce43d905b",
"image_manifest_sha256": "sha256:10c2bd0309edcea7cdc78723f9b8a8e893995c1b014d8305c33cf7b38876a075",
"queue_seconds": 0.4837036971002817,
"host_execution_seconds": 27.311977170407772,
"execution_timeout_seconds": 530,
"program_sha256": "edc2fba210584be16adab1883fd54586fd1a25b976f6ceb9c4969cf7a3473b7c",
"node": "datava270000004",
"monitor_revision": "stale-pid-race-fix-v1",
"pool_queue_seconds": 0.4877773430198431,
"evaluator_gpus": [
6,
7
]
},
{
"program": "original",
"status": "valid",
"correctness": 1.0,
"reward": 0.2735808927990231,
"raw_score": 5482.839041328533,
"score_us": 5482.839041328533,
"wall_seconds": 27.579238647595048,
"official_commit": "6c40e82dab9d5de7416ac873ad5cd3106084aaed",
"manifest_sha256": "3a909e8f95197114714ca34d28c929ca186fece71fc4b5932ff3be67a10815c9",
"result": {
"success": true,
"error": "",
"system": {
"gpu": "NVIDIA H100 80GB HBM3",
"device_count": 1,
"cpu": "Intel(R) Xeon(R) Platinum 8480C",
"runtime": "CUDA",
"platform": "Linux-5.15.0-1098-azure-x86_64-with-glibc2.39",
"torch": "2.7.1+cu128",
"hostname": "fffdd913906a",
"requeues": 0
},
"runs": {
"test": {
"start": "2026-11-05 10:59:41.513643",
"end": "2026-11-05 10:59:53.532433",
"compilation": null,
"run": {
"success": true,
"passed": true,
"command": "python3 eval.py test /tmp/tmpuzlkfokp",
"stdout": "",
"stderr": "",
"exit_code": 0,
"duration": 8.913430538028479,
"result": {
"test-count": "18",
"test.0.spec": "seqlen: 32; bs: 1; dim: 128; hiddendim: 128; seed: 9371; nomask: True; distribution: normal",
"test.0.status": "pass",
"test.0.message": "Maximum error: 0.0021506547927856445",
"test.1.spec": "seqlen: 32; bs: 1; dim: 128; hiddendim: 128; seed: 1092; nomask: False; distribution: normal",
"test.1.status": "pass",
"test.1.message": "Maximum error: 0.002777099609375",
"test.2.spec": "seqlen: 64; bs: 2; dim: 256; hiddendim: 128; seed: 2291; nomask: True; distribution: normal",
"test.2.status": "pass",
"test.2.message": "Maximum error: 0.0018999576568603516",
"test.3.spec": "seqlen: 64; bs: 2; dim: 256; hiddendim: 128; seed: 210284; nomask: False; distribution: normal",
"test.3.status": "pass",
"test.3.message": "Maximum error: 0.002408742904663086",
"test.4.spec": "seqlen: 128; bs: 1; dim: 768; hiddendim: 128; seed: 81934; nomask: True; distribution: normal",
"test.4.status": "pass",
"test.4.message": "Maximum error: 0.0017777681350708008",
"test.5.spec": "seqlen: 256; bs: 1; dim: 128; hiddendim: 128; seed: 1932; nomask: True; distribution: normal",
"test.5.status": "pass",
"test.5.message": "Maximum error: 0.0038526058197021484",
"test.6.spec": "seqlen: 256; bs: 1; dim: 128; hiddendim: 128; seed: 10432; nomask: False; distribution: normal",
"test.6.status": "pass",
"test.6.message": "Maximum error: 0.002830982208251953",
"test.7.spec": "seqlen: 768; bs: 2; dim: 128; hiddendim: 128; seed: 731; nomask: True; distribution: normal",
"test.7.status": "pass",
"test.7.message": "Maximum error: 0.004423856735229492",
"test.8.spec": "seqlen: 1024; bs: 1; dim: 384; hiddendim: 128; seed: 53121; nomask: False; distribution: normal",
"test.8.status": "pass",
"test.8.message": "Maximum error: 0.00204622745513916",
"test.9.spec": "seqlen: 1024; bs: 1; dim: 768; hiddendim: 128; seed: 31; nomask: True; distribution: normal",
"test.9.status": "pass",
"test.9.message": "Maximum error: 0.001607358455657959",
"test.10.spec": "seqlen: 1024; bs: 1; dim: 768; hiddendim: 128; seed: 4921; nomask: False; distribution: normal",
"test.10.status": "pass",
"test.10.message": "Maximum error: 0.002584695816040039",
"test.11.spec": "seqlen: 32; bs: 1; dim: 128; hiddendim: 128; seed: 937321; nomask: True; distribution: cauchy",
"test.11.status": "pass",
"test.11.message": "Maximum error: 0.002385735511779785",
"test.12.spec": "seqlen: 64; bs: 2; dim: 256; hiddendim: 128; seed: 2291; nomask: True; distribution: cauchy",
"test.12.status": "pass",
"test.12.message": "Maximum error: 0.002347707748413086",
"test.13.spec": "seqlen: 128; bs: 1; dim: 768; hiddendim: 128; seed: 8134; nomask: True; distribution: cauchy",
"test.13.status": "pass",
"test.13.message": "Maximum error: 0.0014344453811645508",
"test.14.spec": "seqlen: 256; bs: 1; dim: 128; hiddendim: 128; seed: 932; nomask: True; distribution: cauchy",
"test.14.status": "pass",
"test.14.message": "Maximum error: 0.002925872802734375",
"test.15.spec": "seqlen: 768; bs: 2; dim: 128; hiddendim: 128; seed: 31; nomask: True; distribution: cauchy",
"test.15.status": "pass",
"test.15.message": "Maximum error: 0.003274202346801758",
"test.16.spec": "seqlen: 1024; bs: 1; dim: 384; hiddendim: 128; seed: 5321; nomask: False; distribution: cauchy",
"test.16.status": "pass",
"test.16.message": "Maximum error: 0.001819014549255371",
"test.17.spec": "seqlen: 1024; bs: 1; dim: 768; hiddendim: 128; seed: 491; nomask: False; distribution: cauchy",
"test.17.status": "pass",
"test.17.message": "Maximum error: 0.0016510486602783203",
"check": "pass"
}
},
"profile": null
},
"leaderboard": {
"start": "2026-11-05 10:59:53.532541",
"end": "2026-11-05 11:00:05.264538",
"compilation": null,
"run": {
"success": true,
"passed": true,
"command": "python3 eval.py leaderboard /tmp/tmpe22e2kvp",
"stdout": "",
"stderr": "",
"exit_code": 0,
"duration": 8.489543667063117,
"result": {
"benchmark-count": "7",
"benchmark.0.spec": "seqlen: 256; bs: 2; dim: 128; hiddendim: 128; seed: 9371; nomask: True; distribution: normal",
"benchmark.0.runs": "24",
"benchmark.0.mean": "1254465.3366009395",
"benchmark.0.std": "6046.224580521765",
"benchmark.0.err": "1234.1804243792992",
"benchmark.0.best": "1247295.9756851196",
"benchmark.0.worst": "1268704.0567398071",
"benchmark.1.spec": "seqlen: 768; bs: 1; dim: 128; hiddendim: 128; seed: 381; nomask: True; distribution: cauchy",
"benchmark.1.runs": "3",
"benchmark.1.mean": "8542709.032694498",
"benchmark.1.std": "8969.233497536574",
"benchmark.1.err": "5178.389374227349",
"benchmark.1.best": "8532511.711120605",
"benchmark.1.worst": "8549375.534057617",
"benchmark.2.spec": "seqlen: 256; bs: 2; dim: 384; hiddendim: 128; seed: 2301; nomask: False; distribution: normal",
"benchmark.2.runs": "10",
"benchmark.2.mean": "1640451.192855835",
"benchmark.2.std": "4782.600514541117",
"benchmark.2.err": "1512.391076464317",
"benchmark.2.best": "1636479.9737930298",
"benchmark.2.worst": "1652863.9793395996",
"benchmark.3.spec": "seqlen: 512; bs: 1; dim: 128; hiddendim: 128; seed: 12819; nomask: True; distribution: normal",
"benchmark.3.runs": "8",
"benchmark.3.mean": "2636548.012495041",
"benchmark.3.std": "7089.02954310995",
"benchmark.3.err": "2506.3504309824093",
"benchmark.3.best": "2628159.999847412",
"benchmark.3.worst": "2647968.053817749",
"benchmark.4.spec": "seqlen: 1024; bs: 1; dim: 128; hiddendim: 128; seed: 381; nomask: True; distribution: cauchy",
"benchmark.4.runs": "3",
"benchmark.4.mean": "16387200.037638346",
"benchmark.4.std": "24449.40670968853",
"benchmark.4.err": "14115.871545365317",
"benchmark.4.best": "16362623.21472168",
"benchmark.4.worst": "16411520.004272461",
"benchmark.5.spec": "seqlen: 768; bs: 1; dim: 384; hiddendim: 128; seed: 481; nomask: False; distribution: normal",
"benchmark.5.runs": "3",
"benchmark.5.mean": "10195850.690205893",
"benchmark.5.std": "8691.630150317755",
"benchmark.5.err": "5018.1150069826235",
"benchmark.5.best": "10188159.942626953",
"benchmark.5.worst": "10205280.303955078",
"benchmark.6.spec": "seqlen: 1024; bs: 1; dim: 384; hiddendim: 128; seed: 23291; nomask: True; distribution: normal",
"benchmark.6.runs": "3",
"benchmark.6.mean": "19233440.399169922",
"benchmark.6.std": "27268.61009893384",
"benchmark.6.err": "15743.539381046401",
"benchmark.6.best": "19208160.400390625",
"benchmark.6.worst": "19262336.73095703",
"check": "pass"
}
},
"profile": null
}
}
},
"request_id": "1d3884b339f77cdd313e4c2931f486df",
"gpu_index": 7,
"gpu_uuid": "GPU-e6205be5-b1a7-bc81-55de-8f79262cf524",
"image": "sha256:63e5cf4304efec771bdf3ad56291db7ad0168d86ea8d34a7b4629e1ce43d905b",
"canonical_image_config_sha256": "sha256:63e5cf4304efec771bdf3ad56291db7ad0168d86ea8d34a7b4629e1ce43d905b",
"image_manifest_sha256": "sha256:10c2bd0309edcea7cdc78723f9b8a8e893995c1b014d8305c33cf7b38876a075",
"queue_seconds": 0.48289336264133453,
"host_execution_seconds": 28.84198692254722,
"execution_timeout_seconds": 530,
"program_sha256": "d4fe30c2270c484dace78e1462a1d1d3a2e172fe8aefd0212f682e33bdc18174",
"node": "datava270000004",
"monitor_revision": "stale-pid-race-fix-v1",
"pool_queue_seconds": 0.4879708234220743,
"evaluator_gpus": [
6,
7
]
}
],
"new_root_us": 5479.77083400601,
"new_root_source": "actual split Step0 on primary GPU6",
"official_correctness_cases": 18,
"official_timing_cases": 7,
"paid_requests": 0,
"time": 1793876405.8683152
}

Xet Storage Details

Size:
42.6 kB
·
Xet hash:
216a269a76b75d2ed8376b3310b951a05393ea312db63c36110d691a6135d2fe

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.