| { |
| "submission_id": "7b8658aedfdb158782567d09e26a9e819c578ff20c4db1d940e12de77cd9d6d2", |
| "arch_id": "arch_f17d92b32a8c79f7", |
| "owner_hotkey": "462c4a7dfe3091db3333dfb6cd5967e2b1d6f07bd1ceeb60300dcd63d13d3976", |
| "bpb": 3.7180674325750696, |
| "n_params": 106980502, |
| "tokens_seen": 2048, |
| "wall_clock_seconds": 20539.00746917725, |
| "battery": { |
| "tier": "public", |
| "groups": { |
| "g1": { |
| "module": "g1_intrinsic", |
| "status": "ok", |
| "metrics": { |
| "g1.bpb.val": 3.7180674450035034, |
| "g1.bpb.fresh": 4.490946593024322, |
| "g1.bpb.key_token": 5.835137702656671, |
| "g1.posloss.p0_128": 3.981469353984971, |
| "g1.bpb.domain.code": 3.048406180661858, |
| "g1.bpb.domain.math": 4.08552483348228, |
| "g1.bpb.domain.news": 5.367073807343638, |
| "g1.bpb.multidomain": 4.325325867950423, |
| "g1.bpb.domain.prose": 4.800298650313917, |
| "g1.posloss.p128_256": 3.6657491348531672, |
| "g1.posloss.p256_384": 3.6666124634651536, |
| "g1.posloss.p384_512": 3.684105031419292, |
| "g1.posloss.p512_end": 3.631590617330868, |
| "g1.bits_per_byte.val": 0.9333457081310936, |
| "g1.bits_per_byte.fresh": 1.2095837878739255, |
| "g1.assets.domains_present": 4.0, |
| "g1.bits_per_byte.key_token": 0.13152751503557092, |
| "g1.bits_per_byte.domain.code": 1.5197987035964795, |
| "g1.bits_per_byte.domain.math": 1.451242832264349, |
| "g1.bits_per_byte.domain.news": 1.4909604377215282, |
| "g1.bits_per_byte.multidomain": 1.433540428766053, |
| "g1.bits_per_byte.domain.prose": 1.2721597414818553 |
| } |
| }, |
| "g2": { |
| "module": "g2_downstream", |
| "status": "ok", |
| "metrics": { |
| "g2.core.mean_nll": 4.128923485933247, |
| "g2.piqa.acc_norm": 0.63, |
| "g2.boolq.acc_norm": 0.63, |
| "g2.lambada.acc_norm": 0.955, |
| "g2.arc_easy.acc_norm": 0.335, |
| "g2.core.mean_acc_norm": 0.504375, |
| "g2.hellaswag.acc_norm": 0.36, |
| "g2.openbookqa.acc_norm": 0.31, |
| "g2.winogrande.acc_norm": 0.52, |
| "g2.assets.tasks_present": 8.0, |
| "g2.arc_challenge.acc_norm": 0.295 |
| } |
| }, |
| "g3": { |
| "module": "g3_recall", |
| "status": "ok", |
| "metrics": { |
| "g3.copy.acc": 0.4444444444444445, |
| "g3.mqar.acc": 0.22916666666666663, |
| "g3.copy.g0.acc": 0.4166666666666667, |
| "g3.mqar.n4.acc": 0.375, |
| "g3.mqar.n8.acc": 0.25, |
| "g3.copy.g16.acc": 0.5, |
| "g3.copy.g64.acc": 0.4166666666666667, |
| "g3.mqar.n16.acc": 0.0625, |
| "g3.recall.mean_acc": 0.6684027777777778, |
| "g3.recall.mean_nll": 5.947218637065828, |
| "g3.passkey.base.acc": 1.0, |
| "g3.induction.base.acc": 1.0 |
| } |
| }, |
| "g4": { |
| "module": "g4_reasoning", |
| "status": "ok", |
| "metrics": { |
| "g4.kk.acc": 0.75, |
| "g4.s5.acc": 0.0, |
| "g4.mod.acc": 0.375, |
| "g4.bool.acc": 0.5, |
| "g4.dyck.acc": 0.638888888888889, |
| "g4.arith.acc": 0.3, |
| "g4.kk.n3.acc": 0.75, |
| "g4.proof.acc": 0.625, |
| "g4.s5.k5.acc": 0.0, |
| "g4.mod.id.acc": 0.25, |
| "g4.s5.k10.acc": 0.0, |
| "g4.dyck.k1.acc": 0.8333333333333334, |
| "g4.dyck.k2.acc": 0.75, |
| "g4.mod.ood.acc": 0.5, |
| "g4.arith.t1.acc": 0.75, |
| "g4.arith.t2.acc": 0.25, |
| "g4.arith.t3.acc": 0.0, |
| "g4.proof.d0.acc": 0.75, |
| "g4.proof.d1.acc": 0.5, |
| "g4.proof.d2.acc": 1.0, |
| "g4.proof.d3.acc": 0.25, |
| "g4.bool.base.acc": 0.5, |
| "g4.arith.noop.acc": 0.25, |
| "g4.mod.id_ood_gap": -0.25, |
| "g4.dyck.k2long.acc": 0.3333333333333333, |
| "g4.arith.extrap.acc": 0.25, |
| "g4.reasoning.mean_acc": 0.4555555555555556, |
| "g4.reasoning.mean_nll": 6.31392470209135 |
| } |
| }, |
| "g5": { |
| "module": "g5_longctx", |
| "status": "ok", |
| "metrics": { |
| "g5.lstar": 0.0, |
| "g5.retention": 3.428571428571429, |
| "g5.ruler.acc": 0.19525, |
| "g5.babilong.acc": 0.2, |
| "g5.helmet_rag.n": 20.0, |
| "g5.ruler.qa.acc": 0.25, |
| "g5.ruler.vt.acc": 0.12, |
| "g5.natural_mcq.n": 40.0, |
| "g5.ruler.qa.pack": 1.0, |
| "g5.helmet_rag.acc": 0.15, |
| "g5.mean.L4096.acc": 0.0875, |
| "g5.mean.L8192.acc": 0.3175, |
| "g5.natural.staged": 1.0, |
| "g5.ruler.mean_nll": 3.762060427080064, |
| "g5.mean.L16384.acc": 0.16999999999999998, |
| "g5.mean.L32768.acc": 0.175, |
| "g5.mean.L65536.acc": 0.3, |
| "g5.natural_mcq.acc": 0.325, |
| "g5.ruler.L4096.acc": 0.075, |
| "g5.ruler.L8192.acc": 0.335, |
| "g5.ruler.shot_rate": 0.8181818181818182, |
| "g5.babilong.qa1.acc": 0.0, |
| "g5.babilong.qa2.acc": 0.3333333333333333, |
| "g5.babilong.qa3.acc": 0.16666666666666666, |
| "g5.babilong.qa4.acc": 0.3333333333333333, |
| "g5.babilong.qa5.acc": 0.16666666666666666, |
| "g5.ruler.L16384.acc": 0.14, |
| "g5.ruler.L32768.acc": 0.175, |
| "g5.ruler.L65536.acc": 0.3, |
| "g5.ruler.mean_depth": 0.4692217355300894, |
| "g5.babilong.mean_nll": 10.280345492892796, |
| "g5.helmet_rag.rank_n": 20.0, |
| "g5.ruler.niah_mk.acc": 0.2, |
| "g5.ruler.niah_mq.acc": 0.09375, |
| "g5.ruler.niah_mv.acc": 0.3125, |
| "g5.babilong.L4096.acc": 0.1, |
| "g5.babilong.L8192.acc": 0.3, |
| "g5.ruler.qa.L4096.acc": 0.0, |
| "g5.ruler.qa.L8192.acc": 1.0, |
| "g5.ruler.vt.L4096.acc": 0.0, |
| "g5.ruler.vt.L8192.acc": 0.3, |
| "g5.babilong.L16384.acc": 0.2, |
| "g5.babilong.mean_depth": 0.5756925183491818, |
| "g5.babilong.mean_shots": 2.0, |
| "g5.helmet_rag.gold_nll": 4.511707195772065, |
| "g5.helmet_rag.rank_acc": 0.5, |
| "g5.ruler.L4096.len_err": 0.004150390625, |
| "g5.ruler.L8192.len_err": 0.002734375, |
| "g5.ruler.evidence_miss": 0.0, |
| "g5.ruler.qa.L16384.acc": 0.0, |
| "g5.ruler.qa.L32768.acc": 0.0, |
| "g5.ruler.vt.L16384.acc": 0.2, |
| "g5.ruler.vt.L32768.acc": 0.0, |
| "g5.ruler.vt.L65536.acc": 0.1, |
| "g5.babilong.filler_pack": 1.0, |
| "g5.helmet_rag.L8192.acc": 0.15, |
| "g5.natural_mcq.mean_nll": 3.679664521258111, |
| "g5.ruler.L16384.len_err": 0.00224609375, |
| "g5.ruler.L32768.len_err": 0.0, |
| "g5.ruler.L65536.len_err": 0.0, |
| "g5.natural_mcq.L8192.acc": 0.325, |
| "g5.babilong.L4096.len_err": 0.0, |
| "g5.babilong.L8192.len_err": 0.0, |
| "g5.babilong.evidence_miss": 0.0, |
| "g5.babilong.qa1.L4096.acc": 0.0, |
| "g5.babilong.qa1.L8192.acc": 0.0, |
| "g5.babilong.qa2.L4096.acc": 0.0, |
| "g5.babilong.qa2.L8192.acc": 0.5, |
| "g5.babilong.qa3.L4096.acc": 0.5, |
| "g5.babilong.qa3.L8192.acc": 0.0, |
| "g5.babilong.qa4.L4096.acc": 0.0, |
| "g5.babilong.qa4.L8192.acc": 0.5, |
| "g5.babilong.qa5.L4096.acc": 0.0, |
| "g5.babilong.qa5.L8192.acc": 0.5, |
| "g5.babilong.L16384.len_err": 0.0, |
| "g5.babilong.qa1.L16384.acc": 0.0, |
| "g5.babilong.qa2.L16384.acc": 0.5, |
| "g5.babilong.qa3.L16384.acc": 0.0, |
| "g5.babilong.qa4.L16384.acc": 0.5, |
| "g5.babilong.qa5.L16384.acc": 0.0, |
| "g5.ruler.niah_mk.L4096.acc": 0.0, |
| "g5.ruler.niah_mk.L8192.acc": 0.0, |
| "g5.ruler.niah_mq.L4096.acc": 0.125, |
| "g5.ruler.niah_mq.L8192.acc": 0.0, |
| "g5.ruler.niah_mv.L4096.acc": 0.25, |
| "g5.ruler.niah_mv.L8192.acc": 0.375, |
| "g5.ruler.niah_mk.L16384.acc": 0.0, |
| "g5.ruler.niah_mk.L32768.acc": 0.5, |
| "g5.ruler.niah_mk.L65536.acc": 0.5, |
| "g5.ruler.niah_mq.L16384.acc": 0.125, |
| "g5.ruler.niah_mq.L32768.acc": 0.125, |
| "g5.ruler.niah_mv.L16384.acc": 0.375, |
| "g5.ruler.niah_mv.L32768.acc": 0.25, |
| "g5.natural.pool_rows.helmet_rag": 96.0, |
| "g5.natural.pool_rows.natural_mcq": 64.0 |
| } |
| }, |
| "g6": { |
| "module": "g6_curve", |
| "status": "ok", |
| "metrics": { |
| "g6.stub": 1.0, |
| "g6.points": 0.0 |
| } |
| }, |
| "g7": { |
| "module": "g7_inference", |
| "status": "ok", |
| "metrics": { |
| "g7.vram.peak_gb": 3.81773056, |
| "g7.tpot.L1024.ms": 11.54466256593878, |
| "g7.tpot.L4096.ms": 11.588581441174028, |
| "g7.ttft.L1024.ms": 11.557281017303469, |
| "g7.ttft.L4096.ms": 11.51505101006478, |
| "g7.tpot.L16384.ms": 11.5575027793966, |
| "g7.tpot.L32768.ms": 11.551143341421266, |
| "g7.ttft.L16384.ms": 11.513719975482672, |
| "g7.ttft.L32768.ms": 11.49219903163612, |
| "g7.quant.w8a8.dbpb": 0.0037219625386475386, |
| "g7.quant.w4a16.dbpb": 0.22044160045714412, |
| "g7.energy.j_per_token": 0.0009267897161566004, |
| "g7.throughput.b1.toks": 90.11773760248344, |
| "g7.throughput.b32.toks": 151.57566520175138, |
| "g7.state.analytic_unavailable": 1.0, |
| "g7.state.bytes_per_token.measured": 8.0 |
| } |
| }, |
| "g8": { |
| "module": "g8_stability", |
| "status": "ok", |
| "metrics": { |
| "g8.mup.stub": 1.0, |
| "g8.spikes.count": 0.0, |
| "g8.mup.stability": 0.0, |
| "g8.spikes.per_1k_steps": 0.0, |
| "g8.divergence.probe_nan_frac": 0.0, |
| "g8.divergence.series_nan_frac": 0.0, |
| "g8.mup.stub_reason_width_knob_unsupported": 1.0 |
| } |
| } |
| }, |
| "metrics": { |
| "org.g5.lstar": 0.0, |
| "org.g2.obqa_acc": { |
| "value": 0.31, |
| "clusters": { |
| "g2/openbookqa": 0.31 |
| } |
| }, |
| "org.g2.piqa_acc": { |
| "value": 0.63, |
| "clusters": { |
| "g2/piqa": 0.63 |
| } |
| }, |
| "org.g3.mqar_acc": { |
| "value": 0.22916666666666663, |
| "clusters": { |
| "mqar/n4": 0.375, |
| "mqar/n8": 0.25, |
| "mqar/n16": 0.0625 |
| } |
| }, |
| "org.g4.dyck_acc": { |
| "value": 0.638888888888889, |
| "clusters": { |
| "dyck/k1/len10": 0.8333333333333334, |
| "dyck/k2/len10": 0.75, |
| "dyck/k2/len28": 0.3333333333333333 |
| } |
| }, |
| "org.g2.boolq_acc": { |
| "value": 0.63, |
| "clusters": { |
| "g2/boolq": 0.63 |
| } |
| }, |
| "org.g5.ruler_acc": { |
| "value": 0.19525, |
| "clusters": { |
| "qa@4096": 0.0, |
| "qa@8192": 1.0, |
| "vt@4096": 0.0, |
| "vt@8192": 0.30000000000000004, |
| "qa@16384": 0.0, |
| "qa@32768": 0.0, |
| "vt@16384": 0.2, |
| "vt@32768": 0.0, |
| "vt@65536": 0.1, |
| "niah_mk@4096": 0.0, |
| "niah_mk@8192": 0.0, |
| "niah_mq@4096": 0.125, |
| "niah_mq@8192": 0.0, |
| "niah_mv@4096": 0.25, |
| "niah_mv@8192": 0.375, |
| "niah_mk@16384": 0.0, |
| "niah_mk@32768": 0.5, |
| "niah_mk@65536": 0.5, |
| "niah_mq@16384": 0.125, |
| "niah_mq@32768": 0.125, |
| "niah_mv@16384": 0.375, |
| "niah_mv@32768": 0.25 |
| } |
| }, |
| "org.g2.lambada_acc": { |
| "value": 0.955, |
| "clusters": { |
| "g2/lambada": 0.955 |
| } |
| }, |
| "org.g3.copying_acc": { |
| "value": 0.4444444444444445, |
| "clusters": { |
| "copy/len16/gap0": 0.4166666666666667, |
| "copy/len16/gap16": 0.5, |
| "copy/len16/gap64": 0.4166666666666667 |
| } |
| }, |
| "org.g3.passkey_acc": { |
| "value": 1.0, |
| "clusters": { |
| "passkey/f40": 1.0 |
| } |
| }, |
| "org.g4.modular_acc": { |
| "value": 0.375, |
| "clusters": { |
| "mod/p97/id": 0.25, |
| "mod/p97/ood": 0.5 |
| } |
| }, |
| "org.g7.tpot_ms_32k": 11.551143341421266, |
| "org.g7.ttft_ms_32k": { |
| "value": 11.49219903163612, |
| "clusters": { |
| "L32768": 11.49219903163612 |
| } |
| }, |
| "org.g2.arc_easy_acc": { |
| "value": 0.335, |
| "clusters": { |
| "g2/arc_easy": 0.335 |
| } |
| }, |
| "org.g5.babilong_acc": { |
| "value": 0.2, |
| "clusters": { |
| "qa1@4096": 0.0, |
| "qa1@8192": 0.0, |
| "qa2@4096": 0.0, |
| "qa2@8192": 0.5, |
| "qa3@4096": 0.5, |
| "qa3@8192": 0.0, |
| "qa4@4096": 0.0, |
| "qa4@8192": 0.5, |
| "qa5@4096": 0.0, |
| "qa5@8192": 0.5, |
| "qa1@16384": 0.0, |
| "qa2@16384": 0.5, |
| "qa3@16384": 0.0, |
| "qa4@16384": 0.5, |
| "qa5@16384": 0.0 |
| } |
| }, |
| "org.g2.hellaswag_acc": { |
| "value": 0.36, |
| "clusters": { |
| "g2/hellaswag": 0.36 |
| } |
| }, |
| "org.g3.induction_acc": { |
| "value": 1.0, |
| "clusters": { |
| "induction/gap8/r2": 1.0 |
| } |
| }, |
| "org.g2.winogrande_acc": { |
| "value": 0.52, |
| "clusters": { |
| "g2/winogrande": 0.52 |
| } |
| }, |
| "org.g4.arithmetic_acc": { |
| "value": 0.3, |
| "clusters": { |
| "arith/t1": 0.75, |
| "arith/t2": 0.25, |
| "arith/t3": 0.0, |
| "arith/t2/noop": 0.25, |
| "arith/t2/extrap": 0.25 |
| } |
| }, |
| "org.g5.helmet_rag_acc": { |
| "value": 0.15, |
| "clusters": { |
| "helmet_rag@nq": 0.0, |
| "helmet_rag@popqa": 0.0, |
| "helmet_rag@hotpotqa": 0.3333333333333333, |
| "helmet_rag@triviaqa": 0.4 |
| } |
| }, |
| "org.g4.proofwriter_acc": { |
| "value": 0.625, |
| "clusters": { |
| "proof/d0/neg": 0.0, |
| "proof/d0/pos": 1.0, |
| "proof/d1/neg": 0.0, |
| "proof/d1/pos": 1.0, |
| "proof/d2/pos": 1.0, |
| "proof/d3/neg": 0.0, |
| "proof/d3/pos": 1.0 |
| } |
| }, |
| "org.g5.natural_mcq_acc": { |
| "value": 0.325, |
| "clusters": { |
| "natural_mcq@Multi-Document QA": 0.3, |
| "natural_mcq@Single-Document QA": 0.4375, |
| "natural_mcq@Long In-context Learning": 0.5, |
| "natural_mcq@Code Repository Understanding": 0.14285714285714285, |
| "natural_mcq@Long-dialogue History Understanding": 0.2 |
| } |
| }, |
| "org.g7.joules_per_token": 0.0009267897161566004, |
| "org.g8.loss_spike_score": 1.0, |
| "org.g8.mup_lr_stability": 0.0, |
| "org.g2.arc_challenge_acc": { |
| "value": 0.295, |
| "clusters": { |
| "g2/arc_challenge": 0.295 |
| } |
| }, |
| "org.g4.boolean_logic_acc": { |
| "value": 0.5, |
| "clusters": { |
| "bool/d2": 0.5 |
| } |
| }, |
| "org.g7.throughput_toks_s": { |
| "value": 151.57566520175138, |
| "clusters": { |
| "b32": 151.57566520175138 |
| } |
| }, |
| "org.g1.bits_per_byte_code": { |
| "value": 1.5197987035964795, |
| "clusters": { |
| "domain/code": 1.5197987035964795 |
| } |
| }, |
| "org.g1.bits_per_byte_math": { |
| "value": 1.451242832264349, |
| "clusters": { |
| "domain/math": 1.451242832264349 |
| } |
| }, |
| "org.g4.knights_knaves_acc": { |
| "value": 0.75, |
| "clusters": { |
| "kk/n3": 0.75 |
| } |
| }, |
| "org.g1.bits_per_byte_prose": { |
| "value": 1.2721597414818553, |
| "clusters": { |
| "domain/prose": 1.2721597414818553 |
| } |
| }, |
| "org.g1.bits_per_byte_key_token": 0.13152751503557092, |
| "org.g1.bits_per_byte_fresh_crawl": { |
| "value": 1.2095837878739255, |
| "clusters": { |
| "fresh": 1.2095837878739255 |
| } |
| }, |
| "org.g7.state_bytes_per_token_32k": 8.0 |
| }, |
| "mirrors": [ |
| { |
| "group": "g4", |
| "metric": "org.g4.arithmetic_acc", |
| "mirror": { |
| "value": 0.0, |
| "clusters": { |
| "arith#0": 0.0, |
| "arith#1": 0.0, |
| "arith#2": 0.0, |
| "arith#3": 0.0 |
| } |
| }, |
| "public": { |
| "value": 0.0, |
| "clusters": { |
| "arith#0": 0.0, |
| "arith#1": 0.0, |
| "arith#2": 0.0, |
| "arith#3": 0.0 |
| } |
| } |
| }, |
| { |
| "group": "g4", |
| "metric": "org.g4.boolean_logic_acc", |
| "mirror": { |
| "value": 0.75, |
| "clusters": { |
| "boolean_expr#0": 1.0, |
| "boolean_expr#1": 0.0, |
| "boolean_expr#2": 1.0, |
| "boolean_expr#3": 1.0 |
| } |
| }, |
| "public": { |
| "value": 0.75, |
| "clusters": { |
| "boolean_expr#0": 1.0, |
| "boolean_expr#1": 1.0, |
| "boolean_expr#2": 1.0, |
| "boolean_expr#3": 0.0 |
| } |
| } |
| }, |
| { |
| "group": "g4", |
| "metric": "org.g4.dyck_acc", |
| "mirror": { |
| "value": 0.5, |
| "clusters": { |
| "dyck#0": 0.6666666666666666, |
| "dyck#1": 0.3333333333333333, |
| "dyck#2": 0.3333333333333333, |
| "dyck#3": 0.6666666666666666 |
| } |
| }, |
| "public": { |
| "value": 0.5833333333333333, |
| "clusters": { |
| "dyck#0": 0.3333333333333333, |
| "dyck#1": 0.6666666666666666, |
| "dyck#2": 0.6666666666666666, |
| "dyck#3": 0.6666666666666666 |
| } |
| } |
| }, |
| { |
| "group": "g4", |
| "metric": "org.g4.modular_acc", |
| "mirror": { |
| "value": 0.25, |
| "clusters": { |
| "modular#0": 0.0, |
| "modular#1": 0.0, |
| "modular#2": 0.0, |
| "modular#3": 1.0 |
| } |
| }, |
| "public": { |
| "value": 0.5, |
| "clusters": { |
| "modular#0": 1.0, |
| "modular#1": 0.0, |
| "modular#2": 0.0, |
| "modular#3": 1.0 |
| } |
| } |
| }, |
| { |
| "group": "g4", |
| "metric": "org.g4.knights_knaves_acc", |
| "mirror": { |
| "value": 0.75, |
| "clusters": { |
| "knights_knaves#0": 1.0, |
| "knights_knaves#1": 1.0, |
| "knights_knaves#2": 0.0, |
| "knights_knaves#3": 1.0 |
| } |
| }, |
| "public": { |
| "value": 0.5, |
| "clusters": { |
| "knights_knaves#0": 1.0, |
| "knights_knaves#1": 1.0, |
| "knights_knaves#2": 0.0, |
| "knights_knaves#3": 0.0 |
| } |
| } |
| }, |
| { |
| "group": "g4", |
| "metric": "org.g4.proofwriter_acc", |
| "mirror": { |
| "value": 0.5, |
| "clusters": { |
| "proofwriter#0": 0.0, |
| "proofwriter#1": 1.0, |
| "proofwriter#2": 0.0, |
| "proofwriter#3": 1.0 |
| } |
| }, |
| "public": { |
| "value": 0.5, |
| "clusters": { |
| "proofwriter#0": 0.0, |
| "proofwriter#1": 1.0, |
| "proofwriter#2": 1.0, |
| "proofwriter#3": 0.0 |
| } |
| } |
| }, |
| { |
| "group": "g2", |
| "metric": "org.g2.lambada_acc", |
| "mirror": { |
| "value": 0.75, |
| "clusters": { |
| "lambada#0": 0.0, |
| "lambada#1": 1.0, |
| "lambada#2": 1.0, |
| "lambada#3": 1.0 |
| } |
| }, |
| "public": { |
| "value": 1.0, |
| "clusters": { |
| "lambada#0": 1.0, |
| "lambada#1": 1.0, |
| "lambada#2": 1.0, |
| "lambada#3": 1.0 |
| } |
| } |
| }, |
| { |
| "group": "g2", |
| "metric": "org.g2.hellaswag_acc", |
| "mirror": { |
| "value": 0.75, |
| "clusters": { |
| "hellaswag#0": 1.0, |
| "hellaswag#1": 1.0, |
| "hellaswag#2": 0.0, |
| "hellaswag#3": 1.0 |
| } |
| }, |
| "public": { |
| "value": 1.0, |
| "clusters": { |
| "hellaswag#0": 1.0, |
| "hellaswag#1": 1.0, |
| "hellaswag#2": 1.0, |
| "hellaswag#3": 1.0 |
| } |
| } |
| }, |
| { |
| "group": "g2", |
| "metric": "org.g2.piqa_acc", |
| "mirror": { |
| "value": 0.5, |
| "clusters": { |
| "piqa#0": 1.0, |
| "piqa#1": 0.0, |
| "piqa#2": 0.0, |
| "piqa#3": 1.0 |
| } |
| }, |
| "public": { |
| "value": 0.75, |
| "clusters": { |
| "piqa#0": 0.0, |
| "piqa#1": 1.0, |
| "piqa#2": 1.0, |
| "piqa#3": 1.0 |
| } |
| } |
| }, |
| { |
| "group": "g2", |
| "metric": "org.g2.arc_easy_acc", |
| "mirror": { |
| "value": 0.75, |
| "clusters": { |
| "arc_easy#0": 1.0, |
| "arc_easy#1": 1.0, |
| "arc_easy#2": 1.0, |
| "arc_easy#3": 0.0 |
| } |
| }, |
| "public": { |
| "value": 0.5, |
| "clusters": { |
| "arc_easy#0": 0.0, |
| "arc_easy#1": 1.0, |
| "arc_easy#2": 1.0, |
| "arc_easy#3": 0.0 |
| } |
| } |
| }, |
| { |
| "group": "g2", |
| "metric": "org.g2.arc_challenge_acc", |
| "mirror": { |
| "value": 0.75, |
| "clusters": { |
| "arc_challenge#0": 1.0, |
| "arc_challenge#1": 1.0, |
| "arc_challenge#2": 0.0, |
| "arc_challenge#3": 1.0 |
| } |
| }, |
| "public": { |
| "value": 0.0, |
| "clusters": { |
| "arc_challenge#0": 0.0, |
| "arc_challenge#1": 0.0, |
| "arc_challenge#2": 0.0, |
| "arc_challenge#3": 0.0 |
| } |
| } |
| }, |
| { |
| "group": "g2", |
| "metric": "org.g2.winogrande_acc", |
| "mirror": { |
| "value": 0.5, |
| "clusters": { |
| "winogrande#0": 0.0, |
| "winogrande#1": 1.0, |
| "winogrande#2": 0.0, |
| "winogrande#3": 1.0 |
| } |
| }, |
| "public": { |
| "value": 0.75, |
| "clusters": { |
| "winogrande#0": 1.0, |
| "winogrande#1": 0.0, |
| "winogrande#2": 1.0, |
| "winogrande#3": 1.0 |
| } |
| } |
| }, |
| { |
| "group": "g2", |
| "metric": "org.g2.boolq_acc", |
| "mirror": { |
| "value": 0.75, |
| "clusters": { |
| "boolq#0": 0.0, |
| "boolq#1": 1.0, |
| "boolq#2": 1.0, |
| "boolq#3": 1.0 |
| } |
| }, |
| "public": { |
| "value": 0.5, |
| "clusters": { |
| "boolq#0": 1.0, |
| "boolq#1": 0.0, |
| "boolq#2": 1.0, |
| "boolq#3": 0.0 |
| } |
| } |
| }, |
| { |
| "group": "g2", |
| "metric": "org.g2.obqa_acc", |
| "mirror": { |
| "value": 0.5, |
| "clusters": { |
| "openbookqa#0": 1.0, |
| "openbookqa#1": 0.0, |
| "openbookqa#2": 0.0, |
| "openbookqa#3": 1.0 |
| } |
| }, |
| "public": { |
| "value": 0.75, |
| "clusters": { |
| "openbookqa#0": 1.0, |
| "openbookqa#1": 1.0, |
| "openbookqa#2": 1.0, |
| "openbookqa#3": 0.0 |
| } |
| } |
| }, |
| { |
| "group": "g5", |
| "metric": "org.g5.natural_mcq_acc", |
| "mirror": { |
| "value": 0.5, |
| "clusters": { |
| "Multi-Document QA": 1.0, |
| "Single-Document QA": 0.0, |
| "Long In-context Learning": 0.0, |
| "Long-dialogue History Understanding": 1.0 |
| } |
| }, |
| "public": { |
| "value": 0.25, |
| "clusters": { |
| "Multi-Document QA": 0.0, |
| "Single-Document QA": 0.0, |
| "Long In-context Learning": 1.0 |
| } |
| } |
| }, |
| { |
| "group": "g5", |
| "metric": "org.g5.helmet_rag_acc", |
| "mirror": { |
| "value": 0.0, |
| "clusters": { |
| "popqa": 0.0, |
| "hotpotqa": 0.0 |
| } |
| }, |
| "public": { |
| "value": 0.5, |
| "clusters": { |
| "nq": 0.0, |
| "popqa": 0.5, |
| "triviaqa": 1.0 |
| } |
| } |
| } |
| ] |
| }, |
| "eval_tier": "public", |
| "flow": "v3" |
| } |
