| { |
| "date": "2026-09-27", |
| "training": false, |
| "models": { |
| "balanced-skills-1920": { |
| "bundle": "sft_skill_balance_v1/step_0001920_628a0a65099b", |
| "sha256": "4cac7f733e6ef63230e26e2aa66418bf0d9661ef55f46fa1e01f2ac602b17407", |
| "results": { |
| "multiple-choice": { |
| "completed": 15428, |
| "expected": 15428, |
| "status": "complete", |
| "tasks": { |
| "arc_challenge": { |
| "acc": 0.25341296928327645, |
| "acc_norm": 0.3003412969283277, |
| "completed": 1172, |
| "expected": 1172 |
| }, |
| "arc_easy": { |
| "acc": 0.5904882154882155, |
| "acc_norm": 0.5707070707070707, |
| "completed": 2376, |
| "expected": 2376 |
| }, |
| "hellaswag": { |
| "acc": 0.30999800836486757, |
| "acc_norm": 0.34534953196574386, |
| "completed": 10042, |
| "expected": 10042 |
| }, |
| "piqa": { |
| "acc": 0.6474428726877041, |
| "acc_norm": 0.6430903155603918, |
| "completed": 1838, |
| "expected": 1838 |
| } |
| }, |
| "training": false |
| }, |
| "ifeval": { |
| "completed": 541, |
| "expected": 541, |
| "loose": { |
| "instruction_accuracy": 0.2637889688249401, |
| "instruction_correct": 220, |
| "instruction_total": 834, |
| "prompt_accuracy": 0.14417744916820702, |
| "prompt_correct": 78, |
| "prompt_total": 541 |
| }, |
| "status": "complete", |
| "stop_counts": { |
| "context_limit": 0, |
| "eos": 539, |
| "length": 2 |
| }, |
| "strict": { |
| "instruction_accuracy": 0.2637889688249401, |
| "instruction_correct": 220, |
| "instruction_total": 834, |
| "prompt_accuracy": 0.14417744916820702, |
| "prompt_correct": 78, |
| "prompt_total": 541 |
| }, |
| "training": false |
| } |
| } |
| }, |
| "constraint-completion-2048": { |
| "bundle": "sft_constraint_completion_1920_v1/step_0002048_1267d9f4cf7a", |
| "sha256": "e4e8a0de28e2141f6a580b49cc950aa64492c9f94d0d3aa98e6a26a2f439764b", |
| "results": { |
| "multiple-choice": { |
| "completed": 15428, |
| "expected": 15428, |
| "status": "complete", |
| "tasks": { |
| "arc_challenge": { |
| "acc": 0.25426621160409557, |
| "acc_norm": 0.31399317406143346, |
| "completed": 1172, |
| "expected": 1172 |
| }, |
| "arc_easy": { |
| "acc": 0.5871212121212122, |
| "acc_norm": 0.5677609427609428, |
| "completed": 2376, |
| "expected": 2376 |
| }, |
| "hellaswag": { |
| "acc": 0.31029675363473413, |
| "acc_norm": 0.3472415853415654, |
| "completed": 10042, |
| "expected": 10042 |
| }, |
| "piqa": { |
| "acc": 0.64689880304679, |
| "acc_norm": 0.6381936887921654, |
| "completed": 1838, |
| "expected": 1838 |
| } |
| }, |
| "training": false |
| }, |
| "ifeval": { |
| "completed": 541, |
| "expected": 541, |
| "loose": { |
| "instruction_accuracy": 0.29136690647482016, |
| "instruction_correct": 243, |
| "instruction_total": 834, |
| "prompt_accuracy": 0.16820702402957485, |
| "prompt_correct": 91, |
| "prompt_total": 541 |
| }, |
| "status": "complete", |
| "stop_counts": { |
| "context_limit": 0, |
| "eos": 532, |
| "length": 9 |
| }, |
| "strict": { |
| "instruction_accuracy": 0.28776978417266186, |
| "instruction_correct": 240, |
| "instruction_total": 834, |
| "prompt_accuracy": 0.16635859519408502, |
| "prompt_correct": 90, |
| "prompt_total": 541 |
| }, |
| "training": false |
| } |
| } |
| } |
| }, |
| "protocol": { |
| "harness_reference_revision": "b954108c9baaaa934b4ad842033b31a97ee30816", |
| "datasets": { |
| "arc_easy": { |
| "repo": "allenai/ai2_arc", |
| "revision": "210d026faf9955653af8916fad021475a3f00453", |
| "file": "ARC-Easy/test-00000-of-00001.parquet", |
| "split": "test", |
| "rows": 2376, |
| "sha256": "4160597d618ae851c7eb04e281574f3f654776216ac6b6641588d64527b47177" |
| }, |
| "arc_challenge": { |
| "repo": "allenai/ai2_arc", |
| "revision": "210d026faf9955653af8916fad021475a3f00453", |
| "file": "ARC-Challenge/test-00000-of-00001.parquet", |
| "split": "test", |
| "rows": 1172, |
| "sha256": "62f03257e737aed263f55c6abf87c7bb0028a44a6bdd2a26eb1279eb42c1d1e9" |
| }, |
| "piqa": { |
| "repo": "baber/piqa", |
| "revision": "142f6d7367fd9877f0fb3b5734ea6a545f54cdd1", |
| "file": "piqa_validation.parquet", |
| "split": "validation", |
| "rows": 1838, |
| "sha256": "2b17ee1215b1ceb2ddabf4c90a05c5a8b833bef9d207960f57d111c43a2080f3" |
| }, |
| "hellaswag": { |
| "repo": "Rowan/hellaswag", |
| "revision": "218ec52e09a7e7462a5400043bb9a69a41d06b76", |
| "file": "data/validation-00000-of-00001.parquet", |
| "split": "validation", |
| "rows": 10042, |
| "sha256": "899813071e1e95efafec90f856e1987d2150fa4d020fc005df6962c259f660cd" |
| }, |
| "ifeval": { |
| "repo": "google/IFEval", |
| "revision": "966cd89545d6b6acfd7638bc708b98261ca58e84", |
| "file": "ifeval_input_data.jsonl", |
| "split": "train", |
| "rows": 541, |
| "sha256": "6a85310ca8ce15eff755aa08a3a4ff931c7e273e7515ebb3c492ea85fd8288f2" |
| } |
| }, |
| "multiple_choice_normalization": "candidate log-likelihood divided by candidate character length", |
| "few_shot_examples": 0, |
| "ifeval": { |
| "prompt_template": "User: {prompt}\nAssistant:", |
| "max_context": 2048, |
| "max_new_tokens": 1280, |
| "greedy": true, |
| "repetition_penalty": 1 |
| } |
| }, |
| "limitations": [ |
| "Custom MLX evaluator, not an official leaderboard submission.", |
| "Public suites repeatedly inspected during development, not an untouched final test.", |
| "No full contamination audit or statistical-significance claim.", |
| "Instruction compliance does not establish correct content or conversational quality." |
| ] |
| } |
|
|
