| {"step": 150, "eval": {"datasets": {"jevbench": {"intelligence": {"score": 51.3356, "per_type": {"choice": {"n": 139, "n_invalid": 0, "weighted_accuracy": 0.840426, "chance_level": 0.23914, "chance_corrected": 0.601285, "difficulty_weighted": true}, "noul": {"n": 74, "n_invalid": 0, "weighted_accuracy": 0.851282, "chance_level": 0.5, "chance_corrected": 0.351282, "difficulty_weighted": true}, "score": {"n": 18, "n_invalid": 0, "weighted_accuracy": 0.833333, "chance_level": 0.245833, "chance_corrected": 0.5875, "difficulty_weighted": true}}, "n_items": 231, "n_invalid": 0, "chance_mode": "subtract", "type_weights": {"choice": 0.3333333333333333, "noul": 0.3333333333333333, "score": 0.3333333333333333}, "raw": 51.3356, "gap_penalty": 0.0, "public_raw": 50.374, "sealed_raw": 51.9929, "pseudo_sealed": true, "equating_offset": 0.0}, "calibration": {"score": 90.6626, "ece": 0.073239, "mean_fidelity": 0.88649, "ece_weight": 0.5, "n_items": 231, "n_valid_probs": 231, "label_only": false, "raw": 90.6626, "public_raw": 90.3745, "sealed_raw": 90.9128, "equating_offset": 0.0}, "n_items": 231, "n_invalid": 0, "equating": {"applied": false, "reason": "no anchor file; scores stay on the raw draw scale"}, "legacy": {"n_questions": 231, "accuracy": 0.883117, "nll": 0.373094, "per_type": {"choice": {"n": 139, "accuracy": 0.892086}, "noul": {"n": 74, "accuracy": 0.864865}, "score": {"n": 18, "accuracy": 0.888889}}, "per_difficulty": {"easy": {"n": 48, "accuracy": 1.0}, "standard": {"n": 72, "accuracy": 0.972222}, "hard": {"n": 94, "accuracy": 0.808511}, "judge": {"n": 17, "accuracy": 0.588235}}}, "dataset_weight": 0.7}, "dev": {"intelligence": {"score": 59.0104, "per_type": {"choice": {"n": 1580, "n_invalid": 0, "weighted_accuracy": 0.956962, "chance_level": 0.199043, "chance_corrected": 0.757919, "difficulty_weighted": true}, "noul": {"n": 870, "n_invalid": 0, "weighted_accuracy": 0.847126, "chance_level": 0.5, "chance_corrected": 0.347126, "difficulty_weighted": true}, "score": {"n": 750, "n_invalid": 0, "weighted_accuracy": 0.906667, "chance_level": 0.2414, "chance_corrected": 0.665267, "difficulty_weighted": true}}, "n_items": 3200, "n_invalid": 0, "chance_mode": "subtract", "type_weights": {"choice": 0.3333333333333333, "noul": 0.3333333333333333, "score": 0.3333333333333333}, "raw": 59.0104, "gap_penalty": 0.0, "public_raw": 59.0104, "sealed_raw": null, "pseudo_sealed": false, "equating_offset": 0.0}, "calibration": {"score": 95.4772, "ece": 0.014542, "mean_fidelity": 0.924086, "ece_weight": 0.5, "n_items": 3200, "n_valid_probs": 3200, "label_only": false, "raw": 95.4772, "public_raw": 95.4772, "sealed_raw": null, "equating_offset": 0.0}, "n_items": 3200, "n_invalid": 0, "equating": {"applied": false, "reason": "no anchor file; scores stay on the raw draw scale"}, "legacy": {"n_questions": 3200, "accuracy": 0.915312, "nll": 0.241772, "per_type": {"choice": {"n": 1580, "accuracy": 0.956962}, "noul": {"n": 870, "accuracy": 0.847126}, "score": {"n": 750, "accuracy": 0.906667}}, "per_difficulty": {"standard": {"n": 3200, "accuracy": 0.915312}}}, "dataset_weight": 0.3}}, "intelligence": 53.638, "calibration": 92.107, "composite": 66.461, "dataset_weights": {"jevbench": 0.7, "dev": 0.3}, "metric_weights": {"intelligence": 1.0, "calibration": 0.5}}, "eval_seconds": 229.8} | |