{ "datasets_distributed": false, "processed_for_qev_comparative_evaluation": true, "official_unmodified_benchmark_scores": false, "public_comparison_context": {"state": 384, "per_question_row": 1024, "packed": 2048, "truncate": false, "selection": "fixed successful item-ID intersection across compared original models"}, "datasets": { "mmluprox-lite": { "source_url": "https://huggingface.co/datasets/li-lab/MMLU-ProX-Lite", "source_revision_recorded_by_manifest": "e82aafb9460529687d3c7e51b401d8dd1dd309dd", "source_license": "MIT", "split": "test", "languages": 29, "source_questions": 17052, "reported_questions": 16384, "processing": "question to state; valid options to choice criteria; cot_content excluded; blank/N/A choices removed; original question/answer and option order retained and validated; fixed common item-ID subset", "processed_sha256": "46b4e624eba8f41d002ea215c5ff03661fbfc5d31837e7833fbd574f6976b348" }, "belebele": { "source_url": "https://huggingface.co/datasets/facebook/belebele", "source_revision_recorded_by_manifest": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2", "source_license": "CC-BY-SA-4.0", "split": "test", "languages": 29, "source_questions": 26100, "reported_questions": 24622, "processing": "flores_passage to state; original question to instructions; four mc_answers to choice criteria; correct_answer_num mapped to label; fixed common item-ID subset; processed passages not redistributed", "processed_sha256": "fcae61bceeb6953c62b163dd495614bd2a28e04871f82b9a05a14ed86604e8e8" }, "private-typed-decision": { "source": "QueryPie project private-general-v1", "public_download": false, "label_generation": "deterministic code/rule solver; seed 20261006", "translation": "assistant-authored deterministic EN to KO/JA template localization; labels, numbers and option keys preserved; automatic integrity checks; no native-speaker review", "source_states": 450, "source_questions": 900, "reported_states": 270, "reported_questions": 540, "questions_per_language": {"en": 180, "ko": 180, "ja": 180}, "processing": "choice/noul/score questions; exclude the entire state if q1/q2 type, instructions and criteria are identical; identical selected IDs used for every model", "processed_sha256": "5c061e647377b909fde9368de002944ba1ca7e9bb0963facc5ad8e9482b1b5d2" }, "dlp": { "source": "QueryPie internal held-out multilingual DLP test corpus", "public_download": false, "documents": 3167, "positive": 1281, "negative": 1886, "documents_by_language": {"en": 1065, "ja": 1038, "ko": 1064}, "processing": "original text/IDs/labels retained; document to state; English noul true/false instruction; any ground-truth finding means DLP positive; no finding-span/count/type extraction metric", "threshold": 0.5, "processed_sha256": "77cd5fbd3ef74695912f008c0838dd5fd66462278167259fc4e73a182c3a2e97" } }, "composite_definition": "equal-weight mean of question-level accuracies of MMLU-ProX-Lite, Belebele and private Typed-Decision", "new_inference_for_card_update": false, "raw_data_and_item_level_predictions_included": false }