File size: 3,329 Bytes
96adfbe
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
{
  "datasets_distributed": false,
  "processed_for_qev_comparative_evaluation": true,
  "official_unmodified_benchmark_scores": false,
  "public_comparison_context": {"state": 384, "per_question_row": 1024, "packed": 2048, "truncate": false, "selection": "fixed successful item-ID intersection across compared original models"},
  "datasets": {
    "mmluprox-lite": {
      "source_url": "https://huggingface.co/datasets/li-lab/MMLU-ProX-Lite",
      "source_revision_recorded_by_manifest": "e82aafb9460529687d3c7e51b401d8dd1dd309dd",
      "source_license": "MIT",
      "split": "test",
      "languages": 29,
      "source_questions": 17052,
      "reported_questions": 16384,
      "processing": "question to state; valid options to choice criteria; cot_content excluded; blank/N/A choices removed; original question/answer and option order retained and validated; fixed common item-ID subset",
      "processed_sha256": "46b4e624eba8f41d002ea215c5ff03661fbfc5d31837e7833fbd574f6976b348"
    },
    "belebele": {
      "source_url": "https://huggingface.co/datasets/facebook/belebele",
      "source_revision_recorded_by_manifest": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2",
      "source_license": "CC-BY-SA-4.0",
      "split": "test",
      "languages": 29,
      "source_questions": 26100,
      "reported_questions": 24622,
      "processing": "flores_passage to state; original question to instructions; four mc_answers to choice criteria; correct_answer_num mapped to label; fixed common item-ID subset; processed passages not redistributed",
      "processed_sha256": "fcae61bceeb6953c62b163dd495614bd2a28e04871f82b9a05a14ed86604e8e8"
    },
    "private-typed-decision": {
      "source": "QueryPie project private-general-v1",
      "public_download": false,
      "label_generation": "deterministic code/rule solver; seed 20261006",
      "translation": "assistant-authored deterministic EN to KO/JA template localization; labels, numbers and option keys preserved; automatic integrity checks; no native-speaker review",
      "source_states": 450,
      "source_questions": 900,
      "reported_states": 270,
      "reported_questions": 540,
      "questions_per_language": {"en": 180, "ko": 180, "ja": 180},
      "processing": "choice/noul/score questions; exclude the entire state if q1/q2 type, instructions and criteria are identical; identical selected IDs used for every model",
      "processed_sha256": "5c061e647377b909fde9368de002944ba1ca7e9bb0963facc5ad8e9482b1b5d2"
    },
    "dlp": {
      "source": "QueryPie internal held-out multilingual DLP test corpus",
      "public_download": false,
      "documents": 3167,
      "positive": 1281,
      "negative": 1886,
      "documents_by_language": {"en": 1065, "ja": 1038, "ko": 1064},
      "processing": "original text/IDs/labels retained; document to state; English noul true/false instruction; any ground-truth finding means DLP positive; no finding-span/count/type extraction metric",
      "threshold": 0.5,
      "processed_sha256": "77cd5fbd3ef74695912f008c0838dd5fd66462278167259fc4e73a182c3a2e97"
    }
  },
  "composite_definition": "equal-weight mean of question-level accuracies of MMLU-ProX-Lite, Belebele and private Typed-Decision",
  "new_inference_for_card_update": false,
  "raw_data_and_item_level_predictions_included": false
}