eval/analysis/contamination_audit.json

{
  "method": "Row-level text-overlap audit between training source splits and public panel rows.",
  "sources": {
    "mmlu_auxiliary_train": 98370,
    "arc_train": 3361,
    "hellaswag_train": 39799
  },
  "overlap": {
    "mmlu": {
      "n": 14042,
      "mmlu_auxiliary_train": {
        "question": 19,
        "question_and_options": 0
      },
      "arc_train": {
        "question": 1,
        "question_and_options": 0
      },
      "hellaswag_train": {
        "question": 0,
        "question_and_options": 0
      }
    },
    "arc_easy": {
      "n": 2376,
      "mmlu_auxiliary_train": {
        "question": 8,
        "question_and_options": 3
      },
      "arc_train": {
        "question": 7,
        "question_and_options": 3
      },
      "hellaswag_train": {
        "question": 0,
        "question_and_options": 0
      }
    },
    "arc_challenge": {
      "n": 1172,
      "mmlu_auxiliary_train": {
        "question": 9,
        "question_and_options": 0
      },
      "arc_train": {
        "question": 9,
        "question_and_options": 0
      },
      "hellaswag_train": {
        "question": 0,
        "question_and_options": 0
      }
    },
    "hellaswag": {
      "n": 10042,
      "mmlu_auxiliary_train": {
        "question": 0,
        "question_and_options": 0
      },
      "arc_train": {
        "question": 0,
        "question_and_options": 0
      },
      "hellaswag_train": {
        "question": 23,
        "question_and_options": 0
      }
    }
  }
}