{
  "passed": true,
  "models": {
    "qev": {
      "evaluation_sha256": "18e600d2762c06c4ecc7aa07a08055a1a67ca121ec53776e081faed74526ca82",
      "predictions_sha256": "2b404c1dccf9459438ebb9303929b1368678307a5d72fdbd2fb82b44ebbba100"
    },
    "laya-base": {
      "evaluation_sha256": "5c69751621bd4a5f00262943ddbc087b3cc68fd00d957ab709547c7a17cf9a95",
      "predictions_sha256": "c769919177ae6fb1248e21d0dde16c6b67c473bc068c37631a95f3dceafb9d70"
    },
    "laya-specialist": {
      "evaluation_sha256": "3ebfc3b3a38fb1d30616feea28376e60ebcc4b0ae935cf3a84ba4a95e21ff79c",
      "predictions_sha256": "885cb7317dbe422bcb2b05ff2d93f50273ab3d373797d59d556cadf851964cfe"
    }
  },
  "paired_accuracy": {
    "laya-base": {
      "typed_decisions": {
        "qev_minus_reference_accuracy": 0.4095,
        "paired_group_bootstrap_95_ci": [
          0.3745,
          0.4435
        ],
        "groups": 400,
        "questions": 2000,
        "bootstrap_replicates": 10000,
        "seed": 20261001,
        "descriptive_not_multiple_comparison_adjusted": true,
        "qev_only_correct": 993,
        "reference_only_correct": 174
      },
      "ag_news": {
        "qev_minus_reference_accuracy": -0.125,
        "paired_group_bootstrap_95_ci": [
          -0.1625,
          -0.0875
        ],
        "groups": 400,
        "questions": 400,
        "bootstrap_replicates": 10000,
        "seed": 20261001,
        "descriptive_not_multiple_comparison_adjusted": true,
        "qev_only_correct": 10,
        "reference_only_correct": 60
      },
      "emotion": {
        "qev_minus_reference_accuracy": -0.085,
        "paired_group_bootstrap_95_ci": [
          -0.1325,
          -0.035
        ],
        "groups": 400,
        "questions": 400,
        "bootstrap_replicates": 10000,
        "seed": 20261001,
        "descriptive_not_multiple_comparison_adjusted": true,
        "qev_only_correct": 34,
        "reference_only_correct": 68
      }
    },
    "laya-specialist": {
      "typed_decisions": {
        "qev_minus_reference_accuracy": 0.0005,
        "paired_group_bootstrap_95_ci": [
          -0.0185,
          0.0195
        ],
        "groups": 400,
        "questions": 2000,
        "bootstrap_replicates": 10000,
        "seed": 20261001,
        "descriptive_not_multiple_comparison_adjusted": true,
        "qev_only_correct": 166,
        "reference_only_correct": 165
      },
      "ag_news": {
        "qev_minus_reference_accuracy": -0.1275,
        "paired_group_bootstrap_95_ci": [
          -0.1675,
          -0.09
        ],
        "groups": 400,
        "questions": 400,
        "bootstrap_replicates": 10000,
        "seed": 20261001,
        "descriptive_not_multiple_comparison_adjusted": true,
        "qev_only_correct": 10,
        "reference_only_correct": 61
      },
      "emotion": {
        "qev_minus_reference_accuracy": -0.0975,
        "paired_group_bootstrap_95_ci": [
          -0.1475,
          -0.0475
        ],
        "groups": 400,
        "questions": 400,
        "bootstrap_replicates": 10000,
        "seed": 20261001,
        "descriptive_not_multiple_comparison_adjusted": true,
        "qev_only_correct": 33,
        "reference_only_correct": 72
      }
    }
  },
  "requests_sha256": "585850d575d9dbdf943683936dda7e07b9e5525b60875906926e9915c557d1d6",
  "input_protocol_sha256": "98b677278157506b94c52c5d449d1f6b805279914e92542781952b42477de4aa",
  "aggregator_sha256": "e5802fd38ccdee3d85718faa1c5ac664f0173ed3e84115342445a6712f09d7d2",
  "same_cases_questions_labels_and_metrics": true,
  "all_input_truncation_counts_zero": true,
  "new_training_or_test_temperature_fitting": false,
  "scope": "Paired release regression and two task-held-out English suites, not general parity"
}