{
  "rows": {
    "train": 288,
    "validation": 144,
    "test": 144
  },
  "zero_shot_verify": {
    "validation": {
      "categorical": "89/126",
      "categorical_accuracy": 0.7063492063492064,
      "completion_found": "21/24",
      "false_completions": "0/48",
      "unknown": "1/24",
      "score_mae": 0.28145276563851634
    },
    "test": {
      "categorical": "85/126",
      "categorical_accuracy": 0.6746031746031746,
      "completion_found": "22/24",
      "false_completions": "0/48",
      "unknown": "0/24",
      "score_mae": 0.42307032748062895
    }
  },
  "candidates": [
    {
      "l2": 0.0001,
      "validation_nll": 0.010317740071604856,
      "validation": {
        "categorical": "125/126",
        "categorical_accuracy": 0.9920634920634921,
        "completion_found": "24/24",
        "false_completions": "0/48",
        "unknown": "24/24",
        "score_mae": 0.0004602126554537731
      },
      "train": {
        "categorical": "246/246",
        "categorical_accuracy": 1.0,
        "completion_found": "48/48",
        "false_completions": "0/96",
        "unknown": "48/48",
        "score_mae": 4.989935418984657e-05
      }
    },
    {
      "l2": 0.001,
      "validation_nll": 0.010762290137645349,
      "validation": {
        "categorical": "125/126",
        "categorical_accuracy": 0.9920634920634921,
        "completion_found": "24/24",
        "false_completions": "0/48",
        "unknown": "24/24",
        "score_mae": 0.0018140511359126523
      },
      "train": {
        "categorical": "246/246",
        "categorical_accuracy": 1.0,
        "completion_found": "48/48",
        "false_completions": "0/96",
        "unknown": "48/48",
        "score_mae": 0.0003728628331370325
      }
    },
    {
      "l2": 0.01,
      "validation_nll": 0.012511289827795442,
      "validation": {
        "categorical": "125/126",
        "categorical_accuracy": 0.9920634920634921,
        "completion_found": "24/24",
        "false_completions": "0/48",
        "unknown": "24/24",
        "score_mae": 0.007914584885551698
      },
      "train": {
        "categorical": "246/246",
        "categorical_accuracy": 1.0,
        "completion_found": "48/48",
        "false_completions": "0/96",
        "unknown": "48/48",
        "score_mae": 0.0027551349573122588
      }
    },
    {
      "l2": 0.03,
      "validation_nll": 0.015579639968586186,
      "validation": {
        "categorical": "125/126",
        "categorical_accuracy": 0.9920634920634921,
        "completion_found": "24/24",
        "false_completions": "0/48",
        "unknown": "24/24",
        "score_mae": 0.015240309919228874
      },
      "train": {
        "categorical": "246/246",
        "categorical_accuracy": 1.0,
        "completion_found": "48/48",
        "false_completions": "0/96",
        "unknown": "48/48",
        "score_mae": 0.0068053722403096945
      }
    },
    {
      "l2": 0.1,
      "validation_nll": 0.02337755861566644,
      "validation": {
        "categorical": "125/126",
        "categorical_accuracy": 0.9920634920634921,
        "completion_found": "24/24",
        "false_completions": "0/48",
        "unknown": "24/24",
        "score_mae": 0.03018260889306569
      },
      "train": {
        "categorical": "246/246",
        "categorical_accuracy": 1.0,
        "completion_found": "48/48",
        "false_completions": "0/96",
        "unknown": "48/48",
        "score_mae": 0.01768083592835426
      }
    },
    {
      "l2": 0.3,
      "validation_nll": 0.03863210478984501,
      "validation": {
        "categorical": "125/126",
        "categorical_accuracy": 0.9920634920634921,
        "completion_found": "24/24",
        "false_completions": "0/48",
        "unknown": "24/24",
        "score_mae": 0.05465809329027363
      },
      "train": {
        "categorical": "246/246",
        "categorical_accuracy": 1.0,
        "completion_found": "48/48",
        "false_completions": "0/96",
        "unknown": "48/48",
        "score_mae": 0.04015321627012537
      }
    },
    {
      "l2": 1.0,
      "validation_nll": 0.07221928214550037,
      "validation": {
        "categorical": "126/126",
        "categorical_accuracy": 1.0,
        "completion_found": "24/24",
        "false_completions": "0/48",
        "unknown": "24/24",
        "score_mae": 0.10031443697789943
      },
      "train": {
        "categorical": "244/246",
        "categorical_accuracy": 0.991869918699187,
        "completion_found": "48/48",
        "false_completions": "0/96",
        "unknown": "47/48",
        "score_mae": 0.09034251097401509
      }
    }
  ],
  "selected_l2": 1.0,
  "test": {
    "categorical": "115/126",
    "categorical_accuracy": 0.9126984126984127,
    "completion_found": "24/24",
    "false_completions": "0/48",
    "unknown": "24/24",
    "score_mae": 0.1781372601465551
  },
  "margin_weight": 0.9012869778818261,
  "type_bias": {
    "choice": 6.874923957629764e-17,
    "noul": -1.9873536525856503e-17,
    "score": -7.646542884907737e-17
  }
}