Archive verified research artifact: summary.json

artifacts/experiments/runs/test-qwen3-mmqa_ti-s17/summary.json

@@ -0,0 +1,154 @@
+{
+  "checkpoint": null,
+  "dataset": "mmqa_ti",
+  "method": "qwen3",
+  "scores": [
+    {
+      "complete": true,
+      "dataset": "mmqa_ti",
+      "evaluator_sha256": "1b95323ffdf36a0e8a8afef28c598e6fd560f6e63aa8d48709e51607e84d6100",
+      "groups": {
+        "modality:image": {
+          "em": 21.518987341772153,
+          "f1": 24.151898734177212,
+          "n": 79
+        },
+        "modality:image+text": {
+          "em": 8.695652173913043,
+          "f1": 8.695652173913043,
+          "n": 23
+        },
+        "modality:text": {
+          "em": 29.878048780487806,
+          "f1": 36.396341463414636,
+          "n": 164
+        },
+        "type:Compare(Compose(TableQ,ImageQ),Compose(TableQ,TextQ))": {
+          "em": 0.0,
+          "f1": 0.0,
+          "n": 4
+        },
+        "type:Compose(ImageQ,TextQ)": {
+          "em": 14.285714285714286,
+          "f1": 14.285714285714286,
+          "n": 7
+        },
+        "type:Compose(TextQ,ImageListQ)": {
+          "em": 8.333333333333334,
+          "f1": 8.333333333333334,
+          "n": 12
+        },
+        "type:ImageListQ": {
+          "em": 0.0,
+          "f1": 4.40625,
+          "n": 32
+        },
+        "type:ImageQ": {
+          "em": 36.170212765957444,
+          "f1": 37.59574468085107,
+          "n": 47
+        },
+        "type:TextQ": {
+          "em": 29.878048780487806,
+          "f1": 36.396341463414636,
+          "n": 164
+        }
+      },
+      "input_sha256": "098c9d84dca21b07da69903ff247c5af18232b5c8a66ff157cef35006e1e9da1",
+      "metric_scale": "0-100",
+      "n_expected": 266,
+      "n_missing": 0,
+      "n_predictions": 266,
+      "parse_success_count": 266,
+      "prediction_sha256": "fe83c246052a3393b8a143143522f8f66c4f6efe493ab7e27f7886f75cedf22a",
+      "protocol_sha256": "ca6b0397b6532489a12ad40a3a1d18408c0f52c45a20c15fae188d3a59eae462",
+      "scores": {
+        "em": 25.56390977443609,
+        "f1": 30.36466165413534
+      },
+      "split": "val_calibration",
+      "status_counts": {
+        "ok": 266
+      },
+      "target_sha256": "fe1557c98813401676c470515634238527d397080aee9f96f09c30a47e5fe907"
+    },
+    {
+      "complete": true,
+      "dataset": "mmqa_ti",
+      "evaluator_sha256": "1b95323ffdf36a0e8a8afef28c598e6fd560f6e63aa8d48709e51607e84d6100",
+      "groups": {
+        "modality:image": {
+          "em": 23.40966921119593,
+          "f1": 25.38167938931298,
+          "n": 393
+        },
+        "modality:image+text": {
+          "em": 20.338983050847457,
+          "f1": 22.228813559322035,
+          "n": 118
+        },
+        "modality:text": {
+          "em": 25.73170731707317,
+          "f1": 31.079268292682926,
+          "n": 820
+        },
+        "type:Compare(Compose(TableQ,ImageQ),Compose(TableQ,TextQ))": {
+          "em": 0.0,
+          "f1": 0.0,
+          "n": 22
+        },
+        "type:Compose(ImageQ,TextQ)": {
+          "em": 54.54545454545455,
+          "f1": 54.54545454545455,
+          "n": 33
+        },
+        "type:Compose(TextQ,ImageListQ)": {
+          "em": 8.19672131147541,
+          "f1": 11.852459016393443,
+          "n": 61
+        },
+        "type:ImageListQ": {
+          "em": 2.5,
+          "f1": 6.6125,
+          "n": 160
+        },
+        "type:ImageQ": {
+          "em": 37.76824034334764,
+          "f1": 38.27038626609442,
+          "n": 233
+        },
+        "type:Intersect(ImageListQ,TextQ)": {
+          "em": 50.0,
+          "f1": 50.0,
+          "n": 2
+        },
+        "type:TextQ": {
+          "em": 25.73170731707317,
+          "f1": 31.079268292682926,
+          "n": 820
+        }
+      },
+      "input_sha256": "bb92409d451ad352f22a819754ff924c2c92bbfec79df241aa382897095badad",
+      "metric_scale": "0-100",
+      "n_expected": 1331,
+      "n_missing": 0,
+      "n_predictions": 1331,
+      "parse_success_count": 1322,
+      "prediction_sha256": "b017a0688449783f446dd7b8334770af4bdb52b6b5adec65f3e29d22d57ccad5",
+      "protocol_sha256": "ca6b0397b6532489a12ad40a3a1d18408c0f52c45a20c15fae188d3a59eae462",
+      "scores": {
+        "em": 24.567993989481593,
+        "f1": 28.612321562734785
+      },
+      "split": "test",
+      "status_counts": {
+        "ok": 1331
+      },
+      "target_sha256": "f9ac83d22210f0300be901ec863f8233e4438dcd99660a7ae628d7d9a290c5a8"
+    }
+  ],
+  "seed": 17,
+  "test_feedback_for_training": false,
+  "test_used": true,
+  "validation_run": "experiments/runs/qwen3-mmqa_ti-s17"
+}