Archive verified research artifact: summary.json

artifacts/experiments/runs/test-mira-mmqa_ti-s17/summary.json

@@ -0,0 +1,154 @@
+{
+  "checkpoint": "experiments/runs/gate-search-mira-mmqa_ti-transfer_p2_r1-s17/epoch2.pt",
+  "dataset": "mmqa_ti",
+  "method": "mira",
+  "scores": [
+    {
+      "complete": true,
+      "dataset": "mmqa_ti",
+      "evaluator_sha256": "1b95323ffdf36a0e8a8afef28c598e6fd560f6e63aa8d48709e51607e84d6100",
+      "groups": {
+        "modality:image": {
+          "em": 26.582278481012658,
+          "f1": 31.20253164556962,
+          "n": 79
+        },
+        "modality:image+text": {
+          "em": 4.3478260869565215,
+          "f1": 4.3478260869565215,
+          "n": 23
+        },
+        "modality:text": {
+          "em": 28.658536585365855,
+          "f1": 35.43292682926829,
+          "n": 164
+        },
+        "type:Compare(Compose(TableQ,ImageQ),Compose(TableQ,TextQ))": {
+          "em": 0.0,
+          "f1": 0.0,
+          "n": 4
+        },
+        "type:Compose(ImageQ,TextQ)": {
+          "em": 14.285714285714286,
+          "f1": 14.285714285714286,
+          "n": 7
+        },
+        "type:Compose(TextQ,ImageListQ)": {
+          "em": 0.0,
+          "f1": 0.0,
+          "n": 12
+        },
+        "type:ImageListQ": {
+          "em": 0.0,
+          "f1": 9.3125,
+          "n": 32
+        },
+        "type:ImageQ": {
+          "em": 44.680851063829785,
+          "f1": 46.1063829787234,
+          "n": 47
+        },
+        "type:TextQ": {
+          "em": 28.658536585365855,
+          "f1": 35.43292682926829,
+          "n": 164
+        }
+      },
+      "input_sha256": "098c9d84dca21b07da69903ff247c5af18232b5c8a66ff157cef35006e1e9da1",
+      "metric_scale": "0-100",
+      "n_expected": 266,
+      "n_missing": 0,
+      "n_predictions": 266,
+      "parse_success_count": 266,
+      "prediction_sha256": "5a2141b3cfc56a404497ec147a7a9b73fe4cc91d5ec17366a174d44cf9f52a2e",
+      "protocol_sha256": "ca6b0397b6532489a12ad40a3a1d18408c0f52c45a20c15fae188d3a59eae462",
+      "scores": {
+        "em": 25.93984962406015,
+        "f1": 31.48872180451128
+      },
+      "split": "val_calibration",
+      "status_counts": {
+        "ok": 266
+      },
+      "target_sha256": "fe1557c98813401676c470515634238527d397080aee9f96f09c30a47e5fe907"
+    },
+    {
+      "complete": true,
+      "dataset": "mmqa_ti",
+      "evaluator_sha256": "1b95323ffdf36a0e8a8afef28c598e6fd560f6e63aa8d48709e51607e84d6100",
+      "groups": {
+        "modality:image": {
+          "em": 23.918575063613233,
+          "f1": 26.1323155216285,
+          "n": 393
+        },
+        "modality:image+text": {
+          "em": 20.338983050847457,
+          "f1": 21.889830508474578,
+          "n": 118
+        },
+        "modality:text": {
+          "em": 26.829268292682926,
+          "f1": 31.882926829268293,
+          "n": 820
+        },
+        "type:Compare(Compose(TableQ,ImageQ),Compose(TableQ,TextQ))": {
+          "em": 4.545454545454546,
+          "f1": 4.545454545454546,
+          "n": 22
+        },
+        "type:Compose(ImageQ,TextQ)": {
+          "em": 45.45454545454545,
+          "f1": 45.45454545454545,
+          "n": 33
+        },
+        "type:Compose(TextQ,ImageListQ)": {
+          "em": 11.475409836065573,
+          "f1": 14.475409836065573,
+          "n": 61
+        },
+        "type:ImageListQ": {
+          "em": 3.125,
+          "f1": 7.83125,
+          "n": 160
+        },
+        "type:ImageQ": {
+          "em": 38.197424892703864,
+          "f1": 38.69957081545064,
+          "n": 233
+        },
+        "type:Intersect(ImageListQ,TextQ)": {
+          "em": 50.0,
+          "f1": 50.0,
+          "n": 2
+        },
+        "type:TextQ": {
+          "em": 26.829268292682926,
+          "f1": 31.882926829268293,
+          "n": 820
+        }
+      },
+      "input_sha256": "bb92409d451ad352f22a819754ff924c2c92bbfec79df241aa382897095badad",
+      "metric_scale": "0-100",
+      "n_expected": 1331,
+      "n_missing": 0,
+      "n_predictions": 1331,
+      "parse_success_count": 1323,
+      "prediction_sha256": "002910799a30c04efdd241f3808e14fc16c6b0bd26b60cf26d1c44485f9e5da3",
+      "protocol_sha256": "ca6b0397b6532489a12ad40a3a1d18408c0f52c45a20c15fae188d3a59eae462",
+      "scores": {
+        "em": 25.39444027047333,
+        "f1": 29.29902329075883
+      },
+      "split": "test",
+      "status_counts": {
+        "ok": 1331
+      },
+      "target_sha256": "f9ac83d22210f0300be901ec863f8233e4438dcd99660a7ae628d7d9a290c5a8"
+    }
+  ],
+  "seed": 17,
+  "test_feedback_for_training": false,
+  "test_used": true,
+  "validation_run": "experiments/runs/gate-search-mira-mmqa_ti-transfer_p2_r1-s17"
+}