Archive verified research artifact: summary.json

artifacts/experiments/runs/reranker-mmqa_ti-s17/summary.json

@@ -0,0 +1,75 @@
+{
+  "decoder_trainable": false,
+  "scores": [
+    {
+      "complete": true,
+      "dataset": "mmqa_ti",
+      "groups": {
+        "modality:image": {
+          "em": 27.70700636942675,
+          "f1": 32.66242038216561,
+          "n": 314
+        },
+        "modality:image+text": {
+          "em": 26.31578947368421,
+          "f1": 35.67368421052632,
+          "n": 95
+        },
+        "modality:text": {
+          "em": 67.6829268292683,
+          "f1": 76.70121951219512,
+          "n": 656
+        },
+        "type:Compare(Compose(TableQ,ImageQ),Compose(TableQ,TextQ))": {
+          "em": 17.647058823529413,
+          "f1": 17.647058823529413,
+          "n": 17
+        },
+        "type:Compose(ImageQ,TextQ)": {
+          "em": 24.0,
+          "f1": 24.0,
+          "n": 25
+        },
+        "type:Compose(TextQ,ImageListQ)": {
+          "em": 28.0,
+          "f1": 45.78,
+          "n": 50
+        },
+        "type:ImageListQ": {
+          "em": 3.90625,
+          "f1": 12.796875,
+          "n": 128
+        },
+        "type:ImageQ": {
+          "em": 44.086021505376344,
+          "f1": 46.333333333333336,
+          "n": 186
+        },
+        "type:Intersect(ImageListQ,TextQ)": {
+          "em": 66.66666666666667,
+          "f1": 66.66666666666667,
+          "n": 3
+        },
+        "type:TextQ": {
+          "em": 67.6829268292683,
+          "f1": 76.70121951219512,
+          "n": 656
+        }
+      },
+      "metric_scale": "0-100",
+      "n_expected": 1065,
+      "n_missing": 0,
+      "n_predictions": 1065,
+      "parse_success_count": 1064,
+      "scores": {
+        "em": 52.20657276995305,
+        "f1": 60.057276995305166
+      },
+      "status_counts": {
+        "ok": 1065
+      }
+    }
+  ],
+  "seed_is_not_an_independent_train": true,
+  "trainable_method": false
+}