Archive verified research artifact: summary.json

artifacts/experiments/runs/test-mira-scienceqa-s17/summary.json

@@ -0,0 +1,417 @@
+{
+  "checkpoint": "experiments/runs/gate-search-mira-scienceqa-transfer_p2_r1-s17/epoch3.pt",
+  "dataset": "scienceqa",
+  "method": "mira",
+  "scores": [
+    {
+      "complete": true,
+      "dataset": "scienceqa",
+      "evaluator_sha256": "1b95323ffdf36a0e8a8afef28c598e6fd560f6e63aa8d48709e51607e84d6100",
+      "groups": {
+        "context:image": {
+          "accuracy": 83.05489260143199,
+          "n": 419
+        },
+        "context:none": {
+          "accuracy": 91.1660777385159,
+          "n": 283
+        },
+        "context:text": {
+          "accuracy": 88.78048780487805,
+          "n": 410
+        },
+        "context:text_image": {
+          "accuracy": 82.57575757575758,
+          "n": 264
+        },
+        "grade:grade1": {
+          "accuracy": 80.0,
+          "n": 5
+        },
+        "grade:grade10": {
+          "accuracy": 95.0,
+          "n": 20
+        },
+        "grade:grade11": {
+          "accuracy": 94.73684210526316,
+          "n": 19
+        },
+        "grade:grade12": {
+          "accuracy": 88.88888888888889,
+          "n": 18
+        },
+        "grade:grade2": {
+          "accuracy": 93.93939393939394,
+          "n": 66
+        },
+        "grade:grade3": {
+          "accuracy": 90.98360655737704,
+          "n": 122
+        },
+        "grade:grade4": {
+          "accuracy": 92.56756756756756,
+          "n": 148
+        },
+        "grade:grade5": {
+          "accuracy": 91.869918699187,
+          "n": 123
+        },
+        "grade:grade6": {
+          "accuracy": 76.0,
+          "n": 100
+        },
+        "grade:grade7": {
+          "accuracy": 85.18518518518519,
+          "n": 108
+        },
+        "grade:grade8": {
+          "accuracy": 86.13861386138613,
+          "n": 101
+        },
+        "grade:grade9": {
+          "accuracy": 94.44444444444444,
+          "n": 18
+        },
+        "grade_band:1-6": {
+          "accuracy": 89.18439716312056,
+          "n": 564
+        },
+        "grade_band:7-12": {
+          "accuracy": 87.67605633802818,
+          "n": 284
+        },
+        "subject:language science": {
+          "accuracy": 86.82926829268293,
+          "n": 205
+        },
+        "subject:natural science": {
+          "accuracy": 89.85200845665962,
+          "n": 473
+        },
+        "subject:social science": {
+          "accuracy": 87.6470588235294,
+          "n": 170
+        },
+        "topic:biology": {
+          "accuracy": 93.71428571428571,
+          "n": 175
+        },
+        "topic:capitalization": {
+          "accuracy": 100.0,
+          "n": 8
+        },
+        "topic:chemistry": {
+          "accuracy": 78.43137254901961,
+          "n": 51
+        },
+        "topic:civics": {
+          "accuracy": 100.0,
+          "n": 4
+        },
+        "topic:earth-science": {
+          "accuracy": 92.85714285714286,
+          "n": 42
+        },
+        "topic:economics": {
+          "accuracy": 96.15384615384616,
+          "n": 26
+        },
+        "topic:figurative-language": {
+          "accuracy": 96.55172413793103,
+          "n": 58
+        },
+        "topic:geography": {
+          "accuracy": 85.83333333333333,
+          "n": 120
+        },
+        "topic:grammar": {
+          "accuracy": 91.66666666666667,
+          "n": 12
+        },
+        "topic:phonological-awareness": {
+          "accuracy": 66.66666666666667,
+          "n": 6
+        },
+        "topic:physics": {
+          "accuracy": 83.58208955223881,
+          "n": 134
+        },
+        "topic:pronouns": {
+          "accuracy": 100.0,
+          "n": 1
+        },
+        "topic:punctuation": {
+          "accuracy": 100.0,
+          "n": 20
+        },
+        "topic:reading-comprehension": {
+          "accuracy": 100.0,
+          "n": 2
+        },
+        "topic:reference-skills": {
+          "accuracy": 56.666666666666664,
+          "n": 30
+        },
+        "topic:science-and-engineering-practices": {
+          "accuracy": 100.0,
+          "n": 39
+        },
+        "topic:units-and-measurement": {
+          "accuracy": 96.875,
+          "n": 32
+        },
+        "topic:us-history": {
+          "accuracy": 84.21052631578948,
+          "n": 19
+        },
+        "topic:verbs": {
+          "accuracy": 100.0,
+          "n": 4
+        },
+        "topic:vocabulary": {
+          "accuracy": 62.5,
+          "n": 8
+        },
+        "topic:word-study": {
+          "accuracy": 100.0,
+          "n": 2
+        },
+        "topic:world-history": {
+          "accuracy": 100.0,
+          "n": 1
+        },
+        "topic:writing-strategies": {
+          "accuracy": 88.88888888888889,
+          "n": 54
+        }
+      },
+      "input_sha256": "906ff9ed3e02c499fe1f0724b906d80e028c860d1e6827d24ebe6d15910ff554",
+      "metric_scale": "0-100",
+      "n_expected": 848,
+      "n_missing": 0,
+      "n_predictions": 848,
+      "parse_success_count": 848,
+      "prediction_sha256": "1392d7835087789f620816fc80149c697ca6115c266794fb5960dd1ed2b067e5",
+      "protocol_sha256": "ca6b0397b6532489a12ad40a3a1d18408c0f52c45a20c15fae188d3a59eae462",
+      "scores": {
+        "accuracy": 88.67924528301887
+      },
+      "split": "val_calibration",
+      "status_counts": {
+        "ok": 848
+      },
+      "target_sha256": "a0c00e8c336814ef546a3f098aba5de64ae78aedb24f66571ad1b9a930c0e9b5"
+    },
+    {
+      "complete": true,
+      "dataset": "scienceqa",
+      "evaluator_sha256": "1b95323ffdf36a0e8a8afef28c598e6fd560f6e63aa8d48709e51607e84d6100",
+      "groups": {
+        "context:image": {
+          "accuracy": 82.30044620723848,
+          "n": 2017
+        },
+        "context:none": {
+          "accuracy": 93.58885017421603,
+          "n": 1435
+        },
+        "context:text": {
+          "accuracy": 86.90127077223852,
+          "n": 2046
+        },
+        "context:text_image": {
+          "accuracy": 79.23627684964201,
+          "n": 1257
+        },
+        "grade:grade1": {
+          "accuracy": 88.23529411764706,
+          "n": 17
+        },
+        "grade:grade10": {
+          "accuracy": 95.7983193277311,
+          "n": 119
+        },
+        "grade:grade11": {
+          "accuracy": 94.28571428571429,
+          "n": 105
+        },
+        "grade:grade12": {
+          "accuracy": 94.18604651162791,
+          "n": 86
+        },
+        "grade:grade2": {
+          "accuracy": 88.39590443686006,
+          "n": 293
+        },
+        "grade:grade3": {
+          "accuracy": 90.43887147335423,
+          "n": 638
+        },
+        "grade:grade4": {
+          "accuracy": 93.01634472511144,
+          "n": 673
+        },
+        "grade:grade5": {
+          "accuracy": 89.28571428571429,
+          "n": 644
+        },
+        "grade:grade6": {
+          "accuracy": 85.40305010893246,
+          "n": 459
+        },
+        "grade:grade7": {
+          "accuracy": 87.27915194346289,
+          "n": 566
+        },
+        "grade:grade8": {
+          "accuracy": 83.42967244701349,
+          "n": 519
+        },
+        "grade:grade9": {
+          "accuracy": 98.36065573770492,
+          "n": 122
+        },
+        "grade_band:1-6": {
+          "accuracy": 89.72099853157121,
+          "n": 2724
+        },
+        "grade_band:7-12": {
+          "accuracy": 88.39815425181278,
+          "n": 1517
+        },
+        "subject:language science": {
+          "accuracy": 91.0909090909091,
+          "n": 1100
+        },
+        "subject:natural science": {
+          "accuracy": 88.09946714031972,
+          "n": 2252
+        },
+        "subject:social science": {
+          "accuracy": 89.87626546681665,
+          "n": 889
+        },
+        "topic:biology": {
+          "accuracy": 94.25,
+          "n": 800
+        },
+        "topic:capitalization": {
+          "accuracy": 100.0,
+          "n": 33
+        },
+        "topic:chemistry": {
+          "accuracy": 86.85446009389672,
+          "n": 213
+        },
+        "topic:civics": {
+          "accuracy": 100.0,
+          "n": 28
+        },
+        "topic:culture": {
+          "accuracy": 100.0,
+          "n": 2
+        },
+        "topic:earth-science": {
+          "accuracy": 93.21266968325791,
+          "n": 221
+        },
+        "topic:economics": {
+          "accuracy": 100.0,
+          "n": 155
+        },
+        "topic:figurative-language": {
+          "accuracy": 100.0,
+          "n": 267
+        },
+        "topic:geography": {
+          "accuracy": 86.09271523178808,
+          "n": 604
+        },
+        "topic:grammar": {
+          "accuracy": 91.42857142857143,
+          "n": 70
+        },
+        "topic:literacy-in-science": {
+          "accuracy": 100.0,
+          "n": 5
+        },
+        "topic:phonological-awareness": {
+          "accuracy": 88.23529411764706,
+          "n": 17
+        },
+        "topic:physics": {
+          "accuracy": 72.12317666126418,
+          "n": 617
+        },
+        "topic:pronouns": {
+          "accuracy": 100.0,
+          "n": 2
+        },
+        "topic:punctuation": {
+          "accuracy": 100.0,
+          "n": 100
+        },
+        "topic:reading-comprehension": {
+          "accuracy": 85.0,
+          "n": 20
+        },
+        "topic:reference-skills": {
+          "accuracy": 60.416666666666664,
+          "n": 144
+        },
+        "topic:science-and-engineering-practices": {
+          "accuracy": 100.0,
+          "n": 189
+        },
+        "topic:units-and-measurement": {
+          "accuracy": 96.61835748792271,
+          "n": 207
+        },
+        "topic:us-history": {
+          "accuracy": 94.44444444444444,
+          "n": 90
+        },
+        "topic:verbs": {
+          "accuracy": 96.15384615384616,
+          "n": 52
+        },
+        "topic:vocabulary": {
+          "accuracy": 83.87096774193549,
+          "n": 31
+        },
+        "topic:word-study": {
+          "accuracy": 88.88888888888889,
+          "n": 9
+        },
+        "topic:world-history": {
+          "accuracy": 90.0,
+          "n": 10
+        },
+        "topic:writing-strategies": {
+          "accuracy": 93.80281690140845,
+          "n": 355
+        }
+      },
+      "input_sha256": "ca4a53185df6168e7b4802446ccc14e6bad7e8254b70808b32f46fc8af9334d7",
+      "metric_scale": "0-100",
+      "n_expected": 4241,
+      "n_missing": 0,
+      "n_predictions": 4241,
+      "parse_success_count": 4241,
+      "prediction_sha256": "3d689bee6b38493814ad26d0aa05a607fbb832174648e1501b63cd499640fb86",
+      "protocol_sha256": "ca6b0397b6532489a12ad40a3a1d18408c0f52c45a20c15fae188d3a59eae462",
+      "scores": {
+        "accuracy": 89.24781891063428
+      },
+      "split": "test",
+      "status_counts": {
+        "ok": 4241
+      },
+      "target_sha256": "f8ef56c273ad1ecbd7822d4b670663dfd3950f0df42a0c4d50a7c9c73d189f83"
+    }
+  ],
+  "seed": 17,
+  "test_feedback_for_training": false,
+  "test_used": true,
+  "validation_run": "experiments/runs/gate-search-mira-scienceqa-transfer_p2_r1-s17"
+}