Archive verified research artifact: summary.json

artifacts/experiments/runs/test-qwen3-scienceqa-s17/summary.json

@@ -0,0 +1,417 @@
+{
+  "checkpoint": null,
+  "dataset": "scienceqa",
+  "method": "qwen3",
+  "scores": [
+    {
+      "complete": true,
+      "dataset": "scienceqa",
+      "evaluator_sha256": "1b95323ffdf36a0e8a8afef28c598e6fd560f6e63aa8d48709e51607e84d6100",
+      "groups": {
+        "context:image": {
+          "accuracy": 69.21241050119332,
+          "n": 419
+        },
+        "context:none": {
+          "accuracy": 75.61837455830388,
+          "n": 283
+        },
+        "context:text": {
+          "accuracy": 80.48780487804878,
+          "n": 410
+        },
+        "context:text_image": {
+          "accuracy": 74.24242424242425,
+          "n": 264
+        },
+        "grade:grade1": {
+          "accuracy": 80.0,
+          "n": 5
+        },
+        "grade:grade10": {
+          "accuracy": 75.0,
+          "n": 20
+        },
+        "grade:grade11": {
+          "accuracy": 78.94736842105263,
+          "n": 19
+        },
+        "grade:grade12": {
+          "accuracy": 83.33333333333333,
+          "n": 18
+        },
+        "grade:grade2": {
+          "accuracy": 87.87878787878788,
+          "n": 66
+        },
+        "grade:grade3": {
+          "accuracy": 80.32786885245902,
+          "n": 122
+        },
+        "grade:grade4": {
+          "accuracy": 78.37837837837837,
+          "n": 148
+        },
+        "grade:grade5": {
+          "accuracy": 78.04878048780488,
+          "n": 123
+        },
+        "grade:grade6": {
+          "accuracy": 69.0,
+          "n": 100
+        },
+        "grade:grade7": {
+          "accuracy": 72.22222222222223,
+          "n": 108
+        },
+        "grade:grade8": {
+          "accuracy": 59.40594059405941,
+          "n": 101
+        },
+        "grade:grade9": {
+          "accuracy": 77.77777777777777,
+          "n": 18
+        },
+        "grade_band:1-6": {
+          "accuracy": 78.19148936170212,
+          "n": 564
+        },
+        "grade_band:7-12": {
+          "accuracy": 69.36619718309859,
+          "n": 284
+        },
+        "subject:language science": {
+          "accuracy": 71.21951219512195,
+          "n": 205
+        },
+        "subject:natural science": {
+          "accuracy": 80.76109936575052,
+          "n": 473
+        },
+        "subject:social science": {
+          "accuracy": 64.70588235294117,
+          "n": 170
+        },
+        "topic:biology": {
+          "accuracy": 85.14285714285714,
+          "n": 175
+        },
+        "topic:capitalization": {
+          "accuracy": 100.0,
+          "n": 8
+        },
+        "topic:chemistry": {
+          "accuracy": 76.47058823529412,
+          "n": 51
+        },
+        "topic:civics": {
+          "accuracy": 100.0,
+          "n": 4
+        },
+        "topic:earth-science": {
+          "accuracy": 88.0952380952381,
+          "n": 42
+        },
+        "topic:economics": {
+          "accuracy": 88.46153846153847,
+          "n": 26
+        },
+        "topic:figurative-language": {
+          "accuracy": 77.58620689655173,
+          "n": 58
+        },
+        "topic:geography": {
+          "accuracy": 60.833333333333336,
+          "n": 120
+        },
+        "topic:grammar": {
+          "accuracy": 75.0,
+          "n": 12
+        },
+        "topic:phonological-awareness": {
+          "accuracy": 50.0,
+          "n": 6
+        },
+        "topic:physics": {
+          "accuracy": 67.91044776119404,
+          "n": 134
+        },
+        "topic:pronouns": {
+          "accuracy": 100.0,
+          "n": 1
+        },
+        "topic:punctuation": {
+          "accuracy": 70.0,
+          "n": 20
+        },
+        "topic:reading-comprehension": {
+          "accuracy": 50.0,
+          "n": 2
+        },
+        "topic:reference-skills": {
+          "accuracy": 40.0,
+          "n": 30
+        },
+        "topic:science-and-engineering-practices": {
+          "accuracy": 94.87179487179488,
+          "n": 39
+        },
+        "topic:units-and-measurement": {
+          "accuracy": 90.625,
+          "n": 32
+        },
+        "topic:us-history": {
+          "accuracy": 52.63157894736842,
+          "n": 19
+        },
+        "topic:verbs": {
+          "accuracy": 100.0,
+          "n": 4
+        },
+        "topic:vocabulary": {
+          "accuracy": 62.5,
+          "n": 8
+        },
+        "topic:word-study": {
+          "accuracy": 50.0,
+          "n": 2
+        },
+        "topic:world-history": {
+          "accuracy": 0.0,
+          "n": 1
+        },
+        "topic:writing-strategies": {
+          "accuracy": 79.62962962962963,
+          "n": 54
+        }
+      },
+      "input_sha256": "906ff9ed3e02c499fe1f0724b906d80e028c860d1e6827d24ebe6d15910ff554",
+      "metric_scale": "0-100",
+      "n_expected": 848,
+      "n_missing": 0,
+      "n_predictions": 848,
+      "parse_success_count": 848,
+      "prediction_sha256": "846bff60170e1010c252a797f7695d1c4cfd0cdec477419c544abff4225f1a66",
+      "protocol_sha256": "ca6b0397b6532489a12ad40a3a1d18408c0f52c45a20c15fae188d3a59eae462",
+      "scores": {
+        "accuracy": 75.23584905660377
+      },
+      "split": "val_calibration",
+      "status_counts": {
+        "ok": 848
+      },
+      "target_sha256": "a0c00e8c336814ef546a3f098aba5de64ae78aedb24f66571ad1b9a930c0e9b5"
+    },
+    {
+      "complete": true,
+      "dataset": "scienceqa",
+      "evaluator_sha256": "1b95323ffdf36a0e8a8afef28c598e6fd560f6e63aa8d48709e51607e84d6100",
+      "groups": {
+        "context:image": {
+          "accuracy": 70.25285076846802,
+          "n": 2017
+        },
+        "context:none": {
+          "accuracy": 73.86759581881533,
+          "n": 1435
+        },
+        "context:text": {
+          "accuracy": 79.66764418377322,
+          "n": 2046
+        },
+        "context:text_image": {
+          "accuracy": 73.1901352426412,
+          "n": 1257
+        },
+        "grade:grade1": {
+          "accuracy": 76.47058823529412,
+          "n": 17
+        },
+        "grade:grade10": {
+          "accuracy": 73.94957983193277,
+          "n": 119
+        },
+        "grade:grade11": {
+          "accuracy": 73.33333333333333,
+          "n": 105
+        },
+        "grade:grade12": {
+          "accuracy": 72.09302325581395,
+          "n": 86
+        },
+        "grade:grade2": {
+          "accuracy": 79.86348122866895,
+          "n": 293
+        },
+        "grade:grade3": {
+          "accuracy": 79.62382445141066,
+          "n": 638
+        },
+        "grade:grade4": {
+          "accuracy": 81.72362555720653,
+          "n": 673
+        },
+        "grade:grade5": {
+          "accuracy": 76.3975155279503,
+          "n": 644
+        },
+        "grade:grade6": {
+          "accuracy": 72.98474945533769,
+          "n": 459
+        },
+        "grade:grade7": {
+          "accuracy": 70.6713780918728,
+          "n": 566
+        },
+        "grade:grade8": {
+          "accuracy": 65.89595375722543,
+          "n": 519
+        },
+        "grade:grade9": {
+          "accuracy": 70.49180327868852,
+          "n": 122
+        },
+        "grade_band:1-6": {
+          "accuracy": 78.26725403817915,
+          "n": 2724
+        },
+        "grade_band:7-12": {
+          "accuracy": 69.54515491100857,
+          "n": 1517
+        },
+        "subject:language science": {
+          "accuracy": 72.18181818181819,
+          "n": 1100
+        },
+        "subject:natural science": {
+          "accuracy": 78.73001776198934,
+          "n": 2252
+        },
+        "subject:social science": {
+          "accuracy": 69.74128233970754,
+          "n": 889
+        },
+        "topic:biology": {
+          "accuracy": 84.125,
+          "n": 800
+        },
+        "topic:capitalization": {
+          "accuracy": 100.0,
+          "n": 33
+        },
+        "topic:chemistry": {
+          "accuracy": 72.30046948356808,
+          "n": 213
+        },
+        "topic:civics": {
+          "accuracy": 92.85714285714286,
+          "n": 28
+        },
+        "topic:culture": {
+          "accuracy": 50.0,
+          "n": 2
+        },
+        "topic:earth-science": {
+          "accuracy": 77.37556561085972,
+          "n": 221
+        },
+        "topic:economics": {
+          "accuracy": 93.54838709677419,
+          "n": 155
+        },
+        "topic:figurative-language": {
+          "accuracy": 73.03370786516854,
+          "n": 267
+        },
+        "topic:geography": {
+          "accuracy": 65.56291390728477,
+          "n": 604
+        },
+        "topic:grammar": {
+          "accuracy": 61.42857142857143,
+          "n": 70
+        },
+        "topic:literacy-in-science": {
+          "accuracy": 80.0,
+          "n": 5
+        },
+        "topic:phonological-awareness": {
+          "accuracy": 52.94117647058823,
+          "n": 17
+        },
+        "topic:physics": {
+          "accuracy": 65.31604538087521,
+          "n": 617
+        },
+        "topic:pronouns": {
+          "accuracy": 100.0,
+          "n": 2
+        },
+        "topic:punctuation": {
+          "accuracy": 75.0,
+          "n": 100
+        },
+        "topic:reading-comprehension": {
+          "accuracy": 95.0,
+          "n": 20
+        },
+        "topic:reference-skills": {
+          "accuracy": 47.916666666666664,
+          "n": 144
+        },
+        "topic:science-and-engineering-practices": {
+          "accuracy": 98.41269841269842,
+          "n": 189
+        },
+        "topic:units-and-measurement": {
+          "accuracy": 87.92270531400966,
+          "n": 207
+        },
+        "topic:us-history": {
+          "accuracy": 48.888888888888886,
+          "n": 90
+        },
+        "topic:verbs": {
+          "accuracy": 92.3076923076923,
+          "n": 52
+        },
+        "topic:vocabulary": {
+          "accuracy": 51.61290322580645,
+          "n": 31
+        },
+        "topic:word-study": {
+          "accuracy": 66.66666666666667,
+          "n": 9
+        },
+        "topic:world-history": {
+          "accuracy": 80.0,
+          "n": 10
+        },
+        "topic:writing-strategies": {
+          "accuracy": 78.59154929577464,
+          "n": 355
+        }
+      },
+      "input_sha256": "ca4a53185df6168e7b4802446ccc14e6bad7e8254b70808b32f46fc8af9334d7",
+      "metric_scale": "0-100",
+      "n_expected": 4241,
+      "n_missing": 0,
+      "n_predictions": 4241,
+      "parse_success_count": 4241,
+      "prediction_sha256": "ad7ebe7650a2549a878d240ad64ede7b97d77902121bf39eedd5ac3df0045e68",
+      "protocol_sha256": "ca6b0397b6532489a12ad40a3a1d18408c0f52c45a20c15fae188d3a59eae462",
+      "scores": {
+        "accuracy": 75.1473709030889
+      },
+      "split": "test",
+      "status_counts": {
+        "ok": 4241
+      },
+      "target_sha256": "f8ef56c273ad1ecbd7822d4b670663dfd3950f0df42a0c4d50a7c9c73d189f83"
+    }
+  ],
+  "seed": 17,
+  "test_feedback_for_training": false,
+  "test_used": true,
+  "validation_run": "experiments/runs/qwen3-scienceqa-s17"
+}