Upload extrapolation artifact: 020305 RL OP11-14 step200 score report

020305/evals/rl_ckpt18809_op11_14_step200_merged_hf/graph_results/rl_ckpt18809_op11_14_step200_mergedhf_paper_eval_r69_score_report.jsonl

@@ -0,0 +1 @@
+{"gold_dir": "/mnt/hdfs/output/work/rl_ckpt18809_op11_14_step200_mergedhf_paper_eval_r69/gold_graphs", "generated_path": "/mnt/hdfs/output/work/rl_ckpt18809_op11_14_step200_mergedhf_paper_eval_r69", "results_dir": "/mnt/hdfs/output/work/rl_ckpt18809_op11_14_step200_mergedhf_paper_eval_r69/graph_results", "per_file": [{"file": "/mnt/hdfs/output/work/rl_ckpt18809_op11_14_step200_mergedhf_paper_eval_r69/rl_ckpt18809_op11_14_step200_merged_hf_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 790340, "value_mismatch": 313149, "dependency_mismatch": 183128, "answer_mismatch": 229583, "extra_nodes": 72100, "missing_nodes": 32682, "perfect_with_extra": 13550, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "perfect": 259730, "dependency_mismatch": 63122, "value_mismatch": 102696, "answer_mismatch": 74660, "extra_nodes": 26516, "missing_nodes": 12177, "perfect_with_extra": 4475}, "teachers_in_school": {"total": 390656, "perfect": 267416, "dependency_mismatch": 60220, "value_mismatch": 106701, "answer_mismatch": 77633, "extra_nodes": 22587, "missing_nodes": 9893, "perfect_with_extra": 4165}, "movie_festival_awards": {"total": 382336, "perfect": 263194, "dependency_mismatch": 59786, "value_mismatch": 103752, "answer_mismatch": 77290, "extra_nodes": 22997, "missing_nodes": 10612, "perfect_with_extra": 4910}}, "results_path": "/mnt/hdfs/output/work/rl_ckpt18809_op11_14_step200_mergedhf_paper_eval_r69/graph_results/rl_ckpt18809_op11_14_step200_merged_hf_id_generations.jsonl.graph_results.jsonl"}, {"file": "/mnt/hdfs/output/work/rl_ckpt18809_op11_14_step200_mergedhf_paper_eval_r69/rl_ckpt18809_op11_14_step200_merged_hf_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 223439, "value_mismatch": 1001097, "dependency_mismatch": 971604, "answer_mismatch": 806909, "extra_nodes": 62883, "missing_nodes": 623379, "perfect_with_extra": 111, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "value_mismatch": 339494, "dependency_mismatch": 332502, "answer_mismatch": 273067, "extra_nodes": 23301, "perfect": 72424, "missing_nodes": 214704, "perfect_with_extra": 35}, "teachers_in_school": {"total": 430464, "perfect": 76276, "dependency_mismatch": 323586, "missing_nodes": 208066, "value_mismatch": 336386, "answer_mismatch": 271249, "extra_nodes": 20164, "perfect_with_extra": 64}, "movie_festival_awards": {"total": 418816, "perfect": 74739, "value_mismatch": 325217, "answer_mismatch": 262593, "dependency_mismatch": 315516, "extra_nodes": 19418, "perfect_with_extra": 12, "missing_nodes": 200609}}, "results_path": "/mnt/hdfs/output/work/rl_ckpt18809_op11_14_step200_mergedhf_paper_eval_r69/graph_results/rl_ckpt18809_op11_14_step200_merged_hf_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 1013779, "value_mismatch": 1314246, "dependency_mismatch": 1154732, "answer_mismatch": 1036492, "extra_nodes": 134983, "missing_nodes": 656061, "perfect_with_extra": 13661, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "/mnt/hdfs/output/work/rl_ckpt18809_op11_14_step200_mergedhf_paper_eval_r69/graph_results/rl_ckpt18809_op11_14_step200_mergedhf_paper_eval_r69_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "perfect": 332154, "dependency_mismatch": 395624, "value_mismatch": 442190, "answer_mismatch": 347727, "extra_nodes": 49817, "missing_nodes": 226881, "perfect_with_extra": 4510}, "teachers_in_school": {"total": 821120, "perfect": 343692, "dependency_mismatch": 383806, "value_mismatch": 443087, "answer_mismatch": 348882, "extra_nodes": 42751, "missing_nodes": 217959, "perfect_with_extra": 4229}, "movie_festival_awards": {"total": 801152, "perfect": 337933, "dependency_mismatch": 375302, "value_mismatch": 428969, "answer_mismatch": 339883, "extra_nodes": 42415, "missing_nodes": 211221, "perfect_with_extra": 4922}}}}