Upload extrapolation artifact: 020305 RL OP15-20 step200 score report

020305/evals/rl_ckpt18809_op15_20_step200_merged_hf/graph_results/rl_ckpt18809_op15_20_step200_mergedhf_paper_eval_r71_score_report.jsonl

@@ -0,0 +1 @@
+{"gold_dir": "/mnt/hdfs/output/work/rl_ckpt18809_op15_20_step200_mergedhf_paper_eval_r71/gold_graphs", "generated_path": "/mnt/hdfs/output/work/rl_ckpt18809_op15_20_step200_mergedhf_paper_eval_r71", "results_dir": "/mnt/hdfs/output/work/rl_ckpt18809_op15_20_step200_mergedhf_paper_eval_r71/graph_results", "per_file": [{"file": "/mnt/hdfs/output/work/rl_ckpt18809_op15_20_step200_mergedhf_paper_eval_r71/rl_ckpt18809_op15_20_step200_merged_hf_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 259710, "value_mismatch": 833615, "dependency_mismatch": 628307, "answer_mismatch": 703943, "extra_nodes": 380777, "missing_nodes": 172057, "perfect_with_extra": 1681, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "value_mismatch": 284148, "dependency_mismatch": 217109, "answer_mismatch": 237556, "extra_nodes": 134301, "missing_nodes": 65133, "perfect": 75972, "perfect_with_extra": 240}, "teachers_in_school": {"total": 390656, "value_mismatch": 288738, "answer_mismatch": 246731, "dependency_mismatch": 219499, "extra_nodes": 139447, "missing_nodes": 62591, "perfect": 81838, "perfect_with_extra": 389}, "movie_festival_awards": {"total": 382336, "value_mismatch": 260729, "dependency_mismatch": 191699, "answer_mismatch": 219656, "extra_nodes": 107029, "missing_nodes": 44333, "perfect": 101900, "perfect_with_extra": 1052}}, "results_path": "/mnt/hdfs/output/work/rl_ckpt18809_op15_20_step200_mergedhf_paper_eval_r71/graph_results/rl_ckpt18809_op15_20_step200_merged_hf_id_generations.jsonl.graph_results.jsonl"}, {"file": "/mnt/hdfs/output/work/rl_ckpt18809_op15_20_step200_mergedhf_paper_eval_r71/rl_ckpt18809_op15_20_step200_merged_hf_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 5225, "value_mismatch": 1270005, "dependency_mismatch": 1229380, "answer_mismatch": 1074260, "extra_nodes": 375783, "missing_nodes": 1036557, "perfect_with_extra": 27, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "value_mismatch": 428035, "dependency_mismatch": 414700, "answer_mismatch": 360409, "extra_nodes": 134079, "missing_nodes": 351881, "perfect": 1296, "perfect_with_extra": 11}, "teachers_in_school": {"total": 430464, "perfect": 1258, "dependency_mismatch": 415438, "value_mismatch": 427878, "answer_mismatch": 362880, "extra_nodes": 139570, "missing_nodes": 353125, "perfect_with_extra": 6}, "movie_festival_awards": {"total": 418816, "value_mismatch": 414092, "dependency_mismatch": 399242, "answer_mismatch": 350971, "extra_nodes": 102134, "missing_nodes": 331551, "perfect": 2671, "perfect_with_extra": 10}}, "results_path": "/mnt/hdfs/output/work/rl_ckpt18809_op15_20_step200_mergedhf_paper_eval_r71/graph_results/rl_ckpt18809_op15_20_step200_merged_hf_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 264935, "value_mismatch": 2103620, "dependency_mismatch": 1857687, "answer_mismatch": 1778203, "extra_nodes": 756560, "missing_nodes": 1208614, "perfect_with_extra": 1708, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "/mnt/hdfs/output/work/rl_ckpt18809_op15_20_step200_mergedhf_paper_eval_r71/graph_results/rl_ckpt18809_op15_20_step200_mergedhf_paper_eval_r71_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "value_mismatch": 712183, "dependency_mismatch": 631809, "answer_mismatch": 597965, "extra_nodes": 268380, "missing_nodes": 417014, "perfect": 77268, "perfect_with_extra": 251}, "teachers_in_school": {"total": 821120, "value_mismatch": 716616, "answer_mismatch": 609611, "dependency_mismatch": 634937, "extra_nodes": 279017, "missing_nodes": 415716, "perfect": 83096, "perfect_with_extra": 395}, "movie_festival_awards": {"total": 801152, "value_mismatch": 674821, "dependency_mismatch": 590941, "answer_mismatch": 570627, "extra_nodes": 209163, "missing_nodes": 375884, "perfect": 104571, "perfect_with_extra": 1062}}}}