Upload extrapolation artifact: 050302 pretrain checkpoint-18586 score report

050302/evals/pretrain_050302_ckpt18586/graph_results/pretrain_050302_ckpt18586_paper_eval_r42_score_report.jsonl

@@ -0,0 +1 @@
+{"gold_dir": "/mnt/hdfs/output/work/pretrain_050302_ckpt18586_paper_eval_r42/gold_graphs", "generated_path": "/mnt/hdfs/output/work/pretrain_050302_ckpt18586_paper_eval_r42", "results_dir": "/mnt/hdfs/output/work/pretrain_050302_ckpt18586_paper_eval_r42/graph_results", "per_file": [{"file": "/mnt/hdfs/output/work/pretrain_050302_ckpt18586_paper_eval_r42/pretrain_050302_ckpt18586_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 671429, "value_mismatch": 456460, "dependency_mismatch": 243628, "answer_mismatch": 392675, "extra_nodes": 70111, "missing_nodes": 68026, "perfect_with_extra": 5077, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "value_mismatch": 147862, "dependency_mismatch": 78297, "answer_mismatch": 127048, "perfect": 222964, "extra_nodes": 23228, "missing_nodes": 23291, "perfect_with_extra": 1816}, "teachers_in_school": {"total": 390656, "perfect": 225779, "value_mismatch": 156890, "answer_mismatch": 135617, "dependency_mismatch": 81865, "extra_nodes": 22063, "missing_nodes": 20246, "perfect_with_extra": 1032}, "movie_festival_awards": {"total": 382336, "perfect": 222686, "value_mismatch": 151708, "answer_mismatch": 130010, "dependency_mismatch": 83466, "missing_nodes": 24489, "extra_nodes": 24820, "perfect_with_extra": 2229}}, "results_path": "/mnt/hdfs/output/work/pretrain_050302_ckpt18586_paper_eval_r42/graph_results/pretrain_050302_ckpt18586_id_generations.jsonl.graph_results.jsonl"}, {"file": "/mnt/hdfs/output/work/pretrain_050302_ckpt18586_paper_eval_r42/pretrain_050302_ckpt18586_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 81595, "value_mismatch": 1182036, "dependency_mismatch": 1156042, "answer_mismatch": 1056881, "extra_nodes": 114455, "missing_nodes": 1013622, "perfect_with_extra": 25, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "value_mismatch": 398325, "dependency_mismatch": 389681, "answer_mismatch": 356007, "missing_nodes": 342710, "extra_nodes": 38732, "perfect": 27715, "perfect_with_extra": 6}, "teachers_in_school": {"total": 430464, "perfect": 27891, "dependency_mismatch": 387382, "value_mismatch": 397913, "answer_mismatch": 354635, "extra_nodes": 38204, "missing_nodes": 338866, "perfect_with_extra": 9}, "movie_festival_awards": {"total": 418816, "perfect": 25989, "value_mismatch": 385798, "answer_mismatch": 346239, "dependency_mismatch": 378979, "missing_nodes": 332046, "extra_nodes": 37519, "perfect_with_extra": 10}}, "results_path": "/mnt/hdfs/output/work/pretrain_050302_ckpt18586_paper_eval_r42/graph_results/pretrain_050302_ckpt18586_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 753024, "value_mismatch": 1638496, "dependency_mismatch": 1399670, "answer_mismatch": 1449556, "extra_nodes": 184566, "missing_nodes": 1081648, "perfect_with_extra": 5102, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "/mnt/hdfs/output/work/pretrain_050302_ckpt18586_paper_eval_r42/graph_results/pretrain_050302_ckpt18586_paper_eval_r42_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "value_mismatch": 546187, "dependency_mismatch": 467978, "answer_mismatch": 483055, "perfect": 250679, "extra_nodes": 61960, "missing_nodes": 366001, "perfect_with_extra": 1822}, "teachers_in_school": {"total": 821120, "perfect": 253670, "value_mismatch": 554803, "answer_mismatch": 490252, "dependency_mismatch": 469247, "extra_nodes": 60267, "missing_nodes": 359112, "perfect_with_extra": 1041}, "movie_festival_awards": {"total": 801152, "perfect": 248675, "value_mismatch": 537506, "answer_mismatch": 476249, "dependency_mismatch": 462445, "missing_nodes": 356535, "extra_nodes": 62339, "perfect_with_extra": 2239}}}}