Upload extrapolation artifact: 020305 pretrain checkpoint-18809 score report

020305/evals/pretrain_ckpt18809/graph_results/pretrain_ckpt18809_paper_eval_r36_score_report.jsonl

@@ -0,0 +1 @@
+{"gold_dir": "/mnt/hdfs/output/work/pretrain_ckpt18809_paper_eval_r36/gold_graphs", "generated_path": "/mnt/hdfs/output/work/pretrain_ckpt18809_paper_eval_r36", "results_dir": "/mnt/hdfs/output/work/pretrain_ckpt18809_paper_eval_r36/graph_results", "per_file": [{"file": "/mnt/hdfs/output/work/pretrain_ckpt18809_paper_eval_r36/pretrain_ckpt18809_id_generations.jsonl", "total": 1152000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 310596, "value_mismatch": 792452, "dependency_mismatch": 500347, "answer_mismatch": 734934, "extra_nodes": 224467, "missing_nodes": 95580, "perfect_with_extra": 1356, "unused_gold_entries": 10000, "template_stats": {"crazy_zootopia": {"total": 379008, "value_mismatch": 265559, "dependency_mismatch": 172154, "answer_mismatch": 245194, "missing_nodes": 36385, "extra_nodes": 79426, "perfect": 96980, "perfect_with_extra": 165}, "teachers_in_school": {"total": 390656, "value_mismatch": 275405, "answer_mismatch": 256324, "dependency_mismatch": 175152, "perfect": 99103, "missing_nodes": 34915, "extra_nodes": 83390, "perfect_with_extra": 532}, "movie_festival_awards": {"total": 382336, "value_mismatch": 251488, "dependency_mismatch": 153041, "answer_mismatch": 233416, "extra_nodes": 61651, "missing_nodes": 24280, "perfect": 114513, "perfect_with_extra": 659}}, "results_path": "/mnt/hdfs/output/work/pretrain_ckpt18809_paper_eval_r36/graph_results/pretrain_ckpt18809_id_generations.jsonl.graph_results.jsonl"}, {"file": "/mnt/hdfs/output/work/pretrain_ckpt18809_paper_eval_r36/pretrain_ckpt18809_ood_generations.jsonl", "total": 1280000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 6782, "value_mismatch": 1266595, "dependency_mismatch": 1204520, "answer_mismatch": 1151441, "extra_nodes": 186680, "missing_nodes": 999107, "perfect_with_extra": 6, "unused_gold_entries": 9000, "template_stats": {"crazy_zootopia": {"total": 430720, "value_mismatch": 426873, "dependency_mismatch": 406779, "answer_mismatch": 389577, "extra_nodes": 67107, "missing_nodes": 336283, "perfect": 1953, "perfect_with_extra": 4}, "teachers_in_school": {"total": 430464, "value_mismatch": 426076, "dependency_mismatch": 405845, "answer_mismatch": 384768, "perfect": 2159, "extra_nodes": 69877, "missing_nodes": 337021}, "movie_festival_awards": {"total": 418816, "value_mismatch": 413646, "answer_mismatch": 377096, "dependency_mismatch": 391896, "extra_nodes": 49696, "missing_nodes": 325803, "perfect": 2670, "perfect_with_extra": 2}}, "results_path": "/mnt/hdfs/output/work/pretrain_ckpt18809_paper_eval_r36/graph_results/pretrain_ckpt18809_ood_generations.jsonl.graph_results.jsonl"}], "aggregate": {"files": 2, "total": 2432000, "missing_solution": 0, "missing_gold": 0, "parse_fail": 0, "perfect": 317378, "value_mismatch": 2059047, "dependency_mismatch": 1704867, "answer_mismatch": 1886375, "extra_nodes": 411147, "missing_nodes": 1094687, "perfect_with_extra": 1362, "skipped": 0, "unused_gold_entries": 0, "aggregated": true, "report_path": "/mnt/hdfs/output/work/pretrain_ckpt18809_paper_eval_r36/graph_results/pretrain_ckpt18809_paper_eval_r36_score_report.jsonl", "template_stats": {"crazy_zootopia": {"total": 809728, "value_mismatch": 692432, "dependency_mismatch": 578933, "answer_mismatch": 634771, "missing_nodes": 372668, "extra_nodes": 146533, "perfect": 98933, "perfect_with_extra": 169}, "teachers_in_school": {"total": 821120, "value_mismatch": 701481, "answer_mismatch": 641092, "dependency_mismatch": 580997, "perfect": 101262, "missing_nodes": 371936, "extra_nodes": 153267, "perfect_with_extra": 532}, "movie_festival_awards": {"total": 801152, "value_mismatch": 665134, "dependency_mismatch": 544937, "answer_mismatch": 610512, "extra_nodes": 111347, "missing_nodes": 350083, "perfect": 117183, "perfect_with_extra": 661}}}}